switchroom 0.18.17 → 0.18.18

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. package/dist/agent-scheduler/index.js +13 -0
  2. package/dist/auth-broker/index.js +13 -0
  3. package/dist/cli/notion-write-pretool.mjs +13 -0
  4. package/dist/cli/switchroom.js +605 -479
  5. package/dist/host-control/main.js +17 -1
  6. package/dist/vault/approvals/kernel-server.js +13 -0
  7. package/dist/vault/broker/server.js +13 -0
  8. package/package.json +1 -1
  9. package/telegram-plugin/bridge/bridge.ts +7 -1
  10. package/telegram-plugin/dist/bridge/bridge.js +26 -1
  11. package/telegram-plugin/dist/gateway/gateway.js +1401 -431
  12. package/telegram-plugin/dist/server.js +26 -1
  13. package/telegram-plugin/fleet-fallback-resume.ts +26 -3
  14. package/telegram-plugin/gateway/approval-hold.ts +49 -0
  15. package/telegram-plugin/gateway/bridge-dead-watchdog.ts +61 -18
  16. package/telegram-plugin/gateway/gateway.ts +362 -71
  17. package/telegram-plugin/gateway/linear-activity.ts +20 -4
  18. package/telegram-plugin/gateway/premium-recovery-wiring.ts +122 -0
  19. package/telegram-plugin/gateway/session-model-file.ts +103 -0
  20. package/telegram-plugin/gateway/tier-downgrade-wiring.ts +121 -0
  21. package/telegram-plugin/gateway/unhandled-rejection-policy.ts +14 -1
  22. package/telegram-plugin/llm-error-present.ts +436 -0
  23. package/telegram-plugin/operator-events.ts +7 -1
  24. package/telegram-plugin/permission-title.ts +172 -10
  25. package/telegram-plugin/premium-recovery.ts +101 -0
  26. package/telegram-plugin/raw-error-scrub.ts +73 -0
  27. package/telegram-plugin/retry-api-call.ts +8 -2
  28. package/telegram-plugin/send-gate-degraded.test.ts +152 -1
  29. package/telegram-plugin/send-gate-observability.test.ts +140 -0
  30. package/telegram-plugin/send-gate-observability.ts +65 -20
  31. package/telegram-plugin/send-gate.test.ts +143 -1
  32. package/telegram-plugin/send-gate.ts +212 -19
  33. package/telegram-plugin/session-tail.ts +16 -0
  34. package/telegram-plugin/shared/local-time.ts +69 -0
  35. package/telegram-plugin/tests/approval-hold-harness.ts +6 -6
  36. package/telegram-plugin/tests/approval-hold-outcome.test.ts +10 -2
  37. package/telegram-plugin/tests/bridge-dead-watchdog.test.ts +61 -0
  38. package/telegram-plugin/tests/fleet-fallback-resume.test.ts +39 -0
  39. package/telegram-plugin/tests/flood-windows-persistence.test.ts +3 -2
  40. package/telegram-plugin/tests/linear-create-issue.test.ts +30 -2
  41. package/telegram-plugin/tests/llm-error-present.test.ts +380 -0
  42. package/telegram-plugin/tests/permission-title.test.ts +167 -4
  43. package/telegram-plugin/tests/premium-recovery-wiring.test.ts +150 -0
  44. package/telegram-plugin/tests/premium-recovery.test.ts +165 -0
  45. package/telegram-plugin/tests/reaction-gate-routing.test.ts +6 -1
  46. package/telegram-plugin/tests/retry-api-call.test.ts +21 -0
  47. package/telegram-plugin/tests/tier-downgrade-wiring.test.ts +165 -0
  48. package/telegram-plugin/tests/tier-downgrade.test.ts +141 -0
  49. package/telegram-plugin/tests/unhandled-rejection-policy.test.ts +27 -1
  50. package/telegram-plugin/tests/worker-activity-feed.test.ts +5 -2
  51. package/telegram-plugin/tests/worker-feed-coalesce.test.ts +492 -0
  52. package/telegram-plugin/tier-downgrade.ts +198 -0
  53. package/telegram-plugin/tool-activity-summary.ts +99 -0
  54. package/telegram-plugin/worker-activity-feed.ts +509 -409
@@ -0,0 +1,101 @@
1
+ /**
2
+ * premium-recovery.ts — "your premium model is servable again" ping (pure layer).
3
+ *
4
+ * Companion to tier-downgrade.ts. When a premium `/model` selection (e.g.
5
+ * `/model fable`) is walled fleet-wide, the tier-downgrade tier resumes the
6
+ * turn on the configured default and tells the user to re-issue `/model fable`
7
+ * once it frees up (there is NO automatic revert — the override is session-
8
+ * scoped and dies on the downgrade restart). This module closes that loop with
9
+ * a HEADS-UP: the moment the premium tier can be served again, ping the chat
10
+ * ONCE with a one-tap "switch back" button.
11
+ *
12
+ * DETERMINISTIC recovery signal (P0 deterministic-controls — never model-
13
+ * decided). The signal is the auth-broker's own per-account eligibility, read
14
+ * off `list-state` (the SAME state the gateway's `runQuotaWatch` tick already
15
+ * polls every 15 min — no new poller, no broker change):
16
+ * - `account.exhausted` — live-authoritative `isAccountExhausted` verdict
17
+ * (5h/7d wall or an unexpired `exhausted_until` mark).
18
+ * - `account.premium_walled` — live-authoritative `isAccountPremiumWalled`
19
+ * verdict; `isModelTierWalled` is a pure timestamp compare
20
+ * (`premium_walled_until > now`) unless a fresher canary read the flagship
21
+ * tier `allowed`. (src/auth/broker/model-tier-quota.ts / account-eligibility.ts.)
22
+ * The tier-downgrade fires from the account-swap `all-blocked` verdict — i.e.
23
+ * EVERY account was exhausted or premium-walled. Recovery is the exact
24
+ * complement: at least ONE account is neither. No clock of our own, no model
25
+ * judgement — the broker already decided; we read its decision.
26
+ *
27
+ * AT-MOST-ONCE. The gateway holds a consume-once `.premium-recovery` marker
28
+ * (session-model-file.ts) recording the dropped premium token + the chats to
29
+ * notify. `decidePremiumRecovery` is a pure predicate; the gateway clears the
30
+ * marker BEFORE sending (never-storm) and additionally takes a fleet-wide
31
+ * `claim-notification` so a bounce or a second gateway tick can't double-fire.
32
+ * The marker is also cleared the instant the user manually re-issues
33
+ * `/model <premium>` (no stale ping).
34
+ *
35
+ * PURE — no I/O, no clock. The gateway does the list-state read, the marker
36
+ * FS, the claim, and the send off these verdicts.
37
+ */
38
+
39
+ /** The minimal per-account eligibility view the recovery predicate needs —
40
+ * both fields are the broker's live-authoritative verdicts from `list-state`. */
41
+ export interface AccountRecoveryView {
42
+ /** `isAccountExhausted` — a 5h/7d wall or an unexpired `exhausted_until` mark. */
43
+ exhausted: boolean
44
+ /** `isAccountPremiumWalled` — the flagship (`7d_oi`) tier wall verdict. */
45
+ premiumWalled: boolean
46
+ }
47
+
48
+ export interface PremiumRecoveryDecision {
49
+ /** Fire exactly one ping now (a marker is pending AND the premium tier is
50
+ * servable again). The gateway clears the marker + claims before sending. */
51
+ fire: boolean
52
+ /** Why not, for the log (never surfaced to the user). */
53
+ reason: 'no-marker' | 'still-walled' | 'recovered'
54
+ }
55
+
56
+ /**
57
+ * Decide whether the premium tier has recovered for a pending downgrade.
58
+ * PURE. Fires iff a marker is pending AND at least one account can serve the
59
+ * premium tier again (neither exhausted nor premium-walled) — the exact
60
+ * complement of the `all-blocked` condition that fired the downgrade.
61
+ */
62
+ export function decidePremiumRecovery(opts: {
63
+ hasMarker: boolean
64
+ accounts: ReadonlyArray<AccountRecoveryView>
65
+ }): PremiumRecoveryDecision {
66
+ if (!opts.hasMarker) return { fire: false, reason: 'no-marker' }
67
+ const servable = opts.accounts.some((a) => !a.exhausted && !a.premiumWalled)
68
+ return servable
69
+ ? { fire: true, reason: 'recovered' }
70
+ : { fire: false, reason: 'still-walled' }
71
+ }
72
+
73
+ export interface PremiumRecoveryPing {
74
+ /** Telegram-formatted (rich-markdown) body. */
75
+ text: string
76
+ /** Inline-keyboard button label. */
77
+ buttonText: string
78
+ }
79
+
80
+ /**
81
+ * The user-facing recovery ping. PURE + deterministic so the wording + button
82
+ * are pinned by a test. Honest: the switch it offers is SESSION-SCOPED (the
83
+ * same `/model` semantics as everywhere else — it reverts on the next restart),
84
+ * so it never promises a durable pin.
85
+ */
86
+ export function renderPremiumRecoveryPing(premiumModel: string): PremiumRecoveryPing {
87
+ return {
88
+ text:
89
+ `✅ \`${premiumModel}\` is available again — tap to switch back to it for this session.`,
90
+ buttonText: `Switch to ${premiumModel}`,
91
+ }
92
+ }
93
+
94
+ /**
95
+ * The fleet-wide `claim-notification` key for a premium-recovery ping. Keyed on
96
+ * agent + the specific premium token so two different dropped models never
97
+ * dedup each other, and a bounce mid-window can't re-send the same one.
98
+ */
99
+ export function premiumRecoveryClaimKey(agent: string, premiumModel: string): string {
100
+ return `premium-recovery:${agent}:${premiumModel}`
101
+ }
@@ -0,0 +1,73 @@
1
+ /**
2
+ * raw-error-scrub.ts — the ONE scrubber that removes raw-API-error bytes from a
3
+ * string before it reaches a user, shared by every surface that might print an
4
+ * error `detail`.
5
+ *
6
+ * Zero dependencies (a leaf module) so both the pure `operator-events.ts`
7
+ * renderer AND `llm-error-present.ts` can import it without a cycle. Anchored on
8
+ * the same markers as `looksLikeRawApiError` (pty-partial-handler.ts): the CLI's
9
+ * Python `· b'{…}'` byte-blob, a trailing `{"type":"error"…}` JSON object, and an
10
+ * `API Error:` prefix.
11
+ */
12
+
13
+ /**
14
+ * Strip the raw-API-error bytes a source can smuggle into an otherwise-human
15
+ * string. Total — never throws; a clean human string passes through unchanged
16
+ * (modulo whitespace tidy-up).
17
+ */
18
+ export function stripRawErrorBytes(raw: string): string {
19
+ if (typeof raw !== 'string' || raw.length === 0) return ''
20
+ let s = raw
21
+ // 1. `API Error:` / `API Error 429:` prefix anywhere.
22
+ s = s.replace(/API Error:?\s*\d*\s*/gi, ' ')
23
+ // 2. Python byte-blob render: ` b'{…}'` or ` b"{…}"` (the CLI's raw-body echo).
24
+ s = s.replace(/\bb'[^']*'/g, ' ')
25
+ s = s.replace(/\bb"[^"]*"/g, ' ')
26
+ // 3. A JSON error object `{"type":"error"…}` (or `'type': 'error'`) and
27
+ // everything after it — these blobs are always trailing on the real lines,
28
+ // and brace-balanced stripping is not worth the fragility.
29
+ s = s.replace(/[·\-\s]*\{[\s\S]*?["']type["']\s*:\s*["']error["'][\s\S]*$/i, ' ')
30
+ // 4. A bare leading/trailing JSON object with no human text around it.
31
+ s = s.replace(/^\s*\{[\s\S]*\}\s*$/g, ' ')
32
+ // 5. Tidy: collapse whitespace, drop dangling separators.
33
+ s = s.replace(/\s+/g, ' ').replace(/[·:\-\s]+$/g, '').replace(/^[·:\-\s]+/g, '').trim()
34
+ return s
35
+ }
36
+
37
+ /**
38
+ * Pull an Anthropic `request_id` out of a raw error string, or undefined. The
39
+ * id lives in the trailing byte-blob (`…,"request_id":"req_abc"}'`) and can sit
40
+ * far into a long body — see `truncateDetailPreservingRequestId`. Total.
41
+ */
42
+ export function extractRequestId(raw: string): string | undefined {
43
+ if (typeof raw !== 'string' || raw.length === 0) return undefined
44
+ const m =
45
+ raw.match(/["']request[_-]?id["']\s*:\s*["']([A-Za-z0-9._-]+)["']/i) ??
46
+ raw.match(/\brequest[_-]?id[=:]\s*([A-Za-z0-9._-]+)/i)
47
+ return m ? m[1] : undefined
48
+ }
49
+
50
+ /**
51
+ * Truncate an error `detail` to `max` chars WITHOUT losing the `request_id`.
52
+ *
53
+ * Why this exists: the bridge forwards operator-event detail truncated to 1000
54
+ * chars (OPERATOR_EVENT_DETAIL_MAX), but the Anthropic `request_id` lives in the
55
+ * trailing byte-blob, which can sit PAST char 1000. A naive `slice(0, 1000)`
56
+ * drops it, so the cross-surface dedup gate silently degrades from the reliable
57
+ * `rid:` EXACT key to the coarse `${kind}:${agent}:${bucket}` key — collapsing
58
+ * two genuinely-distinct same-kind errors within 60s into one card.
59
+ *
60
+ * Fix: extract the id from the FULL body first; if the plain head would drop it,
61
+ * append `request_id=<id>` in the freed tail budget (`extractRequestId` matches
62
+ * that form), keeping the result ≤ `max`. Total — never throws.
63
+ */
64
+ export function truncateDetailPreservingRequestId(detail: string, max: number): string {
65
+ if (typeof detail !== 'string') return ''
66
+ if (detail.length <= max) return detail
67
+ const rid = extractRequestId(detail)
68
+ const head = detail.slice(0, max)
69
+ if (rid == null || head.includes(rid)) return head
70
+ const suffix = ` request_id=${rid}`
71
+ const headBudget = Math.max(0, max - suffix.length)
72
+ return `${detail.slice(0, headBudget)}${suffix}`
73
+ }
@@ -121,8 +121,14 @@ export interface RetryApiCallConfig {
121
121
  * a container restart during an active ban can suppress non-essential
122
122
  * sends (boot cards) instead of feeding the same per-bot-token flood
123
123
  * counter and prolonging the ban. Best-effort; a throw here is swallowed.
124
+ *
125
+ * The call's `opts` are passed through (#3111) so the hook can open a
126
+ * SCOPE-PRECISE send-gate window: this fires even for a SHORT slept-and-retried
127
+ * 429 that never throws `FLOOD_WAIT_ACTIVE`, so without the opts the gateway
128
+ * could only open a blanket `global` window for those. Existing callers that
129
+ * ignore the second argument are unaffected.
124
130
  */
125
- onFloodWait?: (retryAfterSec: number) => void
131
+ onFloodWait?: (retryAfterSec: number, opts?: RetryCallOpts) => void
126
132
  }
127
133
 
128
134
  /**
@@ -353,7 +359,7 @@ export function createRetryApiCall(
353
359
  // Persist the flood window so a restart during the ban can suppress
354
360
  // non-essential sends instead of extending it (#2923 circuit breaker).
355
361
  try {
356
- onFloodWait?.(retryAfter)
362
+ onFloodWait?.(retryAfter, opts)
357
363
  } catch {
358
364
  /* best-effort — never let the breaker hook break the retry path */
359
365
  }
@@ -566,11 +566,162 @@ describe('send-gate PR2: opening a window from a 429, and persistence hook', ()
566
566
  ).rejects.toBe(floodErr)
567
567
 
568
568
  const scopes = opened.map((o) => o.scopeKey).sort()
569
+ // #3111: a chat-bound 429 opens ONLY the chat/group/msg-edit scopes — NOT a
570
+ // coincident `global` window (which would suppress unrelated chats).
569
571
  // H1: msg-edit scope carries the chat_id (msg-edit:7:3), not the bare id.
570
- expect(scopes).toEqual(['chat:7', 'global', 'group:7', 'msg-edit:7:3'])
572
+ expect(scopes).toEqual(['chat:7', 'group:7', 'msg-edit:7:3'])
571
573
  // After the window opens, a later cosmetic on the same chat sheds.
572
574
  const shed = await gate.gate(async () => 'x', { chat_id: '7', priorityClass: 'cosmetic' })
573
575
  expect(shed).toBe(SEND_GATE_SHED)
574
576
  expect(gate.stats().global.shed).toBe(1)
575
577
  })
576
578
  })
579
+
580
+ describe('send-gate #3111: scope-precise flood windows', () => {
581
+ // A structured FLOOD_WAIT_ACTIVE the wrapped `fn` throws to simulate a 429
582
+ // whose remaining window (`untilTs`) exceeds the fail-fast ceiling.
583
+ function floodErrUntil(untilTs: number) {
584
+ const sec = Math.ceil(untilTs / 1000)
585
+ return Object.assign(new Error('FLOOD_WAIT_ACTIVE'), {
586
+ retryAfterSec: sec,
587
+ untilTs,
588
+ error_code: 429 as const,
589
+ parameters: { retry_after: sec },
590
+ original: null,
591
+ })
592
+ }
593
+
594
+ it('a chat:A 429 suppresses only chat:A — an unrelated chat:B critical still sends', async () => {
595
+ const clock = new FakeClock()
596
+ const opened: string[] = []
597
+ const gate = createSendGate({
598
+ enabled: true,
599
+ clock,
600
+ criticalFailFastMs: 60_000,
601
+ onWindowOpen: (scopeKey) => opened.push(scopeKey),
602
+ })
603
+
604
+ // A LONG (10-min) 429 on chat:A opens the flood window via the gate's catch.
605
+ const floodErr = floodErrUntil(600_000)
606
+ await expect(
607
+ gate.gate(
608
+ async () => {
609
+ throw floodErr
610
+ },
611
+ { chat_id: 'A', priorityClass: 'critical' },
612
+ ),
613
+ ).rejects.toBe(floodErr)
614
+
615
+ // Only chat:A was suppressed — pre-#3111 this ALSO opened `global`.
616
+ expect(opened.sort()).toEqual(['chat:A'])
617
+ // The 429 came from `fn`, not a fail-fast admission.
618
+ expect(gate.stats().global.failedFast).toBe(0)
619
+
620
+ // An unrelated chat:B critical is NOT fail-fasted — it sends. Pre-#3111 the
621
+ // coincident global window would have fail-fasted this critical too.
622
+ const bResult = await gate.gate(async () => 'B-sent', {
623
+ chat_id: 'B',
624
+ priorityClass: 'critical',
625
+ })
626
+ expect(bResult).toBe('B-sent')
627
+ expect(gate.stats().global.failedFast).toBe(0)
628
+ expect(gate.stats().global.sent).toBe(1)
629
+
630
+ // A chat:A critical IS fail-fasted — its window exceeds the ceiling.
631
+ let caught: unknown
632
+ await gate
633
+ .gate(async () => 'A-sent', { chat_id: 'A', priorityClass: 'critical' })
634
+ .catch((e) => {
635
+ caught = e
636
+ })
637
+ expect(isFloodWaitActiveError(caught)).toBe(true)
638
+ expect(gate.stats().global.failedFast).toBe(1)
639
+ // chat:B's critical really was the only additional send.
640
+ expect(gate.stats().global.sent).toBe(1)
641
+ })
642
+
643
+ it('a global 429 (no chat scope) still opens the global window and suppresses everywhere', async () => {
644
+ const clock = new FakeClock()
645
+ const opened: string[] = []
646
+ const gate = createSendGate({
647
+ enabled: true,
648
+ clock,
649
+ onWindowOpen: (scopeKey) => opened.push(scopeKey),
650
+ })
651
+
652
+ // A 429 on a call with NO chat_id → the evidence IS global.
653
+ const floodErr = floodErrUntil(HOUR)
654
+ await expect(
655
+ gate.gate(
656
+ async () => {
657
+ throw floodErr
658
+ },
659
+ { priorityClass: 'critical' },
660
+ ),
661
+ ).rejects.toBe(floodErr)
662
+
663
+ expect(opened).toEqual(['global'])
664
+ // An unrelated chat's cosmetic sheds — global covers every scope.
665
+ const shed = await gate.gate(async () => 'x', { chat_id: 'Z', priorityClass: 'cosmetic' })
666
+ expect(shed).toBe(SEND_GATE_SHED)
667
+ expect(gate.stats().global.shed).toBe(1)
668
+ })
669
+
670
+ it('conservativeGlobalFloodScope restores the pre-#3111 always-open-global posture', async () => {
671
+ const clock = new FakeClock()
672
+ const opened: string[] = []
673
+ const gate = createSendGate({
674
+ enabled: true,
675
+ clock,
676
+ conservativeGlobalFloodScope: true,
677
+ onWindowOpen: (scopeKey) => opened.push(scopeKey),
678
+ })
679
+
680
+ const floodErr = floodErrUntil(HOUR)
681
+ await expect(
682
+ gate.gate(
683
+ async () => {
684
+ throw floodErr
685
+ },
686
+ { chat_id: 'A', priorityClass: 'critical' },
687
+ ),
688
+ ).rejects.toBe(floodErr)
689
+
690
+ // Conservative posture: a chat-bound 429 ALSO opens the global window.
691
+ expect(opened.sort()).toEqual(['chat:A', 'global'])
692
+ })
693
+
694
+ it('openScopedFloodWindows (the gateway onFloodWait seam) opens scope-precise windows', async () => {
695
+ const clock = new FakeClock()
696
+ const opened: string[] = []
697
+ const gate = createSendGate({
698
+ enabled: true,
699
+ clock,
700
+ onWindowOpen: (scopeKey) => opened.push(scopeKey),
701
+ })
702
+
703
+ // Simulates the gateway's onFloodWait hook for a SHORT slept-and-retried 429
704
+ // on a supergroup chat — the gate's own FLOOD_WAIT_ACTIVE catch never fires
705
+ // for those, so this seam must carry the scope precision.
706
+ gate.openScopedFloodWindows({ chat_id: '9', chatType: 'supergroup' }, HOUR)
707
+ expect(opened.sort()).toEqual(['chat:9', 'group:9'])
708
+
709
+ // A no-chat-scope 429 opens global.
710
+ opened.length = 0
711
+ gate.openScopedFloodWindows(undefined, HOUR)
712
+ expect(opened).toEqual(['global'])
713
+ })
714
+
715
+ it('openScopedFloodWindows is a pure no-op when the gate is disabled', async () => {
716
+ const clock = new FakeClock()
717
+ const opened: string[] = []
718
+ const gate = createSendGate({
719
+ enabled: false,
720
+ clock,
721
+ onWindowOpen: (scopeKey) => opened.push(scopeKey),
722
+ })
723
+ gate.openScopedFloodWindows({ chat_id: 'A' }, HOUR)
724
+ gate.openScopedFloodWindows(undefined, HOUR)
725
+ expect(opened).toEqual([])
726
+ })
727
+ })
@@ -352,6 +352,146 @@ describe('createFloodWindowObserver — alerting', () => {
352
352
  expect(alerts).toHaveLength(1)
353
353
  })
354
354
 
355
+ // The operator-facing alert timestamps must render in the CONFIGURED local
356
+ // timezone, not UTC. The live incident that prompted this: a "flood ban
357
+ // cleared" card printed `2026-07-12T23:39:24Z to 2026-07-12T23:46:40Z UTC`,
358
+ // which is 9:39am–9:46am AEST in the fleet's configured zone. These assert
359
+ // the local rendering and would FAIL on the old `.toISOString()` output.
360
+ const OBSERVED = Date.parse('2026-07-12T23:39:24Z') // 2026-07-13 09:39 AEST
361
+ const UNTIL = Date.parse('2026-07-12T23:46:40Z') // 2026-07-13 09:46 AEST
362
+
363
+ it('renders the CLEARED card timestamps in the configured local tz (AEST), not UTC', async () => {
364
+ const clock = new TestClock()
365
+ const alerts: string[] = []
366
+ let windows: FloodWindowRecord[] = [
367
+ { scopeKey: 'global', untilTs: UNTIL, retryAfterSrc: '429', observedAt: OBSERVED },
368
+ ]
369
+ const obs = createFloodWindowObserver({
370
+ clock,
371
+ log: () => {},
372
+ stats: () => makeStats(),
373
+ readWindows: () => windows,
374
+ markAlerted: () => {},
375
+ sendAlert: async (t) => {
376
+ alerts.push(t)
377
+ },
378
+ operatorChatId: () => 'op',
379
+ alertThresholdMs: 60_000,
380
+ tz: 'Australia/Melbourne',
381
+ })
382
+
383
+ // Past threshold but global → operator unreachable → defer to close.
384
+ clock.cur = OBSERVED + 61_000
385
+ await obs.tick()
386
+ expect(alerts).toHaveLength(0)
387
+
388
+ // Window closes → deferred cleared card fires, in LOCAL time.
389
+ clock.cur = UNTIL + 1_000
390
+ windows = []
391
+ await obs.tick()
392
+ expect(alerts).toHaveLength(1)
393
+ expect(alerts[0]).toContain('flood ban cleared')
394
+ // Local-tz rendering: 9:39am to 9:46am AEST.
395
+ expect(alerts[0]).toContain('was banned from 9:39am to 9:46am AEST')
396
+ // And explicitly NOT the old UTC ISO output.
397
+ expect(alerts[0]).not.toContain('2026-07-12T23:39:24Z')
398
+ expect(alerts[0]).not.toContain('UTC')
399
+ })
400
+
401
+ it('renders the ACTIVE card clear-time in the configured local tz (AEST), not UTC', async () => {
402
+ const clock = new TestClock()
403
+ const alerts: string[] = []
404
+ let windows: FloodWindowRecord[] = [
405
+ // A far-future untilTs so the window stays open; a non-operator chat so
406
+ // the immediate-delivery branch fires (openAlertText).
407
+ { scopeKey: 'chat:other', untilTs: UNTIL, retryAfterSrc: '429', observedAt: OBSERVED },
408
+ ]
409
+ const obs = createFloodWindowObserver({
410
+ clock,
411
+ log: () => {},
412
+ stats: () => makeStats(),
413
+ readWindows: () => windows,
414
+ markAlerted: (scope, at) => {
415
+ windows = windows.map((w) => (w.scopeKey === scope ? { ...w, alertedAt: at } : w))
416
+ },
417
+ sendAlert: async (t) => {
418
+ alerts.push(t)
419
+ },
420
+ operatorChatId: () => 'op',
421
+ alertThresholdMs: 60_000,
422
+ tz: 'Australia/Melbourne',
423
+ })
424
+
425
+ clock.cur = OBSERVED + 61_000
426
+ await obs.tick()
427
+ expect(alerts).toHaveLength(1)
428
+ expect(alerts[0]).toContain('flood ban active')
429
+ // "expected to clear at 9:46am AEST" — local wall-clock, not UTC ISO.
430
+ expect(alerts[0]).toContain('expected to clear at 9:46am AEST')
431
+ expect(alerts[0]).not.toContain('2026-07-12T23:46:40Z')
432
+ expect(alerts[0]).not.toContain('UTC')
433
+ })
434
+
435
+ it('defaults to UTC wording when no tz is configured (backward compat)', async () => {
436
+ const clock = new TestClock()
437
+ const alerts: string[] = []
438
+ let windows: FloodWindowRecord[] = [
439
+ { scopeKey: 'global', untilTs: UNTIL, retryAfterSrc: '429', observedAt: OBSERVED },
440
+ ]
441
+ const obs = createFloodWindowObserver({
442
+ clock,
443
+ log: () => {},
444
+ stats: () => makeStats(),
445
+ readWindows: () => windows,
446
+ markAlerted: () => {},
447
+ sendAlert: async (t) => {
448
+ alerts.push(t)
449
+ },
450
+ operatorChatId: () => 'op',
451
+ alertThresholdMs: 60_000,
452
+ // no tz → UTC
453
+ })
454
+ clock.cur = OBSERVED + 61_000
455
+ await obs.tick()
456
+ clock.cur = UNTIL + 1_000
457
+ windows = []
458
+ await obs.tick()
459
+ expect(alerts).toHaveLength(1)
460
+ // UTC fallback: 11:39pm to 11:46pm UTC on 2026-07-12.
461
+ expect(alerts[0]).toContain('11:39pm to 11:46pm UTC')
462
+ })
463
+
464
+ it('spans the local date when a window crosses local midnight', async () => {
465
+ const clock = new TestClock()
466
+ const alerts: string[] = []
467
+ // 2026-07-13T13:55:00Z .. 2026-07-13T14:05:00Z = 11:55pm 13 Jul .. 12:05am 14 Jul AEST.
468
+ const obsStart = Date.parse('2026-07-13T13:55:00Z')
469
+ const obsEnd = Date.parse('2026-07-13T14:05:00Z')
470
+ let windows: FloodWindowRecord[] = [
471
+ { scopeKey: 'global', untilTs: obsEnd, retryAfterSrc: '429', observedAt: obsStart },
472
+ ]
473
+ const obs = createFloodWindowObserver({
474
+ clock,
475
+ log: () => {},
476
+ stats: () => makeStats(),
477
+ readWindows: () => windows,
478
+ markAlerted: () => {},
479
+ sendAlert: async (t) => {
480
+ alerts.push(t)
481
+ },
482
+ operatorChatId: () => 'op',
483
+ alertThresholdMs: 60_000,
484
+ tz: 'Australia/Melbourne',
485
+ })
486
+ clock.cur = obsStart + 61_000
487
+ await obs.tick()
488
+ clock.cur = obsEnd + 1_000
489
+ windows = []
490
+ await obs.tick()
491
+ expect(alerts).toHaveLength(1)
492
+ expect(alerts[0]).toContain('13 Jul 11:55pm to 14 Jul 12:05am AEST')
493
+ })
494
+
355
495
  it('does not alert for msg-edit (cosmetic) scopes', async () => {
356
496
  const clock = new TestClock()
357
497
  const alerts: string[] = []
@@ -29,14 +29,15 @@
29
29
  * "was banned from X to Y" alert when the window CLOSES (or as soon as the
30
30
  * operator chat becomes reachable again).
31
31
  *
32
- * STATUS OF THE IMMEDIATE PATH (today vs. after #3111)
33
- * ----------------------------------------------------
34
- * The immediate-delivery branch is present and unit-tested, but it does NOT
35
- * fire in production yet: `openScopedWindowsForOpts` opens a coincident `global`
36
- * window on every 429, so the operator chat is always "covered" while any ban is
37
- * active and every alert defers to close. It becomes live once #3111 makes
38
- * `onFloodWait` scope-precise. Until then the operator is alerted at window
39
- * CLOSE — once per incident, all scopes coalesced into one card (M1, #3112).
32
+ * STATUS OF THE IMMEDIATE PATH (live since #3111)
33
+ * -----------------------------------------------
34
+ * The immediate-delivery branch is present, unit-tested, AND now fires in
35
+ * production: as of #3111 a chat-scoped 429 opens ONLY that chat's window (no
36
+ * coincident `global`), so when a ban covers `chat:<other>` while the operator's
37
+ * own chat is clear, the alert is delivered IMMEDIATELY rather than deferred to
38
+ * close. A genuinely global 429 still opens `global`, so the operator chat is
39
+ * "covered" and the alert defers to close — once per incident, all scopes
40
+ * coalesced into one card (M1, #3112).
40
41
  *
41
42
  * At-most-once is anchored by the persisted `alertedAt` (survives restart). The
42
43
  * deferred close-alert is best-effort at-least-once: if the gateway is down for
@@ -53,6 +54,7 @@
53
54
 
54
55
  import type { Clock, SendGateStats } from './send-gate.js'
55
56
  import type { FloodWindowRecord } from './flood-circuit-breaker.js'
57
+ import { fmtLocalClock, fmtLocalDate, localDay, tzAbbrev } from './shared/local-time.js'
56
58
 
57
59
  /** One-line, human-scannable summary of the gate counters + global fill. */
58
60
  export function formatStatsLine(stats: SendGateStats): string {
@@ -153,6 +155,14 @@ export interface FloodWindowObserverConfig {
153
155
  operatorChatId?: () => string | undefined
154
156
  /** Ms a window must be open before it earns an alert. Default 60_000. */
155
157
  alertThresholdMs?: number
158
+ /**
159
+ * IANA timezone (e.g. `Australia/Melbourne`) for rendering the operator-facing
160
+ * alert timestamps in local wall-clock time instead of UTC. The gateway sources
161
+ * this from `SWITCHROOM_TIMEZONE ?? TZ` (the same env the config cascade bakes
162
+ * into every agent service — see `src/config/timezone.ts`). Defaults to `'UTC'`
163
+ * so a missing tz degrades to the previous behaviour rather than throwing.
164
+ */
165
+ tz?: string
156
166
  }
157
167
 
158
168
  export interface FloodWindowObserver {
@@ -160,11 +170,44 @@ export interface FloodWindowObserver {
160
170
  tick(): Promise<void>
161
171
  }
162
172
 
163
- /** Human-readable ISO-ish timestamp (UTC, second precision) for alert wording. */
164
- function fmtTs(ms: number): string {
165
- // Strip the milliseconds (`.\d{3}`) for second precision this also covers
166
- // the `.000` case, so no separate replace is needed.
167
- return new Date(ms).toISOString().replace(/\.\d{3}/, '')
173
+ // ─── Operator-facing local-time formatting ──────────────────────────────────
174
+ // Operator alerts (openAlertText / closeAlertText) render timestamps in the
175
+ // operator's CONFIGURED timezone, not UTC a "was banned from 9:39am to 9:46am
176
+ // AEST" reads at a glance where a raw `2026-07-12T23:39:24Z UTC` does not. The
177
+ // machine-facing snapshot LOG lines (config.log → gateway-supervisor.log) keep
178
+ // their epoch/ISO wording; only the chat/card text is localized.
179
+ //
180
+ // The four wall-clock primitives (fmtLocalClock / fmtLocalDate / localDay /
181
+ // tzAbbrev) now live in shared/local-time.ts so the humanized LLM-error card
182
+ // renders timestamps through the SAME source of truth. The two composers below
183
+ // (fmtLocalStamp / fmtLocalRange) stay here — they are this module's own wording.
184
+
185
+ /**
186
+ * A single operator-facing local timestamp with tz suffix, e.g. `9:46am AEST`.
187
+ * When `refMs` is on a different local calendar day (or omitted), the date is
188
+ * prepended (`12 Jul 11:52pm AEST`) so a day-spanning window is unambiguous.
189
+ */
190
+ function fmtLocalStamp(ms: number, tz: string, refMs?: number): string {
191
+ const clock = fmtLocalClock(ms, tz)
192
+ const sameDay = refMs != null && localDay(ms, tz) === localDay(refMs, tz)
193
+ const body = sameDay ? clock : `${fmtLocalDate(ms, tz)} ${clock}`
194
+ return `${body} ${tzAbbrev(ms, tz)}`
195
+ }
196
+
197
+ /**
198
+ * A local time RANGE with a single tz suffix, e.g. `9:39am to 9:46am AEST`.
199
+ * Same-local-day windows drop the redundant start date; day-spanning windows
200
+ * carry the date on each end (`12 Jul 11:59pm to 13 Jul 12:04am AEST`).
201
+ */
202
+ function fmtLocalRange(startMs: number, endMs: number, tz: string): string {
203
+ const sameDay = localDay(startMs, tz) === localDay(endMs, tz)
204
+ if (sameDay) {
205
+ return `${fmtLocalClock(startMs, tz)} to ${fmtLocalClock(endMs, tz)} ${tzAbbrev(endMs, tz)}`
206
+ }
207
+ return (
208
+ `${fmtLocalDate(startMs, tz)} ${fmtLocalClock(startMs, tz)} to ` +
209
+ `${fmtLocalDate(endMs, tz)} ${fmtLocalClock(endMs, tz)} ${tzAbbrev(endMs, tz)}`
210
+ )
168
211
  }
169
212
 
170
213
  function fmtDur(ms: number): string {
@@ -186,6 +229,7 @@ export function createFloodWindowObserver(
186
229
  config: FloodWindowObserverConfig,
187
230
  ): FloodWindowObserver {
188
231
  const alertThresholdMs = config.alertThresholdMs ?? 60_000
232
+ const tz = config.tz ?? 'UTC'
189
233
  // Full records seen on the previous tick, keyed by scope (close detection).
190
234
  let lastSeen = new Map<string, FloodWindowRecord>()
191
235
  // First tick after boot: any window already present was loaded from disk
@@ -232,7 +276,7 @@ export function createFloodWindowObserver(
232
276
  const openFor = fmtDur(now - w.observedAt)
233
277
  return (
234
278
  `⚠️ Telegram flood ban active (scope \`${w.scopeKey}\`). ` +
235
- `Open for ${openFor}, expected to clear at ${fmtTs(w.untilTs)} UTC. ` +
279
+ `Open for ${openFor}, expected to clear at ${fmtLocalStamp(w.untilTs, tz, now)}. ` +
236
280
  `Outbound to that scope is being suppressed.`
237
281
  )
238
282
  }
@@ -250,7 +294,7 @@ export function createFloodWindowObserver(
250
294
  const plural = recs.length > 1 ? 's' : ''
251
295
  return (
252
296
  `⚠️ Telegram flood ban cleared (scope${plural} ${scopes}). ` +
253
- `The bot was banned from ${fmtTs(observedAt)} to ${fmtTs(untilTs)} UTC ` +
297
+ `The bot was banned from ${fmtLocalRange(observedAt, untilTs, tz)} ` +
254
298
  `(~${fmtDur(untilTs - observedAt)}). Some outbound messages during that ` +
255
299
  `window were suppressed.`
256
300
  )
@@ -312,11 +356,12 @@ export function createFloodWindowObserver(
312
356
  if (w.alertedAt != null) continue
313
357
  if (now - w.observedAt < alertThresholdMs) continue
314
358
  if (operatorReachable && !coversOperator(w, operatorChatId)) {
315
- // NOTE (M1, #3112): this immediate-delivery branch is currently
316
- // UNREACHABLE for recorder-produced state `openScopedWindowsForOpts`
317
- // opens a coincident `global` window on every 429, so `operatorReachable`
318
- // is always false while any ban is active. It goes live once #3111 makes
319
- // `onFloodWait` scope-precise. Kept + unit-tested as forward-looking.
359
+ // NOTE (M1, #3112 / #3111): this immediate-delivery branch is now LIVE
360
+ // for recorder-produced state. Since #3111 a chat-scoped 429 opens only
361
+ // that chat's window (no coincident `global`), so when a ban covers a
362
+ // chat OTHER than the operator's, `operatorReachable` is true and the
363
+ // alert is delivered here rather than deferred to close. A genuinely
364
+ // global 429 still opens `global`, keeping the operator "covered".
320
365
  // NOTE (L1, #3112): send-first, then persist `alertedAt`. A crash between
321
366
  // the delivered card and the disk write re-alerts on restart — a benign
322
367
  // duplicate. Deliberate: send-first guarantees an alert is never LOST,