switchroom 0.18.17 → 0.18.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-scheduler/index.js +13 -0
- package/dist/auth-broker/index.js +13 -0
- package/dist/cli/notion-write-pretool.mjs +13 -0
- package/dist/cli/switchroom.js +605 -479
- package/dist/host-control/main.js +17 -1
- package/dist/vault/approvals/kernel-server.js +13 -0
- package/dist/vault/broker/server.js +13 -0
- package/package.json +1 -1
- package/telegram-plugin/bridge/bridge.ts +7 -1
- package/telegram-plugin/dist/bridge/bridge.js +26 -1
- package/telegram-plugin/dist/gateway/gateway.js +1401 -431
- package/telegram-plugin/dist/server.js +26 -1
- package/telegram-plugin/fleet-fallback-resume.ts +26 -3
- package/telegram-plugin/gateway/approval-hold.ts +49 -0
- package/telegram-plugin/gateway/bridge-dead-watchdog.ts +61 -18
- package/telegram-plugin/gateway/gateway.ts +362 -71
- package/telegram-plugin/gateway/linear-activity.ts +20 -4
- package/telegram-plugin/gateway/premium-recovery-wiring.ts +122 -0
- package/telegram-plugin/gateway/session-model-file.ts +103 -0
- package/telegram-plugin/gateway/tier-downgrade-wiring.ts +121 -0
- package/telegram-plugin/gateway/unhandled-rejection-policy.ts +14 -1
- package/telegram-plugin/llm-error-present.ts +436 -0
- package/telegram-plugin/operator-events.ts +7 -1
- package/telegram-plugin/permission-title.ts +172 -10
- package/telegram-plugin/premium-recovery.ts +101 -0
- package/telegram-plugin/raw-error-scrub.ts +73 -0
- package/telegram-plugin/retry-api-call.ts +8 -2
- package/telegram-plugin/send-gate-degraded.test.ts +152 -1
- package/telegram-plugin/send-gate-observability.test.ts +140 -0
- package/telegram-plugin/send-gate-observability.ts +65 -20
- package/telegram-plugin/send-gate.test.ts +143 -1
- package/telegram-plugin/send-gate.ts +212 -19
- package/telegram-plugin/session-tail.ts +16 -0
- package/telegram-plugin/shared/local-time.ts +69 -0
- package/telegram-plugin/tests/approval-hold-harness.ts +6 -6
- package/telegram-plugin/tests/approval-hold-outcome.test.ts +10 -2
- package/telegram-plugin/tests/bridge-dead-watchdog.test.ts +61 -0
- package/telegram-plugin/tests/fleet-fallback-resume.test.ts +39 -0
- package/telegram-plugin/tests/flood-windows-persistence.test.ts +3 -2
- package/telegram-plugin/tests/linear-create-issue.test.ts +30 -2
- package/telegram-plugin/tests/llm-error-present.test.ts +380 -0
- package/telegram-plugin/tests/permission-title.test.ts +167 -4
- package/telegram-plugin/tests/premium-recovery-wiring.test.ts +150 -0
- package/telegram-plugin/tests/premium-recovery.test.ts +165 -0
- package/telegram-plugin/tests/reaction-gate-routing.test.ts +6 -1
- package/telegram-plugin/tests/retry-api-call.test.ts +21 -0
- package/telegram-plugin/tests/tier-downgrade-wiring.test.ts +165 -0
- package/telegram-plugin/tests/tier-downgrade.test.ts +141 -0
- package/telegram-plugin/tests/unhandled-rejection-policy.test.ts +27 -1
- package/telegram-plugin/tests/worker-activity-feed.test.ts +5 -2
- package/telegram-plugin/tests/worker-feed-coalesce.test.ts +492 -0
- package/telegram-plugin/tier-downgrade.ts +198 -0
- package/telegram-plugin/tool-activity-summary.ts +99 -0
- package/telegram-plugin/worker-activity-feed.ts +509 -409
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* premium-recovery.ts — "your premium model is servable again" ping (pure layer).
|
|
3
|
+
*
|
|
4
|
+
* Companion to tier-downgrade.ts. When a premium `/model` selection (e.g.
|
|
5
|
+
* `/model fable`) is walled fleet-wide, the tier-downgrade tier resumes the
|
|
6
|
+
* turn on the configured default and tells the user to re-issue `/model fable`
|
|
7
|
+
* once it frees up (there is NO automatic revert — the override is session-
|
|
8
|
+
* scoped and dies on the downgrade restart). This module closes that loop with
|
|
9
|
+
* a HEADS-UP: the moment the premium tier can be served again, ping the chat
|
|
10
|
+
* ONCE with a one-tap "switch back" button.
|
|
11
|
+
*
|
|
12
|
+
* DETERMINISTIC recovery signal (P0 deterministic-controls — never model-
|
|
13
|
+
* decided). The signal is the auth-broker's own per-account eligibility, read
|
|
14
|
+
* off `list-state` (the SAME state the gateway's `runQuotaWatch` tick already
|
|
15
|
+
* polls every 15 min — no new poller, no broker change):
|
|
16
|
+
* - `account.exhausted` — live-authoritative `isAccountExhausted` verdict
|
|
17
|
+
* (5h/7d wall or an unexpired `exhausted_until` mark).
|
|
18
|
+
* - `account.premium_walled` — live-authoritative `isAccountPremiumWalled`
|
|
19
|
+
* verdict; `isModelTierWalled` is a pure timestamp compare
|
|
20
|
+
* (`premium_walled_until > now`) unless a fresher canary read the flagship
|
|
21
|
+
* tier `allowed`. (src/auth/broker/model-tier-quota.ts / account-eligibility.ts.)
|
|
22
|
+
* The tier-downgrade fires from the account-swap `all-blocked` verdict — i.e.
|
|
23
|
+
* EVERY account was exhausted or premium-walled. Recovery is the exact
|
|
24
|
+
* complement: at least ONE account is neither. No clock of our own, no model
|
|
25
|
+
* judgement — the broker already decided; we read its decision.
|
|
26
|
+
*
|
|
27
|
+
* AT-MOST-ONCE. The gateway holds a consume-once `.premium-recovery` marker
|
|
28
|
+
* (session-model-file.ts) recording the dropped premium token + the chats to
|
|
29
|
+
* notify. `decidePremiumRecovery` is a pure predicate; the gateway clears the
|
|
30
|
+
* marker BEFORE sending (never-storm) and additionally takes a fleet-wide
|
|
31
|
+
* `claim-notification` so a bounce or a second gateway tick can't double-fire.
|
|
32
|
+
* The marker is also cleared the instant the user manually re-issues
|
|
33
|
+
* `/model <premium>` (no stale ping).
|
|
34
|
+
*
|
|
35
|
+
* PURE — no I/O, no clock. The gateway does the list-state read, the marker
|
|
36
|
+
* FS, the claim, and the send off these verdicts.
|
|
37
|
+
*/
|
|
38
|
+
|
|
39
|
+
/** The minimal per-account eligibility view the recovery predicate needs —
|
|
40
|
+
* both fields are the broker's live-authoritative verdicts from `list-state`. */
|
|
41
|
+
export interface AccountRecoveryView {
|
|
42
|
+
/** `isAccountExhausted` — a 5h/7d wall or an unexpired `exhausted_until` mark. */
|
|
43
|
+
exhausted: boolean
|
|
44
|
+
/** `isAccountPremiumWalled` — the flagship (`7d_oi`) tier wall verdict. */
|
|
45
|
+
premiumWalled: boolean
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
export interface PremiumRecoveryDecision {
|
|
49
|
+
/** Fire exactly one ping now (a marker is pending AND the premium tier is
|
|
50
|
+
* servable again). The gateway clears the marker + claims before sending. */
|
|
51
|
+
fire: boolean
|
|
52
|
+
/** Why not, for the log (never surfaced to the user). */
|
|
53
|
+
reason: 'no-marker' | 'still-walled' | 'recovered'
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/**
|
|
57
|
+
* Decide whether the premium tier has recovered for a pending downgrade.
|
|
58
|
+
* PURE. Fires iff a marker is pending AND at least one account can serve the
|
|
59
|
+
* premium tier again (neither exhausted nor premium-walled) — the exact
|
|
60
|
+
* complement of the `all-blocked` condition that fired the downgrade.
|
|
61
|
+
*/
|
|
62
|
+
export function decidePremiumRecovery(opts: {
|
|
63
|
+
hasMarker: boolean
|
|
64
|
+
accounts: ReadonlyArray<AccountRecoveryView>
|
|
65
|
+
}): PremiumRecoveryDecision {
|
|
66
|
+
if (!opts.hasMarker) return { fire: false, reason: 'no-marker' }
|
|
67
|
+
const servable = opts.accounts.some((a) => !a.exhausted && !a.premiumWalled)
|
|
68
|
+
return servable
|
|
69
|
+
? { fire: true, reason: 'recovered' }
|
|
70
|
+
: { fire: false, reason: 'still-walled' }
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
export interface PremiumRecoveryPing {
|
|
74
|
+
/** Telegram-formatted (rich-markdown) body. */
|
|
75
|
+
text: string
|
|
76
|
+
/** Inline-keyboard button label. */
|
|
77
|
+
buttonText: string
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/**
|
|
81
|
+
* The user-facing recovery ping. PURE + deterministic so the wording + button
|
|
82
|
+
* are pinned by a test. Honest: the switch it offers is SESSION-SCOPED (the
|
|
83
|
+
* same `/model` semantics as everywhere else — it reverts on the next restart),
|
|
84
|
+
* so it never promises a durable pin.
|
|
85
|
+
*/
|
|
86
|
+
export function renderPremiumRecoveryPing(premiumModel: string): PremiumRecoveryPing {
|
|
87
|
+
return {
|
|
88
|
+
text:
|
|
89
|
+
`✅ \`${premiumModel}\` is available again — tap to switch back to it for this session.`,
|
|
90
|
+
buttonText: `Switch to ${premiumModel}`,
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
/**
|
|
95
|
+
* The fleet-wide `claim-notification` key for a premium-recovery ping. Keyed on
|
|
96
|
+
* agent + the specific premium token so two different dropped models never
|
|
97
|
+
* dedup each other, and a bounce mid-window can't re-send the same one.
|
|
98
|
+
*/
|
|
99
|
+
export function premiumRecoveryClaimKey(agent: string, premiumModel: string): string {
|
|
100
|
+
return `premium-recovery:${agent}:${premiumModel}`
|
|
101
|
+
}
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* raw-error-scrub.ts — the ONE scrubber that removes raw-API-error bytes from a
|
|
3
|
+
* string before it reaches a user, shared by every surface that might print an
|
|
4
|
+
* error `detail`.
|
|
5
|
+
*
|
|
6
|
+
* Zero dependencies (a leaf module) so both the pure `operator-events.ts`
|
|
7
|
+
* renderer AND `llm-error-present.ts` can import it without a cycle. Anchored on
|
|
8
|
+
* the same markers as `looksLikeRawApiError` (pty-partial-handler.ts): the CLI's
|
|
9
|
+
* Python `· b'{…}'` byte-blob, a trailing `{"type":"error"…}` JSON object, and an
|
|
10
|
+
* `API Error:` prefix.
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* Strip the raw-API-error bytes a source can smuggle into an otherwise-human
|
|
15
|
+
* string. Total — never throws; a clean human string passes through unchanged
|
|
16
|
+
* (modulo whitespace tidy-up).
|
|
17
|
+
*/
|
|
18
|
+
export function stripRawErrorBytes(raw: string): string {
|
|
19
|
+
if (typeof raw !== 'string' || raw.length === 0) return ''
|
|
20
|
+
let s = raw
|
|
21
|
+
// 1. `API Error:` / `API Error 429:` prefix anywhere.
|
|
22
|
+
s = s.replace(/API Error:?\s*\d*\s*/gi, ' ')
|
|
23
|
+
// 2. Python byte-blob render: ` b'{…}'` or ` b"{…}"` (the CLI's raw-body echo).
|
|
24
|
+
s = s.replace(/\bb'[^']*'/g, ' ')
|
|
25
|
+
s = s.replace(/\bb"[^"]*"/g, ' ')
|
|
26
|
+
// 3. A JSON error object `{"type":"error"…}` (or `'type': 'error'`) and
|
|
27
|
+
// everything after it — these blobs are always trailing on the real lines,
|
|
28
|
+
// and brace-balanced stripping is not worth the fragility.
|
|
29
|
+
s = s.replace(/[·\-\s]*\{[\s\S]*?["']type["']\s*:\s*["']error["'][\s\S]*$/i, ' ')
|
|
30
|
+
// 4. A bare leading/trailing JSON object with no human text around it.
|
|
31
|
+
s = s.replace(/^\s*\{[\s\S]*\}\s*$/g, ' ')
|
|
32
|
+
// 5. Tidy: collapse whitespace, drop dangling separators.
|
|
33
|
+
s = s.replace(/\s+/g, ' ').replace(/[·:\-\s]+$/g, '').replace(/^[·:\-\s]+/g, '').trim()
|
|
34
|
+
return s
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* Pull an Anthropic `request_id` out of a raw error string, or undefined. The
|
|
39
|
+
* id lives in the trailing byte-blob (`…,"request_id":"req_abc"}'`) and can sit
|
|
40
|
+
* far into a long body — see `truncateDetailPreservingRequestId`. Total.
|
|
41
|
+
*/
|
|
42
|
+
export function extractRequestId(raw: string): string | undefined {
|
|
43
|
+
if (typeof raw !== 'string' || raw.length === 0) return undefined
|
|
44
|
+
const m =
|
|
45
|
+
raw.match(/["']request[_-]?id["']\s*:\s*["']([A-Za-z0-9._-]+)["']/i) ??
|
|
46
|
+
raw.match(/\brequest[_-]?id[=:]\s*([A-Za-z0-9._-]+)/i)
|
|
47
|
+
return m ? m[1] : undefined
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* Truncate an error `detail` to `max` chars WITHOUT losing the `request_id`.
|
|
52
|
+
*
|
|
53
|
+
* Why this exists: the bridge forwards operator-event detail truncated to 1000
|
|
54
|
+
* chars (OPERATOR_EVENT_DETAIL_MAX), but the Anthropic `request_id` lives in the
|
|
55
|
+
* trailing byte-blob, which can sit PAST char 1000. A naive `slice(0, 1000)`
|
|
56
|
+
* drops it, so the cross-surface dedup gate silently degrades from the reliable
|
|
57
|
+
* `rid:` EXACT key to the coarse `${kind}:${agent}:${bucket}` key — collapsing
|
|
58
|
+
* two genuinely-distinct same-kind errors within 60s into one card.
|
|
59
|
+
*
|
|
60
|
+
* Fix: extract the id from the FULL body first; if the plain head would drop it,
|
|
61
|
+
* append `request_id=<id>` in the freed tail budget (`extractRequestId` matches
|
|
62
|
+
* that form), keeping the result ≤ `max`. Total — never throws.
|
|
63
|
+
*/
|
|
64
|
+
export function truncateDetailPreservingRequestId(detail: string, max: number): string {
|
|
65
|
+
if (typeof detail !== 'string') return ''
|
|
66
|
+
if (detail.length <= max) return detail
|
|
67
|
+
const rid = extractRequestId(detail)
|
|
68
|
+
const head = detail.slice(0, max)
|
|
69
|
+
if (rid == null || head.includes(rid)) return head
|
|
70
|
+
const suffix = ` request_id=${rid}`
|
|
71
|
+
const headBudget = Math.max(0, max - suffix.length)
|
|
72
|
+
return `${detail.slice(0, headBudget)}${suffix}`
|
|
73
|
+
}
|
|
@@ -121,8 +121,14 @@ export interface RetryApiCallConfig {
|
|
|
121
121
|
* a container restart during an active ban can suppress non-essential
|
|
122
122
|
* sends (boot cards) instead of feeding the same per-bot-token flood
|
|
123
123
|
* counter and prolonging the ban. Best-effort; a throw here is swallowed.
|
|
124
|
+
*
|
|
125
|
+
* The call's `opts` are passed through (#3111) so the hook can open a
|
|
126
|
+
* SCOPE-PRECISE send-gate window: this fires even for a SHORT slept-and-retried
|
|
127
|
+
* 429 that never throws `FLOOD_WAIT_ACTIVE`, so without the opts the gateway
|
|
128
|
+
* could only open a blanket `global` window for those. Existing callers that
|
|
129
|
+
* ignore the second argument are unaffected.
|
|
124
130
|
*/
|
|
125
|
-
onFloodWait?: (retryAfterSec: number) => void
|
|
131
|
+
onFloodWait?: (retryAfterSec: number, opts?: RetryCallOpts) => void
|
|
126
132
|
}
|
|
127
133
|
|
|
128
134
|
/**
|
|
@@ -353,7 +359,7 @@ export function createRetryApiCall(
|
|
|
353
359
|
// Persist the flood window so a restart during the ban can suppress
|
|
354
360
|
// non-essential sends instead of extending it (#2923 circuit breaker).
|
|
355
361
|
try {
|
|
356
|
-
onFloodWait?.(retryAfter)
|
|
362
|
+
onFloodWait?.(retryAfter, opts)
|
|
357
363
|
} catch {
|
|
358
364
|
/* best-effort — never let the breaker hook break the retry path */
|
|
359
365
|
}
|
|
@@ -566,11 +566,162 @@ describe('send-gate PR2: opening a window from a 429, and persistence hook', ()
|
|
|
566
566
|
).rejects.toBe(floodErr)
|
|
567
567
|
|
|
568
568
|
const scopes = opened.map((o) => o.scopeKey).sort()
|
|
569
|
+
// #3111: a chat-bound 429 opens ONLY the chat/group/msg-edit scopes — NOT a
|
|
570
|
+
// coincident `global` window (which would suppress unrelated chats).
|
|
569
571
|
// H1: msg-edit scope carries the chat_id (msg-edit:7:3), not the bare id.
|
|
570
|
-
expect(scopes).toEqual(['chat:7', '
|
|
572
|
+
expect(scopes).toEqual(['chat:7', 'group:7', 'msg-edit:7:3'])
|
|
571
573
|
// After the window opens, a later cosmetic on the same chat sheds.
|
|
572
574
|
const shed = await gate.gate(async () => 'x', { chat_id: '7', priorityClass: 'cosmetic' })
|
|
573
575
|
expect(shed).toBe(SEND_GATE_SHED)
|
|
574
576
|
expect(gate.stats().global.shed).toBe(1)
|
|
575
577
|
})
|
|
576
578
|
})
|
|
579
|
+
|
|
580
|
+
describe('send-gate #3111: scope-precise flood windows', () => {
|
|
581
|
+
// A structured FLOOD_WAIT_ACTIVE the wrapped `fn` throws to simulate a 429
|
|
582
|
+
// whose remaining window (`untilTs`) exceeds the fail-fast ceiling.
|
|
583
|
+
function floodErrUntil(untilTs: number) {
|
|
584
|
+
const sec = Math.ceil(untilTs / 1000)
|
|
585
|
+
return Object.assign(new Error('FLOOD_WAIT_ACTIVE'), {
|
|
586
|
+
retryAfterSec: sec,
|
|
587
|
+
untilTs,
|
|
588
|
+
error_code: 429 as const,
|
|
589
|
+
parameters: { retry_after: sec },
|
|
590
|
+
original: null,
|
|
591
|
+
})
|
|
592
|
+
}
|
|
593
|
+
|
|
594
|
+
it('a chat:A 429 suppresses only chat:A — an unrelated chat:B critical still sends', async () => {
|
|
595
|
+
const clock = new FakeClock()
|
|
596
|
+
const opened: string[] = []
|
|
597
|
+
const gate = createSendGate({
|
|
598
|
+
enabled: true,
|
|
599
|
+
clock,
|
|
600
|
+
criticalFailFastMs: 60_000,
|
|
601
|
+
onWindowOpen: (scopeKey) => opened.push(scopeKey),
|
|
602
|
+
})
|
|
603
|
+
|
|
604
|
+
// A LONG (10-min) 429 on chat:A opens the flood window via the gate's catch.
|
|
605
|
+
const floodErr = floodErrUntil(600_000)
|
|
606
|
+
await expect(
|
|
607
|
+
gate.gate(
|
|
608
|
+
async () => {
|
|
609
|
+
throw floodErr
|
|
610
|
+
},
|
|
611
|
+
{ chat_id: 'A', priorityClass: 'critical' },
|
|
612
|
+
),
|
|
613
|
+
).rejects.toBe(floodErr)
|
|
614
|
+
|
|
615
|
+
// Only chat:A was suppressed — pre-#3111 this ALSO opened `global`.
|
|
616
|
+
expect(opened.sort()).toEqual(['chat:A'])
|
|
617
|
+
// The 429 came from `fn`, not a fail-fast admission.
|
|
618
|
+
expect(gate.stats().global.failedFast).toBe(0)
|
|
619
|
+
|
|
620
|
+
// An unrelated chat:B critical is NOT fail-fasted — it sends. Pre-#3111 the
|
|
621
|
+
// coincident global window would have fail-fasted this critical too.
|
|
622
|
+
const bResult = await gate.gate(async () => 'B-sent', {
|
|
623
|
+
chat_id: 'B',
|
|
624
|
+
priorityClass: 'critical',
|
|
625
|
+
})
|
|
626
|
+
expect(bResult).toBe('B-sent')
|
|
627
|
+
expect(gate.stats().global.failedFast).toBe(0)
|
|
628
|
+
expect(gate.stats().global.sent).toBe(1)
|
|
629
|
+
|
|
630
|
+
// A chat:A critical IS fail-fasted — its window exceeds the ceiling.
|
|
631
|
+
let caught: unknown
|
|
632
|
+
await gate
|
|
633
|
+
.gate(async () => 'A-sent', { chat_id: 'A', priorityClass: 'critical' })
|
|
634
|
+
.catch((e) => {
|
|
635
|
+
caught = e
|
|
636
|
+
})
|
|
637
|
+
expect(isFloodWaitActiveError(caught)).toBe(true)
|
|
638
|
+
expect(gate.stats().global.failedFast).toBe(1)
|
|
639
|
+
// chat:B's critical really was the only additional send.
|
|
640
|
+
expect(gate.stats().global.sent).toBe(1)
|
|
641
|
+
})
|
|
642
|
+
|
|
643
|
+
it('a global 429 (no chat scope) still opens the global window and suppresses everywhere', async () => {
|
|
644
|
+
const clock = new FakeClock()
|
|
645
|
+
const opened: string[] = []
|
|
646
|
+
const gate = createSendGate({
|
|
647
|
+
enabled: true,
|
|
648
|
+
clock,
|
|
649
|
+
onWindowOpen: (scopeKey) => opened.push(scopeKey),
|
|
650
|
+
})
|
|
651
|
+
|
|
652
|
+
// A 429 on a call with NO chat_id → the evidence IS global.
|
|
653
|
+
const floodErr = floodErrUntil(HOUR)
|
|
654
|
+
await expect(
|
|
655
|
+
gate.gate(
|
|
656
|
+
async () => {
|
|
657
|
+
throw floodErr
|
|
658
|
+
},
|
|
659
|
+
{ priorityClass: 'critical' },
|
|
660
|
+
),
|
|
661
|
+
).rejects.toBe(floodErr)
|
|
662
|
+
|
|
663
|
+
expect(opened).toEqual(['global'])
|
|
664
|
+
// An unrelated chat's cosmetic sheds — global covers every scope.
|
|
665
|
+
const shed = await gate.gate(async () => 'x', { chat_id: 'Z', priorityClass: 'cosmetic' })
|
|
666
|
+
expect(shed).toBe(SEND_GATE_SHED)
|
|
667
|
+
expect(gate.stats().global.shed).toBe(1)
|
|
668
|
+
})
|
|
669
|
+
|
|
670
|
+
it('conservativeGlobalFloodScope restores the pre-#3111 always-open-global posture', async () => {
|
|
671
|
+
const clock = new FakeClock()
|
|
672
|
+
const opened: string[] = []
|
|
673
|
+
const gate = createSendGate({
|
|
674
|
+
enabled: true,
|
|
675
|
+
clock,
|
|
676
|
+
conservativeGlobalFloodScope: true,
|
|
677
|
+
onWindowOpen: (scopeKey) => opened.push(scopeKey),
|
|
678
|
+
})
|
|
679
|
+
|
|
680
|
+
const floodErr = floodErrUntil(HOUR)
|
|
681
|
+
await expect(
|
|
682
|
+
gate.gate(
|
|
683
|
+
async () => {
|
|
684
|
+
throw floodErr
|
|
685
|
+
},
|
|
686
|
+
{ chat_id: 'A', priorityClass: 'critical' },
|
|
687
|
+
),
|
|
688
|
+
).rejects.toBe(floodErr)
|
|
689
|
+
|
|
690
|
+
// Conservative posture: a chat-bound 429 ALSO opens the global window.
|
|
691
|
+
expect(opened.sort()).toEqual(['chat:A', 'global'])
|
|
692
|
+
})
|
|
693
|
+
|
|
694
|
+
it('openScopedFloodWindows (the gateway onFloodWait seam) opens scope-precise windows', async () => {
|
|
695
|
+
const clock = new FakeClock()
|
|
696
|
+
const opened: string[] = []
|
|
697
|
+
const gate = createSendGate({
|
|
698
|
+
enabled: true,
|
|
699
|
+
clock,
|
|
700
|
+
onWindowOpen: (scopeKey) => opened.push(scopeKey),
|
|
701
|
+
})
|
|
702
|
+
|
|
703
|
+
// Simulates the gateway's onFloodWait hook for a SHORT slept-and-retried 429
|
|
704
|
+
// on a supergroup chat — the gate's own FLOOD_WAIT_ACTIVE catch never fires
|
|
705
|
+
// for those, so this seam must carry the scope precision.
|
|
706
|
+
gate.openScopedFloodWindows({ chat_id: '9', chatType: 'supergroup' }, HOUR)
|
|
707
|
+
expect(opened.sort()).toEqual(['chat:9', 'group:9'])
|
|
708
|
+
|
|
709
|
+
// A no-chat-scope 429 opens global.
|
|
710
|
+
opened.length = 0
|
|
711
|
+
gate.openScopedFloodWindows(undefined, HOUR)
|
|
712
|
+
expect(opened).toEqual(['global'])
|
|
713
|
+
})
|
|
714
|
+
|
|
715
|
+
it('openScopedFloodWindows is a pure no-op when the gate is disabled', async () => {
|
|
716
|
+
const clock = new FakeClock()
|
|
717
|
+
const opened: string[] = []
|
|
718
|
+
const gate = createSendGate({
|
|
719
|
+
enabled: false,
|
|
720
|
+
clock,
|
|
721
|
+
onWindowOpen: (scopeKey) => opened.push(scopeKey),
|
|
722
|
+
})
|
|
723
|
+
gate.openScopedFloodWindows({ chat_id: 'A' }, HOUR)
|
|
724
|
+
gate.openScopedFloodWindows(undefined, HOUR)
|
|
725
|
+
expect(opened).toEqual([])
|
|
726
|
+
})
|
|
727
|
+
})
|
|
@@ -352,6 +352,146 @@ describe('createFloodWindowObserver — alerting', () => {
|
|
|
352
352
|
expect(alerts).toHaveLength(1)
|
|
353
353
|
})
|
|
354
354
|
|
|
355
|
+
// The operator-facing alert timestamps must render in the CONFIGURED local
|
|
356
|
+
// timezone, not UTC. The live incident that prompted this: a "flood ban
|
|
357
|
+
// cleared" card printed `2026-07-12T23:39:24Z to 2026-07-12T23:46:40Z UTC`,
|
|
358
|
+
// which is 9:39am–9:46am AEST in the fleet's configured zone. These assert
|
|
359
|
+
// the local rendering and would FAIL on the old `.toISOString()` output.
|
|
360
|
+
const OBSERVED = Date.parse('2026-07-12T23:39:24Z') // 2026-07-13 09:39 AEST
|
|
361
|
+
const UNTIL = Date.parse('2026-07-12T23:46:40Z') // 2026-07-13 09:46 AEST
|
|
362
|
+
|
|
363
|
+
it('renders the CLEARED card timestamps in the configured local tz (AEST), not UTC', async () => {
|
|
364
|
+
const clock = new TestClock()
|
|
365
|
+
const alerts: string[] = []
|
|
366
|
+
let windows: FloodWindowRecord[] = [
|
|
367
|
+
{ scopeKey: 'global', untilTs: UNTIL, retryAfterSrc: '429', observedAt: OBSERVED },
|
|
368
|
+
]
|
|
369
|
+
const obs = createFloodWindowObserver({
|
|
370
|
+
clock,
|
|
371
|
+
log: () => {},
|
|
372
|
+
stats: () => makeStats(),
|
|
373
|
+
readWindows: () => windows,
|
|
374
|
+
markAlerted: () => {},
|
|
375
|
+
sendAlert: async (t) => {
|
|
376
|
+
alerts.push(t)
|
|
377
|
+
},
|
|
378
|
+
operatorChatId: () => 'op',
|
|
379
|
+
alertThresholdMs: 60_000,
|
|
380
|
+
tz: 'Australia/Melbourne',
|
|
381
|
+
})
|
|
382
|
+
|
|
383
|
+
// Past threshold but global → operator unreachable → defer to close.
|
|
384
|
+
clock.cur = OBSERVED + 61_000
|
|
385
|
+
await obs.tick()
|
|
386
|
+
expect(alerts).toHaveLength(0)
|
|
387
|
+
|
|
388
|
+
// Window closes → deferred cleared card fires, in LOCAL time.
|
|
389
|
+
clock.cur = UNTIL + 1_000
|
|
390
|
+
windows = []
|
|
391
|
+
await obs.tick()
|
|
392
|
+
expect(alerts).toHaveLength(1)
|
|
393
|
+
expect(alerts[0]).toContain('flood ban cleared')
|
|
394
|
+
// Local-tz rendering: 9:39am to 9:46am AEST.
|
|
395
|
+
expect(alerts[0]).toContain('was banned from 9:39am to 9:46am AEST')
|
|
396
|
+
// And explicitly NOT the old UTC ISO output.
|
|
397
|
+
expect(alerts[0]).not.toContain('2026-07-12T23:39:24Z')
|
|
398
|
+
expect(alerts[0]).not.toContain('UTC')
|
|
399
|
+
})
|
|
400
|
+
|
|
401
|
+
it('renders the ACTIVE card clear-time in the configured local tz (AEST), not UTC', async () => {
|
|
402
|
+
const clock = new TestClock()
|
|
403
|
+
const alerts: string[] = []
|
|
404
|
+
let windows: FloodWindowRecord[] = [
|
|
405
|
+
// A far-future untilTs so the window stays open; a non-operator chat so
|
|
406
|
+
// the immediate-delivery branch fires (openAlertText).
|
|
407
|
+
{ scopeKey: 'chat:other', untilTs: UNTIL, retryAfterSrc: '429', observedAt: OBSERVED },
|
|
408
|
+
]
|
|
409
|
+
const obs = createFloodWindowObserver({
|
|
410
|
+
clock,
|
|
411
|
+
log: () => {},
|
|
412
|
+
stats: () => makeStats(),
|
|
413
|
+
readWindows: () => windows,
|
|
414
|
+
markAlerted: (scope, at) => {
|
|
415
|
+
windows = windows.map((w) => (w.scopeKey === scope ? { ...w, alertedAt: at } : w))
|
|
416
|
+
},
|
|
417
|
+
sendAlert: async (t) => {
|
|
418
|
+
alerts.push(t)
|
|
419
|
+
},
|
|
420
|
+
operatorChatId: () => 'op',
|
|
421
|
+
alertThresholdMs: 60_000,
|
|
422
|
+
tz: 'Australia/Melbourne',
|
|
423
|
+
})
|
|
424
|
+
|
|
425
|
+
clock.cur = OBSERVED + 61_000
|
|
426
|
+
await obs.tick()
|
|
427
|
+
expect(alerts).toHaveLength(1)
|
|
428
|
+
expect(alerts[0]).toContain('flood ban active')
|
|
429
|
+
// "expected to clear at 9:46am AEST" — local wall-clock, not UTC ISO.
|
|
430
|
+
expect(alerts[0]).toContain('expected to clear at 9:46am AEST')
|
|
431
|
+
expect(alerts[0]).not.toContain('2026-07-12T23:46:40Z')
|
|
432
|
+
expect(alerts[0]).not.toContain('UTC')
|
|
433
|
+
})
|
|
434
|
+
|
|
435
|
+
it('defaults to UTC wording when no tz is configured (backward compat)', async () => {
|
|
436
|
+
const clock = new TestClock()
|
|
437
|
+
const alerts: string[] = []
|
|
438
|
+
let windows: FloodWindowRecord[] = [
|
|
439
|
+
{ scopeKey: 'global', untilTs: UNTIL, retryAfterSrc: '429', observedAt: OBSERVED },
|
|
440
|
+
]
|
|
441
|
+
const obs = createFloodWindowObserver({
|
|
442
|
+
clock,
|
|
443
|
+
log: () => {},
|
|
444
|
+
stats: () => makeStats(),
|
|
445
|
+
readWindows: () => windows,
|
|
446
|
+
markAlerted: () => {},
|
|
447
|
+
sendAlert: async (t) => {
|
|
448
|
+
alerts.push(t)
|
|
449
|
+
},
|
|
450
|
+
operatorChatId: () => 'op',
|
|
451
|
+
alertThresholdMs: 60_000,
|
|
452
|
+
// no tz → UTC
|
|
453
|
+
})
|
|
454
|
+
clock.cur = OBSERVED + 61_000
|
|
455
|
+
await obs.tick()
|
|
456
|
+
clock.cur = UNTIL + 1_000
|
|
457
|
+
windows = []
|
|
458
|
+
await obs.tick()
|
|
459
|
+
expect(alerts).toHaveLength(1)
|
|
460
|
+
// UTC fallback: 11:39pm to 11:46pm UTC on 2026-07-12.
|
|
461
|
+
expect(alerts[0]).toContain('11:39pm to 11:46pm UTC')
|
|
462
|
+
})
|
|
463
|
+
|
|
464
|
+
it('spans the local date when a window crosses local midnight', async () => {
|
|
465
|
+
const clock = new TestClock()
|
|
466
|
+
const alerts: string[] = []
|
|
467
|
+
// 2026-07-13T13:55:00Z .. 2026-07-13T14:05:00Z = 11:55pm 13 Jul .. 12:05am 14 Jul AEST.
|
|
468
|
+
const obsStart = Date.parse('2026-07-13T13:55:00Z')
|
|
469
|
+
const obsEnd = Date.parse('2026-07-13T14:05:00Z')
|
|
470
|
+
let windows: FloodWindowRecord[] = [
|
|
471
|
+
{ scopeKey: 'global', untilTs: obsEnd, retryAfterSrc: '429', observedAt: obsStart },
|
|
472
|
+
]
|
|
473
|
+
const obs = createFloodWindowObserver({
|
|
474
|
+
clock,
|
|
475
|
+
log: () => {},
|
|
476
|
+
stats: () => makeStats(),
|
|
477
|
+
readWindows: () => windows,
|
|
478
|
+
markAlerted: () => {},
|
|
479
|
+
sendAlert: async (t) => {
|
|
480
|
+
alerts.push(t)
|
|
481
|
+
},
|
|
482
|
+
operatorChatId: () => 'op',
|
|
483
|
+
alertThresholdMs: 60_000,
|
|
484
|
+
tz: 'Australia/Melbourne',
|
|
485
|
+
})
|
|
486
|
+
clock.cur = obsStart + 61_000
|
|
487
|
+
await obs.tick()
|
|
488
|
+
clock.cur = obsEnd + 1_000
|
|
489
|
+
windows = []
|
|
490
|
+
await obs.tick()
|
|
491
|
+
expect(alerts).toHaveLength(1)
|
|
492
|
+
expect(alerts[0]).toContain('13 Jul 11:55pm to 14 Jul 12:05am AEST')
|
|
493
|
+
})
|
|
494
|
+
|
|
355
495
|
it('does not alert for msg-edit (cosmetic) scopes', async () => {
|
|
356
496
|
const clock = new TestClock()
|
|
357
497
|
const alerts: string[] = []
|
|
@@ -29,14 +29,15 @@
|
|
|
29
29
|
* "was banned from X to Y" alert when the window CLOSES (or as soon as the
|
|
30
30
|
* operator chat becomes reachable again).
|
|
31
31
|
*
|
|
32
|
-
* STATUS OF THE IMMEDIATE PATH (
|
|
33
|
-
*
|
|
34
|
-
* The immediate-delivery branch is present
|
|
35
|
-
*
|
|
36
|
-
*
|
|
37
|
-
*
|
|
38
|
-
*
|
|
39
|
-
*
|
|
32
|
+
* STATUS OF THE IMMEDIATE PATH (live since #3111)
|
|
33
|
+
* -----------------------------------------------
|
|
34
|
+
* The immediate-delivery branch is present, unit-tested, AND now fires in
|
|
35
|
+
* production: as of #3111 a chat-scoped 429 opens ONLY that chat's window (no
|
|
36
|
+
* coincident `global`), so when a ban covers `chat:<other>` while the operator's
|
|
37
|
+
* own chat is clear, the alert is delivered IMMEDIATELY rather than deferred to
|
|
38
|
+
* close. A genuinely global 429 still opens `global`, so the operator chat is
|
|
39
|
+
* "covered" and the alert defers to close — once per incident, all scopes
|
|
40
|
+
* coalesced into one card (M1, #3112).
|
|
40
41
|
*
|
|
41
42
|
* At-most-once is anchored by the persisted `alertedAt` (survives restart). The
|
|
42
43
|
* deferred close-alert is best-effort at-least-once: if the gateway is down for
|
|
@@ -53,6 +54,7 @@
|
|
|
53
54
|
|
|
54
55
|
import type { Clock, SendGateStats } from './send-gate.js'
|
|
55
56
|
import type { FloodWindowRecord } from './flood-circuit-breaker.js'
|
|
57
|
+
import { fmtLocalClock, fmtLocalDate, localDay, tzAbbrev } from './shared/local-time.js'
|
|
56
58
|
|
|
57
59
|
/** One-line, human-scannable summary of the gate counters + global fill. */
|
|
58
60
|
export function formatStatsLine(stats: SendGateStats): string {
|
|
@@ -153,6 +155,14 @@ export interface FloodWindowObserverConfig {
|
|
|
153
155
|
operatorChatId?: () => string | undefined
|
|
154
156
|
/** Ms a window must be open before it earns an alert. Default 60_000. */
|
|
155
157
|
alertThresholdMs?: number
|
|
158
|
+
/**
|
|
159
|
+
* IANA timezone (e.g. `Australia/Melbourne`) for rendering the operator-facing
|
|
160
|
+
* alert timestamps in local wall-clock time instead of UTC. The gateway sources
|
|
161
|
+
* this from `SWITCHROOM_TIMEZONE ?? TZ` (the same env the config cascade bakes
|
|
162
|
+
* into every agent service — see `src/config/timezone.ts`). Defaults to `'UTC'`
|
|
163
|
+
* so a missing tz degrades to the previous behaviour rather than throwing.
|
|
164
|
+
*/
|
|
165
|
+
tz?: string
|
|
156
166
|
}
|
|
157
167
|
|
|
158
168
|
export interface FloodWindowObserver {
|
|
@@ -160,11 +170,44 @@ export interface FloodWindowObserver {
|
|
|
160
170
|
tick(): Promise<void>
|
|
161
171
|
}
|
|
162
172
|
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
173
|
+
// ─── Operator-facing local-time formatting ──────────────────────────────────
|
|
174
|
+
// Operator alerts (openAlertText / closeAlertText) render timestamps in the
|
|
175
|
+
// operator's CONFIGURED timezone, not UTC — a "was banned from 9:39am to 9:46am
|
|
176
|
+
// AEST" reads at a glance where a raw `2026-07-12T23:39:24Z UTC` does not. The
|
|
177
|
+
// machine-facing snapshot LOG lines (config.log → gateway-supervisor.log) keep
|
|
178
|
+
// their epoch/ISO wording; only the chat/card text is localized.
|
|
179
|
+
//
|
|
180
|
+
// The four wall-clock primitives (fmtLocalClock / fmtLocalDate / localDay /
|
|
181
|
+
// tzAbbrev) now live in shared/local-time.ts so the humanized LLM-error card
|
|
182
|
+
// renders timestamps through the SAME source of truth. The two composers below
|
|
183
|
+
// (fmtLocalStamp / fmtLocalRange) stay here — they are this module's own wording.
|
|
184
|
+
|
|
185
|
+
/**
|
|
186
|
+
* A single operator-facing local timestamp with tz suffix, e.g. `9:46am AEST`.
|
|
187
|
+
* When `refMs` is on a different local calendar day (or omitted), the date is
|
|
188
|
+
* prepended (`12 Jul 11:52pm AEST`) so a day-spanning window is unambiguous.
|
|
189
|
+
*/
|
|
190
|
+
function fmtLocalStamp(ms: number, tz: string, refMs?: number): string {
|
|
191
|
+
const clock = fmtLocalClock(ms, tz)
|
|
192
|
+
const sameDay = refMs != null && localDay(ms, tz) === localDay(refMs, tz)
|
|
193
|
+
const body = sameDay ? clock : `${fmtLocalDate(ms, tz)} ${clock}`
|
|
194
|
+
return `${body} ${tzAbbrev(ms, tz)}`
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
/**
|
|
198
|
+
* A local time RANGE with a single tz suffix, e.g. `9:39am to 9:46am AEST`.
|
|
199
|
+
* Same-local-day windows drop the redundant start date; day-spanning windows
|
|
200
|
+
* carry the date on each end (`12 Jul 11:59pm to 13 Jul 12:04am AEST`).
|
|
201
|
+
*/
|
|
202
|
+
function fmtLocalRange(startMs: number, endMs: number, tz: string): string {
|
|
203
|
+
const sameDay = localDay(startMs, tz) === localDay(endMs, tz)
|
|
204
|
+
if (sameDay) {
|
|
205
|
+
return `${fmtLocalClock(startMs, tz)} to ${fmtLocalClock(endMs, tz)} ${tzAbbrev(endMs, tz)}`
|
|
206
|
+
}
|
|
207
|
+
return (
|
|
208
|
+
`${fmtLocalDate(startMs, tz)} ${fmtLocalClock(startMs, tz)} to ` +
|
|
209
|
+
`${fmtLocalDate(endMs, tz)} ${fmtLocalClock(endMs, tz)} ${tzAbbrev(endMs, tz)}`
|
|
210
|
+
)
|
|
168
211
|
}
|
|
169
212
|
|
|
170
213
|
function fmtDur(ms: number): string {
|
|
@@ -186,6 +229,7 @@ export function createFloodWindowObserver(
|
|
|
186
229
|
config: FloodWindowObserverConfig,
|
|
187
230
|
): FloodWindowObserver {
|
|
188
231
|
const alertThresholdMs = config.alertThresholdMs ?? 60_000
|
|
232
|
+
const tz = config.tz ?? 'UTC'
|
|
189
233
|
// Full records seen on the previous tick, keyed by scope (close detection).
|
|
190
234
|
let lastSeen = new Map<string, FloodWindowRecord>()
|
|
191
235
|
// First tick after boot: any window already present was loaded from disk
|
|
@@ -232,7 +276,7 @@ export function createFloodWindowObserver(
|
|
|
232
276
|
const openFor = fmtDur(now - w.observedAt)
|
|
233
277
|
return (
|
|
234
278
|
`⚠️ Telegram flood ban active (scope \`${w.scopeKey}\`). ` +
|
|
235
|
-
`Open for ${openFor}, expected to clear at ${
|
|
279
|
+
`Open for ${openFor}, expected to clear at ${fmtLocalStamp(w.untilTs, tz, now)}. ` +
|
|
236
280
|
`Outbound to that scope is being suppressed.`
|
|
237
281
|
)
|
|
238
282
|
}
|
|
@@ -250,7 +294,7 @@ export function createFloodWindowObserver(
|
|
|
250
294
|
const plural = recs.length > 1 ? 's' : ''
|
|
251
295
|
return (
|
|
252
296
|
`⚠️ Telegram flood ban cleared (scope${plural} ${scopes}). ` +
|
|
253
|
-
`The bot was banned from ${
|
|
297
|
+
`The bot was banned from ${fmtLocalRange(observedAt, untilTs, tz)} ` +
|
|
254
298
|
`(~${fmtDur(untilTs - observedAt)}). Some outbound messages during that ` +
|
|
255
299
|
`window were suppressed.`
|
|
256
300
|
)
|
|
@@ -312,11 +356,12 @@ export function createFloodWindowObserver(
|
|
|
312
356
|
if (w.alertedAt != null) continue
|
|
313
357
|
if (now - w.observedAt < alertThresholdMs) continue
|
|
314
358
|
if (operatorReachable && !coversOperator(w, operatorChatId)) {
|
|
315
|
-
// NOTE (M1, #3112): this immediate-delivery branch is
|
|
316
|
-
//
|
|
317
|
-
//
|
|
318
|
-
//
|
|
319
|
-
//
|
|
359
|
+
// NOTE (M1, #3112 / #3111): this immediate-delivery branch is now LIVE
|
|
360
|
+
// for recorder-produced state. Since #3111 a chat-scoped 429 opens only
|
|
361
|
+
// that chat's window (no coincident `global`), so when a ban covers a
|
|
362
|
+
// chat OTHER than the operator's, `operatorReachable` is true and the
|
|
363
|
+
// alert is delivered here rather than deferred to close. A genuinely
|
|
364
|
+
// global 429 still opens `global`, keeping the operator "covered".
|
|
320
365
|
// NOTE (L1, #3112): send-first, then persist `alertedAt`. A crash between
|
|
321
366
|
// the delivered card and the disk write re-alerts on restart — a benign
|
|
322
367
|
// duplicate. Deliberate: send-first guarantees an alert is never LOST,
|