switchroom 0.18.17 → 0.18.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-scheduler/index.js +13 -0
- package/dist/auth-broker/index.js +13 -0
- package/dist/cli/notion-write-pretool.mjs +13 -0
- package/dist/cli/switchroom.js +605 -479
- package/dist/host-control/main.js +17 -1
- package/dist/vault/approvals/kernel-server.js +13 -0
- package/dist/vault/broker/server.js +13 -0
- package/package.json +1 -1
- package/telegram-plugin/bridge/bridge.ts +7 -1
- package/telegram-plugin/dist/bridge/bridge.js +26 -1
- package/telegram-plugin/dist/gateway/gateway.js +1544 -619
- package/telegram-plugin/dist/server.js +32 -1
- package/telegram-plugin/fleet-fallback-resume.ts +26 -3
- package/telegram-plugin/format.ts +137 -213
- package/telegram-plugin/gateway/approval-hold.ts +49 -0
- package/telegram-plugin/gateway/bridge-dead-watchdog.ts +61 -18
- package/telegram-plugin/gateway/gateway.ts +399 -85
- package/telegram-plugin/gateway/linear-activity.ts +20 -4
- package/telegram-plugin/gateway/outbound-send-path.ts +9 -7
- package/telegram-plugin/gateway/premium-recovery-wiring.ts +122 -0
- package/telegram-plugin/gateway/session-model-file.ts +103 -0
- package/telegram-plugin/gateway/tier-downgrade-wiring.ts +121 -0
- package/telegram-plugin/gateway/unhandled-rejection-policy.ts +14 -1
- package/telegram-plugin/llm-error-present.ts +474 -0
- package/telegram-plugin/operator-events.ts +7 -1
- package/telegram-plugin/permission-title.ts +172 -10
- package/telegram-plugin/premium-recovery.ts +101 -0
- package/telegram-plugin/raw-error-scrub.ts +73 -0
- package/telegram-plugin/retry-api-call.ts +8 -2
- package/telegram-plugin/send-gate-degraded.test.ts +152 -1
- package/telegram-plugin/send-gate-observability.test.ts +140 -0
- package/telegram-plugin/send-gate-observability.ts +65 -20
- package/telegram-plugin/send-gate.test.ts +143 -1
- package/telegram-plugin/send-gate.ts +212 -19
- package/telegram-plugin/session-tail.ts +16 -0
- package/telegram-plugin/shared/local-time.ts +69 -0
- package/telegram-plugin/stream-reply-handler.ts +5 -14
- package/telegram-plugin/tests/approval-hold-harness.ts +6 -6
- package/telegram-plugin/tests/approval-hold-outcome.test.ts +10 -2
- package/telegram-plugin/tests/bridge-dead-watchdog.test.ts +61 -0
- package/telegram-plugin/tests/fleet-fallback-resume.test.ts +39 -0
- package/telegram-plugin/tests/flood-windows-persistence.test.ts +3 -2
- package/telegram-plugin/tests/format-consistency.test.ts +68 -53
- package/telegram-plugin/tests/formatting-parse-regression.test.ts +5 -6
- package/telegram-plugin/tests/formatting-torture-set.ts +1 -1
- package/telegram-plugin/tests/gateway-outbound-redact.test.ts +26 -0
- package/telegram-plugin/tests/linear-create-issue.test.ts +30 -2
- package/telegram-plugin/tests/llm-error-present.test.ts +481 -0
- package/telegram-plugin/tests/outbound-send-path.test.ts +4 -3
- package/telegram-plugin/tests/paragraph-normalizer.test.ts +42 -100
- package/telegram-plugin/tests/permission-title.test.ts +167 -4
- package/telegram-plugin/tests/premium-recovery-wiring.test.ts +150 -0
- package/telegram-plugin/tests/premium-recovery.test.ts +165 -0
- package/telegram-plugin/tests/reaction-gate-routing.test.ts +6 -1
- package/telegram-plugin/tests/retry-api-call.test.ts +21 -0
- package/telegram-plugin/tests/stream-reply-handler.test.ts +9 -12
- package/telegram-plugin/tests/telegram-format.test.ts +86 -31
- package/telegram-plugin/tests/tier-downgrade-wiring.test.ts +165 -0
- package/telegram-plugin/tests/tier-downgrade.test.ts +141 -0
- package/telegram-plugin/tests/turn-flush-safety.test.ts +17 -21
- package/telegram-plugin/tests/unhandled-rejection-policy.test.ts +27 -1
- package/telegram-plugin/tests/worker-activity-feed.test.ts +5 -2
- package/telegram-plugin/tests/worker-feed-coalesce.test.ts +492 -0
- package/telegram-plugin/tier-downgrade.ts +198 -0
- package/telegram-plugin/tool-activity-summary.ts +99 -0
- package/telegram-plugin/turn-flush-safety.ts +4 -3
- package/telegram-plugin/worker-activity-feed.ts +509 -409
|
@@ -304,6 +304,21 @@ export interface SendGateConfig {
|
|
|
304
304
|
* `initialWindows` applied at construction (those already came from disk).
|
|
305
305
|
*/
|
|
306
306
|
onWindowOpen?: (scopeKey: string, untilTs: number) => void
|
|
307
|
+
/**
|
|
308
|
+
* Conservative flood-scope posture (#3111). When `false` (the DEFAULT), a 429
|
|
309
|
+
* on a call bound to a specific chat opens ONLY that chat's (and, for groups,
|
|
310
|
+
* that group's) flood window — NOT a blanket `global` window — so a genuinely
|
|
311
|
+
* chat-scoped 429 (e.g. a per-group 20/min limit) cannot fail-fast CRITICAL
|
|
312
|
+
* sends to every other chat. `global` is opened only when the 429 carries no
|
|
313
|
+
* finer scope (no `chat_id`), i.e. the evidence is genuinely global.
|
|
314
|
+
*
|
|
315
|
+
* When `true`, the pre-#3111 posture is restored: EVERY 429 additionally opens
|
|
316
|
+
* the `global` window. Telegram 429s are per-bot-token, so a flood often IS
|
|
317
|
+
* global; an operator who prefers to over-suppress rather than risk a missed
|
|
318
|
+
* global ban can opt back in via
|
|
319
|
+
* `SWITCHROOM_TG_SEND_GATE_CONSERVATIVE_GLOBAL=1`.
|
|
320
|
+
*/
|
|
321
|
+
conservativeGlobalFloodScope?: boolean
|
|
307
322
|
}
|
|
308
323
|
|
|
309
324
|
/**
|
|
@@ -462,22 +477,58 @@ export interface SendGate {
|
|
|
462
477
|
* by PR 2 on a 429 and at boot from the persisted `flood-wait.json`.
|
|
463
478
|
*/
|
|
464
479
|
openFloodWindow(scopeKey: string, untilTs: number): void
|
|
480
|
+
/**
|
|
481
|
+
* Open the SCOPE-PRECISE flood windows a 429 on a call with these `opts`
|
|
482
|
+
* implies (#3111). Used by the gateway's `onFloodWait` hook, which sees a
|
|
483
|
+
* SHORT slept-and-retried 429 the gate's own `FLOOD_WAIT_ACTIVE` catch never
|
|
484
|
+
* observes. Opens `global` only when `opts` carries no `chat_id` (genuinely
|
|
485
|
+
* global evidence) or the conservative posture is on — so a chat-scoped 429
|
|
486
|
+
* does not suppress unrelated chats. A pure no-op when the gate is disabled.
|
|
487
|
+
*/
|
|
488
|
+
openScopedFloodWindows(opts: SendGateOpts | undefined, untilTs: number): void
|
|
465
489
|
/** Snapshot of counters + current bucket fill. */
|
|
466
490
|
stats(): SendGateStats
|
|
467
491
|
}
|
|
468
492
|
|
|
469
493
|
const GROUP_TYPES = new Set<ChatType>(['group', 'supergroup'])
|
|
470
494
|
|
|
495
|
+
/**
|
|
496
|
+
* Compile-time default rate limits for the send gate's token buckets and edit
|
|
497
|
+
* floor — the values `createSendGate` falls back to when a field is omitted.
|
|
498
|
+
*
|
|
499
|
+
* Exposed so the config plumbing (`channels.telegram.send_gate.*` →
|
|
500
|
+
* `SWITCHROOM_TG_SEND_GATE_*` env → `sendGateConfigFromEnv`) defaults to the
|
|
501
|
+
* SAME numbers, and so a test can PIN them: a future accidental change to any
|
|
502
|
+
* default fails that pin. Changing a value here is a real behaviour change for
|
|
503
|
+
* every install — do it deliberately, never as a drive-by edit.
|
|
504
|
+
*/
|
|
505
|
+
export const SEND_GATE_DEFAULTS = {
|
|
506
|
+
/** Global bucket sustained rate (tokens/sec). */
|
|
507
|
+
globalPerSec: 25,
|
|
508
|
+
/** Global burst capacity (headroom under Telegram's ~30/s). */
|
|
509
|
+
globalBurst: 4,
|
|
510
|
+
/** Per-chat sustained rate (tokens/sec). */
|
|
511
|
+
perChatPerSec: 1,
|
|
512
|
+
/** Per-chat burst capacity. */
|
|
513
|
+
perChatBurst: 3,
|
|
514
|
+
/** Per-group sustained rate (tokens/min). */
|
|
515
|
+
perGroupPerMin: 18,
|
|
516
|
+
/** Per-group burst capacity (headroom under 20/min). */
|
|
517
|
+
perGroupBurst: 2,
|
|
518
|
+
/** Minimum ms between edits of the same message_id. */
|
|
519
|
+
editFloorMs: 1500,
|
|
520
|
+
} as const
|
|
521
|
+
|
|
471
522
|
export function createSendGate(config: SendGateConfig): SendGate {
|
|
472
523
|
const enabled = config.enabled
|
|
473
524
|
const clock = config.clock ?? systemClock
|
|
474
|
-
const globalPerSec = config.globalPerSec ??
|
|
475
|
-
const globalBurst = config.globalBurst ??
|
|
476
|
-
const perChatPerSec = config.perChatPerSec ??
|
|
477
|
-
const perChatBurst = config.perChatBurst ??
|
|
478
|
-
const perGroupPerMin = config.perGroupPerMin ??
|
|
479
|
-
const perGroupBurst = config.perGroupBurst ??
|
|
480
|
-
const editFloorMs = config.editFloorMs ??
|
|
525
|
+
const globalPerSec = config.globalPerSec ?? SEND_GATE_DEFAULTS.globalPerSec
|
|
526
|
+
const globalBurst = config.globalBurst ?? SEND_GATE_DEFAULTS.globalBurst
|
|
527
|
+
const perChatPerSec = config.perChatPerSec ?? SEND_GATE_DEFAULTS.perChatPerSec
|
|
528
|
+
const perChatBurst = config.perChatBurst ?? SEND_GATE_DEFAULTS.perChatBurst
|
|
529
|
+
const perGroupPerMin = config.perGroupPerMin ?? SEND_GATE_DEFAULTS.perGroupPerMin
|
|
530
|
+
const perGroupBurst = config.perGroupBurst ?? SEND_GATE_DEFAULTS.perGroupBurst
|
|
531
|
+
const editFloorMs = config.editFloorMs ?? SEND_GATE_DEFAULTS.editFloorMs
|
|
481
532
|
const messageStateTtlMs = config.messageStateTtlMs ?? 60_000
|
|
482
533
|
const maxMessageStates = config.maxMessageStates ?? 5_000
|
|
483
534
|
const usefulTtlMs = config.usefulTtlMs ?? 120_000
|
|
@@ -485,6 +536,7 @@ export function createSendGate(config: SendGateConfig): SendGate {
|
|
|
485
536
|
const criticalJitterMaxMs = config.criticalJitterMaxMs ?? 250
|
|
486
537
|
const jitter = config.jitter ?? Math.random
|
|
487
538
|
const onWindowOpen = config.onWindowOpen
|
|
539
|
+
const conservativeGlobalFloodScope = config.conservativeGlobalFloodScope ?? false
|
|
488
540
|
|
|
489
541
|
const counters: BucketCounters = {
|
|
490
542
|
sent: 0,
|
|
@@ -597,18 +649,35 @@ export function createSendGate(config: SendGateConfig): SendGate {
|
|
|
597
649
|
}
|
|
598
650
|
|
|
599
651
|
/**
|
|
600
|
-
* Open
|
|
601
|
-
*
|
|
602
|
-
*
|
|
603
|
-
*
|
|
604
|
-
*
|
|
652
|
+
* Open the flood windows a 429 on this call implies at `untilTs`, at the
|
|
653
|
+
* FINEST scope the evidence supports (#3111). Called when a wrapped send
|
|
654
|
+
* surfaces a `FLOOD_WAIT_ACTIVE` (a real 429 or a pre-call short-circuit), and
|
|
655
|
+
* by the gateway's `onFloodWait` hook for a SHORT slept-and-retried 429 (via
|
|
656
|
+
* the public `openScopedFloodWindows`), so the ban is remembered at the finest
|
|
657
|
+
* scope we know (part3-design §7).
|
|
658
|
+
*
|
|
659
|
+
* SCOPE PRECISION (#3111): a call bound to a specific `chat_id` opens ONLY that
|
|
660
|
+
* chat's (and, for groups, that group's) window — NOT a blanket `global`
|
|
661
|
+
* window that would suppress unrelated chats. `global` is opened only when the
|
|
662
|
+
* call carries NO finer scope (no `chat_id`) — i.e. the evidence is genuinely
|
|
663
|
+
* global — OR when `conservativeGlobalFloodScope` restores the pre-#3111
|
|
664
|
+
* always-open-global posture. Telegram 429s are per-bot-token so a flood is
|
|
665
|
+
* OFTEN global, but a genuinely chat-scoped 429 (e.g. a per-group 20/min
|
|
666
|
+
* limit) must not fail-fast CRITICAL sends to every other chat. The
|
|
667
|
+
* per-message (`msg-edit:`) window is additive whenever a `messageId` is
|
|
668
|
+
* present. Monotonic/idempotent: `applyWindow` only ever extends.
|
|
605
669
|
*/
|
|
606
670
|
function openScopedWindowsForOpts(opts: SendGateOpts | undefined, untilTs: number): void {
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
671
|
+
const hasChatScope = opts?.chat_id != null && opts.chat_id !== ''
|
|
672
|
+
// Open `global` only when the 429 has no finer scope (genuinely global), or
|
|
673
|
+
// when the operator opted into the conservative always-global posture.
|
|
674
|
+
if (!hasChatScope || conservativeGlobalFloodScope) {
|
|
675
|
+
applyWindow('global', untilTs, true)
|
|
676
|
+
}
|
|
677
|
+
if (hasChatScope) {
|
|
678
|
+
applyWindow(`chat:${opts!.chat_id}`, untilTs, true)
|
|
679
|
+
if (opts!.chatType && GROUP_TYPES.has(opts!.chatType)) {
|
|
680
|
+
applyWindow(`group:${opts!.chat_id}`, untilTs, true)
|
|
612
681
|
}
|
|
613
682
|
}
|
|
614
683
|
if (opts?.messageId != null) {
|
|
@@ -1067,7 +1136,18 @@ export function createSendGate(config: SendGateConfig): SendGate {
|
|
|
1067
1136
|
}
|
|
1068
1137
|
}
|
|
1069
1138
|
|
|
1070
|
-
|
|
1139
|
+
/**
|
|
1140
|
+
* Public, scope-precise window opener for the gateway's `onFloodWait` hook
|
|
1141
|
+
* (#3111). Flag-OFF is a pure no-op (mirrors `openFloodWindow`'s M3 guard) so
|
|
1142
|
+
* a 429 never mutates suppression or writes `flood-windows.json` while the gate
|
|
1143
|
+
* is disabled.
|
|
1144
|
+
*/
|
|
1145
|
+
function openScopedFloodWindows(opts: SendGateOpts | undefined, untilTs: number): void {
|
|
1146
|
+
if (!enabled) return
|
|
1147
|
+
openScopedWindowsForOpts(opts, untilTs)
|
|
1148
|
+
}
|
|
1149
|
+
|
|
1150
|
+
return { gate, openFloodWindow, openScopedFloodWindows, stats }
|
|
1071
1151
|
}
|
|
1072
1152
|
|
|
1073
1153
|
/**
|
|
@@ -1081,6 +1161,119 @@ export function createSendGate(config: SendGateConfig): SendGate {
|
|
|
1081
1161
|
export function sendGateEnabledFromEnv(env: NodeJS.ProcessEnv = process.env): boolean {
|
|
1082
1162
|
const v = env.SWITCHROOM_TELEGRAM_SEND_GATE
|
|
1083
1163
|
if (v == null) return true
|
|
1164
|
+
return !isOffValue(v)
|
|
1165
|
+
}
|
|
1166
|
+
|
|
1167
|
+
/** Shared parse for the default-on kill-switch grammar: `0/false/off/no`. */
|
|
1168
|
+
function isOffValue(v: string): boolean {
|
|
1084
1169
|
const t = v.trim().toLowerCase()
|
|
1085
|
-
return
|
|
1170
|
+
return t === '0' || t === 'false' || t === 'off' || t === 'no'
|
|
1171
|
+
}
|
|
1172
|
+
|
|
1173
|
+
/** Parse a strictly-positive finite number env var; blank/NaN/≤0 → undefined. */
|
|
1174
|
+
function parsePositiveNumber(raw: string | undefined): number | undefined {
|
|
1175
|
+
if (raw == null || raw.trim() === '') return undefined
|
|
1176
|
+
const n = Number(raw)
|
|
1177
|
+
return Number.isFinite(n) && n > 0 ? n : undefined
|
|
1178
|
+
}
|
|
1179
|
+
|
|
1180
|
+
/** Parse a positive integer env var; blank/NaN/non-int/≤0 → undefined. */
|
|
1181
|
+
function parsePositiveInt(raw: string | undefined): number | undefined {
|
|
1182
|
+
if (raw == null || raw.trim() === '') return undefined
|
|
1183
|
+
const n = Number(raw)
|
|
1184
|
+
return Number.isInteger(n) && n > 0 ? n : undefined
|
|
1185
|
+
}
|
|
1186
|
+
|
|
1187
|
+
/** Parse a non-negative integer env var; blank/NaN/non-int/<0 → undefined. */
|
|
1188
|
+
function parseNonNegativeInt(raw: string | undefined): number | undefined {
|
|
1189
|
+
if (raw == null || raw.trim() === '') return undefined
|
|
1190
|
+
const n = Number(raw)
|
|
1191
|
+
return Number.isInteger(n) && n >= 0 ? n : undefined
|
|
1192
|
+
}
|
|
1193
|
+
|
|
1194
|
+
/**
|
|
1195
|
+
* Parse a boolean env flag using the repo's on/off grammar
|
|
1196
|
+
* (`1/true/on/yes` → true, `0/false/off/no` → false). Blank or unrecognised →
|
|
1197
|
+
* undefined so the caller keeps its built-in default (#3111).
|
|
1198
|
+
*/
|
|
1199
|
+
function parseBoolFlag(raw: string | undefined): boolean | undefined {
|
|
1200
|
+
if (raw == null || raw.trim() === '') return undefined
|
|
1201
|
+
const t = raw.trim().toLowerCase()
|
|
1202
|
+
if (t === '1' || t === 'true' || t === 'on' || t === 'yes') return true
|
|
1203
|
+
if (isOffValue(t)) return false
|
|
1204
|
+
return undefined
|
|
1205
|
+
}
|
|
1206
|
+
|
|
1207
|
+
/**
|
|
1208
|
+
* The resolved rate-limit + enable slice `createSendGate` consumes, sourced
|
|
1209
|
+
* from env (which the scaffold populates from `channels.telegram.send_gate.*`).
|
|
1210
|
+
* Only fields the operator SET are present; an unset rate field is absent so
|
|
1211
|
+
* `createSendGate` applies its `SEND_GATE_DEFAULTS` fallback (unset ⇒ today's
|
|
1212
|
+
* exact behaviour).
|
|
1213
|
+
*/
|
|
1214
|
+
export type SendGateEnvConfig = Pick<SendGateConfig, 'enabled'> &
|
|
1215
|
+
Partial<
|
|
1216
|
+
Pick<
|
|
1217
|
+
SendGateConfig,
|
|
1218
|
+
| 'globalPerSec'
|
|
1219
|
+
| 'globalBurst'
|
|
1220
|
+
| 'perChatPerSec'
|
|
1221
|
+
| 'perChatBurst'
|
|
1222
|
+
| 'perGroupPerMin'
|
|
1223
|
+
| 'perGroupBurst'
|
|
1224
|
+
| 'editFloorMs'
|
|
1225
|
+
| 'conservativeGlobalFloodScope'
|
|
1226
|
+
>
|
|
1227
|
+
>
|
|
1228
|
+
|
|
1229
|
+
/**
|
|
1230
|
+
* Resolve the send gate's `enabled` flag + tunable rate limits from env.
|
|
1231
|
+
*
|
|
1232
|
+
* ENABLED PRECEDENCE (least-surprising, documented):
|
|
1233
|
+
* 1. `SWITCHROOM_TELEGRAM_SEND_GATE` — the operator break-glass valve. When
|
|
1234
|
+
* EXPLICITLY set it ALWAYS wins (unchanged kill-switch semantics), on or
|
|
1235
|
+
* off, regardless of config. An operator disabling the gate in an incident
|
|
1236
|
+
* must not be overridden by YAML.
|
|
1237
|
+
* 2. `SWITCHROOM_TG_SEND_GATE_ENABLED` — from `channels.telegram.send_gate.
|
|
1238
|
+
* enabled`. Decides only when the valve above is unset.
|
|
1239
|
+
* 3. Both unset ⇒ `true` (the gate is ON by default, unchanged).
|
|
1240
|
+
*
|
|
1241
|
+
* The rate knobs come solely from config (there was never an env override for
|
|
1242
|
+
* them). Config values are validated LOUDLY at load by the Zod schema; this
|
|
1243
|
+
* parse is a defensive net — a hand-set malformed env falls back to undefined
|
|
1244
|
+
* so `createSendGate` uses its built-in default rather than wedging sends.
|
|
1245
|
+
*/
|
|
1246
|
+
export function sendGateConfigFromEnv(
|
|
1247
|
+
env: NodeJS.ProcessEnv = process.env,
|
|
1248
|
+
): SendGateEnvConfig {
|
|
1249
|
+
const killSwitch = env.SWITCHROOM_TELEGRAM_SEND_GATE
|
|
1250
|
+
let enabled: boolean
|
|
1251
|
+
if (killSwitch != null && killSwitch.trim() !== '') {
|
|
1252
|
+
enabled = !isOffValue(killSwitch)
|
|
1253
|
+
} else {
|
|
1254
|
+
const cfg = env.SWITCHROOM_TG_SEND_GATE_ENABLED
|
|
1255
|
+
enabled = cfg != null && cfg.trim() !== '' ? !isOffValue(cfg) : true
|
|
1256
|
+
}
|
|
1257
|
+
|
|
1258
|
+
const out: SendGateEnvConfig = { enabled }
|
|
1259
|
+
const globalPerSec = parsePositiveNumber(env.SWITCHROOM_TG_SEND_GATE_GLOBAL_PER_SEC)
|
|
1260
|
+
if (globalPerSec !== undefined) out.globalPerSec = globalPerSec
|
|
1261
|
+
const globalBurst = parsePositiveInt(env.SWITCHROOM_TG_SEND_GATE_GLOBAL_BURST)
|
|
1262
|
+
if (globalBurst !== undefined) out.globalBurst = globalBurst
|
|
1263
|
+
const perChatPerSec = parsePositiveNumber(env.SWITCHROOM_TG_SEND_GATE_PER_CHAT_PER_SEC)
|
|
1264
|
+
if (perChatPerSec !== undefined) out.perChatPerSec = perChatPerSec
|
|
1265
|
+
const perChatBurst = parsePositiveInt(env.SWITCHROOM_TG_SEND_GATE_PER_CHAT_BURST)
|
|
1266
|
+
if (perChatBurst !== undefined) out.perChatBurst = perChatBurst
|
|
1267
|
+
const perGroupPerMin = parsePositiveNumber(env.SWITCHROOM_TG_SEND_GATE_PER_GROUP_PER_MIN)
|
|
1268
|
+
if (perGroupPerMin !== undefined) out.perGroupPerMin = perGroupPerMin
|
|
1269
|
+
const perGroupBurst = parsePositiveInt(env.SWITCHROOM_TG_SEND_GATE_PER_GROUP_BURST)
|
|
1270
|
+
if (perGroupBurst !== undefined) out.perGroupBurst = perGroupBurst
|
|
1271
|
+
const editFloorMs = parseNonNegativeInt(env.SWITCHROOM_TG_SEND_GATE_EDIT_FLOOR_MS)
|
|
1272
|
+
if (editFloorMs !== undefined) out.editFloorMs = editFloorMs
|
|
1273
|
+
// #3111 break-glass: restore the pre-#3111 always-open-global flood posture.
|
|
1274
|
+
// Unset ⇒ scope-precise default (global opened only on genuinely global 429s).
|
|
1275
|
+
const conservativeGlobal = parseBoolFlag(env.SWITCHROOM_TG_SEND_GATE_CONSERVATIVE_GLOBAL)
|
|
1276
|
+
if (conservativeGlobal !== undefined) out.conservativeGlobalFloodScope = conservativeGlobal
|
|
1277
|
+
|
|
1278
|
+
return out
|
|
1086
1279
|
}
|
|
@@ -333,6 +333,22 @@ export function projectTranscriptLine(line: string): SessionEvent[] {
|
|
|
333
333
|
const message = obj.message as Record<string, unknown> | undefined
|
|
334
334
|
const content = message?.content as Array<Record<string, unknown>> | undefined
|
|
335
335
|
if (!Array.isArray(content)) return []
|
|
336
|
+
// #llm-error-surfacing — DETERMINISTIC TERMINAL SOURCE. Claude Code writes a
|
|
337
|
+
// usage-limit / API error as a SYNTHETIC ASSISTANT MESSAGE
|
|
338
|
+
// (`isApiErrorMessage: true`) whose `content[].text` is the raw error string
|
|
339
|
+
// ("You've hit your limit · resets … b'{\"type\":\"error\"…}'"). That text
|
|
340
|
+
// used to fan out to the reply/answer passthrough AND the turn-end "done"
|
|
341
|
+
// card as if it were the model's answer. It is NOT an answer — it is an
|
|
342
|
+
// error the operator-event pipeline (detectErrorInTranscriptLine →
|
|
343
|
+
// humanized card) owns. Suppress the text/thinking events here at the ONE
|
|
344
|
+
// authoritative source so neither downstream surface can relay the raw
|
|
345
|
+
// bytes; the humanized card is rendered independently from the same line.
|
|
346
|
+
if (obj.isApiErrorMessage === true) {
|
|
347
|
+
const mainModel = message?.model
|
|
348
|
+
return typeof mainModel === 'string' && !isModelSentinel(mainModel)
|
|
349
|
+
? [{ kind: 'model', model: mainModel }]
|
|
350
|
+
: []
|
|
351
|
+
}
|
|
336
352
|
const events: SessionEvent[] = []
|
|
337
353
|
// Live model capture: `message.model` is the exact resolved model that
|
|
338
354
|
// served THIS assistant API call. Emit it FIRST (before the content events)
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* shared/local-time.ts — operator-facing local wall-clock formatting.
|
|
3
|
+
*
|
|
4
|
+
* These four primitives render an epoch-ms instant in an operator's CONFIGURED
|
|
5
|
+
* IANA timezone (e.g. `Australia/Melbourne`) instead of UTC, so a card reading
|
|
6
|
+
* "clears ~4:52pm AEST" is legible at a glance where a raw `…T04:52:00Z UTC`
|
|
7
|
+
* is not. They were originally file-private in `send-gate-observability.ts`
|
|
8
|
+
* (#3084 / PR 3205); extracted here so BOTH the flood-window observer and the
|
|
9
|
+
* humanized LLM-error card (`llm-error-present.ts`) render timestamps through
|
|
10
|
+
* ONE source of truth rather than two drifting copies.
|
|
11
|
+
*
|
|
12
|
+
* Pure — no I/O, no clock, no env. Every function is total (never throws) and
|
|
13
|
+
* degrades gracefully: an unknown tz falls back to the offset form or the tz
|
|
14
|
+
* string itself. `tz='UTC'` reproduces the pre-extraction behaviour exactly, so
|
|
15
|
+
* `send-gate-observability`'s snapshot-locked output is byte-identical.
|
|
16
|
+
*/
|
|
17
|
+
|
|
18
|
+
/** Lowercased wall-clock time in `tz`, e.g. `9:39am`. */
|
|
19
|
+
export function fmtLocalClock(ms: number, tz: string): string {
|
|
20
|
+
return new Intl.DateTimeFormat('en-US', {
|
|
21
|
+
timeZone: tz,
|
|
22
|
+
hour: 'numeric',
|
|
23
|
+
minute: '2-digit',
|
|
24
|
+
hour12: true,
|
|
25
|
+
})
|
|
26
|
+
.format(new Date(ms))
|
|
27
|
+
.replace(/\s([AP])M$/, (_m, p: string) => `${p.toLowerCase()}m`)
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/** Compact local date, e.g. `12 Jul` — used only to disambiguate day-spanning windows. */
|
|
31
|
+
export function fmtLocalDate(ms: number, tz: string): string {
|
|
32
|
+
return new Intl.DateTimeFormat('en-GB', {
|
|
33
|
+
timeZone: tz,
|
|
34
|
+
day: 'numeric',
|
|
35
|
+
month: 'short',
|
|
36
|
+
}).format(new Date(ms))
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/** Calendar day (`YYYY-MM-DD`) in `tz`, for a timezone-correct "same day?" test. */
|
|
40
|
+
export function localDay(ms: number, tz: string): string {
|
|
41
|
+
return new Intl.DateTimeFormat('en-CA', {
|
|
42
|
+
timeZone: tz,
|
|
43
|
+
year: 'numeric',
|
|
44
|
+
month: '2-digit',
|
|
45
|
+
day: '2-digit',
|
|
46
|
+
}).format(new Date(ms))
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/**
|
|
50
|
+
* Short tz abbreviation for `tz` at `ms`, e.g. `AEST`, `EDT`, `BST`. ICU only
|
|
51
|
+
* surfaces the common alpha abbreviation for a locale whose region matches the
|
|
52
|
+
* zone, so we try a small locale list and take the first genuine abbreviation;
|
|
53
|
+
* zones with no common abbreviation (e.g. Asia/Kolkata) fall back to the offset
|
|
54
|
+
* form (`GMT+5:30`), and `UTC` stays `UTC`.
|
|
55
|
+
*/
|
|
56
|
+
export function tzAbbrev(ms: number, tz: string): string {
|
|
57
|
+
const at = new Date(ms)
|
|
58
|
+
for (const loc of ['en-US', 'en-AU', 'en-GB']) {
|
|
59
|
+
const v = new Intl.DateTimeFormat(loc, { timeZone: tz, timeZoneName: 'short' })
|
|
60
|
+
.formatToParts(at)
|
|
61
|
+
.find((p) => p.type === 'timeZoneName')?.value
|
|
62
|
+
if (v && !/^(?:GMT|UTC)/i.test(v)) return v
|
|
63
|
+
}
|
|
64
|
+
return (
|
|
65
|
+
new Intl.DateTimeFormat('en-US', { timeZone: tz, timeZoneName: 'short' })
|
|
66
|
+
.formatToParts(at)
|
|
67
|
+
.find((p) => p.type === 'timeZoneName')?.value ?? tz
|
|
68
|
+
)
|
|
69
|
+
}
|
|
@@ -171,14 +171,6 @@ export interface StreamReplyDeps {
|
|
|
171
171
|
* after normalizePunctuation. Optional for backward compat.
|
|
172
172
|
*/
|
|
173
173
|
stripExcessBold?: (text: string) => string
|
|
174
|
-
/**
|
|
175
|
-
* Insert a visible blank-line spacer into each prose `\n\n` gap so the rich
|
|
176
|
-
* GFM renderer shows a real empty line between paragraphs (the rich engine
|
|
177
|
-
* otherwise renders `\n\n` tight — the post-#2669 paragraph-spacing
|
|
178
|
-
* regression). Applied only on the rich path (never on `format:'text'`).
|
|
179
|
-
* Optional for backward compat; omitted → no spacers added.
|
|
180
|
-
*/
|
|
181
|
-
addParagraphSpacers?: (text: string) => string
|
|
182
174
|
/** Validates the chat id against the access list. Throws on deny. */
|
|
183
175
|
assertAllowedChat: (chatId: string) => void
|
|
184
176
|
/** Resolves the effective thread id (explicit, last-inbound, or undefined). */
|
|
@@ -362,12 +354,11 @@ export async function handleStreamReply(
|
|
|
362
354
|
// markdown→HTML / MarkdownV2 rendering happens here anymore — the raw
|
|
363
355
|
// text IS the wire payload.
|
|
364
356
|
const literalText = format === 'text'
|
|
365
|
-
//
|
|
366
|
-
//
|
|
367
|
-
//
|
|
368
|
-
//
|
|
369
|
-
let effectiveText: string =
|
|
370
|
-
!literalText && deps.addParagraphSpacers ? deps.addParagraphSpacers(rawText) : rawText
|
|
357
|
+
// No paragraph-spacer pass: the NBSP spacer (#2669) was removed in the
|
|
358
|
+
// follow-up because the Bot API 10.1 rich GFM renderer already shows a `\n\n`
|
|
359
|
+
// gap as one blank line — the spacer double-gapped every paragraph. The raw
|
|
360
|
+
// text (already normalized upstream) is the effective text on both paths.
|
|
361
|
+
let effectiveText: string = rawText
|
|
371
362
|
|
|
372
363
|
// Inline status-accent header (issue #320 fallback). Prepended so it
|
|
373
364
|
// leads the body. Since stream_reply callers pass the full text snapshot
|
|
@@ -42,6 +42,7 @@ import {
|
|
|
42
42
|
safeActionForRecord,
|
|
43
43
|
holdReasonFor,
|
|
44
44
|
heldRetryBackoffMs,
|
|
45
|
+
applyDeliveredHoldReset,
|
|
45
46
|
type UndeliverableMark,
|
|
46
47
|
type BlockedApprovalStore,
|
|
47
48
|
} from '../gateway/approval-hold.js'
|
|
@@ -280,12 +281,11 @@ export function createHarness(opts: { cap?: number; targets?: string[] } = {}):
|
|
|
280
281
|
const live = pending.get(requestId)
|
|
281
282
|
if (live && sent) {
|
|
282
283
|
live.cards.push({ chatId, messageId: sent.message_id })
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
live.startedAt = clock.now()
|
|
284
|
+
// Drive the SAME reset the gateway drives (#3128) — no private copy.
|
|
285
|
+
// Deleting `startedAt = now` from `applyDeliveredHoldReset` turns the
|
|
286
|
+
// `(d2)` full-window outcome test RED, because there is one shared
|
|
287
|
+
// implementation, not a mirror the harness silently compensates with.
|
|
288
|
+
if (applyDeliveredHoldReset(live, clock.now())) {
|
|
289
289
|
reconcile()
|
|
290
290
|
}
|
|
291
291
|
}
|
|
@@ -289,8 +289,16 @@ describe('gateway wiring — the leash', () => {
|
|
|
289
289
|
expect(sweep).not.toContain('if (now - v.startedAt > ttl)')
|
|
290
290
|
})
|
|
291
291
|
|
|
292
|
-
it('successful (re)delivery resets startedAt', () => {
|
|
293
|
-
|
|
292
|
+
it('successful (re)delivery resets startedAt through the SHARED reset (#3128)', () => {
|
|
293
|
+
// This pin is no longer the safety net for the reset — the behavioural `(d2)`
|
|
294
|
+
// test above is, and it now drives the same `applyDeliveredHoldReset` the
|
|
295
|
+
// gateway does, so deleting `startedAt = now` from that one shared function
|
|
296
|
+
// turns `(d2)` RED. This only guards the WIRING: that the gateway routes
|
|
297
|
+
// delivery through the shared reset and can't regrow a private inline copy
|
|
298
|
+
// that drifts from what the harness exercises.
|
|
299
|
+
expect(GATEWAY_SRC).toContain('applyDeliveredHoldReset(live, Date.now())')
|
|
300
|
+
// …and must NOT carry its own inline copy of the reset.
|
|
301
|
+
expect(GATEWAY_SRC).not.toContain('live.startedAt = Date.now()')
|
|
294
302
|
})
|
|
295
303
|
|
|
296
304
|
it('the missed-approvals re-offer no longer drops a card that never landed', () => {
|
|
@@ -99,6 +99,9 @@ function makeWatchdog(overrides: Partial<BridgeDeadWatchdogOpts> = {}) {
|
|
|
99
99
|
const state = { sessionAlive: true, shuttingDown: false, escalateResult: true }
|
|
100
100
|
const wd = createBridgeDeadWatchdog({
|
|
101
101
|
graceMs: 90_000,
|
|
102
|
+
// The gateway serves AGENT — only AGENT's own bridge drives the
|
|
103
|
+
// watchdog (#3086). Overridable per-test.
|
|
104
|
+
selfAgentName: AGENT,
|
|
102
105
|
isSessionAlive: () => state.sessionAlive,
|
|
103
106
|
isShuttingDown: () => state.shuttingDown,
|
|
104
107
|
escalate: (reason) => {
|
|
@@ -386,6 +389,64 @@ describe('cron / anonymous identity gating', () => {
|
|
|
386
389
|
})
|
|
387
390
|
})
|
|
388
391
|
|
|
392
|
+
// ─── #3086: secondary/relay identity gating ──────────────────────────────────
|
|
393
|
+
//
|
|
394
|
+
// Repro of the klanker incident: a short-lived `overlord-relay` IPC client
|
|
395
|
+
// registers and disconnects against klanker's gateway socket while klanker's
|
|
396
|
+
// OWN bridge is registered and actively replying. Pre-fix, isRealBridgeIdentity
|
|
397
|
+
// treated the relay (named, non-cron) as the real bridge, so its disconnect
|
|
398
|
+
// flipped bridgeRegistered=false, re-armed the grace window, and 90s later the
|
|
399
|
+
// watchdog SIGTERM'd a healthy container — killing the in-flight claude session.
|
|
400
|
+
describe('#3086 secondary/relay identity gating', () => {
|
|
401
|
+
const RELAY = 'overlord-relay' // a DIFFERENT agent name than AGENT
|
|
402
|
+
|
|
403
|
+
it('a relay disconnecting while the primary bridge is alive does NOT re-arm or escalate', () => {
|
|
404
|
+
const { wd, harness, escalations } = makeWatchdog() // selfAgentName === AGENT
|
|
405
|
+
wd.arm()
|
|
406
|
+
wd.noteBridgeRegistered(AGENT) // klanker's own bridge — healthy
|
|
407
|
+
expect(harness.liveCount()).toBe(0) // grace timer stood down
|
|
408
|
+
// The relay connects then disconnects (flood-ban workaround churn).
|
|
409
|
+
wd.noteBridgeRegistered(RELAY)
|
|
410
|
+
wd.noteBridgeDisconnected(RELAY)
|
|
411
|
+
// Must NOT re-arm: the primary bridge never went away.
|
|
412
|
+
expect(harness.liveCount()).toBe(0)
|
|
413
|
+
expect(escalations).toEqual([])
|
|
414
|
+
expect(wd.hasEscalated()).toBe(false)
|
|
415
|
+
})
|
|
416
|
+
|
|
417
|
+
it('a relay register does NOT satisfy the watchdog (only the primary bridge does)', () => {
|
|
418
|
+
const { wd, harness, escalations } = makeWatchdog()
|
|
419
|
+
wd.arm()
|
|
420
|
+
wd.noteBridgeRegistered(RELAY) // relay is not THIS gateway's bridge
|
|
421
|
+
expect(harness.liveCount()).toBe(1) // still armed — primary still missing
|
|
422
|
+
harness.fireLatest()
|
|
423
|
+
// Primary bridge genuinely absent → the real failure still escalates.
|
|
424
|
+
expect(escalations).toEqual([BRIDGE_DEAD_RESTART_REASON])
|
|
425
|
+
})
|
|
426
|
+
|
|
427
|
+
it('a genuine PRIMARY bridge death still escalates (fix does not over-suppress)', () => {
|
|
428
|
+
const { wd, harness, escalations } = makeWatchdog()
|
|
429
|
+
wd.arm()
|
|
430
|
+
wd.noteBridgeRegistered(AGENT) // primary registers
|
|
431
|
+
expect(harness.liveCount()).toBe(0)
|
|
432
|
+
wd.noteBridgeDisconnected(AGENT) // primary dies for good, never reconnects
|
|
433
|
+
expect(harness.liveCount()).toBe(1) // re-armed
|
|
434
|
+
harness.fireLatest()
|
|
435
|
+
expect(escalations).toEqual([BRIDGE_DEAD_RESTART_REASON])
|
|
436
|
+
})
|
|
437
|
+
|
|
438
|
+
it('fallback: with no selfAgentName, any named non-cron client still counts (pre-#3086)', () => {
|
|
439
|
+
const { wd, harness, escalations } = makeWatchdog({ selfAgentName: '' })
|
|
440
|
+
wd.arm()
|
|
441
|
+
wd.noteBridgeRegistered(RELAY) // no identity to match against → treated as real
|
|
442
|
+
expect(harness.liveCount()).toBe(0) // satisfied the watchdog
|
|
443
|
+
wd.noteBridgeDisconnected(RELAY)
|
|
444
|
+
expect(harness.liveCount()).toBe(1) // re-armed
|
|
445
|
+
harness.fireLatest()
|
|
446
|
+
expect(escalations).toEqual([BRIDGE_DEAD_RESTART_REASON])
|
|
447
|
+
})
|
|
448
|
+
})
|
|
449
|
+
|
|
389
450
|
// ─── Crash-log tail ──────────────────────────────────────────────────────────
|
|
390
451
|
|
|
391
452
|
describe('readFreshCrashLogTail', () => {
|
|
@@ -122,6 +122,45 @@ describe('createFleetFallbackResumeGate — staleness guard', () => {
|
|
|
122
122
|
})
|
|
123
123
|
})
|
|
124
124
|
|
|
125
|
+
describe('createFleetFallbackResumeGate — peek / arm seams (tier-downgrade LOW-3 + LOW-9a)', () => {
|
|
126
|
+
it('peek() evaluates the verdict WITHOUT arming the single-flight latch', () => {
|
|
127
|
+
const clk = fakeClock()
|
|
128
|
+
const gate = createFleetFallbackResumeGate({ nowFn: clk.now })
|
|
129
|
+
// Peeking 'resume' must not record an arm time — a following peek/decide is
|
|
130
|
+
// still 'resume'. This is what lets the gateway do a fallible carrier write
|
|
131
|
+
// between the peek and the arm without leaving a phantom armed latch (LOW-3).
|
|
132
|
+
expect(gate.peek(clk.now())).toBe('resume')
|
|
133
|
+
expect(gate.inspect().lastResumedAtMs).toBe(Number.NEGATIVE_INFINITY)
|
|
134
|
+
expect(gate.peek(clk.now())).toBe('resume')
|
|
135
|
+
// The real restart never fires (write threw) → nothing armed → a genuine
|
|
136
|
+
// later swap still resumes.
|
|
137
|
+
expect(gate.decide(clk.now())).toBe('resume')
|
|
138
|
+
})
|
|
139
|
+
|
|
140
|
+
it('arm() commits the single-flight window so a later peek reports skip-inflight (LOW-9a)', () => {
|
|
141
|
+
const clk = fakeClock()
|
|
142
|
+
const gate = createFleetFallbackResumeGate({ nowFn: clk.now, singleFlightMs: 60_000 })
|
|
143
|
+
// Turn 1: peek 'resume' → do the write → arm.
|
|
144
|
+
expect(gate.peek(clk.now())).toBe('resume')
|
|
145
|
+
gate.arm()
|
|
146
|
+
// Turn 2 (concurrent, same process, 1s later): a resume restart is already
|
|
147
|
+
// armed, so peek reports skip-inflight → the gateway suppresses the give-up.
|
|
148
|
+
clk.advance(1_000)
|
|
149
|
+
expect(gate.peek(clk.now())).toBe('skip-inflight')
|
|
150
|
+
})
|
|
151
|
+
|
|
152
|
+
it('peek(skip-inflight/skip-stale) never arms; only arm() records the window', () => {
|
|
153
|
+
const clk = fakeClock()
|
|
154
|
+
const gate = createFleetFallbackResumeGate({ nowFn: clk.now })
|
|
155
|
+
// A stale peek must not arm (mirrors decide()'s stale behaviour).
|
|
156
|
+
expect(gate.peek(clk.now() - (DEFAULT_RESUME_MAX_AGE_MS + 1))).toBe('skip-stale')
|
|
157
|
+
expect(gate.inspect().lastResumedAtMs).toBe(Number.NEGATIVE_INFINITY)
|
|
158
|
+
// decide() remains equivalent to peek+arm-on-resume.
|
|
159
|
+
expect(gate.decide(clk.now())).toBe('resume')
|
|
160
|
+
expect(gate.peek(clk.now())).toBe('skip-inflight')
|
|
161
|
+
})
|
|
162
|
+
})
|
|
163
|
+
|
|
125
164
|
describe('createFleetFallbackResumeGate — reset / inspect seams', () => {
|
|
126
165
|
it('reset() clears the single-flight arm', () => {
|
|
127
166
|
const clk = fakeClock()
|
|
@@ -103,9 +103,10 @@ describe('#3084 scoped flood-window persistence', () => {
|
|
|
103
103
|
),
|
|
104
104
|
).rejects.toBe(floodErr)
|
|
105
105
|
|
|
106
|
-
// The window is on disk.
|
|
106
|
+
// The window is on disk. #3111: a chat-bound 429 persists ONLY the chat
|
|
107
|
+
// scope — no coincident `global` window that would suppress unrelated chats.
|
|
107
108
|
const persisted = readFloodWindows(winPath, 0)
|
|
108
|
-
expect(persisted.map((w) => w.scopeKey).sort()).toEqual(['chat:7'
|
|
109
|
+
expect(persisted.map((w) => w.scopeKey).sort()).toEqual(['chat:7'])
|
|
109
110
|
|
|
110
111
|
// Gate 2 (simulated restart): built BEFORE any outbound call from the
|
|
111
112
|
// persisted windows. A cosmetic send on chat:7 must still shed.
|