switchroom 0.18.11 → 0.18.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-scheduler/index.js +37 -5
- package/dist/auth-broker/index.js +116 -78
- package/dist/cli/hindsight-mental-model-pretool.mjs +39 -0
- package/dist/cli/ms-365-write-pretool.mjs +31 -8
- package/dist/cli/notion-write-pretool.mjs +38 -6
- package/dist/cli/skill-validate-pretool.mjs +144 -2847
- package/dist/cli/switchroom.js +3529 -4543
- package/dist/cli/ui/index.html +163 -17
- package/dist/host-control/main.js +605 -2847
- package/dist/vault/approvals/kernel-server.js +120 -13
- package/dist/vault/broker/server.js +259 -157
- package/package.json +3 -4
- package/profiles/_base/start.sh.hbs +65 -0
- package/profiles/_shared/vault-protocol.md.hbs +3 -1
- package/profiles/coding/CLAUDE.md.hbs +1 -1
- package/profiles/default/CLAUDE.md.hbs +2 -2
- package/profiles/executive-assistant/CLAUDE.md.hbs +1 -1
- package/profiles/health-coach/CLAUDE.md.hbs +1 -1
- package/telegram-plugin/bridge/bridge.ts +37 -0
- package/telegram-plugin/bridge/inbound-dedup.ts +101 -0
- package/telegram-plugin/dist/bridge/bridge.js +122 -4
- package/telegram-plugin/dist/gateway/gateway.js +4213 -3288
- package/telegram-plugin/dist/server.js +139 -5
- package/telegram-plugin/flood-circuit-breaker.ts +493 -21
- package/telegram-plugin/format.ts +19 -0
- package/telegram-plugin/gateway/approval-hold.ts +602 -0
- package/telegram-plugin/gateway/auth-command.ts +92 -2
- package/telegram-plugin/gateway/auth-loopback-relay.ts +670 -0
- package/telegram-plugin/gateway/boot-card.ts +12 -5
- package/telegram-plugin/gateway/callback-query-handlers.ts +88 -1
- package/telegram-plugin/gateway/config-approval-handler.ts +6 -1
- package/telegram-plugin/gateway/disconnect-flush.ts +19 -0
- package/telegram-plugin/gateway/dm-pin-sweep.test.ts +251 -0
- package/telegram-plugin/gateway/dm-pin-sweep.ts +178 -0
- package/telegram-plugin/gateway/gateway.ts +1695 -230
- package/telegram-plugin/gateway/hostd-dispatch.ts +23 -0
- package/telegram-plugin/gateway/idle-clear.ts +90 -6
- package/telegram-plugin/gateway/inbound-delivery-machine-shadow.ts +26 -5
- package/telegram-plugin/gateway/inject-handler.ts +8 -0
- package/telegram-plugin/gateway/ipc-protocol.ts +46 -3
- package/telegram-plugin/gateway/ipc-server.ts +43 -0
- package/telegram-plugin/gateway/mental-model-propose-resolve.ts +145 -37
- package/telegram-plugin/gateway/model-command.ts +9 -3
- package/telegram-plugin/gateway/pending-session-command.ts +13 -1
- package/telegram-plugin/gateway/permission-ttl-sweep.ts +66 -0
- package/telegram-plugin/gateway/pre-approval-check.ts +74 -0
- package/telegram-plugin/gateway/queued-card-store.ts +217 -0
- package/telegram-plugin/gateway/session-model-file.ts +26 -1
- package/telegram-plugin/gateway/turn-end-gate-backstop.ts +59 -0
- package/telegram-plugin/gateway/turn-end-gate.ts +95 -0
- package/telegram-plugin/gateway/turn-typing-loop.ts +10 -2
- package/telegram-plugin/gateway/unhandled-rejection-policy.ts +13 -0
- package/telegram-plugin/history.ts +51 -0
- package/telegram-plugin/hooks/dispatch-claim-scan.mjs +259 -0
- package/telegram-plugin/hooks/dispatch-claim-stop.mjs +129 -0
- package/telegram-plugin/hooks/hooks.json +9 -0
- package/telegram-plugin/inline-keyboard-callbacks.ts +303 -2
- package/telegram-plugin/model-unavailable.ts +41 -11
- package/telegram-plugin/operator-events.ts +23 -0
- package/telegram-plugin/outbound-field-redact.ts +69 -0
- package/telegram-plugin/package.json +0 -1
- package/telegram-plugin/permission-rule.ts +1 -0
- package/telegram-plugin/permission-title.ts +1 -0
- package/telegram-plugin/render/render.ts +32 -14
- package/telegram-plugin/retry-api-call.ts +212 -2
- package/telegram-plugin/scoped-approval.ts +11 -2
- package/telegram-plugin/secret-detect/chunker.ts +18 -4
- package/telegram-plugin/secret-detect/index.ts +12 -56
- package/telegram-plugin/send-gate-degraded.test.ts +574 -0
- package/telegram-plugin/send-gate-observability.test.ts +470 -0
- package/telegram-plugin/send-gate-observability.ts +355 -0
- package/telegram-plugin/send-gate.test.ts +717 -0
- package/telegram-plugin/send-gate.ts +1056 -0
- package/telegram-plugin/session-tail.ts +82 -7
- package/telegram-plugin/shared/bot-runtime.ts +17 -5
- package/telegram-plugin/shared/gw-trace-gate.ts +105 -0
- package/telegram-plugin/status-pin-driver.ts +52 -7
- package/telegram-plugin/status-pin.ts +81 -0
- package/telegram-plugin/subagent-watcher.ts +173 -18
- package/telegram-plugin/tests/activity-card-wiring.test.ts +18 -5
- package/telegram-plugin/tests/approval-hold-harness.ts +425 -0
- package/telegram-plugin/tests/approval-hold-outcome.test.ts +327 -0
- package/telegram-plugin/tests/approval-hold-record.test.ts +531 -0
- package/telegram-plugin/tests/approval-hold-redeliver.test.ts +602 -0
- package/telegram-plugin/tests/auth-loopback-relay.test.ts +533 -0
- package/telegram-plugin/tests/boot-card-flood-suppress.test.ts +53 -7
- package/telegram-plugin/tests/busy-key-reaper.test.ts +1 -0
- package/telegram-plugin/tests/callback-query-handlers.test.ts +65 -0
- package/telegram-plugin/tests/dispatch-claim-scan.test.ts +250 -0
- package/telegram-plugin/tests/flood-breaker-blindness.test.ts +213 -0
- package/telegram-plugin/tests/flood-windows-persistence.test.ts +224 -0
- package/telegram-plugin/tests/gateway-boot-marker-clear.test.ts +3 -3
- package/telegram-plugin/tests/gateway-disconnect-flush.test.ts +29 -1
- package/telegram-plugin/tests/gateway-loopback-paste-redact.test.ts +66 -0
- package/telegram-plugin/tests/gateway-outbound-redact.test.ts +57 -0
- package/telegram-plugin/tests/gw-trace-gate.test.ts +105 -0
- package/telegram-plugin/tests/history.test.ts +115 -0
- package/telegram-plugin/tests/idle-clear.test.ts +233 -3
- package/telegram-plugin/tests/inbound-dedup.test.ts +93 -0
- package/telegram-plugin/tests/inbound-message-types.test.ts +5 -1
- package/telegram-plugin/tests/inline-keyboard-callbacks.test.ts +448 -0
- package/telegram-plugin/tests/ipc-server-check-pre-approved.test.ts +194 -0
- package/telegram-plugin/tests/mental-model-propose-resolve.test.ts +123 -0
- package/telegram-plugin/tests/missed-approvals-wiring.test.ts +1 -1
- package/telegram-plugin/tests/model-command.test.ts +14 -0
- package/telegram-plugin/tests/operator-events-session-tail.test.ts +74 -0
- package/telegram-plugin/tests/outbound-field-redact.test.ts +107 -0
- package/telegram-plugin/tests/pending-session-command.test.ts +21 -0
- package/telegram-plugin/tests/permission-card-routing.test.ts +30 -5
- package/telegram-plugin/tests/permission-no-repeat-wiring.test.ts +8 -7
- package/telegram-plugin/tests/permission-rearm-wiring.test.ts +1 -1
- package/telegram-plugin/tests/pre-approval-check.test.ts +148 -0
- package/telegram-plugin/tests/queued-card-store.test.ts +232 -0
- package/telegram-plugin/tests/reaction-flush-turn-gated.test.ts +100 -0
- package/telegram-plugin/tests/reaction-gate-routing.test.ts +173 -0
- package/telegram-plugin/tests/render/render.test.ts +88 -0
- package/telegram-plugin/tests/retry-api-call.test.ts +398 -0
- package/telegram-plugin/tests/scoped-approval.test.ts +27 -0
- package/telegram-plugin/tests/secret-detect-chunk-overlap.test.ts +65 -0
- package/telegram-plugin/tests/secret-detect-oauth-code.test.ts +5 -4
- package/telegram-plugin/tests/session-model-file.test.ts +50 -0
- package/telegram-plugin/tests/session-tail-sidecar-reap.test.ts +268 -0
- package/telegram-plugin/tests/status-pin-boot-recovery.test.ts +35 -14
- package/telegram-plugin/tests/status-pin.test.ts +275 -1
- package/telegram-plugin/tests/subagent-watcher-deferral-log-ratelimit.test.ts +316 -0
- package/telegram-plugin/tests/subagent-watcher-fd-leak.test.ts +275 -0
- package/telegram-plugin/tests/turn-end-gate-backstop.test.ts +92 -0
- package/telegram-plugin/tests/turn-end-gate.test.ts +137 -0
- package/telegram-plugin/tests/typing-emitter.test.ts +586 -0
- package/telegram-plugin/tests/unhandled-rejection-policy.test.ts +20 -0
- package/telegram-plugin/tests/worktree-watch-cwds.test.ts +215 -1
- package/telegram-plugin/typing-emitter.ts +224 -0
- package/telegram-plugin/uat/scenarios/jtbd-feel-like-a-colleague-dm.test.ts +136 -0
- package/telegram-plugin/welcome-text.ts +42 -0
- package/telegram-plugin/worktree-watch-cwds.ts +194 -5
- package/vendor/hindsight-memory/scripts/drain_pending.py +22 -6
- package/vendor/hindsight-memory/scripts/lib/client.py +12 -5
- package/vendor/hindsight-memory/scripts/lib/directives.py +38 -3
- package/vendor/hindsight-memory/scripts/lib/pending.py +36 -9
- package/vendor/hindsight-memory/scripts/session_end.py +14 -3
- package/vendor/hindsight-memory/scripts/session_start.py +21 -0
- package/vendor/hindsight-memory/scripts/tests/test_directives.py +38 -0
- package/vendor/hindsight-memory/tests/test_drain_pending.py +68 -0
- package/vendor/hindsight-memory/tests/test_pending.py +44 -0
- package/vendor/hindsight-memory/tests/test_session_end_pending.py +38 -0
- package/vendor/hindsight-memory/tests/test_session_start_drain.py +155 -0
- package/telegram-plugin/channel-envelope-safety.test.ts +0 -56
- package/telegram-plugin/channel-envelope-safety.ts +0 -56
- package/telegram-plugin/secret-detect/secretlint-source.ts +0 -95
- package/telegram-plugin/tests/secret-detect-secretlint.test.ts +0 -105
|
@@ -0,0 +1,586 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* #3084 — the typing indicator must be structurally incapable of earning a
|
|
3
|
+
* per-bot flood ban.
|
|
4
|
+
*
|
|
5
|
+
* On 2026-07-11 `overlord` emitted 8,729 sendChatAction calls (55% of ALL
|
|
6
|
+
* outbound volume, bursting 200-300/min into ONE DM) to deliver 203 messages,
|
|
7
|
+
* and Telegram banned the bot token for 4.6 hours (429 retry_after=16739s).
|
|
8
|
+
* Root cause: both typing loops fire an action IMMEDIATELY on (re)start, and
|
|
9
|
+
* the tool-use wrapper restarts the loop on every tool call — so the ping rate
|
|
10
|
+
* equalled the agent's TOOL-CALL rate and the "4 s interval" was decorative.
|
|
11
|
+
*
|
|
12
|
+
* These tests assert the OUTCOME the fix owes: no matter how the loops are
|
|
13
|
+
* driven, at most ~one chat action per chat key per refresh window.
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'
|
|
17
|
+
import { mkdtempSync, rmSync } from 'node:fs'
|
|
18
|
+
import { tmpdir } from 'node:os'
|
|
19
|
+
import { join } from 'node:path'
|
|
20
|
+
import {
|
|
21
|
+
createTypingEmitter,
|
|
22
|
+
TYPING_FLOOR_MS,
|
|
23
|
+
TYPING_REFRESH_MS,
|
|
24
|
+
} from '../typing-emitter.js'
|
|
25
|
+
import { createTurnTypingLoop } from '../gateway/turn-typing-loop.js'
|
|
26
|
+
import { createRetryApiCall, isFloodWaitActiveError } from '../retry-api-call.js'
|
|
27
|
+
import {
|
|
28
|
+
floodStatePath,
|
|
29
|
+
makeFloodWaitRecorder,
|
|
30
|
+
suppressNonEssentialSendMs,
|
|
31
|
+
} from '../flood-circuit-breaker.js'
|
|
32
|
+
import { errors } from './fake-bot-api.js'
|
|
33
|
+
|
|
34
|
+
const chatKey = (chatId: string, threadId: number | null) =>
|
|
35
|
+
`${chatId}:${threadId ?? '_'}`
|
|
36
|
+
|
|
37
|
+
/** Deterministic clock — no real timers anywhere in the emitter tests. */
|
|
38
|
+
function fakeClock(start = 1_000_000) {
|
|
39
|
+
let t = start
|
|
40
|
+
return {
|
|
41
|
+
now: () => t,
|
|
42
|
+
advance: (ms: number) => {
|
|
43
|
+
t += ms
|
|
44
|
+
},
|
|
45
|
+
set: (ms: number) => {
|
|
46
|
+
t = ms
|
|
47
|
+
},
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
interface Sent {
|
|
52
|
+
chatId: string
|
|
53
|
+
threadId: number | null
|
|
54
|
+
action: string
|
|
55
|
+
at: number
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
function harness(opts: { suppressed?: () => boolean } = {}) {
|
|
59
|
+
const clock = fakeClock()
|
|
60
|
+
const sent: Sent[] = []
|
|
61
|
+
const emitter = createTypingEmitter({
|
|
62
|
+
chatKey,
|
|
63
|
+
now: clock.now,
|
|
64
|
+
isSuppressed: opts.suppressed ?? (() => false),
|
|
65
|
+
send: (chatId, threadId, action) =>
|
|
66
|
+
sent.push({ chatId, threadId, action, at: clock.now() }),
|
|
67
|
+
})
|
|
68
|
+
return { clock, sent, emitter }
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/** The invariant the ban owes us: no two emissions on one key inside the floor. */
|
|
72
|
+
function assertFloorHeld(sent: Sent[], floorMs = TYPING_FLOOR_MS): void {
|
|
73
|
+
const lastByKey = new Map<string, number>()
|
|
74
|
+
for (const s of sent) {
|
|
75
|
+
const k = chatKey(s.chatId, s.threadId)
|
|
76
|
+
const prev = lastByKey.get(k)
|
|
77
|
+
if (prev != null) {
|
|
78
|
+
expect(s.at - prev).toBeGreaterThanOrEqual(floorMs)
|
|
79
|
+
}
|
|
80
|
+
lastByKey.set(k, s.at)
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
describe('typing emitter — per-chat-key emission floor (#3084)', () => {
|
|
85
|
+
it('50 loop restarts in 2 s emit exactly ONE chat action (the regression)', () => {
|
|
86
|
+
const { clock, sent, emitter } = harness()
|
|
87
|
+
|
|
88
|
+
// Mirrors the production driver: the tool-use typing wrapper restarts the
|
|
89
|
+
// loop on every tool call, and each restart fired an immediate ping.
|
|
90
|
+
for (let i = 0; i < 50; i++) {
|
|
91
|
+
emitter.emit('chat-1', null, 'typing')
|
|
92
|
+
clock.advance(40) // 50 restarts across 2 s — the observed burst shape
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
expect(sent).toHaveLength(1)
|
|
96
|
+
expect(sent[0]?.at).toBe(1_000_000)
|
|
97
|
+
assertFloorHeld(sent)
|
|
98
|
+
})
|
|
99
|
+
|
|
100
|
+
it('a cold start still pings IMMEDIATELY — the UX intent survives', () => {
|
|
101
|
+
const { sent, emitter } = harness()
|
|
102
|
+
expect(emitter.emit('chat-1')).toBe(true)
|
|
103
|
+
expect(sent).toHaveLength(1)
|
|
104
|
+
})
|
|
105
|
+
|
|
106
|
+
it('the floor releases after the window, so the indicator stays lit', () => {
|
|
107
|
+
const { clock, sent, emitter } = harness()
|
|
108
|
+
emitter.emit('chat-1')
|
|
109
|
+
clock.advance(TYPING_FLOOR_MS - 1)
|
|
110
|
+
expect(emitter.emit('chat-1')).toBe(false)
|
|
111
|
+
clock.advance(1)
|
|
112
|
+
expect(emitter.emit('chat-1')).toBe(true)
|
|
113
|
+
expect(sent).toHaveLength(2)
|
|
114
|
+
})
|
|
115
|
+
|
|
116
|
+
it('the floor is UNDER the refresh cadence, so an on-time refresh is never eaten', () => {
|
|
117
|
+
const { clock, sent, emitter } = harness()
|
|
118
|
+
// A loop refreshing on its own 4 s cadence must never be dropped, or the
|
|
119
|
+
// chat goes dark (Telegram's typing action expires at ~5 s).
|
|
120
|
+
for (let i = 0; i < 15; i++) {
|
|
121
|
+
expect(emitter.emit('chat-1')).toBe(true)
|
|
122
|
+
clock.advance(TYPING_REFRESH_MS)
|
|
123
|
+
}
|
|
124
|
+
expect(sent).toHaveLength(15)
|
|
125
|
+
expect(TYPING_FLOOR_MS).toBeLessThan(TYPING_REFRESH_MS)
|
|
126
|
+
})
|
|
127
|
+
|
|
128
|
+
it('two chats (and two topics in one chat) are independent lanes', () => {
|
|
129
|
+
const { sent, emitter } = harness()
|
|
130
|
+
expect(emitter.emit('chat-1', null)).toBe(true)
|
|
131
|
+
expect(emitter.emit('chat-2', null)).toBe(true)
|
|
132
|
+
expect(emitter.emit('chat-1', 77)).toBe(true) // supergroup topic = own lane
|
|
133
|
+
expect(emitter.emit('chat-1', null)).toBe(false) // same lane, inside floor
|
|
134
|
+
expect(sent).toHaveLength(3)
|
|
135
|
+
})
|
|
136
|
+
|
|
137
|
+
it('emits NOTHING while a flood window is open (typing is non-essential)', () => {
|
|
138
|
+
let floodOpen = true
|
|
139
|
+
const { clock, sent, emitter } = harness({ suppressed: () => floodOpen })
|
|
140
|
+
|
|
141
|
+
for (let i = 0; i < 40; i++) {
|
|
142
|
+
expect(emitter.emit('chat-1')).toBe(false)
|
|
143
|
+
clock.advance(1000)
|
|
144
|
+
}
|
|
145
|
+
expect(sent).toHaveLength(0)
|
|
146
|
+
|
|
147
|
+
floodOpen = false
|
|
148
|
+
expect(emitter.emit('chat-1')).toBe(true)
|
|
149
|
+
expect(sent).toHaveLength(1)
|
|
150
|
+
})
|
|
151
|
+
|
|
152
|
+
it('checks the flood state at most once per floor per chat (no read storm)', () => {
|
|
153
|
+
const suppressed = vi.fn(() => false)
|
|
154
|
+
const clock = fakeClock()
|
|
155
|
+
const emitter = createTypingEmitter({
|
|
156
|
+
chatKey,
|
|
157
|
+
now: clock.now,
|
|
158
|
+
isSuppressed: suppressed,
|
|
159
|
+
send: () => {},
|
|
160
|
+
})
|
|
161
|
+
for (let i = 0; i < 50; i++) {
|
|
162
|
+
emitter.emit('chat-1')
|
|
163
|
+
clock.advance(40)
|
|
164
|
+
}
|
|
165
|
+
// 2 s of restarts, floor 3.5 s → one window claimed → one flood check.
|
|
166
|
+
expect(suppressed).toHaveBeenCalledTimes(1)
|
|
167
|
+
})
|
|
168
|
+
|
|
169
|
+
it('a backwards clock step does not wedge the indicator off', () => {
|
|
170
|
+
const { clock, sent, emitter } = harness()
|
|
171
|
+
emitter.emit('chat-1')
|
|
172
|
+
clock.set(500) // NTP step backwards
|
|
173
|
+
expect(emitter.emit('chat-1')).toBe(true)
|
|
174
|
+
expect(sent).toHaveLength(2)
|
|
175
|
+
})
|
|
176
|
+
})
|
|
177
|
+
|
|
178
|
+
/**
|
|
179
|
+
* A 60 s turn, the REAL `createTurnTypingLoop`, and a faithful reproduction of
|
|
180
|
+
* the tool-use loop as gateway.ts drives it (stop → restart → fire immediately
|
|
181
|
+
* → re-arm a fixed interval), both emitting through ONE emitter.
|
|
182
|
+
*
|
|
183
|
+
* The bounds below are ABSOLUTE — derived from the incident and from Telegram's
|
|
184
|
+
* action expiry, never from the implementation constants. A test whose bound is
|
|
185
|
+
* expressed in terms of the thing it tests cannot fail when that thing is
|
|
186
|
+
* removed: with the floor disabled, `ceil(60_000 / floorMs)` is Infinity. These
|
|
187
|
+
* numbers go red on the unfixed code (~135 emissions in 60 s), which is the
|
|
188
|
+
* only property that makes them worth having.
|
|
189
|
+
*/
|
|
190
|
+
const TURN_MS = 60_000
|
|
191
|
+
/** 60 s at the enforced floor is ~17 emissions. 20 is the slack-adjusted cap;
|
|
192
|
+
* the pre-fix code emits ~135 (one per tool call) and fails this loudly. */
|
|
193
|
+
const MAX_EMISSIONS_PER_TURN = 20
|
|
194
|
+
/** Telegram's `typing` chat action expires at ~5 s. A longer gap = a DARK chat
|
|
195
|
+
* mid-turn, which breaches know-what-my-agent-is-doing ("stays present from
|
|
196
|
+
* receipt to turn end"). This is the assertion the catch-up timer exists for. */
|
|
197
|
+
const MAX_DARK_MS = 5000
|
|
198
|
+
|
|
199
|
+
interface Schedule {
|
|
200
|
+
/** ms into the turn before the first tool call fires. */
|
|
201
|
+
phase: number
|
|
202
|
+
/** how long each tool call runs (loop stops on the tool result). */
|
|
203
|
+
dur: number
|
|
204
|
+
/** quiet gap between tool calls (agent thinking / streaming text). */
|
|
205
|
+
gap: number
|
|
206
|
+
/** number of serial tool calls. */
|
|
207
|
+
count: number
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
/**
|
|
211
|
+
* A VIRTUAL CLOCK + event queue — no `vi` fake timers anywhere in this file.
|
|
212
|
+
*
|
|
213
|
+
* This suite is DUAL-RUN: CI's `bun-test-run` executes the whole
|
|
214
|
+
* `telegram-plugin/tests` directory, and bun's `vi` shim has no
|
|
215
|
+
* `setSystemTime` / `useFakeTimers` (CLAUDE.md: dual-run files must avoid
|
|
216
|
+
* bun-incompatible vitest APIs). A test that throws on startup is not a weaker
|
|
217
|
+
* test, it is NOT A TEST — and this file is the entire guard for #3084, so it
|
|
218
|
+
* has to bite in BOTH runners.
|
|
219
|
+
*
|
|
220
|
+
* The emitter already takes injected `now` / `schedule` / `cancel`, so it needs
|
|
221
|
+
* no timer mocking at all. `createTurnTypingLoop` reaches for the global
|
|
222
|
+
* `setInterval`, so `withVirtualTimers` swaps the global for the queue below for
|
|
223
|
+
* the duration of one simulation and restores it after. Fully deterministic and
|
|
224
|
+
* runner-agnostic.
|
|
225
|
+
*/
|
|
226
|
+
interface VTimer {
|
|
227
|
+
id: number
|
|
228
|
+
at: number
|
|
229
|
+
fn: () => void
|
|
230
|
+
every: number | null
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
function createVirtualClock(start = 0) {
|
|
234
|
+
let t = start
|
|
235
|
+
let seq = 1
|
|
236
|
+
const timers = new Map<number, VTimer>()
|
|
237
|
+
|
|
238
|
+
function schedule(fn: () => void, ms: number, every: number | null = null): number {
|
|
239
|
+
const id = seq++
|
|
240
|
+
timers.set(id, { id, at: t + Math.max(0, ms), fn, every })
|
|
241
|
+
return id
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
return {
|
|
245
|
+
now: () => t,
|
|
246
|
+
schedule: (fn: () => void, ms: number) => schedule(fn, ms),
|
|
247
|
+
scheduleEvery: (fn: () => void, ms: number) => schedule(fn, ms, ms),
|
|
248
|
+
cancel: (h: unknown) => {
|
|
249
|
+
timers.delete(h as number)
|
|
250
|
+
},
|
|
251
|
+
/** Run every timer due at or before `target`, in (time, insertion) order. */
|
|
252
|
+
advanceTo(target: number): void {
|
|
253
|
+
for (;;) {
|
|
254
|
+
let next: VTimer | null = null
|
|
255
|
+
for (const tm of timers.values()) {
|
|
256
|
+
if (tm.at > target) continue
|
|
257
|
+
if (next === null || tm.at < next.at || (tm.at === next.at && tm.id < next.id)) {
|
|
258
|
+
next = tm
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
if (next === null) break
|
|
262
|
+
t = next.at
|
|
263
|
+
if (next.every != null) next.at = t + next.every
|
|
264
|
+
else timers.delete(next.id)
|
|
265
|
+
next.fn()
|
|
266
|
+
}
|
|
267
|
+
t = target
|
|
268
|
+
},
|
|
269
|
+
advanceBy(ms: number): void {
|
|
270
|
+
this.advanceTo(t + ms)
|
|
271
|
+
},
|
|
272
|
+
pending: () => timers.size,
|
|
273
|
+
}
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
type VirtualClock = ReturnType<typeof createVirtualClock>
|
|
277
|
+
|
|
278
|
+
/** Run `body` with the GLOBAL setInterval/clearInterval bound to `clock`. */
|
|
279
|
+
function withVirtualTimers<T>(clock: VirtualClock, body: () => T): T {
|
|
280
|
+
const realSetInterval = globalThis.setInterval
|
|
281
|
+
const realClearInterval = globalThis.clearInterval
|
|
282
|
+
globalThis.setInterval = ((fn: () => void, ms: number) =>
|
|
283
|
+
clock.scheduleEvery(fn, ms)) as unknown as typeof globalThis.setInterval
|
|
284
|
+
globalThis.clearInterval = ((h: unknown) =>
|
|
285
|
+
clock.cancel(h)) as unknown as typeof globalThis.clearInterval
|
|
286
|
+
try {
|
|
287
|
+
return body()
|
|
288
|
+
} finally {
|
|
289
|
+
globalThis.setInterval = realSetInterval
|
|
290
|
+
globalThis.clearInterval = realClearInterval
|
|
291
|
+
}
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
/** Run one schedule; return the emission times over the turn. */
|
|
295
|
+
function simulateTurn(sch: Schedule): number[] {
|
|
296
|
+
const clock = createVirtualClock(0)
|
|
297
|
+
const sent: number[] = []
|
|
298
|
+
|
|
299
|
+
return withVirtualTimers(clock, () => {
|
|
300
|
+
const emitter = createTypingEmitter({
|
|
301
|
+
chatKey,
|
|
302
|
+
now: clock.now,
|
|
303
|
+
schedule: clock.schedule,
|
|
304
|
+
cancel: clock.cancel,
|
|
305
|
+
send: () => sent.push(clock.now()),
|
|
306
|
+
})
|
|
307
|
+
|
|
308
|
+
// Loop A — the turn-level loop: fixed cadence for the WHOLE turn. The REAL
|
|
309
|
+
// factory, driving the REAL global setInterval (bound to the virtual clock).
|
|
310
|
+
const turnLoop = createTurnTypingLoop({
|
|
311
|
+
chatKey,
|
|
312
|
+
sendChatAction: (chatId, threadId) => {
|
|
313
|
+
emitter.emit(chatId, threadId, 'typing')
|
|
314
|
+
},
|
|
315
|
+
refreshMs: TYPING_REFRESH_MS,
|
|
316
|
+
})
|
|
317
|
+
|
|
318
|
+
// Loop B — the tool-use loop, as gateway.ts drives it: restarts (and fires
|
|
319
|
+
// immediately) per serial tool call, stops on the tool result when the
|
|
320
|
+
// ref-count hits zero.
|
|
321
|
+
let toolInterval: unknown = null
|
|
322
|
+
const startToolLoop = () => {
|
|
323
|
+
if (toolInterval != null) clearInterval(toolInterval as ReturnType<typeof setInterval>)
|
|
324
|
+
const send = () => emitter.emit('chat-1', null, 'typing')
|
|
325
|
+
send()
|
|
326
|
+
toolInterval = setInterval(send, TYPING_REFRESH_MS)
|
|
327
|
+
}
|
|
328
|
+
const stopToolLoop = () => {
|
|
329
|
+
if (toolInterval != null) clearInterval(toolInterval as ReturnType<typeof setInterval>)
|
|
330
|
+
toolInterval = null
|
|
331
|
+
}
|
|
332
|
+
|
|
333
|
+
// Build the tool start/stop timeline, then advance the clock through it.
|
|
334
|
+
const events: Array<{ at: number; fn: () => void }> = []
|
|
335
|
+
let cursor = sch.phase
|
|
336
|
+
for (let i = 0; i < sch.count && cursor < TURN_MS; i++) {
|
|
337
|
+
const start = cursor
|
|
338
|
+
const end = Math.min(start + sch.dur, TURN_MS)
|
|
339
|
+
events.push({ at: start, fn: startToolLoop })
|
|
340
|
+
events.push({ at: end, fn: stopToolLoop })
|
|
341
|
+
cursor = end + sch.gap
|
|
342
|
+
}
|
|
343
|
+
events.sort((a, b) => a.at - b.at)
|
|
344
|
+
|
|
345
|
+
turnLoop.start('chat-1', null) // turn enqueue — the cold-start ping
|
|
346
|
+
for (const ev of events) {
|
|
347
|
+
clock.advanceTo(ev.at)
|
|
348
|
+
ev.fn()
|
|
349
|
+
}
|
|
350
|
+
clock.advanceTo(TURN_MS)
|
|
351
|
+
|
|
352
|
+
turnLoop.stop('chat-1', null) // canonical turn-end
|
|
353
|
+
emitter.cancelPending('chat-1', null) // …as the gateway does
|
|
354
|
+
stopToolLoop()
|
|
355
|
+
|
|
356
|
+
// Nothing may fire after the turn ends — no zombie "typing…" on a dead turn.
|
|
357
|
+
const afterTurn = sent.length
|
|
358
|
+
clock.advanceTo(TURN_MS + 30_000)
|
|
359
|
+
expect(sent.length).toBe(afterTurn)
|
|
360
|
+
|
|
361
|
+
return sent
|
|
362
|
+
})
|
|
363
|
+
}
|
|
364
|
+
|
|
365
|
+
/** Longest stretch of the turn with no live "typing…", including the head. */
|
|
366
|
+
function maxDarkMs(sent: number[]): number {
|
|
367
|
+
let worst = sent.length > 0 ? sent[0]! : TURN_MS
|
|
368
|
+
for (let i = 1; i < sent.length; i++) worst = Math.max(worst, sent[i]! - sent[i - 1]!)
|
|
369
|
+
return worst
|
|
370
|
+
}
|
|
371
|
+
|
|
372
|
+
describe('typing emitter — BOTH loops share one floor (#3084)', () => {
|
|
373
|
+
it('the reviewer counterexample (phase=3500 dur=3400 n=6) never goes dark', () => {
|
|
374
|
+
// Before the catch-up timer this schedule left a 7000 ms gap (emissions at
|
|
375
|
+
// 21000 → 28000): a tool ping eats the turn loop's tick, the tool loop then
|
|
376
|
+
// stops, and the turn loop's NEXT fixed tick is floor+refresh away. ~2 s of
|
|
377
|
+
// a dead indicator mid-turn. Red before the fix, green after.
|
|
378
|
+
const sent = simulateTurn({ phase: 3500, dur: 3400, gap: 300, count: 6 })
|
|
379
|
+
expect(maxDarkMs(sent)).toBeLessThanOrEqual(MAX_DARK_MS)
|
|
380
|
+
expect(sent.length).toBeLessThanOrEqual(MAX_EMISSIONS_PER_TURN)
|
|
381
|
+
})
|
|
382
|
+
|
|
383
|
+
it('a 120-tool-call storm over 60 s emits <=20 actions (the ban shape)', () => {
|
|
384
|
+
// The incident shape: serial tool calls firing every ~500 ms into ONE DM for
|
|
385
|
+
// a whole minute. Pre-fix this emitted ~136 actions (one per tool call, the
|
|
386
|
+
// 8,729-pings-for-203-messages ratio). The cap is absolute, so it goes red
|
|
387
|
+
// the moment the floor stops holding.
|
|
388
|
+
const sent = simulateTurn({ phase: 0, dur: 250, gap: 250, count: 120 })
|
|
389
|
+
expect(sent.length).toBeLessThanOrEqual(MAX_EMISSIONS_PER_TURN)
|
|
390
|
+
expect(sent.length).toBeGreaterThan(10) // still lit for the whole minute
|
|
391
|
+
expect(maxDarkMs(sent)).toBeLessThanOrEqual(MAX_DARK_MS)
|
|
392
|
+
})
|
|
393
|
+
|
|
394
|
+
it('ADVERSARIAL sweep: no schedule floods, and no schedule goes dark', () => {
|
|
395
|
+
// Sweep the tool-call phase/duration/gap/count space — the point is that the
|
|
396
|
+
// bound holds across schedules, not on one hand-picked one.
|
|
397
|
+
let worstDark = 0
|
|
398
|
+
let worstDarkSch: Schedule | null = null
|
|
399
|
+
let worstCount = 0
|
|
400
|
+
let worstCountSch: Schedule | null = null
|
|
401
|
+
let schedules = 0
|
|
402
|
+
|
|
403
|
+
for (let phase = 0; phase <= 4000; phase += 250) {
|
|
404
|
+
for (let dur = 200; dur <= 4200; dur += 400) {
|
|
405
|
+
for (const gap of [150, 900, 2600]) {
|
|
406
|
+
for (const count of [3, 6, 12]) {
|
|
407
|
+
const sch: Schedule = { phase, dur, gap, count }
|
|
408
|
+
const sent = simulateTurn(sch)
|
|
409
|
+
schedules++
|
|
410
|
+
|
|
411
|
+
const dark = maxDarkMs(sent)
|
|
412
|
+
if (dark > worstDark) {
|
|
413
|
+
worstDark = dark
|
|
414
|
+
worstDarkSch = sch
|
|
415
|
+
}
|
|
416
|
+
if (sent.length > worstCount) {
|
|
417
|
+
worstCount = sent.length
|
|
418
|
+
worstCountSch = sch
|
|
419
|
+
}
|
|
420
|
+
// Floor still holds under every schedule — the flood cap is intact.
|
|
421
|
+
assertFloorHeld(
|
|
422
|
+
sent.map((at) => ({ chatId: 'chat-1', threadId: null, action: 'typing', at })),
|
|
423
|
+
)
|
|
424
|
+
}
|
|
425
|
+
}
|
|
426
|
+
}
|
|
427
|
+
}
|
|
428
|
+
|
|
429
|
+
expect(schedules).toBeGreaterThan(500)
|
|
430
|
+
// The two outcomes, both absolute — neither is expressed in terms of the
|
|
431
|
+
// constants under test, so both go red when the mechanism is removed.
|
|
432
|
+
expect(
|
|
433
|
+
worstDark,
|
|
434
|
+
`worst dark gap ${worstDark}ms on ${JSON.stringify(worstDarkSch)}`,
|
|
435
|
+
).toBeLessThanOrEqual(MAX_DARK_MS) // never a dark chat
|
|
436
|
+
expect(
|
|
437
|
+
worstCount,
|
|
438
|
+
`worst emission count ${worstCount} on ${JSON.stringify(worstCountSch)}`,
|
|
439
|
+
).toBeLessThanOrEqual(MAX_EMISSIONS_PER_TURN) // never a flood
|
|
440
|
+
// …and the indicator is genuinely live, not merely "not flooding".
|
|
441
|
+
expect(worstCount).toBeGreaterThan(10)
|
|
442
|
+
})
|
|
443
|
+
|
|
444
|
+
it('a dropped tick arms exactly ONE coalesced catch-up, and never leaks', () => {
|
|
445
|
+
const clock = createVirtualClock(0)
|
|
446
|
+
const sent: Sent[] = []
|
|
447
|
+
const emitter = createTypingEmitter({
|
|
448
|
+
chatKey,
|
|
449
|
+
now: clock.now,
|
|
450
|
+
schedule: clock.schedule,
|
|
451
|
+
cancel: clock.cancel,
|
|
452
|
+
send: (chatId, threadId, action) =>
|
|
453
|
+
sent.push({ chatId, threadId, action, at: clock.now() }),
|
|
454
|
+
})
|
|
455
|
+
|
|
456
|
+
emitter.emit('chat-1') // t=0, sent
|
|
457
|
+
for (let i = 0; i < 20; i++) emitter.emit('chat-1') // 20 drops, ONE window
|
|
458
|
+
expect(emitter.pendingCatchUps()).toBe(1) // coalesced — never 20 timers
|
|
459
|
+
|
|
460
|
+
clock.advanceBy(TYPING_FLOOR_MS)
|
|
461
|
+
expect(sent.map((s) => s.at)).toEqual([0, TYPING_FLOOR_MS]) // catch-up fired
|
|
462
|
+
expect(emitter.pendingCatchUps()).toBe(0) // …and did not re-arm itself
|
|
463
|
+
|
|
464
|
+
// Turn-end cancellation: a drop in the last moments of a turn must not
|
|
465
|
+
// resurrect "typing…" after the reply has landed.
|
|
466
|
+
emitter.emit('chat-1')
|
|
467
|
+
expect(emitter.pendingCatchUps()).toBe(1)
|
|
468
|
+
emitter.cancelPending('chat-1')
|
|
469
|
+
expect(emitter.pendingCatchUps()).toBe(0)
|
|
470
|
+
clock.advanceBy(10_000)
|
|
471
|
+
expect(sent).toHaveLength(2) // nothing fired after the turn ended
|
|
472
|
+
expect(clock.pending()).toBe(0) // no timer leaked
|
|
473
|
+
})
|
|
474
|
+
|
|
475
|
+
it('a flood window cancels the catch-up instead of deferring a ping into the ban', () => {
|
|
476
|
+
const clock = createVirtualClock(0)
|
|
477
|
+
let floodOpen = false
|
|
478
|
+
const sent: Sent[] = []
|
|
479
|
+
const emitter = createTypingEmitter({
|
|
480
|
+
chatKey,
|
|
481
|
+
now: clock.now,
|
|
482
|
+
isSuppressed: () => floodOpen,
|
|
483
|
+
schedule: clock.schedule,
|
|
484
|
+
cancel: clock.cancel,
|
|
485
|
+
send: (chatId, threadId, action) =>
|
|
486
|
+
sent.push({ chatId, threadId, action, at: clock.now() }),
|
|
487
|
+
})
|
|
488
|
+
|
|
489
|
+
emitter.emit('chat-1')
|
|
490
|
+
emitter.emit('chat-1') // dropped by the floor → catch-up armed
|
|
491
|
+
expect(emitter.pendingCatchUps()).toBe(1)
|
|
492
|
+
|
|
493
|
+
floodOpen = true
|
|
494
|
+
clock.advanceBy(TYPING_FLOOR_MS) // catch-up fires INTO the ban…
|
|
495
|
+
expect(sent).toHaveLength(1) // …and is suppressed, not sent
|
|
496
|
+
expect(emitter.pendingCatchUps()).toBe(0)
|
|
497
|
+
|
|
498
|
+
clock.advanceBy(60_000)
|
|
499
|
+
expect(sent).toHaveLength(1)
|
|
500
|
+
})
|
|
501
|
+
})
|
|
502
|
+
|
|
503
|
+
describe('typing sends record 429s to the flood breaker (#3084 / #2923)', () => {
|
|
504
|
+
let dir: string
|
|
505
|
+
let path: string
|
|
506
|
+
|
|
507
|
+
beforeEach(() => {
|
|
508
|
+
dir = mkdtempSync(join(tmpdir(), 'typing-flood-'))
|
|
509
|
+
path = floodStatePath(dir)
|
|
510
|
+
})
|
|
511
|
+
afterEach(() => {
|
|
512
|
+
rmSync(dir, { recursive: true, force: true })
|
|
513
|
+
})
|
|
514
|
+
|
|
515
|
+
/** The gateway's `nonEssentialApiCall`: records the flood, never retries. */
|
|
516
|
+
const nonEssentialApiCall = () =>
|
|
517
|
+
createRetryApiCall({
|
|
518
|
+
maxRetries: 1,
|
|
519
|
+
sleep: async () => {},
|
|
520
|
+
onFloodWait: makeFloodWaitRecorder(path),
|
|
521
|
+
})
|
|
522
|
+
|
|
523
|
+
it('a 429 on a typing ping is RECORDED, not swallowed', async () => {
|
|
524
|
+
const call = nonEssentialApiCall()
|
|
525
|
+
const fn = vi.fn().mockRejectedValue(errors.floodWait(16739))
|
|
526
|
+
|
|
527
|
+
await expect(call(fn, { verb: 'sendChatAction' })).rejects.toThrow()
|
|
528
|
+
|
|
529
|
+
// Recorded: the breaker now knows the bot is banned — before the fix the
|
|
530
|
+
// typing path called bot.api raw with `.catch(() => {})`, so the breaker
|
|
531
|
+
// was blind to 429s from the LARGEST emitter of them.
|
|
532
|
+
expect(suppressNonEssentialSendMs(path, Date.now())).toBeGreaterThan(0)
|
|
533
|
+
})
|
|
534
|
+
|
|
535
|
+
it('a typing ping is NEVER retried (a retry is what feeds the ban)', async () => {
|
|
536
|
+
const call = nonEssentialApiCall()
|
|
537
|
+
const fn = vi.fn().mockRejectedValue(errors.floodWait(16739))
|
|
538
|
+
await expect(call(fn)).rejects.toThrow()
|
|
539
|
+
expect(fn).toHaveBeenCalledTimes(1)
|
|
540
|
+
})
|
|
541
|
+
|
|
542
|
+
it('composes with #3094: a long ban surfaces FLOOD_WAIT_ACTIVE, still ONE call', async () => {
|
|
543
|
+
// #3094 bounded the in-process flood sleep: above the ceiling retryApiCall
|
|
544
|
+
// throws the FLOOD_WAIT_ACTIVE marker instead of sleeping out a multi-hour
|
|
545
|
+
// retry_after. For a typing ping that changes nothing that matters — the
|
|
546
|
+
// window is still recorded, the API is still hit exactly once, nothing is
|
|
547
|
+
// slept. The gateway's onRejected handler drops the marker, so a
|
|
548
|
+
// fire-and-forget ping can't surface it as an unhandled rejection.
|
|
549
|
+
const call = nonEssentialApiCall()
|
|
550
|
+
const fn = vi.fn().mockRejectedValue(errors.floodWait(16739))
|
|
551
|
+
|
|
552
|
+
const err = await call(fn).then(
|
|
553
|
+
() => null,
|
|
554
|
+
(e: unknown) => e,
|
|
555
|
+
)
|
|
556
|
+
expect(isFloodWaitActiveError(err)).toBe(true)
|
|
557
|
+
expect(fn).toHaveBeenCalledTimes(1)
|
|
558
|
+
expect(suppressNonEssentialSendMs(path, Date.now())).toBeGreaterThan(0)
|
|
559
|
+
})
|
|
560
|
+
|
|
561
|
+
it('the recorded window then suppresses further typing end-to-end', async () => {
|
|
562
|
+
const call = nonEssentialApiCall()
|
|
563
|
+
const sent: Sent[] = []
|
|
564
|
+
const clock = fakeClock()
|
|
565
|
+
const emitter = createTypingEmitter({
|
|
566
|
+
chatKey,
|
|
567
|
+
now: clock.now,
|
|
568
|
+
isSuppressed: () => suppressNonEssentialSendMs(path, Date.now()) > 0,
|
|
569
|
+
send: (chatId, threadId, action) =>
|
|
570
|
+
sent.push({ chatId, threadId, action, at: clock.now() }),
|
|
571
|
+
})
|
|
572
|
+
|
|
573
|
+
expect(emitter.emit('chat-1')).toBe(true) // healthy: pings
|
|
574
|
+
|
|
575
|
+
// The bot earns a flood ban on some other send.
|
|
576
|
+
await expect(
|
|
577
|
+
call(vi.fn().mockRejectedValue(errors.floodWait(600))),
|
|
578
|
+
).rejects.toThrow()
|
|
579
|
+
|
|
580
|
+
clock.advance(TYPING_FLOOR_MS)
|
|
581
|
+
expect(emitter.emit('chat-1')).toBe(false) // banned: silent
|
|
582
|
+
clock.advance(TYPING_FLOOR_MS)
|
|
583
|
+
expect(emitter.emit('chat-1')).toBe(false)
|
|
584
|
+
expect(sent).toHaveLength(1)
|
|
585
|
+
})
|
|
586
|
+
})
|
|
@@ -11,6 +11,7 @@
|
|
|
11
11
|
import { describe, it, expect } from 'bun:test'
|
|
12
12
|
import { GrammyError } from 'grammy'
|
|
13
13
|
import { classifyRejection } from '../gateway/unhandled-rejection-policy.js'
|
|
14
|
+
import { FLOOD_WAIT_ACTIVE } from '../retry-api-call.js'
|
|
14
15
|
|
|
15
16
|
// ── Real GrammyError fixtures ──────────────────────────────────────────────
|
|
16
17
|
|
|
@@ -221,3 +222,22 @@ describe('classifyRejection — duck typing via injected detector', () => {
|
|
|
221
222
|
expect(result).toBe('shutdown')
|
|
222
223
|
})
|
|
223
224
|
})
|
|
225
|
+
|
|
226
|
+
describe('classifyRejection — FLOOD_WAIT_ACTIVE marker (#3084)', () => {
|
|
227
|
+
// retry-api-call throws this instead of sleeping through a multi-hour per-bot
|
|
228
|
+
// flood ban (overlord logged retry_after = 16739s on 2026-07-11). It's a plain
|
|
229
|
+
// Error, not a GrammyError, so without an explicit entry it would fall into
|
|
230
|
+
// the `!isGrammy → shutdown` branch — a crash on rate-limiting, which fires
|
|
231
|
+
// MORE sends into the open window on reboot and extends the ban.
|
|
232
|
+
it('returns "log_only" for a leaked FLOOD_WAIT_ACTIVE marker', () => {
|
|
233
|
+
const err = Object.assign(new Error(FLOOD_WAIT_ACTIVE), {
|
|
234
|
+
retryAfterSec: 16739,
|
|
235
|
+
untilTs: Date.now() + 16739 * 1000,
|
|
236
|
+
})
|
|
237
|
+
expect(classifyRejection(err)).toBe('log_only')
|
|
238
|
+
})
|
|
239
|
+
|
|
240
|
+
it('still returns "shutdown" for an unrelated plain Error', () => {
|
|
241
|
+
expect(classifyRejection(new Error('FLOOD_WAIT_ACTIVE-ish but not it'))).toBe('shutdown')
|
|
242
|
+
})
|
|
243
|
+
})
|