switchroom 0.18.11 → 0.18.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-scheduler/index.js +37 -5
- package/dist/auth-broker/index.js +116 -78
- package/dist/cli/hindsight-mental-model-pretool.mjs +39 -0
- package/dist/cli/ms-365-write-pretool.mjs +31 -8
- package/dist/cli/notion-write-pretool.mjs +38 -6
- package/dist/cli/skill-validate-pretool.mjs +144 -2847
- package/dist/cli/switchroom.js +3529 -4543
- package/dist/cli/ui/index.html +163 -17
- package/dist/host-control/main.js +605 -2847
- package/dist/vault/approvals/kernel-server.js +120 -13
- package/dist/vault/broker/server.js +259 -157
- package/package.json +3 -4
- package/profiles/_base/start.sh.hbs +65 -0
- package/profiles/_shared/vault-protocol.md.hbs +3 -1
- package/profiles/coding/CLAUDE.md.hbs +1 -1
- package/profiles/default/CLAUDE.md.hbs +2 -2
- package/profiles/executive-assistant/CLAUDE.md.hbs +1 -1
- package/profiles/health-coach/CLAUDE.md.hbs +1 -1
- package/telegram-plugin/bridge/bridge.ts +37 -0
- package/telegram-plugin/bridge/inbound-dedup.ts +101 -0
- package/telegram-plugin/dist/bridge/bridge.js +122 -4
- package/telegram-plugin/dist/gateway/gateway.js +4213 -3288
- package/telegram-plugin/dist/server.js +139 -5
- package/telegram-plugin/flood-circuit-breaker.ts +493 -21
- package/telegram-plugin/format.ts +19 -0
- package/telegram-plugin/gateway/approval-hold.ts +602 -0
- package/telegram-plugin/gateway/auth-command.ts +92 -2
- package/telegram-plugin/gateway/auth-loopback-relay.ts +670 -0
- package/telegram-plugin/gateway/boot-card.ts +12 -5
- package/telegram-plugin/gateway/callback-query-handlers.ts +88 -1
- package/telegram-plugin/gateway/config-approval-handler.ts +6 -1
- package/telegram-plugin/gateway/disconnect-flush.ts +19 -0
- package/telegram-plugin/gateway/dm-pin-sweep.test.ts +251 -0
- package/telegram-plugin/gateway/dm-pin-sweep.ts +178 -0
- package/telegram-plugin/gateway/gateway.ts +1695 -230
- package/telegram-plugin/gateway/hostd-dispatch.ts +23 -0
- package/telegram-plugin/gateway/idle-clear.ts +90 -6
- package/telegram-plugin/gateway/inbound-delivery-machine-shadow.ts +26 -5
- package/telegram-plugin/gateway/inject-handler.ts +8 -0
- package/telegram-plugin/gateway/ipc-protocol.ts +46 -3
- package/telegram-plugin/gateway/ipc-server.ts +43 -0
- package/telegram-plugin/gateway/mental-model-propose-resolve.ts +145 -37
- package/telegram-plugin/gateway/model-command.ts +9 -3
- package/telegram-plugin/gateway/pending-session-command.ts +13 -1
- package/telegram-plugin/gateway/permission-ttl-sweep.ts +66 -0
- package/telegram-plugin/gateway/pre-approval-check.ts +74 -0
- package/telegram-plugin/gateway/queued-card-store.ts +217 -0
- package/telegram-plugin/gateway/session-model-file.ts +26 -1
- package/telegram-plugin/gateway/turn-end-gate-backstop.ts +59 -0
- package/telegram-plugin/gateway/turn-end-gate.ts +95 -0
- package/telegram-plugin/gateway/turn-typing-loop.ts +10 -2
- package/telegram-plugin/gateway/unhandled-rejection-policy.ts +13 -0
- package/telegram-plugin/history.ts +51 -0
- package/telegram-plugin/hooks/dispatch-claim-scan.mjs +259 -0
- package/telegram-plugin/hooks/dispatch-claim-stop.mjs +129 -0
- package/telegram-plugin/hooks/hooks.json +9 -0
- package/telegram-plugin/inline-keyboard-callbacks.ts +303 -2
- package/telegram-plugin/model-unavailable.ts +41 -11
- package/telegram-plugin/operator-events.ts +23 -0
- package/telegram-plugin/outbound-field-redact.ts +69 -0
- package/telegram-plugin/package.json +0 -1
- package/telegram-plugin/permission-rule.ts +1 -0
- package/telegram-plugin/permission-title.ts +1 -0
- package/telegram-plugin/render/render.ts +32 -14
- package/telegram-plugin/retry-api-call.ts +212 -2
- package/telegram-plugin/scoped-approval.ts +11 -2
- package/telegram-plugin/secret-detect/chunker.ts +18 -4
- package/telegram-plugin/secret-detect/index.ts +12 -56
- package/telegram-plugin/send-gate-degraded.test.ts +574 -0
- package/telegram-plugin/send-gate-observability.test.ts +470 -0
- package/telegram-plugin/send-gate-observability.ts +355 -0
- package/telegram-plugin/send-gate.test.ts +717 -0
- package/telegram-plugin/send-gate.ts +1056 -0
- package/telegram-plugin/session-tail.ts +82 -7
- package/telegram-plugin/shared/bot-runtime.ts +17 -5
- package/telegram-plugin/shared/gw-trace-gate.ts +105 -0
- package/telegram-plugin/status-pin-driver.ts +52 -7
- package/telegram-plugin/status-pin.ts +81 -0
- package/telegram-plugin/subagent-watcher.ts +173 -18
- package/telegram-plugin/tests/activity-card-wiring.test.ts +18 -5
- package/telegram-plugin/tests/approval-hold-harness.ts +425 -0
- package/telegram-plugin/tests/approval-hold-outcome.test.ts +327 -0
- package/telegram-plugin/tests/approval-hold-record.test.ts +531 -0
- package/telegram-plugin/tests/approval-hold-redeliver.test.ts +602 -0
- package/telegram-plugin/tests/auth-loopback-relay.test.ts +533 -0
- package/telegram-plugin/tests/boot-card-flood-suppress.test.ts +53 -7
- package/telegram-plugin/tests/busy-key-reaper.test.ts +1 -0
- package/telegram-plugin/tests/callback-query-handlers.test.ts +65 -0
- package/telegram-plugin/tests/dispatch-claim-scan.test.ts +250 -0
- package/telegram-plugin/tests/flood-breaker-blindness.test.ts +213 -0
- package/telegram-plugin/tests/flood-windows-persistence.test.ts +224 -0
- package/telegram-plugin/tests/gateway-boot-marker-clear.test.ts +3 -3
- package/telegram-plugin/tests/gateway-disconnect-flush.test.ts +29 -1
- package/telegram-plugin/tests/gateway-loopback-paste-redact.test.ts +66 -0
- package/telegram-plugin/tests/gateway-outbound-redact.test.ts +57 -0
- package/telegram-plugin/tests/gw-trace-gate.test.ts +105 -0
- package/telegram-plugin/tests/history.test.ts +115 -0
- package/telegram-plugin/tests/idle-clear.test.ts +233 -3
- package/telegram-plugin/tests/inbound-dedup.test.ts +93 -0
- package/telegram-plugin/tests/inbound-message-types.test.ts +5 -1
- package/telegram-plugin/tests/inline-keyboard-callbacks.test.ts +448 -0
- package/telegram-plugin/tests/ipc-server-check-pre-approved.test.ts +194 -0
- package/telegram-plugin/tests/mental-model-propose-resolve.test.ts +123 -0
- package/telegram-plugin/tests/missed-approvals-wiring.test.ts +1 -1
- package/telegram-plugin/tests/model-command.test.ts +14 -0
- package/telegram-plugin/tests/operator-events-session-tail.test.ts +74 -0
- package/telegram-plugin/tests/outbound-field-redact.test.ts +107 -0
- package/telegram-plugin/tests/pending-session-command.test.ts +21 -0
- package/telegram-plugin/tests/permission-card-routing.test.ts +30 -5
- package/telegram-plugin/tests/permission-no-repeat-wiring.test.ts +8 -7
- package/telegram-plugin/tests/permission-rearm-wiring.test.ts +1 -1
- package/telegram-plugin/tests/pre-approval-check.test.ts +148 -0
- package/telegram-plugin/tests/queued-card-store.test.ts +232 -0
- package/telegram-plugin/tests/reaction-flush-turn-gated.test.ts +100 -0
- package/telegram-plugin/tests/reaction-gate-routing.test.ts +173 -0
- package/telegram-plugin/tests/render/render.test.ts +88 -0
- package/telegram-plugin/tests/retry-api-call.test.ts +398 -0
- package/telegram-plugin/tests/scoped-approval.test.ts +27 -0
- package/telegram-plugin/tests/secret-detect-chunk-overlap.test.ts +65 -0
- package/telegram-plugin/tests/secret-detect-oauth-code.test.ts +5 -4
- package/telegram-plugin/tests/session-model-file.test.ts +50 -0
- package/telegram-plugin/tests/session-tail-sidecar-reap.test.ts +268 -0
- package/telegram-plugin/tests/status-pin-boot-recovery.test.ts +35 -14
- package/telegram-plugin/tests/status-pin.test.ts +275 -1
- package/telegram-plugin/tests/subagent-watcher-deferral-log-ratelimit.test.ts +316 -0
- package/telegram-plugin/tests/subagent-watcher-fd-leak.test.ts +275 -0
- package/telegram-plugin/tests/turn-end-gate-backstop.test.ts +92 -0
- package/telegram-plugin/tests/turn-end-gate.test.ts +137 -0
- package/telegram-plugin/tests/typing-emitter.test.ts +586 -0
- package/telegram-plugin/tests/unhandled-rejection-policy.test.ts +20 -0
- package/telegram-plugin/tests/worktree-watch-cwds.test.ts +215 -1
- package/telegram-plugin/typing-emitter.ts +224 -0
- package/telegram-plugin/uat/scenarios/jtbd-feel-like-a-colleague-dm.test.ts +136 -0
- package/telegram-plugin/welcome-text.ts +42 -0
- package/telegram-plugin/worktree-watch-cwds.ts +194 -5
- package/vendor/hindsight-memory/scripts/drain_pending.py +22 -6
- package/vendor/hindsight-memory/scripts/lib/client.py +12 -5
- package/vendor/hindsight-memory/scripts/lib/directives.py +38 -3
- package/vendor/hindsight-memory/scripts/lib/pending.py +36 -9
- package/vendor/hindsight-memory/scripts/session_end.py +14 -3
- package/vendor/hindsight-memory/scripts/session_start.py +21 -0
- package/vendor/hindsight-memory/scripts/tests/test_directives.py +38 -0
- package/vendor/hindsight-memory/tests/test_drain_pending.py +68 -0
- package/vendor/hindsight-memory/tests/test_pending.py +44 -0
- package/vendor/hindsight-memory/tests/test_session_end_pending.py +38 -0
- package/vendor/hindsight-memory/tests/test_session_start_drain.py +155 -0
- package/telegram-plugin/channel-envelope-safety.test.ts +0 -56
- package/telegram-plugin/channel-envelope-safety.ts +0 -56
- package/telegram-plugin/secret-detect/secretlint-source.ts +0 -95
- package/telegram-plugin/tests/secret-detect-secretlint.test.ts +0 -105
|
@@ -20,7 +20,15 @@
|
|
|
20
20
|
* logic is pure + injectable so it unit tests without a real clock or bot.
|
|
21
21
|
*/
|
|
22
22
|
|
|
23
|
-
import {
|
|
23
|
+
import {
|
|
24
|
+
existsSync,
|
|
25
|
+
readFileSync,
|
|
26
|
+
writeFileSync,
|
|
27
|
+
mkdirSync,
|
|
28
|
+
chmodSync,
|
|
29
|
+
unlinkSync,
|
|
30
|
+
renameSync,
|
|
31
|
+
} from 'node:fs'
|
|
24
32
|
import { dirname, join } from 'node:path'
|
|
25
33
|
|
|
26
34
|
export interface FloodWaitState {
|
|
@@ -35,6 +43,43 @@ export interface FloodWaitState {
|
|
|
35
43
|
/** Default marker filename inside the telegram state dir. */
|
|
36
44
|
export const FLOOD_STATE_FILE = 'flood-wait.json'
|
|
37
45
|
|
|
46
|
+
/**
|
|
47
|
+
* Marker file mode — 0644, deliberately world-READABLE (#3106).
|
|
48
|
+
*
|
|
49
|
+
* The marker was 0600 and that silently defeated the entire breaker. The
|
|
50
|
+
* telegram state dir is shared by processes running under DIFFERENT uids: a
|
|
51
|
+
* `root: true` agent's gateway runs as uid 0 (`src/agents/compose.ts:1967`),
|
|
52
|
+
* a normal agent's as its deterministic uid (`compose.ts:1969`), and the same
|
|
53
|
+
* bind-mounted dir survives a container being flipped between the two. Whoever
|
|
54
|
+
* CREATES the marker owns it; at 0600 nobody else can ever read it again. On
|
|
55
|
+
* the live box `overlord`'s marker is `root:root 0600` while its siblings are
|
|
56
|
+
* agent-uid — the moment that agent runs non-root, every flood probe EACCESes.
|
|
57
|
+
*
|
|
58
|
+
* The payload is three integers (`untilTs`, `retryAfterSec`, `recordedTs`) —
|
|
59
|
+
* no secret, nothing worth 0600. `fleet-health/ledger.json` is the working
|
|
60
|
+
* precedent for a state file multiple uids must read: 0644.
|
|
61
|
+
*/
|
|
62
|
+
export const FLOOD_STATE_MODE = 0o644
|
|
63
|
+
|
|
64
|
+
/**
|
|
65
|
+
* Outcome of a marker read. The whole point of this type is that "no ban" and
|
|
66
|
+
* "I cannot tell" stop being the same value (#3106).
|
|
67
|
+
*
|
|
68
|
+
* - `ok` → a window was read (may be expired; `floodWaitRemainingMs` decides)
|
|
69
|
+
* - `absent` → ENOENT, no marker has ever been written. Genuinely "no ban".
|
|
70
|
+
* - `corrupt` → readable but the JSON/shape is junk. Content problem.
|
|
71
|
+
* - `unreadable` → the file exists and we could NOT read it (EACCES/EPERM/EIO).
|
|
72
|
+
* The breaker is BLIND. This is NOT "no ban".
|
|
73
|
+
*/
|
|
74
|
+
export type FloodReadStatus = 'ok' | 'absent' | 'corrupt' | 'unreadable'
|
|
75
|
+
|
|
76
|
+
export interface FloodReadResult {
|
|
77
|
+
status: FloodReadStatus
|
|
78
|
+
state: FloodWaitState | null
|
|
79
|
+
/** Populated for `unreadable` / `corrupt` — the underlying error message. */
|
|
80
|
+
error?: string
|
|
81
|
+
}
|
|
82
|
+
|
|
38
83
|
/**
|
|
39
84
|
* Resolve the flood-wait marker path from a telegram state dir. Kept as a
|
|
40
85
|
* helper so callers share one location.
|
|
@@ -70,30 +115,110 @@ export function isFloodWaitActive(state: FloodWaitState | null, now: number): bo
|
|
|
70
115
|
return floodWaitRemainingMs(state, now) > 0
|
|
71
116
|
}
|
|
72
117
|
|
|
73
|
-
/**
|
|
74
|
-
|
|
118
|
+
/**
|
|
119
|
+
* Read the persisted marker, reporting WHY there is no state (#3106).
|
|
120
|
+
*
|
|
121
|
+
* No `existsSync` pre-check: it is a `stat`, not an `access`, so it cannot
|
|
122
|
+
* tell "missing" from "unreadable" — and pre-checking would race. We read and
|
|
123
|
+
* classify the error instead. ENOENT (and a missing parent dir, ENOTDIR) is
|
|
124
|
+
* the only honest "no marker". Everything else that isn't a parse/shape
|
|
125
|
+
* failure means the breaker could not see its own state.
|
|
126
|
+
*/
|
|
127
|
+
export function readFloodStateResult(path: string): FloodReadResult {
|
|
128
|
+
let text: string
|
|
75
129
|
try {
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
130
|
+
text = readFileSync(path, 'utf-8')
|
|
131
|
+
} catch (err) {
|
|
132
|
+
const code = (err as NodeJS.ErrnoException | undefined)?.code
|
|
133
|
+
if (code === 'ENOENT' || code === 'ENOTDIR') return { status: 'absent', state: null }
|
|
79
134
|
return {
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
135
|
+
status: 'unreadable',
|
|
136
|
+
state: null,
|
|
137
|
+
error: `${code ?? 'EUNKNOWN'}: ${(err as Error)?.message ?? String(err)}`,
|
|
83
138
|
}
|
|
84
|
-
} catch {
|
|
85
|
-
return null
|
|
86
139
|
}
|
|
140
|
+
try {
|
|
141
|
+
const raw = JSON.parse(text) as Partial<FloodWaitState>
|
|
142
|
+
if (typeof raw.untilTs !== 'number' || !Number.isFinite(raw.untilTs)) {
|
|
143
|
+
return { status: 'corrupt', state: null, error: 'untilTs is not a finite number' }
|
|
144
|
+
}
|
|
145
|
+
return {
|
|
146
|
+
status: 'ok',
|
|
147
|
+
state: {
|
|
148
|
+
untilTs: raw.untilTs,
|
|
149
|
+
retryAfterSec: typeof raw.retryAfterSec === 'number' ? raw.retryAfterSec : 0,
|
|
150
|
+
recordedTs: typeof raw.recordedTs === 'number' ? raw.recordedTs : 0,
|
|
151
|
+
},
|
|
152
|
+
}
|
|
153
|
+
} catch (err) {
|
|
154
|
+
return { status: 'corrupt', state: null, error: (err as Error)?.message ?? String(err) }
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
/**
|
|
159
|
+
* Read persisted flood state; null on absence / parse failure / unreadable.
|
|
160
|
+
*
|
|
161
|
+
* Kept for callers that only want the window. It CANNOT distinguish "no ban"
|
|
162
|
+
* from "cannot tell" — that is the bug #3106 exists to fix — so anything
|
|
163
|
+
* making a suppress/proceed DECISION must use `readFloodStateResult`.
|
|
164
|
+
*/
|
|
165
|
+
export function readFloodState(path: string): FloodWaitState | null {
|
|
166
|
+
return readFloodStateResult(path).state
|
|
87
167
|
}
|
|
88
168
|
|
|
89
|
-
/**
|
|
90
|
-
|
|
169
|
+
/**
|
|
170
|
+
* Persist flood state (best-effort — a write failure must not crash the send
|
|
171
|
+
* path), and SELF-HEAL a marker left unreadable by another uid (#3106).
|
|
172
|
+
*
|
|
173
|
+
* Two heals, both needed because the `mode` option only applies at CREATE time:
|
|
174
|
+
* 1. `chmodSync` to 0644 after every write, so a marker created 0600 by an
|
|
175
|
+
* earlier build (or by a root gateway) becomes readable to the agent uid.
|
|
176
|
+
* 2. On EACCES/EPERM (we can't overwrite a file some other uid owns), unlink
|
|
177
|
+
* and recreate. The state DIR is owned by the agent uid, so a non-root
|
|
178
|
+
* gateway can unlink a root-owned marker inside it even though it cannot
|
|
179
|
+
* write through it. Without this the recorder is as blind as the reader.
|
|
180
|
+
*/
|
|
181
|
+
export function writeFloodState(
|
|
182
|
+
path: string,
|
|
183
|
+
state: FloodWaitState,
|
|
184
|
+
log: (line: string) => void = (l) => process.stderr.write(l),
|
|
185
|
+
): void {
|
|
186
|
+
const payload = JSON.stringify(state)
|
|
91
187
|
try {
|
|
92
188
|
mkdirSync(dirname(path), { recursive: true })
|
|
93
|
-
writeFileSync(path, JSON.stringify(state), { mode: 0o600 })
|
|
94
189
|
} catch {
|
|
95
190
|
/* best-effort */
|
|
96
191
|
}
|
|
192
|
+
try {
|
|
193
|
+
writeFileSync(path, payload, { mode: FLOOD_STATE_MODE })
|
|
194
|
+
try {
|
|
195
|
+
chmodSync(path, FLOOD_STATE_MODE)
|
|
196
|
+
} catch {
|
|
197
|
+
/* not the owner — the read path will report it */
|
|
198
|
+
}
|
|
199
|
+
return
|
|
200
|
+
} catch (err) {
|
|
201
|
+
const code = (err as NodeJS.ErrnoException | undefined)?.code
|
|
202
|
+
if (code !== 'EACCES' && code !== 'EPERM') {
|
|
203
|
+
log(`telegram gateway: flood-breaker: could not persist ${path} (${code ?? 'error'})\n`)
|
|
204
|
+
return
|
|
205
|
+
}
|
|
206
|
+
// Owned by a different uid. Unlink + recreate so the CURRENT uid owns it.
|
|
207
|
+
try {
|
|
208
|
+
unlinkSync(path)
|
|
209
|
+
writeFileSync(path, payload, { mode: FLOOD_STATE_MODE })
|
|
210
|
+
log(
|
|
211
|
+
`telegram gateway: flood-breaker: recreated ${path} — it was owned by another uid ` +
|
|
212
|
+
`and could not be updated (${code}); the breaker was blind (issue #3106)\n`,
|
|
213
|
+
)
|
|
214
|
+
} catch (err2) {
|
|
215
|
+
log(
|
|
216
|
+
`telegram gateway: flood-breaker: BLIND — cannot persist the flood-wait window to ${path} ` +
|
|
217
|
+
`(${code}, and recreate failed: ${(err2 as Error)?.message ?? String(err2)}). ` +
|
|
218
|
+
`Flood bans will NOT be recorded. Fix the file's ownership/mode (issue #3106)\n`,
|
|
219
|
+
)
|
|
220
|
+
}
|
|
221
|
+
}
|
|
97
222
|
}
|
|
98
223
|
|
|
99
224
|
/**
|
|
@@ -104,20 +229,367 @@ export function writeFloodState(path: string, state: FloodWaitState): void {
|
|
|
104
229
|
export function makeFloodWaitRecorder(
|
|
105
230
|
path: string,
|
|
106
231
|
now: () => number = Date.now,
|
|
232
|
+
log: (line: string) => void = (l) => process.stderr.write(l),
|
|
107
233
|
): (retryAfterSec: number) => void {
|
|
108
234
|
return (retryAfterSec: number) => {
|
|
109
235
|
const t = now()
|
|
110
236
|
const next = computeFloodWait(readFloodState(path), retryAfterSec, t)
|
|
111
|
-
writeFloodState(path, next)
|
|
237
|
+
writeFloodState(path, next, log)
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
/**
|
|
242
|
+
* How long a blind (unreadable-marker) breaker suppresses non-essential sends
|
|
243
|
+
* for. Nominal — the caller only checks `> 0`. Not a real ban estimate; we
|
|
244
|
+
* have no idea how long the ban is, that is the whole problem.
|
|
245
|
+
*/
|
|
246
|
+
export const FLOOD_BLIND_SUPPRESS_MS = 60_000
|
|
247
|
+
|
|
248
|
+
/** Throttle for the BLIND warning: once per path per interval, so a per-call probe can't spam. */
|
|
249
|
+
const BLIND_LOG_INTERVAL_MS = 60_000
|
|
250
|
+
const blindLoggedAt = new Map<string, number>()
|
|
251
|
+
|
|
252
|
+
/** Test seam — clear the BLIND-warning throttle between cases. */
|
|
253
|
+
export function resetFloodBlindLogThrottle(): void {
|
|
254
|
+
blindLoggedAt.clear()
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
function warnBlind(path: string, error: string | undefined, now: number, log: (l: string) => void) {
|
|
258
|
+
const last = blindLoggedAt.get(path)
|
|
259
|
+
if (last !== undefined && now - last < BLIND_LOG_INTERVAL_MS) return
|
|
260
|
+
blindLoggedAt.set(path, now)
|
|
261
|
+
log(
|
|
262
|
+
`telegram gateway: flood-breaker: BLIND — cannot read the flood-wait marker ${path} ` +
|
|
263
|
+
`(${error ?? 'unknown error'}). The breaker cannot tell whether a Telegram flood ban is ` +
|
|
264
|
+
`open. Essential sends PROCEED (fail-open); non-essential sends (boot card, typing) are ` +
|
|
265
|
+
`SUPPRESSED. Fix the file's ownership/mode (issue #3106)\n`,
|
|
266
|
+
)
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
/**
|
|
270
|
+
* Build the `floodWaitRemainingMs` probe for `createRetryApiCall` (#3084).
|
|
271
|
+
*
|
|
272
|
+
* Returns the remaining ms of the persisted flood window, so the retry policy
|
|
273
|
+
* can refuse to issue a call INTO a known-open long ban rather than letting
|
|
274
|
+
* every 5-6s card heartbeat fire another request at the flood counter. Reads
|
|
275
|
+
* fresh each call (the window is written by `makeFloodWaitRecorder`, possibly
|
|
276
|
+
* from another code path in the same process).
|
|
277
|
+
*
|
|
278
|
+
* Fails OPEN on absent / corrupt / UNREADABLE — this probe gates EVERY api
|
|
279
|
+
* call including the user's reply, and no marker problem may ever mute the
|
|
280
|
+
* bot. But an unreadable marker is no longer SILENT: it is a blind breaker,
|
|
281
|
+
* and it says so, loudly and repeatedly (throttled to once a minute), because
|
|
282
|
+
* a breaker that cannot see its own state while reporting "all clear" is the
|
|
283
|
+
* worst state this system can be in (#3106).
|
|
284
|
+
*/
|
|
285
|
+
export function makeFloodWaitProbe(
|
|
286
|
+
path: string,
|
|
287
|
+
now: () => number = Date.now,
|
|
288
|
+
log: (line: string) => void = (l) => process.stderr.write(l),
|
|
289
|
+
): () => number {
|
|
290
|
+
return () => {
|
|
291
|
+
const t = now()
|
|
292
|
+
const res = readFloodStateResult(path)
|
|
293
|
+
if (res.status === 'unreadable') {
|
|
294
|
+
warnBlind(path, res.error, t, log)
|
|
295
|
+
return 0 // fail OPEN — never gag an essential send on a permissions error
|
|
296
|
+
}
|
|
297
|
+
return floodWaitRemainingMs(res.state, t)
|
|
112
298
|
}
|
|
113
299
|
}
|
|
114
300
|
|
|
115
301
|
/**
|
|
116
|
-
*
|
|
117
|
-
*
|
|
118
|
-
*
|
|
119
|
-
*
|
|
302
|
+
* Why a non-essential send is being held back.
|
|
303
|
+
*
|
|
304
|
+
* - `flood_wait` → we can see an open ban window. Suppress (that is #2923).
|
|
305
|
+
* - `blind` → we CANNOT read the marker. Suppress.
|
|
306
|
+
*/
|
|
307
|
+
export type NonEssentialSuppression =
|
|
308
|
+
| { suppress: false }
|
|
309
|
+
| { suppress: true; reason: 'flood_wait'; remainingMs: number }
|
|
310
|
+
| { suppress: true; reason: 'blind'; error: string }
|
|
311
|
+
|
|
312
|
+
/**
|
|
313
|
+
* Decide whether a NON-ESSENTIAL send (boot card, config summary, typing
|
|
314
|
+
* indicator) should be held back.
|
|
315
|
+
*
|
|
316
|
+
* This is where "cannot tell" is allowed to fail CLOSED, and the asymmetry is
|
|
317
|
+
* deliberate (#3106):
|
|
318
|
+
*
|
|
319
|
+
* - Absent / corrupt → PROCEED. A missing or junk marker is a content
|
|
320
|
+
* problem, and #3094's posture stands: a broken state file must never
|
|
321
|
+
* become a silent gag.
|
|
322
|
+
* - Unreadable → SUPPRESS. A permissions error means the breaker is blind,
|
|
323
|
+
* and blind is not clear. Restarting into an open ban with a boot card is
|
|
324
|
+
* exactly the amplification #2923 exists to prevent, and the cost of being
|
|
325
|
+
* wrong here is bounded and tiny: the operator misses a courtesy card or a
|
|
326
|
+
* typing bubble. It CANNOT mute the agent — every essential send still
|
|
327
|
+
* goes through the fail-open probe above. That bound is what makes
|
|
328
|
+
* fail-closed safe HERE and unsafe on the probe.
|
|
120
329
|
*/
|
|
121
|
-
export function
|
|
122
|
-
|
|
330
|
+
export function nonEssentialSendSuppression(path: string, now: number): NonEssentialSuppression {
|
|
331
|
+
const res = readFloodStateResult(path)
|
|
332
|
+
if (res.status === 'unreadable') {
|
|
333
|
+
return { suppress: true, reason: 'blind', error: res.error ?? 'unknown error' }
|
|
334
|
+
}
|
|
335
|
+
const remainingMs = floodWaitRemainingMs(res.state, now)
|
|
336
|
+
if (remainingMs > 0) return { suppress: true, reason: 'flood_wait', remainingMs }
|
|
337
|
+
return { suppress: false }
|
|
338
|
+
}
|
|
339
|
+
|
|
340
|
+
/**
|
|
341
|
+
* Ms-shaped shim over `nonEssentialSendSuppression` for callers that only ask
|
|
342
|
+
* "> 0?". A blind breaker returns `FLOOD_BLIND_SUPPRESS_MS` (a nominal
|
|
343
|
+
* non-zero), and warns. Callers that want to explain WHICH reason to the
|
|
344
|
+
* operator should use `nonEssentialSendSuppression` directly.
|
|
345
|
+
*/
|
|
346
|
+
export function suppressNonEssentialSendMs(
|
|
347
|
+
path: string,
|
|
348
|
+
now: number,
|
|
349
|
+
log: (line: string) => void = (l) => process.stderr.write(l),
|
|
350
|
+
): number {
|
|
351
|
+
const s = nonEssentialSendSuppression(path, now)
|
|
352
|
+
if (!s.suppress) return 0
|
|
353
|
+
if (s.reason === 'blind') {
|
|
354
|
+
warnBlind(path, s.error, now, log)
|
|
355
|
+
return FLOOD_BLIND_SUPPRESS_MS
|
|
356
|
+
}
|
|
357
|
+
return s.remainingMs
|
|
358
|
+
}
|
|
359
|
+
|
|
360
|
+
// ─── Restart-proof SCOPED flood windows (#3084 PR 2, part3-design §7) ────────
|
|
361
|
+
//
|
|
362
|
+
// The single-object `flood-wait.json` above records ONE global per-bot window
|
|
363
|
+
// (all #3094's `makeFloodWaitProbe` needs). PR 2's send gate opens windows at
|
|
364
|
+
// finer scopes (`global` | `chat:<id>` | `group:<id>` | `msg-edit:<id>`) and
|
|
365
|
+
// must survive a restart so a container that boots mid-ban does not immediately
|
|
366
|
+
// resend into the open flood — exactly the retry-storm that escalates bans.
|
|
367
|
+
//
|
|
368
|
+
// Rather than overload the single-object schema (which would break #3094's
|
|
369
|
+
// probe), scoped windows live in a SIBLING file `flood-windows.json` — an array
|
|
370
|
+
// of `{ scopeKey, untilTs, retryAfterSrc, observedAt }`. `flood-wait.json` is
|
|
371
|
+
// left untouched, so the existing probe keeps working unchanged.
|
|
372
|
+
|
|
373
|
+
/** One persisted scoped flood window (part3-design §7). */
|
|
374
|
+
export interface FloodWindowRecord {
|
|
375
|
+
/** `global` | `chat:<id>` | `group:<id>` | `msg-edit:<id>`. */
|
|
376
|
+
scopeKey: string
|
|
377
|
+
/** Epoch ms until which the scope admits nothing. */
|
|
378
|
+
untilTs: number
|
|
379
|
+
/** Provenance of the window (`429` retry_after, `boot`, …) for diagnostics. */
|
|
380
|
+
retryAfterSrc: string
|
|
381
|
+
/** Epoch ms the window was (re)recorded. */
|
|
382
|
+
observedAt: number
|
|
383
|
+
/**
|
|
384
|
+
* Epoch ms an operator alert was sent for THIS window (#3084 PR 3, §6). The
|
|
385
|
+
* at-most-once anchor: persisted so a restart mid-window never re-alerts.
|
|
386
|
+
* Unset until the observer alerts.
|
|
387
|
+
*/
|
|
388
|
+
alertedAt?: number
|
|
389
|
+
}
|
|
390
|
+
|
|
391
|
+
/** Sibling file holding the scoped-window array. */
|
|
392
|
+
export const FLOOD_WINDOWS_FILE = 'flood-windows.json'
|
|
393
|
+
|
|
394
|
+
/** Resolve the scoped-windows file path from a telegram state dir. */
|
|
395
|
+
export function floodWindowsPath(stateDir: string): string {
|
|
396
|
+
return join(stateDir, FLOOD_WINDOWS_FILE)
|
|
397
|
+
}
|
|
398
|
+
|
|
399
|
+
/**
|
|
400
|
+
* How long a fail-SAFE conservative global window suppresses for when the
|
|
401
|
+
* scoped-windows file exists but cannot be trusted (unreadable/corrupt/junk).
|
|
402
|
+
*
|
|
403
|
+
* The scoped-windows file gates whether a booting gateway resends into an open
|
|
404
|
+
* ban. If we cannot tell what windows are open, the ONLY safe move is to assume
|
|
405
|
+
* a ban may be open and hold non-essential/coalesced traffic for a bounded
|
|
406
|
+
* conservative window — the exact opposite of `flood-wait.json`'s essential-send
|
|
407
|
+
* probe, which must fail OPEN so a marker problem never gags the user's reply.
|
|
408
|
+
* The asymmetry is deliberate: this file drives boot-time shedding, not the
|
|
409
|
+
* user's reply, so failing safe here is the right call, while failing open
|
|
410
|
+
* resends straight into a ban (H2/M1, #3106 posture).
|
|
411
|
+
*
|
|
412
|
+
* Note the blast radius is NOT only cosmetic: this conservative window is
|
|
413
|
+
* `FLOOD_WINDOWS_CORRUPT_SUPPRESS_MS` (5 min), which exceeds the send gate's
|
|
414
|
+
* `criticalFailFastMs` (60s) ceiling — so while it is open a CRITICAL reply also
|
|
415
|
+
* fail-fasts with a structured `FLOOD_WAIT_ACTIVE` (a retryable error carrying
|
|
416
|
+
* `untilTs`, NOT a silent drop and NOT a hang) for up to 5 min after each boot.
|
|
417
|
+
* That only fires with the send gate flag ON and a genuinely corrupt/unreadable
|
|
418
|
+
* persisted file; the fail-fast is a real signal to the caller, so the posture
|
|
419
|
+
* is defensible — but it is fail-fast criticals, not merely suppressed cosmetics.
|
|
420
|
+
*/
|
|
421
|
+
export const FLOOD_WINDOWS_CORRUPT_SUPPRESS_MS = 5 * 60_000
|
|
422
|
+
|
|
423
|
+
/**
|
|
424
|
+
* Read persisted scoped windows, pruning any whose `untilTs` is already in the
|
|
425
|
+
* past.
|
|
426
|
+
*
|
|
427
|
+
* Fail-SAFE, not fail-open (M1/H2): a genuinely ABSENT file (ENOENT) is the
|
|
428
|
+
* only "no windows" answer. If the file exists but is unreadable (EACCES/EIO),
|
|
429
|
+
* corrupt, or not a JSON array, we CANNOT tell whether a ban is open — so we
|
|
430
|
+
* synthesize a conservative `global` window (`FLOOD_WINDOWS_CORRUPT_SUPPRESS_MS`)
|
|
431
|
+
* and log loudly, rather than booting into an open flood with no windows.
|
|
432
|
+
*/
|
|
433
|
+
export function readFloodWindows(
|
|
434
|
+
path: string,
|
|
435
|
+
now: number,
|
|
436
|
+
log: (line: string) => void = (l) => process.stderr.write(l),
|
|
437
|
+
): FloodWindowRecord[] {
|
|
438
|
+
// ENOENT is the ONLY honest "no windows". Anything else = we can't tell.
|
|
439
|
+
if (!existsSync(path)) return []
|
|
440
|
+
const failSafe = (why: string): FloodWindowRecord[] => {
|
|
441
|
+
log(
|
|
442
|
+
`telegram gateway: flood-breaker: scoped-windows file ${path} is ${why} — ` +
|
|
443
|
+
`failing SAFE: opening a conservative ${FLOOD_WINDOWS_CORRUPT_SUPPRESS_MS}ms global ` +
|
|
444
|
+
`window rather than booting into a possible open ban with no windows (issue #3106)\n`,
|
|
445
|
+
)
|
|
446
|
+
return [
|
|
447
|
+
{
|
|
448
|
+
scopeKey: 'global',
|
|
449
|
+
untilTs: now + FLOOD_WINDOWS_CORRUPT_SUPPRESS_MS,
|
|
450
|
+
retryAfterSrc: `failsafe:${why}`,
|
|
451
|
+
observedAt: now,
|
|
452
|
+
},
|
|
453
|
+
]
|
|
454
|
+
}
|
|
455
|
+
let text: string
|
|
456
|
+
try {
|
|
457
|
+
text = readFileSync(path, 'utf-8')
|
|
458
|
+
} catch (err) {
|
|
459
|
+
const code = (err as NodeJS.ErrnoException | undefined)?.code
|
|
460
|
+
if (code === 'ENOENT' || code === 'ENOTDIR') return [] // vanished between stat and read
|
|
461
|
+
return failSafe(`unreadable (${code ?? 'error'})`)
|
|
462
|
+
}
|
|
463
|
+
let raw: unknown
|
|
464
|
+
try {
|
|
465
|
+
raw = JSON.parse(text)
|
|
466
|
+
} catch {
|
|
467
|
+
return failSafe('corrupt (invalid JSON)')
|
|
468
|
+
}
|
|
469
|
+
if (!Array.isArray(raw)) return failSafe('corrupt (not an array)')
|
|
470
|
+
const out: FloodWindowRecord[] = []
|
|
471
|
+
for (const r of raw) {
|
|
472
|
+
const rec = r as Partial<FloodWindowRecord>
|
|
473
|
+
if (typeof rec.scopeKey !== 'string' || typeof rec.untilTs !== 'number') continue
|
|
474
|
+
if (rec.untilTs <= now) continue // prune expired
|
|
475
|
+
out.push({
|
|
476
|
+
scopeKey: rec.scopeKey,
|
|
477
|
+
untilTs: rec.untilTs,
|
|
478
|
+
retryAfterSrc: typeof rec.retryAfterSrc === 'string' ? rec.retryAfterSrc : 'unknown',
|
|
479
|
+
observedAt: typeof rec.observedAt === 'number' ? rec.observedAt : now,
|
|
480
|
+
// Preserve the at-most-once alert anchor (#3084 PR 3) across reads so a
|
|
481
|
+
// restart mid-window does not re-alert.
|
|
482
|
+
...(typeof rec.alertedAt === 'number' ? { alertedAt: rec.alertedAt } : {}),
|
|
483
|
+
})
|
|
484
|
+
}
|
|
485
|
+
return out
|
|
486
|
+
}
|
|
487
|
+
|
|
488
|
+
/**
|
|
489
|
+
* Write-through a single scoped window (best-effort, atomic rename). Merges
|
|
490
|
+
* with the on-disk set: EXTENDS an existing scope's window (never shortens — a
|
|
491
|
+
* restart must not shrink a ban), prunes expired scopes, and drops the scope
|
|
492
|
+
* entirely if the new window is already in the past. Atomic via temp-file +
|
|
493
|
+
* rename so a concurrent reader never sees a half-written file.
|
|
494
|
+
*/
|
|
495
|
+
export function writeFloodWindow(
|
|
496
|
+
path: string,
|
|
497
|
+
record: FloodWindowRecord,
|
|
498
|
+
now: number,
|
|
499
|
+
): void {
|
|
500
|
+
try {
|
|
501
|
+
mkdirSync(dirname(path), { recursive: true })
|
|
502
|
+
const existing = readFloodWindows(path, now)
|
|
503
|
+
const byScope = new Map<string, FloodWindowRecord>()
|
|
504
|
+
for (const r of existing) byScope.set(r.scopeKey, r)
|
|
505
|
+
if (record.untilTs > now) {
|
|
506
|
+
const prior = byScope.get(record.scopeKey)
|
|
507
|
+
// Monotonic: keep the LATER expiry (never shorten an open window).
|
|
508
|
+
const untilTs = prior && prior.untilTs > record.untilTs ? prior.untilTs : record.untilTs
|
|
509
|
+
// Preserve a prior `alertedAt` when a window is EXTENDED (#3084 PR 3): a
|
|
510
|
+
// fresh 429 recorder call carries no alertedAt, and dropping it would
|
|
511
|
+
// re-arm the at-most-once alert for a window we already alerted on.
|
|
512
|
+
const alertedAt = record.alertedAt ?? prior?.alertedAt
|
|
513
|
+
byScope.set(record.scopeKey, {
|
|
514
|
+
...record,
|
|
515
|
+
untilTs,
|
|
516
|
+
...(alertedAt != null ? { alertedAt } : {}),
|
|
517
|
+
})
|
|
518
|
+
}
|
|
519
|
+
const arr = [...byScope.values()]
|
|
520
|
+
const tmp = `${path}.tmp-${process.pid}`
|
|
521
|
+
// 0o644, NOT 0o600 (H2, #3106): the telegram state dir is shared by
|
|
522
|
+
// processes running under DIFFERENT uids; a 0600 marker created by whoever
|
|
523
|
+
// wrote first is unreadable to every other uid, which silently defeats the
|
|
524
|
+
// restart-proof windows — a restart under a different uid boots blind and
|
|
525
|
+
// resends into the ban. Match `flood-wait.json` (FLOOD_STATE_MODE).
|
|
526
|
+
writeFileSync(tmp, JSON.stringify(arr), { mode: FLOOD_STATE_MODE })
|
|
527
|
+
renameSync(tmp, path)
|
|
528
|
+
// `mode:` only applies at CREATE; chmod so a marker left 0600 by an earlier
|
|
529
|
+
// build (or a root gateway) becomes readable to the agent uid. Best-effort:
|
|
530
|
+
// if we don't own it, the read path fails SAFE rather than silently open.
|
|
531
|
+
try {
|
|
532
|
+
chmodSync(path, FLOOD_STATE_MODE)
|
|
533
|
+
} catch {
|
|
534
|
+
/* not the owner — read path fails safe */
|
|
535
|
+
}
|
|
536
|
+
} catch {
|
|
537
|
+
/* best-effort — a persistence failure must not crash the send path */
|
|
538
|
+
}
|
|
539
|
+
}
|
|
540
|
+
|
|
541
|
+
/**
|
|
542
|
+
* Persist an `alertedAt` marker on the window for `scopeKey` (#3084 PR 3, §6).
|
|
543
|
+
*
|
|
544
|
+
* The at-most-once anchor for the operator flood alert: once written, a restart
|
|
545
|
+
* that re-reads the still-open window sees `alertedAt` set and does not re-alert.
|
|
546
|
+
* Best-effort and a no-op if the scope is no longer open (already pruned) — the
|
|
547
|
+
* window has closed, so at-most-once still holds. Reuses `writeFloodWindow`'s
|
|
548
|
+
* merge (it keeps the LATER `untilTs` and now carries `alertedAt` through).
|
|
549
|
+
*/
|
|
550
|
+
export function markFloodWindowAlerted(
|
|
551
|
+
path: string,
|
|
552
|
+
scopeKey: string,
|
|
553
|
+
alertedAt: number,
|
|
554
|
+
now: number,
|
|
555
|
+
): void {
|
|
556
|
+
const open = readFloodWindows(path, now).find((w) => w.scopeKey === scopeKey)
|
|
557
|
+
if (!open) return
|
|
558
|
+
writeFloodWindow(path, { ...open, alertedAt }, now)
|
|
559
|
+
}
|
|
560
|
+
|
|
561
|
+
/**
|
|
562
|
+
* Build the `onWindowOpen` callback the send gate calls whenever it opens /
|
|
563
|
+
* extends a scope window (on a 429, or at boot). Persists write-through so the
|
|
564
|
+
* window survives a restart (part3-design §7).
|
|
565
|
+
*/
|
|
566
|
+
export function makeFloodWindowRecorder(
|
|
567
|
+
path: string,
|
|
568
|
+
now: () => number = Date.now,
|
|
569
|
+
): (scopeKey: string, untilTs: number, retryAfterSrc?: string) => void {
|
|
570
|
+
return (scopeKey: string, untilTs: number, retryAfterSrc = '429') => {
|
|
571
|
+
const t = now()
|
|
572
|
+
writeFloodWindow(path, { scopeKey, untilTs, retryAfterSrc, observedAt: t }, t)
|
|
573
|
+
}
|
|
574
|
+
}
|
|
575
|
+
|
|
576
|
+
/**
|
|
577
|
+
* Assemble the `initialWindows` the send gate is constructed with at boot
|
|
578
|
+
* (part3-design §7). Combines the global window from the single-object
|
|
579
|
+
* `flood-wait.json` (#3094 / #2923) with every future-dated scoped window from
|
|
580
|
+
* the sibling `flood-windows.json`, pruning expired ones. The gate applies
|
|
581
|
+
* these BEFORE any outbound call so a boot mid-ban does not resend into it.
|
|
582
|
+
*/
|
|
583
|
+
export function loadInitialFloodWindows(
|
|
584
|
+
floodStateFilePath: string,
|
|
585
|
+
floodWindowsFilePath: string,
|
|
586
|
+
now: number,
|
|
587
|
+
): { scopeKey: string; untilTs: number }[] {
|
|
588
|
+
const out: { scopeKey: string; untilTs: number }[] = []
|
|
589
|
+
const global = readFloodState(floodStateFilePath)
|
|
590
|
+
if (global && global.untilTs > now) out.push({ scopeKey: 'global', untilTs: global.untilTs })
|
|
591
|
+
for (const w of readFloodWindows(floodWindowsFilePath, now)) {
|
|
592
|
+
out.push({ scopeKey: w.scopeKey, untilTs: w.untilTs })
|
|
593
|
+
}
|
|
594
|
+
return out
|
|
123
595
|
}
|
|
@@ -71,6 +71,25 @@ export function codeSpanSafe(s: string): string {
|
|
|
71
71
|
return s.replace(/`/g, '`')
|
|
72
72
|
}
|
|
73
73
|
|
|
74
|
+
/**
|
|
75
|
+
* Make a URL safe to interpolate as the destination of a `[label](href)`
|
|
76
|
+
* inline link.
|
|
77
|
+
*
|
|
78
|
+
* In GFM / Bot API 10.1 markdown a link destination written as `(...)` is a
|
|
79
|
+
* bare destination whose parentheses must be BALANCED, or every paren must be
|
|
80
|
+
* backslash-escaped. An href containing a literal `)` (Wikipedia
|
|
81
|
+
* `..._(disambiguation)` URLs, tracking params, generated deep links) can
|
|
82
|
+
* either truncate the URL (a lone `)` closing the destination) or, if we
|
|
83
|
+
* escape only `)`, unbalance the parens and leak a literal backslash into the
|
|
84
|
+
* decoded href. The destination honours C-style backslash escapes, so escape
|
|
85
|
+
* `\` first (so we never double-escape a following escape), then BOTH `(` and
|
|
86
|
+
* `)` — the whole URL is preserved balanced and micromark decodes it back to
|
|
87
|
+
* the original href on round-trip. Bot API 10.1 lists `(`/`)` as escapable.
|
|
88
|
+
*/
|
|
89
|
+
export function escapeLinkHref(href: string): string {
|
|
90
|
+
return href.replace(/\\/g, '\\\\').replace(/\(/g, '\\(').replace(/\)/g, '\\)')
|
|
91
|
+
}
|
|
92
|
+
|
|
74
93
|
/**
|
|
75
94
|
* Repair LLM-side JSON escape bungles.
|
|
76
95
|
*
|