switchroom 0.18.31 → 0.18.33
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-scheduler/index.js +4 -2
- package/dist/auth-broker/index.js +21 -3
- package/dist/cli/notion-write-pretool.mjs +4 -2
- package/dist/cli/switchroom.js +1410 -852
- package/dist/host-control/main.js +22 -4
- package/dist/vault/approvals/kernel-server.js +21 -3
- package/dist/vault/broker/server.js +48 -4
- package/package.json +4 -3
- package/profiles/_base/start.sh.hbs +148 -23
- package/telegram-plugin/dist/gateway/gateway.js +62726 -58282
- package/telegram-plugin/gateway/agent-button-callback-handler.ts +237 -0
- package/telegram-plugin/gateway/ask-callback-handler.ts +92 -0
- package/telegram-plugin/gateway/attachment-message-handlers.ts +152 -0
- package/telegram-plugin/gateway/backstop-delivery.ts +223 -23
- package/telegram-plugin/gateway/boot-card.ts +169 -1
- package/telegram-plugin/gateway/bot-commands-model-effort.ts +209 -0
- package/telegram-plugin/gateway/bot-commands-start-info.ts +108 -0
- package/telegram-plugin/gateway/callback-query-handlers.ts +124 -0
- package/telegram-plugin/gateway/captured-answer-resume.ts +259 -0
- package/telegram-plugin/gateway/card-approval-keyboards.test.ts +28 -0
- package/telegram-plugin/gateway/card-tool-handlers.ts +639 -0
- package/telegram-plugin/gateway/checklist-message-handler.ts +107 -0
- package/telegram-plugin/gateway/delivery-confirm-wiring.ts +133 -0
- package/telegram-plugin/gateway/disconnect-flush.ts +6 -44
- package/telegram-plugin/gateway/gateway-import-clean.test.ts +188 -0
- package/telegram-plugin/gateway/gateway.ts +6403 -13456
- package/telegram-plugin/gateway/inbound-delivery-machine-dispatch.ts +7 -15
- package/telegram-plugin/gateway/inbound-delivery-machine-shadow.ts +35 -68
- package/telegram-plugin/gateway/inbound-interceptors.ts +1133 -0
- package/telegram-plugin/gateway/inbound-router.ts +400 -0
- package/telegram-plugin/gateway/liveness-wiring.ts +440 -0
- package/telegram-plugin/gateway/media-message-handlers.ts +256 -0
- package/telegram-plugin/gateway/mental-model-propose-card.ts +16 -0
- package/telegram-plugin/gateway/model-command.ts +23 -0
- package/telegram-plugin/gateway/narrative-lane.ts +865 -0
- package/telegram-plugin/gateway/obligation-ledger.ts +42 -0
- package/telegram-plugin/gateway/obligation-store.ts +37 -1
- package/telegram-plugin/gateway/obligation-wiring.ts +333 -0
- package/telegram-plugin/gateway/outbound-send-path.ts +2012 -0
- package/telegram-plugin/gateway/photo-message-handler.ts +80 -0
- package/telegram-plugin/gateway/pinned-message-handler.ts +86 -0
- package/telegram-plugin/gateway/secret-request-card.test.ts +46 -0
- package/telegram-plugin/gateway/secret-request-card.ts +45 -0
- package/telegram-plugin/gateway/stream-render.ts +2166 -0
- package/telegram-plugin/gateway/turn-end.ts +606 -0
- package/telegram-plugin/gateway/turn-start-surfaces.ts +298 -0
- package/telegram-plugin/gateway/vault-request-access-card.ts +16 -0
- package/telegram-plugin/gateway/vault-request-save-card.test.ts +49 -0
- package/telegram-plugin/gateway/vault-request-save-card.ts +52 -0
- package/telegram-plugin/gateway/voice-message-handler.ts +123 -0
- package/telegram-plugin/gateway/voice-ondemand-callback-handler.ts +204 -0
- package/telegram-plugin/gateway/worker-feed-dispatch.ts +40 -0
- package/telegram-plugin/narrative-dedup.ts +24 -1
- package/telegram-plugin/narrative-flush.ts +2 -2
- package/telegram-plugin/pending-user-notice.ts +59 -13
- package/telegram-plugin/render/render.ts +25 -1
- package/telegram-plugin/status-no-truncate.ts +13 -0
- package/telegram-plugin/subagent-watcher.ts +297 -31
- package/telegram-plugin/tests/activity-card-wiring.test.ts +8 -3
- package/telegram-plugin/tests/activity-ever-opened-sticky.test.ts +18 -3
- package/telegram-plugin/tests/agent-button-callback-handler.test.ts +149 -0
- package/telegram-plugin/tests/ask-callback-handler.test.ts +118 -0
- package/telegram-plugin/tests/attachment-message-handlers.test.ts +135 -0
- package/telegram-plugin/tests/backstop-delivery.test.ts +167 -0
- package/telegram-plugin/tests/backstop-readback-probe.test.ts +144 -0
- package/telegram-plugin/tests/boot-card-routing.test.ts +139 -0
- package/telegram-plugin/tests/bot-commands-model-effort.test.ts +189 -0
- package/telegram-plugin/tests/bot-commands-start-info.test.ts +240 -0
- package/telegram-plugin/tests/buffer-gate-broadened.test.ts +28 -9
- package/telegram-plugin/tests/busy-ack-wiring.test.ts +6 -1
- package/telegram-plugin/tests/button-tap-turn-gated.test.ts +21 -12
- package/telegram-plugin/tests/callback-query-handlers.test.ts +101 -0
- package/telegram-plugin/tests/captured-answer-resume.test.ts +358 -0
- package/telegram-plugin/tests/card-tool-handlers.test.ts +497 -0
- package/telegram-plugin/tests/catch-all-unhandled-message.test.ts +5 -2
- package/telegram-plugin/tests/checklist-message-handler.test.ts +160 -0
- package/telegram-plugin/tests/emission-authority-facade.test.ts +76 -29
- package/telegram-plugin/tests/emission-authority-ping-gate.test.ts +4 -1
- package/telegram-plugin/tests/emission-determinism-wiring.test.ts +45 -16
- package/telegram-plugin/tests/feed-heartbeat-liveness-open.test.ts +30 -7
- package/telegram-plugin/tests/gateway-boot-side-effect-gating.test.ts +270 -0
- package/telegram-plugin/tests/gateway-boot-smoke.test.ts +150 -0
- package/telegram-plugin/tests/gateway-bot-construction-deferral.test.ts +251 -0
- package/telegram-plugin/tests/gateway-disconnect-flush.test.ts +5 -128
- package/telegram-plugin/tests/gateway-handler-registration-wiring.test.ts +299 -0
- package/telegram-plugin/tests/gateway-loopback-paste-redact.test.ts +44 -29
- package/telegram-plugin/tests/gateway-outbound-redact.test.ts +18 -5
- package/telegram-plugin/tests/gateway-request-secret.test.ts +7 -3
- package/telegram-plugin/tests/gateway-secret-detect.test.ts +20 -10
- package/telegram-plugin/tests/gateway-session-model-relaunch.test.ts +8 -2
- package/telegram-plugin/tests/inbound-delivery-cutover-flip.test.ts +54 -150
- package/telegram-plugin/tests/inbound-delivery-cutover-gate.test.ts +10 -14
- package/telegram-plugin/tests/inbound-delivery-dispatch-equivalence.test.ts +6 -7
- package/telegram-plugin/tests/inbound-delivery-machine-dispatch.test.ts +0 -16
- package/telegram-plugin/tests/inbound-emit-after-intercepts.test.ts +18 -7
- package/telegram-plugin/tests/inbound-message-types.test.ts +52 -16
- package/telegram-plugin/tests/litellm-proxy-auth-misconfig.test.ts +69 -14
- package/telegram-plugin/tests/media-message-handlers.test.ts +276 -0
- package/telegram-plugin/tests/mental-model-propose-callback-gate.test.ts +8 -4
- package/telegram-plugin/tests/model-command.test.ts +30 -0
- package/telegram-plugin/tests/multitopic-routing-wiring.test.ts +36 -12
- package/telegram-plugin/tests/narrative-dedup.test.ts +32 -0
- package/telegram-plugin/tests/narrative-flush.test.ts +6 -2
- package/telegram-plugin/tests/narrative-lane-golden.test.ts +458 -0
- package/telegram-plugin/tests/no-reply-bounded-drain.test.ts +14 -3
- package/telegram-plugin/tests/obligation-ledger.test.ts +40 -0
- package/telegram-plugin/tests/obligation-store.test.ts +43 -0
- package/telegram-plugin/tests/pending-card-durability-wiring.test.ts +18 -8
- package/telegram-plugin/tests/per-topic-current-turn.test.ts +32 -8
- package/telegram-plugin/tests/permission-no-repeat-wiring.test.ts +9 -6
- package/telegram-plugin/tests/photo-message-handler.test.ts +114 -0
- package/telegram-plugin/tests/photo-reroute-wiring.test.ts +5 -2
- package/telegram-plugin/tests/pinned-message-handler.test.ts +108 -0
- package/telegram-plugin/tests/render/render.test.ts +42 -0
- package/telegram-plugin/tests/reply-terminal-reaction.test.ts +6 -2
- package/telegram-plugin/tests/secret-detect-delete-must-surface-failures.test.ts +8 -4
- package/telegram-plugin/tests/secret-detect-fail-closed.test.ts +38 -28
- package/telegram-plugin/tests/secret-detect-oauth-code.test.ts +31 -20
- package/telegram-plugin/tests/send-reply-golden.test.ts +571 -0
- package/telegram-plugin/tests/silence-liveness-wiring.test.ts +22 -8
- package/telegram-plugin/tests/status-pin-service-message-suppression.test.ts +42 -49
- package/telegram-plugin/tests/stop-command.test.ts +22 -12
- package/telegram-plugin/tests/stream-render-golden.test.ts +424 -0
- package/telegram-plugin/tests/subagent-watcher-boot-skip-dead.test.ts +218 -0
- package/telegram-plugin/tests/subagent-watcher-resume-reregister.test.ts +305 -0
- package/telegram-plugin/tests/subagent-watcher-resurrection.test.ts +32 -0
- package/telegram-plugin/tests/subagent-watcher.test.ts +35 -3
- package/telegram-plugin/tests/turn-end-gate-backstop.test.ts +8 -12
- package/telegram-plugin/tests/turn-flush-safety.test.ts +191 -11
- package/telegram-plugin/tests/turn-flush-suppression-wiring.test.ts +117 -0
- package/telegram-plugin/tests/vault-approval-posture.test.ts +8 -2
- package/telegram-plugin/tests/vault-grant-inbound-builders.test.ts +2 -2
- package/telegram-plugin/tests/vault-grant-union.test.ts +4 -1
- package/telegram-plugin/tests/vault-key-regex-allows-slash.test.ts +16 -5
- package/telegram-plugin/tests/vault-request-access-tool.test.ts +10 -5
- package/telegram-plugin/tests/vault-request-access-unlock-resume.test.ts +4 -1
- package/telegram-plugin/tests/vault-subcommands.test.ts +6 -1
- package/telegram-plugin/tests/voice-message-handler.test.ts +111 -0
- package/telegram-plugin/tests/voice-ondemand-callback-handler.test.ts +140 -0
- package/telegram-plugin/tests/worker-activity-feed.test.ts +86 -19
- package/telegram-plugin/tests/worker-feed-coalesce.test.ts +236 -20
- package/telegram-plugin/tests/worker-feed-resume-guard.test.ts +86 -0
- package/telegram-plugin/tool-activity-summary.ts +110 -38
- package/telegram-plugin/turn-flush-safety.ts +80 -14
- package/telegram-plugin/uat/restart-capability.ts +76 -0
- package/telegram-plugin/uat/scenarios/bg-sub-agent-dispatch-dm.test.ts +14 -4
- package/telegram-plugin/uat/scenarios/bridge-flap-resilience-dm.test.ts +11 -1
- package/telegram-plugin/uat/scenarios/cross-turn-pending-progress-dm.test.ts +19 -2
- package/telegram-plugin/uat/scenarios/jtbd-always-on-after-restart-dm.test.ts +6 -12
- package/telegram-plugin/uat/scenarios/jtbd-deliberate-restart-resumes-dm.test.ts +6 -12
- package/telegram-plugin/uat/scenarios/jtbd-interrupted-turn-resumes-dm.test.ts +6 -12
- package/telegram-plugin/uat/scenarios/jtbd-multipart-render-dm.test.ts +47 -13
- package/telegram-plugin/worker-activity-feed.ts +34 -4
- package/telegram-plugin/gateway/busy-key-reaper.ts +0 -113
- package/telegram-plugin/gateway/gate-parity-probe.ts +0 -102
- package/telegram-plugin/tests/busy-key-reaper.test.ts +0 -192
- package/telegram-plugin/tests/fixtures/cutover-killswitch-probe.ts +0 -75
- package/telegram-plugin/tests/gate-parity-probe.test.ts +0 -171
- package/telegram-plugin/tests/parallel-turns-deadlock-fix.test.ts +0 -217
|
@@ -40,6 +40,12 @@ export class BackstopDeliveryLedger {
|
|
|
40
40
|
private chunks = new Map<string, Map<number, number[]>>()
|
|
41
41
|
/** turnId -> chunk indices with an in-flight (pre-ack) send (guard 6). */
|
|
42
42
|
private pending = new Map<string, Set<number>>()
|
|
43
|
+
/** turnId -> chunk indices whose read-back probe CONFIRMED the message
|
|
44
|
+
* actually exists in the chat (#3278). A chunk with a landed message id but
|
|
45
|
+
* no confirmation is `landed-unconfirmed` — the Bot API accepted the send but
|
|
46
|
+
* a server-side silent discard (flood/anti-spam) can return a fresh id for a
|
|
47
|
+
* message the user never sees, so an id alone is NOT proof of delivery. */
|
|
48
|
+
private confirmed = new Map<string, Set<number>>()
|
|
43
49
|
|
|
44
50
|
/**
|
|
45
51
|
* Guard 5 — once-per-turn BACKSTOP double-fire latch. Returns `true` on the
|
|
@@ -99,6 +105,78 @@ export class BackstopDeliveryLedger {
|
|
|
99
105
|
return (this.chunks.get(turnId)?.get(index)?.length ?? 0) > 0
|
|
100
106
|
}
|
|
101
107
|
|
|
108
|
+
/** The landed message id(s) for one chunk (empty when unsent). The read-back
|
|
109
|
+
* probe reads these to confirm every id of the chunk (guard A7 — a
|
|
110
|
+
* length-resplit chunk lands >1 id and all must exist to count confirmed). */
|
|
111
|
+
chunkIds(turnId: string, index: number): number[] {
|
|
112
|
+
return (this.chunks.get(turnId)?.get(index) ?? []).slice()
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
/**
|
|
116
|
+
* #3278 — transition a landed-unconfirmed chunk to `landed-confirmed` after a
|
|
117
|
+
* read-back probe proved the message exists in the chat. Only a confirmed
|
|
118
|
+
* chunk counts toward delivery / `complete`.
|
|
119
|
+
*/
|
|
120
|
+
confirmChunk(turnId: string, index: number): void {
|
|
121
|
+
let set = this.confirmed.get(turnId)
|
|
122
|
+
if (set == null) {
|
|
123
|
+
set = new Set()
|
|
124
|
+
this.confirmed.set(turnId, set)
|
|
125
|
+
}
|
|
126
|
+
set.add(index)
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
/**
|
|
130
|
+
* #3278 — demote a chunk back to `unsent` on a POSITIVE absence read-back
|
|
131
|
+
* (Telegram `400 message to edit not found`): the send was silently dropped,
|
|
132
|
+
* so re-sending it is safe (it never reached the user). Clears the landed ids,
|
|
133
|
+
* the pending marker, and any stale confirmation. NEVER call this on an
|
|
134
|
+
* ambiguous probe (429/5xx/network) — that would risk a duplicate.
|
|
135
|
+
*/
|
|
136
|
+
demoteChunk(turnId: string, index: number): void {
|
|
137
|
+
this.chunks.get(turnId)?.delete(index)
|
|
138
|
+
this.pending.get(turnId)?.delete(index)
|
|
139
|
+
this.confirmed.get(turnId)?.delete(index)
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
/** True only for a `landed-confirmed` chunk (read-back proved it exists). */
|
|
143
|
+
hasConfirmedChunk(turnId: string, index: number): boolean {
|
|
144
|
+
return this.confirmed.get(turnId)?.has(index) ?? false
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
/** Chunk indices that have landed at least one id but are NOT yet confirmed —
|
|
148
|
+
* the set the read-back probe must resolve (confirm / demote / leave). */
|
|
149
|
+
landedUnconfirmedIndices(turnId: string, chunkCount: number): number[] {
|
|
150
|
+
const out: number[] = []
|
|
151
|
+
for (let i = 0; i < chunkCount; i++) {
|
|
152
|
+
if (this.hasChunk(turnId, i) && !this.hasConfirmedChunk(turnId, i)) out.push(i)
|
|
153
|
+
}
|
|
154
|
+
return out
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
/** True IFF every chunk 0..chunkCount-1 is `landed-confirmed`. `chunkCount===0`
|
|
158
|
+
* is never "all confirmed" (nothing was delivered). */
|
|
159
|
+
allConfirmed(turnId: string, chunkCount: number): boolean {
|
|
160
|
+
if (chunkCount <= 0) return false
|
|
161
|
+
for (let i = 0; i < chunkCount; i++) {
|
|
162
|
+
if (!this.hasConfirmedChunk(turnId, i)) return false
|
|
163
|
+
}
|
|
164
|
+
return true
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
/** Landed message ids of CONFIRMED chunks only, in chunk-index order — the set
|
|
168
|
+
* the delivery predicate counts (fresh non-card ids that are proven-present). */
|
|
169
|
+
confirmedIds(turnId: string): number[] {
|
|
170
|
+
const m = this.chunks.get(turnId)
|
|
171
|
+
const set = this.confirmed.get(turnId)
|
|
172
|
+
if (m == null || set == null) return []
|
|
173
|
+
const out: number[] = []
|
|
174
|
+
for (const index of Array.from(m.keys()).sort((a, b) => a - b)) {
|
|
175
|
+
if (set.has(index)) out.push(...(m.get(index) ?? []))
|
|
176
|
+
}
|
|
177
|
+
return out
|
|
178
|
+
}
|
|
179
|
+
|
|
102
180
|
/** All landed message ids for this turn, in chunk-index order. */
|
|
103
181
|
sentIds(turnId: string): number[] {
|
|
104
182
|
const m = this.chunks.get(turnId)
|
|
@@ -137,9 +215,49 @@ export class BackstopDeliveryLedger {
|
|
|
137
215
|
this.latched.delete(turnId)
|
|
138
216
|
this.chunks.delete(turnId)
|
|
139
217
|
this.pending.delete(turnId)
|
|
218
|
+
this.confirmed.delete(turnId)
|
|
140
219
|
}
|
|
141
220
|
}
|
|
142
221
|
|
|
222
|
+
/**
|
|
223
|
+
* #3278 — the tri-state a read-back probe resolves a landed chunk into:
|
|
224
|
+
* - `exists` — a no-op `editMessageText` succeeded or returned "message is
|
|
225
|
+
* not modified" ⇒ the message is present at the chat_id ⇒
|
|
226
|
+
* `landed-confirmed` (counts toward delivery).
|
|
227
|
+
* - `absent` — Telegram `400 message to edit not found` ⇒ positive absence
|
|
228
|
+
* ⇒ demote to `unsent` (safe to re-send — it never landed).
|
|
229
|
+
* - `ambiguous` — 429 / 5xx / network / gate-shed / anything else ⇒ leave
|
|
230
|
+
* `landed-unconfirmed`; NEVER re-send (a re-send would risk a
|
|
231
|
+
* duplicate, and duplicate-risk beats missing-risk here).
|
|
232
|
+
*
|
|
233
|
+
* NOTE (honest limitation, #3278 §1.4): a passing probe proves only that the
|
|
234
|
+
* message EXISTS at that chat_id. It does NOT prove the human's client rendered
|
|
235
|
+
* it, and it does NOT catch a wrong-thread/chat misroute (the edit succeeds
|
|
236
|
+
* because the message exists — just not where the operator is looking).
|
|
237
|
+
* Read-back is necessary-but-not-sufficient; it is paired with the correct
|
|
238
|
+
* chat/thread resolution already pinned by the backstop caller.
|
|
239
|
+
*/
|
|
240
|
+
export type ReadBackResult = 'exists' | 'absent' | 'ambiguous'
|
|
241
|
+
|
|
242
|
+
/**
|
|
243
|
+
* Combine the per-id read-back results of ONE chunk into the chunk's verdict
|
|
244
|
+
* (guard A7 — a length-resplit chunk lands >1 message id and every id must be
|
|
245
|
+
* present for the chunk to count confirmed). Pure:
|
|
246
|
+
* - ANY id `absent` ⇒ `absent` (the chunk is incomplete on the
|
|
247
|
+
* wire → re-send the whole chunk)
|
|
248
|
+
* - ALL ids `exists` ⇒ `exists`
|
|
249
|
+
* - otherwise (some `ambiguous`,
|
|
250
|
+
* none `absent`) ⇒ `ambiguous` (never re-send)
|
|
251
|
+
* - empty id set ⇒ `ambiguous` (nothing to probe → don't
|
|
252
|
+
* fabricate a confirmation OR a demotion)
|
|
253
|
+
*/
|
|
254
|
+
export function combineReadBackResults(perId: readonly ReadBackResult[]): ReadBackResult {
|
|
255
|
+
if (perId.length === 0) return 'ambiguous'
|
|
256
|
+
if (perId.some(r => r === 'absent')) return 'absent'
|
|
257
|
+
if (perId.every(r => r === 'exists')) return 'exists'
|
|
258
|
+
return 'ambiguous'
|
|
259
|
+
}
|
|
260
|
+
|
|
143
261
|
/**
|
|
144
262
|
* Guard 7 — the RECEIPT gate. Given the raw delivered ids and the progress-card
|
|
145
263
|
* message id (or null), return only the FRESH, non-card chat message ids. A
|
|
@@ -175,6 +293,25 @@ export interface BackstopDeliveryDeps {
|
|
|
175
293
|
/** Record the delivered ids + aligned texts to history (called once, after
|
|
176
294
|
* the attempts resolve, with the full landed set). */
|
|
177
295
|
recordOutbound?: (messageIds: number[], texts: string[]) => void
|
|
296
|
+
/**
|
|
297
|
+
* #3278 read-back confirmation. After a chunk lands a message id, a no-op
|
|
298
|
+
* `editMessageText` probe re-confirms the message ACTUALLY exists in the chat
|
|
299
|
+
* (a returned id is only an API accept, not proof of visibility). Returns the
|
|
300
|
+
* chunk's {@link ReadBackResult} — `exists` ⇒ landed-confirmed, `absent` ⇒
|
|
301
|
+
* demote to unsent + re-send, `ambiguous` ⇒ stay landed-unconfirmed and NEVER
|
|
302
|
+
* re-send. Runs once per landed chunk after the send attempts resolve, and is
|
|
303
|
+
* itself flood-window-paced by the caller (never a storm).
|
|
304
|
+
*
|
|
305
|
+
* When OMITTED, a landed chunk is confirmed on landing (pre-#3278 behavior) —
|
|
306
|
+
* still gated by the fresh-non-card receipt filter, so a card-only "delivery"
|
|
307
|
+
* is never counted. This is the hot-path default: read-back is scoped to the
|
|
308
|
+
* RARE backstop delivery, so the reply-tool path issues ZERO probes (§1.3).
|
|
309
|
+
*/
|
|
310
|
+
readBack?: (
|
|
311
|
+
chunkIndex: number,
|
|
312
|
+
messageIds: readonly number[],
|
|
313
|
+
text: string,
|
|
314
|
+
) => Promise<ReadBackResult>
|
|
178
315
|
/** Optional stderr sink for progress/resume logging. */
|
|
179
316
|
stderr?: (s: string) => void
|
|
180
317
|
}
|
|
@@ -204,6 +341,14 @@ export interface BackstopDeliveryResult {
|
|
|
204
341
|
* so the caller can leave the delivery obligation OPEN for the liveness floor
|
|
205
342
|
* to re-present — instead of the old silent `send_failed` drop.
|
|
206
343
|
*
|
|
344
|
+
* #3278 — after each attempt's sends resolve, every `landed-unconfirmed` chunk
|
|
345
|
+
* is read-back-probed via `deps.readBack` (when provided): `exists` confirms it,
|
|
346
|
+
* `absent` demotes it to `unsent` so the NEXT attempt re-sends only that chunk,
|
|
347
|
+
* and `ambiguous` leaves it `landed-unconfirmed` — never re-sent (duplicate-risk
|
|
348
|
+
* beats missing-risk). `delivered` now requires every chunk `landed-confirmed`,
|
|
349
|
+
* so an API-ack'd-but-silently-dropped send (fresh id, absent on read-back) is
|
|
350
|
+
* reported `delivered:false` and the caller leaves the obligation OPEN.
|
|
351
|
+
*
|
|
207
352
|
* `recordOutbound` (when provided) fires ONCE at the end with the full landed
|
|
208
353
|
* set and a `texts` array ALIGNED to the actual sent ids (via `ledger.entries`).
|
|
209
354
|
* The ledger is NOT cleared here — the caller clears it only after success or
|
|
@@ -223,37 +368,92 @@ export async function runBackstopDelivery(
|
|
|
223
368
|
|
|
224
369
|
for (let attempt = 1; attempt <= Math.max(1, maxAttempts); attempt++) {
|
|
225
370
|
attempts = attempt
|
|
371
|
+
// 1. SEND the `unsent` chunks (never-landed OR demoted by a prior absent
|
|
372
|
+
// read-back), resuming in chunk-index order. Guard 6 keeps a chunk that
|
|
373
|
+
// already landed from being re-sent.
|
|
226
374
|
const resume = ledger.unsentIndices(turnId, chunkCount)
|
|
227
|
-
if (resume.length === 0) break // everything already landed
|
|
228
|
-
if (attempt > 1) {
|
|
229
|
-
stderr(
|
|
230
|
-
`telegram gateway: backstop delivery retry ${attempt}/${maxAttempts} — ` +
|
|
231
|
-
`resuming at unsent chunk(s) [${resume.join(', ')}] for turn ${turnId}\n`,
|
|
232
|
-
)
|
|
233
|
-
}
|
|
234
375
|
let attemptThrew = false
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
376
|
+
if (resume.length > 0) {
|
|
377
|
+
if (attempt > 1) {
|
|
378
|
+
stderr(
|
|
379
|
+
`telegram gateway: backstop delivery retry ${attempt}/${maxAttempts} — ` +
|
|
380
|
+
`resuming at unsent chunk(s) [${resume.join(', ')}] for turn ${turnId}\n`,
|
|
381
|
+
)
|
|
382
|
+
}
|
|
383
|
+
try {
|
|
384
|
+
for (const i of resume) {
|
|
385
|
+
if (ledger.hasChunk(turnId, i)) continue
|
|
386
|
+
ledger.markPending(turnId, i)
|
|
387
|
+
const ids = await deps.sendChunk(i, chunks[i])
|
|
388
|
+
ledger.recordChunk(turnId, i, ids) // → landed-unconfirmed
|
|
389
|
+
}
|
|
390
|
+
} catch (err) {
|
|
391
|
+
attemptThrew = true
|
|
392
|
+
stderr(
|
|
393
|
+
`telegram gateway: backstop delivery attempt ${attempt}/${maxAttempts} failed: ` +
|
|
394
|
+
`${err instanceof Error ? err.message : String(err)}\n`,
|
|
395
|
+
)
|
|
242
396
|
}
|
|
243
|
-
} catch (err) {
|
|
244
|
-
attemptThrew = true
|
|
245
|
-
stderr(
|
|
246
|
-
`telegram gateway: backstop delivery attempt ${attempt}/${maxAttempts} failed: ` +
|
|
247
|
-
`${err instanceof Error ? err.message : String(err)}\n`,
|
|
248
|
-
)
|
|
249
397
|
}
|
|
250
|
-
|
|
398
|
+
|
|
399
|
+
// 2. CONFIRM landed-unconfirmed chunks via read-back (#3278). Without a probe
|
|
400
|
+
// wired, landing IS confirmation (pre-#3278) — the fresh-non-card receipt
|
|
401
|
+
// filter below still rejects a card-only "delivery".
|
|
402
|
+
let demotedAny = false
|
|
403
|
+
for (const i of ledger.landedUnconfirmedIndices(turnId, chunkCount)) {
|
|
404
|
+
if (deps.readBack == null) {
|
|
405
|
+
ledger.confirmChunk(turnId, i)
|
|
406
|
+
continue
|
|
407
|
+
}
|
|
408
|
+
let verdict: ReadBackResult
|
|
409
|
+
try {
|
|
410
|
+
verdict = await deps.readBack(i, ledger.chunkIds(turnId, i), chunks[i])
|
|
411
|
+
} catch (err) {
|
|
412
|
+
// A probe adapter that itself throws must NEVER trigger a re-send.
|
|
413
|
+
verdict = 'ambiguous'
|
|
414
|
+
stderr(
|
|
415
|
+
`telegram gateway: backstop read-back probe threw for chunk ${i} ` +
|
|
416
|
+
`(turn ${turnId}) — treating as ambiguous: ` +
|
|
417
|
+
`${err instanceof Error ? err.message : String(err)}\n`,
|
|
418
|
+
)
|
|
419
|
+
}
|
|
420
|
+
if (verdict === 'exists') {
|
|
421
|
+
ledger.confirmChunk(turnId, i)
|
|
422
|
+
} else if (verdict === 'absent') {
|
|
423
|
+
// Positive absence — the send was silently dropped. Safe to re-send.
|
|
424
|
+
ledger.demoteChunk(turnId, i)
|
|
425
|
+
demotedAny = true
|
|
426
|
+
stderr(
|
|
427
|
+
`telegram gateway: backstop read-back — chunk ${i} not found in chat ` +
|
|
428
|
+
`(turn ${turnId}); demoting to unsent for re-send\n`,
|
|
429
|
+
)
|
|
430
|
+
} else {
|
|
431
|
+
// Ambiguous (429/5xx/network/gate-shed) — leave landed-unconfirmed and
|
|
432
|
+
// NEVER re-send it (would risk a duplicate).
|
|
433
|
+
stderr(
|
|
434
|
+
`telegram gateway: backstop read-back — chunk ${i} ambiguous ` +
|
|
435
|
+
`(turn ${turnId}); left landed-unconfirmed, not re-sending\n`,
|
|
436
|
+
)
|
|
437
|
+
}
|
|
438
|
+
}
|
|
439
|
+
|
|
440
|
+
// 3. Fully confirmed ⇒ done.
|
|
441
|
+
if (ledger.allConfirmed(turnId, chunkCount)) break
|
|
442
|
+
// 4. Another attempt ONLY re-sends chunks demoted to `unsent`. If nothing is
|
|
443
|
+
// unsent, the remaining non-confirmed chunks are all `landed-unconfirmed`
|
|
444
|
+
// (ambiguous) — re-sending them would duplicate, so stop here.
|
|
445
|
+
if (ledger.unsentIndices(turnId, chunkCount).length === 0) break
|
|
446
|
+
// 5. Guard against a spin: if this attempt neither sent, threw, nor demoted
|
|
447
|
+
// anything, another identical attempt is pointless.
|
|
448
|
+
if (resume.length === 0 && !attemptThrew && !demotedAny) break
|
|
251
449
|
}
|
|
252
450
|
|
|
253
451
|
const sentIds = ledger.sentIds(turnId)
|
|
452
|
+
// #3278 — delivered IFF every chunk is `landed-confirmed` AND at least one
|
|
453
|
+
// confirmed id is a fresh non-card chat id (the receipt gate, guard 7).
|
|
254
454
|
const delivered =
|
|
255
|
-
|
|
256
|
-
backstopReceiptIds(
|
|
455
|
+
ledger.allConfirmed(turnId, chunkCount) &&
|
|
456
|
+
backstopReceiptIds(ledger.confirmedIds(turnId), cardMessageId).length > 0
|
|
257
457
|
const exhausted = !delivered
|
|
258
458
|
|
|
259
459
|
if (deps.recordOutbound && sentIds.length > 0) {
|
|
@@ -68,6 +68,7 @@ import {
|
|
|
68
68
|
type ConfigDiff,
|
|
69
69
|
} from './config-snapshot.js'
|
|
70
70
|
import { join } from 'path'
|
|
71
|
+
import { readFileSync, statSync } from 'fs'
|
|
71
72
|
import { bootCardChatKey, loadBootCardMsgId, saveBootCardMsgId } from './boot-card-msgid.js'
|
|
72
73
|
import { nonEssentialSendSuppression } from '../flood-circuit-breaker.js'
|
|
73
74
|
import { loadConfig as _loadSwitchroomConfig } from '../../src/config/loader.js'
|
|
@@ -278,6 +279,131 @@ const REASON_LABEL: Record<RestartReason, string> = {
|
|
|
278
279
|
fresh: 'fresh start',
|
|
279
280
|
}
|
|
280
281
|
|
|
282
|
+
// ─── Routing-mode row (.routing-mode observability) ─────────────────────────
|
|
283
|
+
|
|
284
|
+
/**
|
|
285
|
+
* Parsed `.routing-mode` state file — one overwrite-per-boot line written by
|
|
286
|
+
* start.sh immediately before `exec claude`:
|
|
287
|
+
*
|
|
288
|
+
* mode=<router-root|passthrough|direct-oauth> base=<url> model=<effective>
|
|
289
|
+
* litellm_ok=<0|1> declared=<mode> ts=<iso>
|
|
290
|
+
*
|
|
291
|
+
* `prevBoot` marks a RENDER-TIMING hazard, not file corruption: the gateway
|
|
292
|
+
* (outer pass) can render the boot card BEFORE the inner pass writes this
|
|
293
|
+
* boot's line (the inner LiteLLM probe can hold the write back for minutes on
|
|
294
|
+
* a degraded proxy). When the file's mtime predates this gateway process
|
|
295
|
+
* start, the value belongs to the PREVIOUS boot and must be labeled as such —
|
|
296
|
+
* never shown as current.
|
|
297
|
+
*/
|
|
298
|
+
export interface RoutingModeInfo {
|
|
299
|
+
/** Landed routing mode this boot: router-root | passthrough | direct-oauth. */
|
|
300
|
+
mode: string
|
|
301
|
+
/** Declared (apply-time) routing intent, when present in the file. */
|
|
302
|
+
declared?: string
|
|
303
|
+
/** Effective launched model recorded alongside the mode. */
|
|
304
|
+
model?: string
|
|
305
|
+
/** LiteLLM liveliness at boot ("1" ok / "0" not confirmed). */
|
|
306
|
+
litellmOk?: string
|
|
307
|
+
/** True when the file predates this gateway boot (stale — prev boot's value). */
|
|
308
|
+
prevBoot: boolean
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
/**
|
|
312
|
+
* Epoch-ms this CONTAINER booted, derived from PID 1's start time
|
|
313
|
+
* (`/proc/stat` btime + `/proc/1/stat` field 22 in USER_HZ ticks). This is the
|
|
314
|
+
* correct staleness threshold for `.routing-mode`: start.sh writes that file
|
|
315
|
+
* ONCE per container boot, before exec-ing claude/the gateway. Keying staleness
|
|
316
|
+
* on the gateway PROCESS start (`Date.now() - process.uptime()*1000`) mislabels
|
|
317
|
+
* a gateway-only respawn — supervisor restarts the gateway process while the
|
|
318
|
+
* container (and its still-current `.routing-mode`) stays up — as "(prev boot)".
|
|
319
|
+
* Container boot time is stable across such respawns (LOW-1). Returns null when
|
|
320
|
+
* /proc is unavailable or unparseable (non-Linux, tests), so callers fall back
|
|
321
|
+
* to the process-start estimate.
|
|
322
|
+
*/
|
|
323
|
+
export function containerBootStartMs(
|
|
324
|
+
fsImpl: { readFileSync: typeof readFileSync } = { readFileSync },
|
|
325
|
+
): number | null {
|
|
326
|
+
try {
|
|
327
|
+
const btimeLine = fsImpl
|
|
328
|
+
.readFileSync('/proc/stat', 'utf-8')
|
|
329
|
+
.split('\n')
|
|
330
|
+
.find((l) => l.startsWith('btime '))
|
|
331
|
+
if (!btimeLine) return null
|
|
332
|
+
const btimeSec = Number(btimeLine.split(/\s+/)[1])
|
|
333
|
+
if (!Number.isFinite(btimeSec)) return null
|
|
334
|
+
// /proc/1/stat field 22 (starttime) is in clock ticks since system boot.
|
|
335
|
+
// Fields 2 (comm) may contain spaces/parens, so split on the last ')'.
|
|
336
|
+
const stat = fsImpl.readFileSync('/proc/1/stat', 'utf-8')
|
|
337
|
+
const afterComm = stat.slice(stat.lastIndexOf(')') + 2)
|
|
338
|
+
const fields = afterComm.split(/\s+/)
|
|
339
|
+
// starttime is field 22 overall → index 19 of the post-comm slice
|
|
340
|
+
// (field 1 pid + field 2 comm consumed; field 3 becomes index 0).
|
|
341
|
+
const startTicks = Number(fields[19])
|
|
342
|
+
if (!Number.isFinite(startTicks)) return null
|
|
343
|
+
const HZ = 100 // USER_HZ on effectively all Linux; sysconf unavailable in node
|
|
344
|
+
return (btimeSec + startTicks / HZ) * 1000
|
|
345
|
+
} catch {
|
|
346
|
+
return null
|
|
347
|
+
}
|
|
348
|
+
}
|
|
349
|
+
|
|
350
|
+
/**
|
|
351
|
+
* Read + parse `<agentDir>/.routing-mode`. Returns null when the file is
|
|
352
|
+
* absent, unreadable, or carries no `mode=` field (tolerant by design — the
|
|
353
|
+
* boot card must render fine on fleets that pre-date the state file).
|
|
354
|
+
*
|
|
355
|
+
* `bootStartMs` is the epoch-ms this CONTAINER booted (see
|
|
356
|
+
* `containerBootStartMs`); a file mtime strictly older than it belongs to a
|
|
357
|
+
* previous container boot and is flagged `prevBoot` so the renderer can label
|
|
358
|
+
* the row instead of presenting stale data as current. Keyed on container boot
|
|
359
|
+
* (not gateway process start) so a gateway-only respawn does not mislabel the
|
|
360
|
+
* still-current file.
|
|
361
|
+
*/
|
|
362
|
+
export function readRoutingMode(
|
|
363
|
+
agentDir: string,
|
|
364
|
+
bootStartMs: number,
|
|
365
|
+
fsImpl: { readFileSync: typeof readFileSync; statSync: typeof statSync } = { readFileSync, statSync },
|
|
366
|
+
): RoutingModeInfo | null {
|
|
367
|
+
const path = join(agentDir, '.routing-mode')
|
|
368
|
+
try {
|
|
369
|
+
const raw = fsImpl.readFileSync(path, 'utf-8')
|
|
370
|
+
const line = raw.split('\n')[0] ?? ''
|
|
371
|
+
const field = (key: string): string | undefined => {
|
|
372
|
+
const m = line.match(new RegExp(`(?:^|\\s)${key}=([^\\s]+)`))
|
|
373
|
+
return m ? m[1] : undefined
|
|
374
|
+
}
|
|
375
|
+
const mode = field('mode')
|
|
376
|
+
if (!mode) return null
|
|
377
|
+
const mtimeMs = fsImpl.statSync(path).mtimeMs
|
|
378
|
+
return {
|
|
379
|
+
mode,
|
|
380
|
+
declared: field('declared'),
|
|
381
|
+
model: field('model'),
|
|
382
|
+
litellmOk: field('litellm_ok'),
|
|
383
|
+
prevBoot: mtimeMs < bootStartMs,
|
|
384
|
+
}
|
|
385
|
+
} catch {
|
|
386
|
+
return null
|
|
387
|
+
}
|
|
388
|
+
}
|
|
389
|
+
|
|
390
|
+
/**
|
|
391
|
+
* Render the routing row for the boot card. Divergence (landed ≠ declared)
|
|
392
|
+
* gets the warning dot; a converged mode renders as a dim informational row —
|
|
393
|
+
* deliberately always visible (2026-07-17 incident: both divergence cases
|
|
394
|
+
* were SILENT; the row is the standing observability surface).
|
|
395
|
+
*/
|
|
396
|
+
export function renderRoutingRow(r: RoutingModeInfo): string {
|
|
397
|
+
const diverged = r.declared != null && r.declared !== '' && r.mode !== r.declared
|
|
398
|
+
const dot = diverged ? DOT.degraded : '🔀'
|
|
399
|
+
const parts: string[] = [escapeMarkdown(r.mode)]
|
|
400
|
+
if (diverged) parts.push(`≠ declared ${escapeMarkdown(r.declared as string)}`)
|
|
401
|
+
if (r.model) parts.push(`\`${r.model}\``)
|
|
402
|
+
if (r.litellmOk != null) parts.push(r.litellmOk === '1' ? 'litellm ok' : 'litellm unconfirmed')
|
|
403
|
+
const suffix = r.prevBoot ? ' _(prev boot)_' : ''
|
|
404
|
+
return `${dot} **Routing** ${parts.join(' · ')}${suffix}`
|
|
405
|
+
}
|
|
406
|
+
|
|
281
407
|
export interface RenderBootCardOpts {
|
|
282
408
|
agentName: string
|
|
283
409
|
/** Lowercase slug used for systemd unit names. Falls back to
|
|
@@ -338,6 +464,14 @@ export interface RenderBootCardOpts {
|
|
|
338
464
|
* persisted snapshot from the prior boot. See `config-snapshot.ts`.
|
|
339
465
|
*/
|
|
340
466
|
configChanges?: ConfigDiff
|
|
467
|
+
/**
|
|
468
|
+
* Parsed `.routing-mode` state (see `readRoutingMode`). When present, a
|
|
469
|
+
* "Routing" row is rendered — always visible, warning-dotted on
|
|
470
|
+
* landed≠declared divergence, suffixed "(prev boot)" when the file
|
|
471
|
+
* predates this gateway boot. Absent/null = row omitted entirely
|
|
472
|
+
* (fleets pre-dating the state file, unreadable file).
|
|
473
|
+
*/
|
|
474
|
+
routing?: RoutingModeInfo | null
|
|
341
475
|
}
|
|
342
476
|
|
|
343
477
|
/**
|
|
@@ -456,8 +590,13 @@ export function renderBootCard(opts: RenderBootCardOpts): string {
|
|
|
456
590
|
}
|
|
457
591
|
}
|
|
458
592
|
|
|
593
|
+
// Routing row (.routing-mode observability) — rendered whenever the state
|
|
594
|
+
// file was readable. Sits with the probe rows so divergence reads like any
|
|
595
|
+
// other degraded surface.
|
|
596
|
+
const routingRows: string[] = opts.routing ? [renderRoutingRow(opts.routing)] : []
|
|
597
|
+
|
|
459
598
|
const sections: string[] = [ack]
|
|
460
|
-
if (degradedRows.length > 0) sections.push('', ...degradedRows)
|
|
599
|
+
if (degradedRows.length > 0 || routingRows.length > 0) sections.push('', ...degradedRows, ...routingRows)
|
|
461
600
|
if (accountRows.length > 0) sections.push('', ...accountRows)
|
|
462
601
|
if (opts.updateOutcomeLine) {
|
|
463
602
|
// PR C: each line of the update-outcome blob is its own row in the
|
|
@@ -616,6 +755,15 @@ export interface RunProbesOpts {
|
|
|
616
755
|
floodStatePath?: string
|
|
617
756
|
/** Injectable clock for the flood-wait check (tests). Defaults to Date.now. */
|
|
618
757
|
nowMs?: () => number
|
|
758
|
+
/**
|
|
759
|
+
* Epoch-ms this gateway boot started — the staleness threshold for the
|
|
760
|
+
* `.routing-mode` row (a file mtime older than this is the PREVIOUS boot's
|
|
761
|
+
* value and is labeled, never shown as current). Defaults to the CONTAINER
|
|
762
|
+
* boot time (`containerBootStartMs`, stable across gateway-only respawns),
|
|
763
|
+
* falling back to the gateway process start when /proc is unavailable. Test
|
|
764
|
+
* override.
|
|
765
|
+
*/
|
|
766
|
+
routingBootStartMs?: number
|
|
619
767
|
}
|
|
620
768
|
|
|
621
769
|
/** Run all six probes concurrently with their own per-probe timeouts.
|
|
@@ -879,6 +1027,23 @@ export async function startBootCard(
|
|
|
879
1027
|
}
|
|
880
1028
|
}
|
|
881
1029
|
|
|
1030
|
+
// Routing-mode row (.routing-mode observability). Read at settle
|
|
1031
|
+
// time — by then a healthy inner pass has usually written this
|
|
1032
|
+
// boot's line; a degraded LiteLLM probe can still delay the write,
|
|
1033
|
+
// in which case the file mtime predates this gateway boot and the
|
|
1034
|
+
// reader flags it prevBoot (rendered "(prev boot)", never as
|
|
1035
|
+
// current). Absent/unreadable file → row omitted.
|
|
1036
|
+
const routingBootStartMs =
|
|
1037
|
+
opts.routingBootStartMs ??
|
|
1038
|
+
containerBootStartMs() ??
|
|
1039
|
+
Date.now() - process.uptime() * 1000
|
|
1040
|
+
let routing: RoutingModeInfo | null = null
|
|
1041
|
+
try {
|
|
1042
|
+
routing = readRoutingMode(opts.agentDir, routingBootStartMs)
|
|
1043
|
+
} catch {
|
|
1044
|
+
routing = null
|
|
1045
|
+
}
|
|
1046
|
+
|
|
882
1047
|
// Render with current probe state and edit if anything changed.
|
|
883
1048
|
let currentText = renderBootCard({
|
|
884
1049
|
agentName: opts.agentName,
|
|
@@ -892,6 +1057,7 @@ export async function startBootCard(
|
|
|
892
1057
|
...(snoozeRows.length > 0 ? { snoozeRows } : {}),
|
|
893
1058
|
...(opts.updateOutcomeLine ? { updateOutcomeLine: opts.updateOutcomeLine } : {}),
|
|
894
1059
|
...(configChanges.length > 0 ? { configChanges } : {}),
|
|
1060
|
+
...(routing ? { routing } : {}),
|
|
895
1061
|
})
|
|
896
1062
|
|
|
897
1063
|
if (currentText !== ackText) {
|
|
@@ -947,6 +1113,8 @@ export async function startBootCard(
|
|
|
947
1113
|
// (computed once in Phase 1). Pass them through unchanged so
|
|
948
1114
|
// the live-agent-status edits keep the config-diff rows.
|
|
949
1115
|
...(configChanges.length > 0 ? { configChanges } : {}),
|
|
1116
|
+
// Routing row is stable for the card lifetime (read once above).
|
|
1117
|
+
...(routing ? { routing } : {}),
|
|
950
1118
|
})
|
|
951
1119
|
|
|
952
1120
|
if (updatedText === currentText) continue
|