switchroom 0.18.30 → 0.18.32
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-scheduler/index.js +4 -2
- package/dist/auth-broker/index.js +4 -2
- package/dist/cli/notion-write-pretool.mjs +4 -2
- package/dist/cli/switchroom.js +708 -255
- package/dist/host-control/main.js +5 -3
- package/dist/vault/approvals/kernel-server.js +4 -2
- package/dist/vault/broker/server.js +4 -2
- package/package.json +3 -2
- package/profiles/_base/start.sh.hbs +142 -7
- package/telegram-plugin/dist/gateway/gateway.js +25870 -25007
- package/telegram-plugin/gateway/backstop-delivery.ts +223 -23
- package/telegram-plugin/gateway/captured-answer-resume.ts +259 -0
- package/telegram-plugin/gateway/disconnect-flush.ts +6 -44
- package/telegram-plugin/gateway/gateway-import-clean.test.ts +188 -0
- package/telegram-plugin/gateway/gateway.ts +5479 -7069
- package/telegram-plugin/gateway/inbound-delivery-machine-dispatch.ts +7 -15
- package/telegram-plugin/gateway/inbound-delivery-machine-shadow.ts +35 -68
- package/telegram-plugin/gateway/obligation-ledger.ts +42 -0
- package/telegram-plugin/gateway/obligation-store.ts +37 -1
- package/telegram-plugin/gateway/outbound-send-path.ts +2012 -0
- package/telegram-plugin/gateway/turn-flush-suppression.ts +82 -0
- package/telegram-plugin/pending-user-notice.ts +59 -13
- package/telegram-plugin/subagent-watcher.ts +111 -28
- package/telegram-plugin/tests/backstop-delivery.test.ts +167 -0
- package/telegram-plugin/tests/backstop-readback-probe.test.ts +144 -0
- package/telegram-plugin/tests/buffer-gate-broadened.test.ts +16 -6
- package/telegram-plugin/tests/button-tap-turn-gated.test.ts +3 -3
- package/telegram-plugin/tests/captured-answer-resume.test.ts +358 -0
- package/telegram-plugin/tests/emission-authority-facade.test.ts +29 -19
- package/telegram-plugin/tests/emission-authority-ping-gate.test.ts +4 -1
- package/telegram-plugin/tests/emission-determinism-wiring.test.ts +18 -7
- package/telegram-plugin/tests/gateway-boot-side-effect-gating.test.ts +249 -0
- package/telegram-plugin/tests/gateway-bot-construction-deferral.test.ts +251 -0
- package/telegram-plugin/tests/gateway-disconnect-flush.test.ts +5 -128
- package/telegram-plugin/tests/gateway-handler-registration-wiring.test.ts +303 -0
- package/telegram-plugin/tests/gateway-outbound-redact.test.ts +10 -3
- package/telegram-plugin/tests/inbound-delivery-cutover-flip.test.ts +54 -150
- package/telegram-plugin/tests/inbound-delivery-cutover-gate.test.ts +10 -14
- package/telegram-plugin/tests/inbound-delivery-dispatch-equivalence.test.ts +6 -7
- package/telegram-plugin/tests/inbound-delivery-machine-dispatch.test.ts +0 -16
- package/telegram-plugin/tests/inbound-emit-after-intercepts.test.ts +4 -4
- package/telegram-plugin/tests/litellm-proxy-auth-misconfig.test.ts +69 -14
- package/telegram-plugin/tests/multitopic-routing-wiring.test.ts +9 -3
- package/telegram-plugin/tests/obligation-ledger.test.ts +40 -0
- package/telegram-plugin/tests/obligation-store.test.ts +43 -0
- package/telegram-plugin/tests/pending-card-durability-wiring.test.ts +2 -1
- package/telegram-plugin/tests/permission-no-repeat-wiring.test.ts +9 -6
- package/telegram-plugin/tests/photo-reroute-wiring.test.ts +5 -2
- package/telegram-plugin/tests/reply-terminal-reaction.test.ts +6 -2
- package/telegram-plugin/tests/secret-detect-oauth-code.test.ts +3 -2
- package/telegram-plugin/tests/send-reply-golden.test.ts +571 -0
- package/telegram-plugin/tests/subagent-watcher-resume-reregister.test.ts +291 -0
- package/telegram-plugin/tests/subagent-watcher-resurrection.test.ts +32 -0
- package/telegram-plugin/tests/turn-end-gate-backstop.test.ts +8 -12
- package/telegram-plugin/tests/turn-flush-safety.test.ts +8 -6
- package/telegram-plugin/tests/turn-flush-suppression-wiring.test.ts +112 -0
- package/telegram-plugin/tests/turn-flush-suppression.test.ts +90 -0
- package/telegram-plugin/tests/vault-grant-inbound-builders.test.ts +2 -2
- package/telegram-plugin/tests/worker-feed-coalesce.test.ts +126 -0
- package/telegram-plugin/tool-activity-summary.ts +27 -3
- package/telegram-plugin/worker-activity-feed.ts +24 -0
- package/telegram-plugin/gateway/busy-key-reaper.ts +0 -113
- package/telegram-plugin/gateway/gate-parity-probe.ts +0 -102
- package/telegram-plugin/tests/busy-key-reaper.test.ts +0 -192
- package/telegram-plugin/tests/fixtures/cutover-killswitch-probe.ts +0 -75
- package/telegram-plugin/tests/gate-parity-probe.test.ts +0 -171
- package/telegram-plugin/tests/parallel-turns-deadlock-fix.test.ts +0 -217
|
@@ -40,6 +40,12 @@ export class BackstopDeliveryLedger {
|
|
|
40
40
|
private chunks = new Map<string, Map<number, number[]>>()
|
|
41
41
|
/** turnId -> chunk indices with an in-flight (pre-ack) send (guard 6). */
|
|
42
42
|
private pending = new Map<string, Set<number>>()
|
|
43
|
+
/** turnId -> chunk indices whose read-back probe CONFIRMED the message
|
|
44
|
+
* actually exists in the chat (#3278). A chunk with a landed message id but
|
|
45
|
+
* no confirmation is `landed-unconfirmed` — the Bot API accepted the send but
|
|
46
|
+
* a server-side silent discard (flood/anti-spam) can return a fresh id for a
|
|
47
|
+
* message the user never sees, so an id alone is NOT proof of delivery. */
|
|
48
|
+
private confirmed = new Map<string, Set<number>>()
|
|
43
49
|
|
|
44
50
|
/**
|
|
45
51
|
* Guard 5 — once-per-turn BACKSTOP double-fire latch. Returns `true` on the
|
|
@@ -99,6 +105,78 @@ export class BackstopDeliveryLedger {
|
|
|
99
105
|
return (this.chunks.get(turnId)?.get(index)?.length ?? 0) > 0
|
|
100
106
|
}
|
|
101
107
|
|
|
108
|
+
/** The landed message id(s) for one chunk (empty when unsent). The read-back
|
|
109
|
+
* probe reads these to confirm every id of the chunk (guard A7 — a
|
|
110
|
+
* length-resplit chunk lands >1 id and all must exist to count confirmed). */
|
|
111
|
+
chunkIds(turnId: string, index: number): number[] {
|
|
112
|
+
return (this.chunks.get(turnId)?.get(index) ?? []).slice()
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
/**
|
|
116
|
+
* #3278 — transition a landed-unconfirmed chunk to `landed-confirmed` after a
|
|
117
|
+
* read-back probe proved the message exists in the chat. Only a confirmed
|
|
118
|
+
* chunk counts toward delivery / `complete`.
|
|
119
|
+
*/
|
|
120
|
+
confirmChunk(turnId: string, index: number): void {
|
|
121
|
+
let set = this.confirmed.get(turnId)
|
|
122
|
+
if (set == null) {
|
|
123
|
+
set = new Set()
|
|
124
|
+
this.confirmed.set(turnId, set)
|
|
125
|
+
}
|
|
126
|
+
set.add(index)
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
/**
|
|
130
|
+
* #3278 — demote a chunk back to `unsent` on a POSITIVE absence read-back
|
|
131
|
+
* (Telegram `400 message to edit not found`): the send was silently dropped,
|
|
132
|
+
* so re-sending it is safe (it never reached the user). Clears the landed ids,
|
|
133
|
+
* the pending marker, and any stale confirmation. NEVER call this on an
|
|
134
|
+
* ambiguous probe (429/5xx/network) — that would risk a duplicate.
|
|
135
|
+
*/
|
|
136
|
+
demoteChunk(turnId: string, index: number): void {
|
|
137
|
+
this.chunks.get(turnId)?.delete(index)
|
|
138
|
+
this.pending.get(turnId)?.delete(index)
|
|
139
|
+
this.confirmed.get(turnId)?.delete(index)
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
/** True only for a `landed-confirmed` chunk (read-back proved it exists). */
|
|
143
|
+
hasConfirmedChunk(turnId: string, index: number): boolean {
|
|
144
|
+
return this.confirmed.get(turnId)?.has(index) ?? false
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
/** Chunk indices that have landed at least one id but are NOT yet confirmed —
|
|
148
|
+
* the set the read-back probe must resolve (confirm / demote / leave). */
|
|
149
|
+
landedUnconfirmedIndices(turnId: string, chunkCount: number): number[] {
|
|
150
|
+
const out: number[] = []
|
|
151
|
+
for (let i = 0; i < chunkCount; i++) {
|
|
152
|
+
if (this.hasChunk(turnId, i) && !this.hasConfirmedChunk(turnId, i)) out.push(i)
|
|
153
|
+
}
|
|
154
|
+
return out
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
/** True IFF every chunk 0..chunkCount-1 is `landed-confirmed`. `chunkCount===0`
|
|
158
|
+
* is never "all confirmed" (nothing was delivered). */
|
|
159
|
+
allConfirmed(turnId: string, chunkCount: number): boolean {
|
|
160
|
+
if (chunkCount <= 0) return false
|
|
161
|
+
for (let i = 0; i < chunkCount; i++) {
|
|
162
|
+
if (!this.hasConfirmedChunk(turnId, i)) return false
|
|
163
|
+
}
|
|
164
|
+
return true
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
/** Landed message ids of CONFIRMED chunks only, in chunk-index order — the set
|
|
168
|
+
* the delivery predicate counts (fresh non-card ids that are proven-present). */
|
|
169
|
+
confirmedIds(turnId: string): number[] {
|
|
170
|
+
const m = this.chunks.get(turnId)
|
|
171
|
+
const set = this.confirmed.get(turnId)
|
|
172
|
+
if (m == null || set == null) return []
|
|
173
|
+
const out: number[] = []
|
|
174
|
+
for (const index of Array.from(m.keys()).sort((a, b) => a - b)) {
|
|
175
|
+
if (set.has(index)) out.push(...(m.get(index) ?? []))
|
|
176
|
+
}
|
|
177
|
+
return out
|
|
178
|
+
}
|
|
179
|
+
|
|
102
180
|
/** All landed message ids for this turn, in chunk-index order. */
|
|
103
181
|
sentIds(turnId: string): number[] {
|
|
104
182
|
const m = this.chunks.get(turnId)
|
|
@@ -137,9 +215,49 @@ export class BackstopDeliveryLedger {
|
|
|
137
215
|
this.latched.delete(turnId)
|
|
138
216
|
this.chunks.delete(turnId)
|
|
139
217
|
this.pending.delete(turnId)
|
|
218
|
+
this.confirmed.delete(turnId)
|
|
140
219
|
}
|
|
141
220
|
}
|
|
142
221
|
|
|
222
|
+
/**
|
|
223
|
+
* #3278 — the tri-state a read-back probe resolves a landed chunk into:
|
|
224
|
+
* - `exists` — a no-op `editMessageText` succeeded or returned "message is
|
|
225
|
+
* not modified" ⇒ the message is present at the chat_id ⇒
|
|
226
|
+
* `landed-confirmed` (counts toward delivery).
|
|
227
|
+
* - `absent` — Telegram `400 message to edit not found` ⇒ positive absence
|
|
228
|
+
* ⇒ demote to `unsent` (safe to re-send — it never landed).
|
|
229
|
+
* - `ambiguous` — 429 / 5xx / network / gate-shed / anything else ⇒ leave
|
|
230
|
+
* `landed-unconfirmed`; NEVER re-send (a re-send would risk a
|
|
231
|
+
* duplicate, and duplicate-risk beats missing-risk here).
|
|
232
|
+
*
|
|
233
|
+
* NOTE (honest limitation, #3278 §1.4): a passing probe proves only that the
|
|
234
|
+
* message EXISTS at that chat_id. It does NOT prove the human's client rendered
|
|
235
|
+
* it, and it does NOT catch a wrong-thread/chat misroute (the edit succeeds
|
|
236
|
+
* because the message exists — just not where the operator is looking).
|
|
237
|
+
* Read-back is necessary-but-not-sufficient; it is paired with the correct
|
|
238
|
+
* chat/thread resolution already pinned by the backstop caller.
|
|
239
|
+
*/
|
|
240
|
+
export type ReadBackResult = 'exists' | 'absent' | 'ambiguous'
|
|
241
|
+
|
|
242
|
+
/**
|
|
243
|
+
* Combine the per-id read-back results of ONE chunk into the chunk's verdict
|
|
244
|
+
* (guard A7 — a length-resplit chunk lands >1 message id and every id must be
|
|
245
|
+
* present for the chunk to count confirmed). Pure:
|
|
246
|
+
* - ANY id `absent` ⇒ `absent` (the chunk is incomplete on the
|
|
247
|
+
* wire → re-send the whole chunk)
|
|
248
|
+
* - ALL ids `exists` ⇒ `exists`
|
|
249
|
+
* - otherwise (some `ambiguous`,
|
|
250
|
+
* none `absent`) ⇒ `ambiguous` (never re-send)
|
|
251
|
+
* - empty id set ⇒ `ambiguous` (nothing to probe → don't
|
|
252
|
+
* fabricate a confirmation OR a demotion)
|
|
253
|
+
*/
|
|
254
|
+
export function combineReadBackResults(perId: readonly ReadBackResult[]): ReadBackResult {
|
|
255
|
+
if (perId.length === 0) return 'ambiguous'
|
|
256
|
+
if (perId.some(r => r === 'absent')) return 'absent'
|
|
257
|
+
if (perId.every(r => r === 'exists')) return 'exists'
|
|
258
|
+
return 'ambiguous'
|
|
259
|
+
}
|
|
260
|
+
|
|
143
261
|
/**
|
|
144
262
|
* Guard 7 — the RECEIPT gate. Given the raw delivered ids and the progress-card
|
|
145
263
|
* message id (or null), return only the FRESH, non-card chat message ids. A
|
|
@@ -175,6 +293,25 @@ export interface BackstopDeliveryDeps {
|
|
|
175
293
|
/** Record the delivered ids + aligned texts to history (called once, after
|
|
176
294
|
* the attempts resolve, with the full landed set). */
|
|
177
295
|
recordOutbound?: (messageIds: number[], texts: string[]) => void
|
|
296
|
+
/**
|
|
297
|
+
* #3278 read-back confirmation. After a chunk lands a message id, a no-op
|
|
298
|
+
* `editMessageText` probe re-confirms the message ACTUALLY exists in the chat
|
|
299
|
+
* (a returned id is only an API accept, not proof of visibility). Returns the
|
|
300
|
+
* chunk's {@link ReadBackResult} — `exists` ⇒ landed-confirmed, `absent` ⇒
|
|
301
|
+
* demote to unsent + re-send, `ambiguous` ⇒ stay landed-unconfirmed and NEVER
|
|
302
|
+
* re-send. Runs once per landed chunk after the send attempts resolve, and is
|
|
303
|
+
* itself flood-window-paced by the caller (never a storm).
|
|
304
|
+
*
|
|
305
|
+
* When OMITTED, a landed chunk is confirmed on landing (pre-#3278 behavior) —
|
|
306
|
+
* still gated by the fresh-non-card receipt filter, so a card-only "delivery"
|
|
307
|
+
* is never counted. This is the hot-path default: read-back is scoped to the
|
|
308
|
+
* RARE backstop delivery, so the reply-tool path issues ZERO probes (§1.3).
|
|
309
|
+
*/
|
|
310
|
+
readBack?: (
|
|
311
|
+
chunkIndex: number,
|
|
312
|
+
messageIds: readonly number[],
|
|
313
|
+
text: string,
|
|
314
|
+
) => Promise<ReadBackResult>
|
|
178
315
|
/** Optional stderr sink for progress/resume logging. */
|
|
179
316
|
stderr?: (s: string) => void
|
|
180
317
|
}
|
|
@@ -204,6 +341,14 @@ export interface BackstopDeliveryResult {
|
|
|
204
341
|
* so the caller can leave the delivery obligation OPEN for the liveness floor
|
|
205
342
|
* to re-present — instead of the old silent `send_failed` drop.
|
|
206
343
|
*
|
|
344
|
+
* #3278 — after each attempt's sends resolve, every `landed-unconfirmed` chunk
|
|
345
|
+
* is read-back-probed via `deps.readBack` (when provided): `exists` confirms it,
|
|
346
|
+
* `absent` demotes it to `unsent` so the NEXT attempt re-sends only that chunk,
|
|
347
|
+
* and `ambiguous` leaves it `landed-unconfirmed` — never re-sent (duplicate-risk
|
|
348
|
+
* beats missing-risk). `delivered` now requires every chunk `landed-confirmed`,
|
|
349
|
+
* so an API-ack'd-but-silently-dropped send (fresh id, absent on read-back) is
|
|
350
|
+
* reported `delivered:false` and the caller leaves the obligation OPEN.
|
|
351
|
+
*
|
|
207
352
|
* `recordOutbound` (when provided) fires ONCE at the end with the full landed
|
|
208
353
|
* set and a `texts` array ALIGNED to the actual sent ids (via `ledger.entries`).
|
|
209
354
|
* The ledger is NOT cleared here — the caller clears it only after success or
|
|
@@ -223,37 +368,92 @@ export async function runBackstopDelivery(
|
|
|
223
368
|
|
|
224
369
|
for (let attempt = 1; attempt <= Math.max(1, maxAttempts); attempt++) {
|
|
225
370
|
attempts = attempt
|
|
371
|
+
// 1. SEND the `unsent` chunks (never-landed OR demoted by a prior absent
|
|
372
|
+
// read-back), resuming in chunk-index order. Guard 6 keeps a chunk that
|
|
373
|
+
// already landed from being re-sent.
|
|
226
374
|
const resume = ledger.unsentIndices(turnId, chunkCount)
|
|
227
|
-
if (resume.length === 0) break // everything already landed
|
|
228
|
-
if (attempt > 1) {
|
|
229
|
-
stderr(
|
|
230
|
-
`telegram gateway: backstop delivery retry ${attempt}/${maxAttempts} — ` +
|
|
231
|
-
`resuming at unsent chunk(s) [${resume.join(', ')}] for turn ${turnId}\n`,
|
|
232
|
-
)
|
|
233
|
-
}
|
|
234
375
|
let attemptThrew = false
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
376
|
+
if (resume.length > 0) {
|
|
377
|
+
if (attempt > 1) {
|
|
378
|
+
stderr(
|
|
379
|
+
`telegram gateway: backstop delivery retry ${attempt}/${maxAttempts} — ` +
|
|
380
|
+
`resuming at unsent chunk(s) [${resume.join(', ')}] for turn ${turnId}\n`,
|
|
381
|
+
)
|
|
382
|
+
}
|
|
383
|
+
try {
|
|
384
|
+
for (const i of resume) {
|
|
385
|
+
if (ledger.hasChunk(turnId, i)) continue
|
|
386
|
+
ledger.markPending(turnId, i)
|
|
387
|
+
const ids = await deps.sendChunk(i, chunks[i])
|
|
388
|
+
ledger.recordChunk(turnId, i, ids) // → landed-unconfirmed
|
|
389
|
+
}
|
|
390
|
+
} catch (err) {
|
|
391
|
+
attemptThrew = true
|
|
392
|
+
stderr(
|
|
393
|
+
`telegram gateway: backstop delivery attempt ${attempt}/${maxAttempts} failed: ` +
|
|
394
|
+
`${err instanceof Error ? err.message : String(err)}\n`,
|
|
395
|
+
)
|
|
242
396
|
}
|
|
243
|
-
} catch (err) {
|
|
244
|
-
attemptThrew = true
|
|
245
|
-
stderr(
|
|
246
|
-
`telegram gateway: backstop delivery attempt ${attempt}/${maxAttempts} failed: ` +
|
|
247
|
-
`${err instanceof Error ? err.message : String(err)}\n`,
|
|
248
|
-
)
|
|
249
397
|
}
|
|
250
|
-
|
|
398
|
+
|
|
399
|
+
// 2. CONFIRM landed-unconfirmed chunks via read-back (#3278). Without a probe
|
|
400
|
+
// wired, landing IS confirmation (pre-#3278) — the fresh-non-card receipt
|
|
401
|
+
// filter below still rejects a card-only "delivery".
|
|
402
|
+
let demotedAny = false
|
|
403
|
+
for (const i of ledger.landedUnconfirmedIndices(turnId, chunkCount)) {
|
|
404
|
+
if (deps.readBack == null) {
|
|
405
|
+
ledger.confirmChunk(turnId, i)
|
|
406
|
+
continue
|
|
407
|
+
}
|
|
408
|
+
let verdict: ReadBackResult
|
|
409
|
+
try {
|
|
410
|
+
verdict = await deps.readBack(i, ledger.chunkIds(turnId, i), chunks[i])
|
|
411
|
+
} catch (err) {
|
|
412
|
+
// A probe adapter that itself throws must NEVER trigger a re-send.
|
|
413
|
+
verdict = 'ambiguous'
|
|
414
|
+
stderr(
|
|
415
|
+
`telegram gateway: backstop read-back probe threw for chunk ${i} ` +
|
|
416
|
+
`(turn ${turnId}) — treating as ambiguous: ` +
|
|
417
|
+
`${err instanceof Error ? err.message : String(err)}\n`,
|
|
418
|
+
)
|
|
419
|
+
}
|
|
420
|
+
if (verdict === 'exists') {
|
|
421
|
+
ledger.confirmChunk(turnId, i)
|
|
422
|
+
} else if (verdict === 'absent') {
|
|
423
|
+
// Positive absence — the send was silently dropped. Safe to re-send.
|
|
424
|
+
ledger.demoteChunk(turnId, i)
|
|
425
|
+
demotedAny = true
|
|
426
|
+
stderr(
|
|
427
|
+
`telegram gateway: backstop read-back — chunk ${i} not found in chat ` +
|
|
428
|
+
`(turn ${turnId}); demoting to unsent for re-send\n`,
|
|
429
|
+
)
|
|
430
|
+
} else {
|
|
431
|
+
// Ambiguous (429/5xx/network/gate-shed) — leave landed-unconfirmed and
|
|
432
|
+
// NEVER re-send it (would risk a duplicate).
|
|
433
|
+
stderr(
|
|
434
|
+
`telegram gateway: backstop read-back — chunk ${i} ambiguous ` +
|
|
435
|
+
`(turn ${turnId}); left landed-unconfirmed, not re-sending\n`,
|
|
436
|
+
)
|
|
437
|
+
}
|
|
438
|
+
}
|
|
439
|
+
|
|
440
|
+
// 3. Fully confirmed ⇒ done.
|
|
441
|
+
if (ledger.allConfirmed(turnId, chunkCount)) break
|
|
442
|
+
// 4. Another attempt ONLY re-sends chunks demoted to `unsent`. If nothing is
|
|
443
|
+
// unsent, the remaining non-confirmed chunks are all `landed-unconfirmed`
|
|
444
|
+
// (ambiguous) — re-sending them would duplicate, so stop here.
|
|
445
|
+
if (ledger.unsentIndices(turnId, chunkCount).length === 0) break
|
|
446
|
+
// 5. Guard against a spin: if this attempt neither sent, threw, nor demoted
|
|
447
|
+
// anything, another identical attempt is pointless.
|
|
448
|
+
if (resume.length === 0 && !attemptThrew && !demotedAny) break
|
|
251
449
|
}
|
|
252
450
|
|
|
253
451
|
const sentIds = ledger.sentIds(turnId)
|
|
452
|
+
// #3278 — delivered IFF every chunk is `landed-confirmed` AND at least one
|
|
453
|
+
// confirmed id is a fresh non-card chat id (the receipt gate, guard 7).
|
|
254
454
|
const delivered =
|
|
255
|
-
|
|
256
|
-
backstopReceiptIds(
|
|
455
|
+
ledger.allConfirmed(turnId, chunkCount) &&
|
|
456
|
+
backstopReceiptIds(ledger.confirmedIds(turnId), cardMessageId).length > 0
|
|
257
457
|
const exhausted = !delivered
|
|
258
458
|
|
|
259
459
|
if (deps.recordOutbound && sentIds.length > 0) {
|
|
@@ -0,0 +1,259 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* captured-answer-resume.ts — durable captured-answer resume for a partially
|
|
3
|
+
* delivered turn-flush backstop (#3282).
|
|
4
|
+
*
|
|
5
|
+
* ── The bug this closes ──────────────────────────────────────────────────────
|
|
6
|
+
* When the turn-flush backstop (`deliverAnswer` → `runBackstopDelivery`) lands
|
|
7
|
+
* chunks 0,1 of a 3-chunk answer but chunk 2 exhausts its bounded retries, the
|
|
8
|
+
* turn-flush IIFE historically (a) cleared the per-chunk `BackstopDeliveryLedger`
|
|
9
|
+
* unconditionally and (b) left the delivery obligation OPEN. The liveness floor
|
|
10
|
+
* then re-presented the obligation as a FRESH must-answer model turn, which
|
|
11
|
+
* REGENERATED the whole answer — re-posting chunks 0,1's content the user already
|
|
12
|
+
* received. Result: the user sees the head of the answer twice after a partial
|
|
13
|
+
* failure (or after a restart mid-delivery). #3282.
|
|
14
|
+
*
|
|
15
|
+
* ── The fix (design-of-record §2.2) ─────────────────────────────────────────
|
|
16
|
+
* Make the re-present a *byte-identical captured-answer resume* of ONLY the
|
|
17
|
+
* not-yet-`landed-confirmed` tail — never a regeneration — so a chunk that
|
|
18
|
+
* already reached the chat is provably never re-posted:
|
|
19
|
+
*
|
|
20
|
+
* 1. On a partial (`!delivered`) turn-flush, persist a {@link
|
|
21
|
+
* CapturedDeliverySnapshot} (the split chunks + each landed chunk's ids and
|
|
22
|
+
* confirmed flag) onto the durable obligation record, keyed by
|
|
23
|
+
* `originTurnId`. This rides the obligation ledger's existing atomic
|
|
24
|
+
* snapshot, so it survives a gateway/container restart.
|
|
25
|
+
* 2. The obligation sweep's `represent` branch becomes source-aware: when the
|
|
26
|
+
* obligation carries a captured-delivery snapshot it drives THIS resume
|
|
27
|
+
* (re-run `runBackstopDelivery` for the SAME `turnId` and the SAME captured
|
|
28
|
+
* chunks, resuming at the first non-`landed-confirmed` index) instead of
|
|
29
|
+
* pushing a fresh-generation inbound. No snapshot ⇒ the genuine "model wrote
|
|
30
|
+
* nothing / never fired a backstop" case falls through to fresh generation
|
|
31
|
+
* unchanged.
|
|
32
|
+
* 3. The in-memory ledger is rehydrated from the snapshot AND reconciled against
|
|
33
|
+
* the durable outbound-text oracle (`hasOutboundWithText`), so a chunk that
|
|
34
|
+
* landed a durable history row is confirmed even if the snapshot's flag was
|
|
35
|
+
* not yet flushed when the process died (crash-idempotency, design §2.3/A5).
|
|
36
|
+
*
|
|
37
|
+
* Chunk identity is the chunk INDEX within `turnId`, stable across the represent
|
|
38
|
+
* boundary and across restart because the resume re-delivers the exact captured
|
|
39
|
+
* chunks (not a regenerated answer). #3278's per-chunk state machine
|
|
40
|
+
* (`unsent → pending → landed-unconfirmed → {landed-confirmed | unsent}`) binds
|
|
41
|
+
* here too: the resume re-PROBES `landed-unconfirmed` chunks (never re-sends
|
|
42
|
+
* them) and re-SENDS only `unsent` ones, so a confirmed chunk is never duplicated
|
|
43
|
+
* by either the in-fire retry OR the cross-represent resume.
|
|
44
|
+
*
|
|
45
|
+
* This module is PURE state + orchestration over injected effects — no Telegram,
|
|
46
|
+
* no SQLite, no gateway module state — so the resume decision is unit-testable
|
|
47
|
+
* (see tests/captured-answer-resume.test.ts). The gateway owns the real
|
|
48
|
+
* `deliver` (via `deliverAnswer`), the durable oracle, and the obligation I/O.
|
|
49
|
+
*/
|
|
50
|
+
|
|
51
|
+
import { hasOutboundWithText } from '../history.js'
|
|
52
|
+
import type { BackstopDeliveryLedger } from './backstop-delivery.js'
|
|
53
|
+
import type { CapturedDeliverySnapshot, Obligation } from './obligation-ledger.js'
|
|
54
|
+
|
|
55
|
+
/**
|
|
56
|
+
* Build the durable {@link CapturedDeliverySnapshot} from the live per-chunk
|
|
57
|
+
* ledger after a partial `runBackstopDelivery`. Captures every LANDED chunk's
|
|
58
|
+
* message ids + confirmed flag so the resume can (a) never re-send a confirmed
|
|
59
|
+
* chunk and (b) re-PROBE a landed-unconfirmed chunk instead of blindly re-sending
|
|
60
|
+
* it. Chunks that never landed (send threw) are simply absent → the resume treats
|
|
61
|
+
* them as `unsent` and re-sends them. Pure.
|
|
62
|
+
*/
|
|
63
|
+
export function buildCapturedDeliverySnapshot(
|
|
64
|
+
ledger: BackstopDeliveryLedger,
|
|
65
|
+
turnId: string,
|
|
66
|
+
chunks: readonly string[],
|
|
67
|
+
): CapturedDeliverySnapshot {
|
|
68
|
+
const chunkStates = ledger.entries(turnId).map(({ index, messageIds }) => ({
|
|
69
|
+
index,
|
|
70
|
+
messageIds,
|
|
71
|
+
confirmed: ledger.hasConfirmedChunk(turnId, index),
|
|
72
|
+
}))
|
|
73
|
+
return { chunks: [...chunks], chunkStates }
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/**
|
|
77
|
+
* Rehydrate a `BackstopDeliveryLedger` for a captured-answer resume from a
|
|
78
|
+
* durable snapshot, reconciled against the durable outbound-text oracle. For each
|
|
79
|
+
* landed chunk in the snapshot:
|
|
80
|
+
* - re-record its landed message id(s) so `unsentIndices` excludes it (the
|
|
81
|
+
* resume never re-sends a chunk that already landed);
|
|
82
|
+
* - mark it `landed-confirmed` when the snapshot flag says so OR the oracle
|
|
83
|
+
* proves its text is a durable outbound row (crash-idempotency, §2.3/A5) —
|
|
84
|
+
* a confirmed chunk is neither re-probed nor re-sent.
|
|
85
|
+
* Chunks absent from the snapshot stay `unsent` (re-sent by the resume). Pure over
|
|
86
|
+
* the injected `hasLanded` oracle.
|
|
87
|
+
*/
|
|
88
|
+
export function hydrateResumeLedger(
|
|
89
|
+
ledger: BackstopDeliveryLedger,
|
|
90
|
+
turnId: string,
|
|
91
|
+
snapshot: CapturedDeliverySnapshot,
|
|
92
|
+
hasLanded: (chunkText: string, index: number) => boolean,
|
|
93
|
+
): void {
|
|
94
|
+
for (const st of snapshot.chunkStates) {
|
|
95
|
+
if (st.messageIds.length === 0) continue // never actually landed → leave unsent
|
|
96
|
+
ledger.recordChunk(turnId, st.index, [...st.messageIds])
|
|
97
|
+
if (st.confirmed || hasLanded(snapshot.chunks[st.index] ?? '', st.index)) {
|
|
98
|
+
ledger.confirmChunk(turnId, st.index)
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
/** The args the module hands the gateway's `deliverAnswer` to run a resume. */
|
|
104
|
+
export interface CapturedResumeArgs {
|
|
105
|
+
chatId: string
|
|
106
|
+
threadId: number | undefined
|
|
107
|
+
text: string
|
|
108
|
+
turnId: string
|
|
109
|
+
cardMessageId: number | null
|
|
110
|
+
replyToMessageId: number | null
|
|
111
|
+
resume: {
|
|
112
|
+
snapshot: CapturedDeliverySnapshot
|
|
113
|
+
hydrate: (ledger: BackstopDeliveryLedger, turnId: string) => void
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
/**
|
|
118
|
+
* The RAW gateway singletons the dispatcher wires. Structural (no gateway class
|
|
119
|
+
* imports) and kept minimal so the gateway injection stays a handful of lines
|
|
120
|
+
* under the gateway.ts line ratchet — the module builds every resume closure
|
|
121
|
+
* (the `deliverAnswer` resume args, the oracle-reconciled hydrate, the supersede
|
|
122
|
+
* record) itself.
|
|
123
|
+
*/
|
|
124
|
+
export interface CapturedResumePorts {
|
|
125
|
+
/** The gateway's `deliverAnswer`, invoked in RESUME mode (byte-identical tail,
|
|
126
|
+
* no card gate). Only `delivered`/`sentIds` are read. */
|
|
127
|
+
deliverAnswer: (args: CapturedResumeArgs) => Promise<{ delivered: boolean; sentIds: number[] }>
|
|
128
|
+
/** The single obligation ledger. `markRepresented` consumes one represent-budget
|
|
129
|
+
* unit + stamps the per-represent grace (bounding the resume ladder exactly
|
|
130
|
+
* like a fresh-generation represent → escalate at the cap); `close` drops the
|
|
131
|
+
* captured snapshot once fully delivered. */
|
|
132
|
+
obligationLedger: {
|
|
133
|
+
markRepresented: (originTurnId: string) => unknown
|
|
134
|
+
close: (originTurnId: string) => unknown
|
|
135
|
+
}
|
|
136
|
+
/** The per-chunk backstop ledger — GC'd for a fully delivered turn. */
|
|
137
|
+
backstopDeliveryLedger: { clear: (turnId: string) => void }
|
|
138
|
+
/** flushedTurnSupersede — feed the resume's fresh chat ids in so a late reply
|
|
139
|
+
* after the resumed tail still corrects the delivered set (design §4). */
|
|
140
|
+
flushedTurnSupersede: {
|
|
141
|
+
record: (
|
|
142
|
+
chatId: string,
|
|
143
|
+
threadId: number | undefined,
|
|
144
|
+
rec: { turnId: string; messageIds: number[]; text: string },
|
|
145
|
+
nowMs: number,
|
|
146
|
+
) => void
|
|
147
|
+
}
|
|
148
|
+
/** True when history is queryable (else the durable-text oracle reconcile that
|
|
149
|
+
* gives crash-idempotency — §2.3/A5 — is skipped). */
|
|
150
|
+
historyEnabled: boolean
|
|
151
|
+
stderr?: (s: string) => void
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
export interface CapturedResumeDispatcher {
|
|
155
|
+
/** Fire a captured-answer resume for `o` (fire-and-forget; the sweep is sync).
|
|
156
|
+
* De-duped per `originTurnId` so two sweeps never launch concurrent resumes;
|
|
157
|
+
* a no-op when `o` carries no non-empty captured snapshot (the caller should
|
|
158
|
+
* then fall through to fresh generation). */
|
|
159
|
+
dispatch: (o: Obligation) => void
|
|
160
|
+
/** In-flight origin ids (test introspection). */
|
|
161
|
+
inFlight: () => string[]
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
/**
|
|
165
|
+
* The obligation-sweep's captured-answer resume driver. Owns the in-flight guard
|
|
166
|
+
* + the deliver→close/leave-open orchestration so the gateway sweep stays a
|
|
167
|
+
* two-line dispatch. On each dispatch it consumes one represent-budget unit
|
|
168
|
+
* (bounding the ladder → escalate), re-delivers only the non-confirmed tail, and:
|
|
169
|
+
* - fully delivered ⇒ record the supersede tail, close the obligation, GC the
|
|
170
|
+
* ledger (the represent ladder stops);
|
|
171
|
+
* - still partial / error ⇒ leave the obligation OPEN so the next eligible
|
|
172
|
+
* sweep (after the per-represent grace) retries, until the represent cap
|
|
173
|
+
* escalates it to the operator nudge — no infinite loop.
|
|
174
|
+
*/
|
|
175
|
+
export function createCapturedResumeDispatcher(ports: CapturedResumePorts): CapturedResumeDispatcher {
|
|
176
|
+
const stderr = ports.stderr ?? (() => {})
|
|
177
|
+
const inFlight = new Set<string>()
|
|
178
|
+
|
|
179
|
+
/** Re-deliver the non-confirmed tail of `o`'s captured answer, byte-identical,
|
|
180
|
+
* by rehydrating the per-chunk ledger from the snapshot reconciled against the
|
|
181
|
+
* durable text oracle. Never regenerates. */
|
|
182
|
+
function deliver(o: Obligation, snapshot: CapturedDeliverySnapshot): Promise<{ delivered: boolean; sentIds: number[] }> {
|
|
183
|
+
return ports.deliverAnswer({
|
|
184
|
+
chatId: o.chatId,
|
|
185
|
+
threadId: o.threadId,
|
|
186
|
+
text: '',
|
|
187
|
+
turnId: o.originTurnId,
|
|
188
|
+
cardMessageId: null,
|
|
189
|
+
replyToMessageId: o.messageId,
|
|
190
|
+
resume: {
|
|
191
|
+
snapshot,
|
|
192
|
+
hydrate: (ledger, turnId) =>
|
|
193
|
+
hydrateResumeLedger(ledger, turnId, snapshot, (txt) =>
|
|
194
|
+
ports.historyEnabled &&
|
|
195
|
+
hasOutboundWithText(o.chatId, txt, o.threadId ?? null, o.openedAt),
|
|
196
|
+
),
|
|
197
|
+
},
|
|
198
|
+
})
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
/** Record the resume's FULL delivered id set (rehydrated-confirmed + freshly
|
|
202
|
+
* re-sent — `runBackstopDelivery` returns `ledger.sentIds`, i.e. all landed
|
|
203
|
+
* ids in index order) into the supersede lane, keyed by the turn id. This
|
|
204
|
+
* REPLACES the partial turn-flush record in place with the complete set, so a
|
|
205
|
+
* late reply for this turn corrects the WHOLE delivered answer (design §4) —
|
|
206
|
+
* no-op when nothing has landed. */
|
|
207
|
+
function supersede(o: Obligation, snapshot: CapturedDeliverySnapshot, sentIds: number[]): void {
|
|
208
|
+
if (sentIds.length === 0) return
|
|
209
|
+
ports.flushedTurnSupersede.record(
|
|
210
|
+
o.chatId,
|
|
211
|
+
o.threadId,
|
|
212
|
+
{ turnId: o.originTurnId, messageIds: sentIds, text: snapshot.chunks.join('') },
|
|
213
|
+
Date.now(),
|
|
214
|
+
)
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
async function run(o: Obligation, snapshot: CapturedDeliverySnapshot): Promise<void> {
|
|
218
|
+
try {
|
|
219
|
+
const { delivered, sentIds } = await deliver(o, snapshot)
|
|
220
|
+
supersede(o, snapshot, sentIds)
|
|
221
|
+
if (delivered) {
|
|
222
|
+
ports.obligationLedger.close(o.originTurnId)
|
|
223
|
+
ports.backstopDeliveryLedger.clear(o.originTurnId)
|
|
224
|
+
stderr(
|
|
225
|
+
`telegram gateway: captured-answer resume delivered — origin=${o.originTurnId} ` +
|
|
226
|
+
`${sentIds.length} chunk(s) confirmed; obligation closed\n`,
|
|
227
|
+
)
|
|
228
|
+
} else {
|
|
229
|
+
// Tail still not confirmed — leave the obligation OPEN for the next paced
|
|
230
|
+
// sweep / escalation (bounded by the represent cap).
|
|
231
|
+
stderr(
|
|
232
|
+
`telegram gateway: captured-answer resume partial — origin=${o.originTurnId} ` +
|
|
233
|
+
`tail still not confirmed; left OPEN for retry\n`,
|
|
234
|
+
)
|
|
235
|
+
}
|
|
236
|
+
} catch (err) {
|
|
237
|
+
stderr(
|
|
238
|
+
`telegram gateway: captured-answer resume error — origin=${o.originTurnId}: ` +
|
|
239
|
+
`${err instanceof Error ? err.message : String(err)}\n`,
|
|
240
|
+
)
|
|
241
|
+
} finally {
|
|
242
|
+
inFlight.delete(o.originTurnId)
|
|
243
|
+
}
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
return {
|
|
247
|
+
dispatch(o: Obligation): void {
|
|
248
|
+
const snapshot = o.capturedDelivery
|
|
249
|
+
if (snapshot == null || snapshot.chunks.length === 0) return
|
|
250
|
+
if (inFlight.has(o.originTurnId)) return
|
|
251
|
+
inFlight.add(o.originTurnId)
|
|
252
|
+
// Consume represent budget + arm the per-represent grace up front so the
|
|
253
|
+
// ladder is bounded and the sweep can't immediately re-dispatch mid-flight.
|
|
254
|
+
ports.obligationLedger.markRepresented(o.originTurnId)
|
|
255
|
+
void run(o, snapshot)
|
|
256
|
+
},
|
|
257
|
+
inFlight: () => [...inFlight],
|
|
258
|
+
}
|
|
259
|
+
}
|
|
@@ -37,17 +37,6 @@ export interface DisconnectFlushDeps<Ctrl extends { finalize: (reason?: 'done' |
|
|
|
37
37
|
activeReactionMsgIds: Map<string, { chatId: string; messageId: number }>
|
|
38
38
|
/** Mirror map: same keys → turn-start timestamps. */
|
|
39
39
|
activeTurnStartedAt: Map<string, number>
|
|
40
|
-
/** PR3b: keys claude has actually been handed (delivered, not just
|
|
41
|
-
* received). Cleared on disconnect for the same reason as
|
|
42
|
-
* activeTurnStartedAt — the bridge just died, every turn it
|
|
43
|
-
* was handed is dead by definition. */
|
|
44
|
-
claudeBusyKeys: Set<string>
|
|
45
|
-
/** #2787: insertion-timestamp map for `claudeBusyKeys`, backing the orphan
|
|
46
|
-
* reaper. MUST stay in lockstep with `claudeBusyKeys` at every clear site —
|
|
47
|
-
* a stale timestamp surviving a clear makes a later re-marked key look
|
|
48
|
-
* >TTL-old and get reaped against a live delivery. Deleted alongside every
|
|
49
|
-
* `claudeBusyKeys.delete`/`.clear` below. */
|
|
50
|
-
claudeBusyKeySince: Map<string, number>
|
|
51
40
|
|
|
52
41
|
/** Open draft-stream handles keyed by chat:thread:replyId. */
|
|
53
42
|
activeDraftStreams: Map<string, Stream>
|
|
@@ -96,8 +85,6 @@ export function flushOnAgentDisconnect<
|
|
|
96
85
|
activeStatusReactions,
|
|
97
86
|
activeReactionMsgIds,
|
|
98
87
|
activeTurnStartedAt,
|
|
99
|
-
claudeBusyKeys,
|
|
100
|
-
claudeBusyKeySince,
|
|
101
88
|
activeDraftStreams,
|
|
102
89
|
clearActiveReactions,
|
|
103
90
|
disposeProgressDriver,
|
|
@@ -125,8 +112,6 @@ export function flushOnAgentDisconnect<
|
|
|
125
112
|
activeStatusReactions.delete(key)
|
|
126
113
|
activeReactionMsgIds.delete(key)
|
|
127
114
|
activeTurnStartedAt.delete(key)
|
|
128
|
-
claudeBusyKeys.delete(key)
|
|
129
|
-
claudeBusyKeySince.delete(key) // #2787: keep orphan-TTL map in lockstep
|
|
130
115
|
}
|
|
131
116
|
clearActiveReactions()
|
|
132
117
|
|
|
@@ -146,8 +131,6 @@ export function flushOnAgentDisconnect<
|
|
|
146
131
|
for (const k of danglingKeys) {
|
|
147
132
|
activeTurnStartedAt.delete(k)
|
|
148
133
|
activeReactionMsgIds.delete(k)
|
|
149
|
-
claudeBusyKeys.delete(k)
|
|
150
|
-
claudeBusyKeySince.delete(k) // #2787: keep orphan-TTL map in lockstep
|
|
151
134
|
}
|
|
152
135
|
log(
|
|
153
136
|
`telegram gateway: disconnect-flush swept ${danglingKeys.length} dangling turn key(s) ` +
|
|
@@ -156,33 +139,12 @@ export function flushOnAgentDisconnect<
|
|
|
156
139
|
onDanglingTurnsSwept?.(danglingKeys)
|
|
157
140
|
}
|
|
158
141
|
|
|
159
|
-
// PR3b orphan-sweep
|
|
160
|
-
//
|
|
161
|
-
//
|
|
162
|
-
//
|
|
163
|
-
//
|
|
164
|
-
//
|
|
165
|
-
// so the sweep loop above wouldn't notice them. Pre-PR3b this was
|
|
166
|
-
// invisible because the fleet gate read activeTurnStartedAt.size —
|
|
167
|
-
// synthetic-only turns never registered. PR3b's claudeBusyKeys.add
|
|
168
|
-
// is the more-accurate "claude is busy on this" gate, which means
|
|
169
|
-
// a synthetic-delivered turn that dies WITHOUT turn_end leaves an
|
|
170
|
-
// orphan that the activeTurnStartedAt-keyed sweep can't see.
|
|
171
|
-
// Cure: clear any leftover busy keys here. Bridge died → every
|
|
172
|
-
// busy key is dead by definition. Same justification as the
|
|
173
|
-
// dangling-sweep above for activeTurnStartedAt.
|
|
174
|
-
if (claudeBusyKeys.size > 0) {
|
|
175
|
-
const orphanCount = claudeBusyKeys.size
|
|
176
|
-
const orphanKeys = [...claudeBusyKeys]
|
|
177
|
-
claudeBusyKeys.clear()
|
|
178
|
-
// #2787: keep the orphan-TTL map in lockstep with the set it shadows.
|
|
179
|
-
for (const k of orphanKeys) claudeBusyKeySince.delete(k)
|
|
180
|
-
log(
|
|
181
|
-
`telegram gateway: disconnect-flush cleared ${orphanCount} orphan claudeBusyKeys ` +
|
|
182
|
-
`entr${orphanCount === 1 ? 'y' : 'ies'} (synthetic-inbound deliveries that never turn_ended)` +
|
|
183
|
-
` keys=${orphanKeys.join(',')}`,
|
|
184
|
-
)
|
|
185
|
-
}
|
|
142
|
+
// (#2996 P1) The PR3b `claudeBusyKeys` orphan-sweep that lived here was
|
|
143
|
+
// deleted with the set itself: the delivery machine's `bridgeDown` event
|
|
144
|
+
// — emitted by the gateway's onClientDisconnected alongside this flush —
|
|
145
|
+
// resets the machine to bridge_dead, which structurally clears the
|
|
146
|
+
// turn-in-flight gate for synthetic-delivered turns that died without a
|
|
147
|
+
// turn_end (the case the sweep existed for).
|
|
186
148
|
|
|
187
149
|
// Stop coalesce timers that could emit into a finalized draft stream, but
|
|
188
150
|
// preserve chats with pendingCompletion=true — those have background
|