switchroom 0.19.30 → 0.19.32

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. package/dist/cli/switchroom.js +1121 -584
  2. package/dist/host-control/main.js +151 -73
  3. package/package.json +1 -1
  4. package/profiles/_base/cron-session.sh.hbs +5 -1
  5. package/profiles/_base/start.sh.hbs +15 -1
  6. package/telegram-plugin/dist/bridge/bridge.js +3 -0
  7. package/telegram-plugin/dist/gateway/gateway.js +1660 -677
  8. package/telegram-plugin/dist/server.js +3 -0
  9. package/telegram-plugin/edit-flood-fuse.ts +70 -20
  10. package/telegram-plugin/gateway/boot-beacon.ts +364 -0
  11. package/telegram-plugin/gateway/boot-sweep-gate.ts +20 -15
  12. package/telegram-plugin/gateway/gateway.ts +87 -88
  13. package/telegram-plugin/gateway/inbound-spool.ts +39 -0
  14. package/telegram-plugin/gateway/narrative-lane.ts +12 -0
  15. package/telegram-plugin/gateway/obligation-store.ts +28 -0
  16. package/telegram-plugin/gateway/stale-pin-sweep-store.ts +221 -0
  17. package/telegram-plugin/gateway/stale-pin-sweep-wiring.ts +211 -0
  18. package/telegram-plugin/gateway/stale-pin-sweep.test.ts +804 -0
  19. package/telegram-plugin/gateway/stale-pin-sweep.ts +1146 -0
  20. package/telegram-plugin/gateway/status-pin-retarget.ts +15 -2
  21. package/telegram-plugin/gateway/status-pin-store.ts +33 -11
  22. package/telegram-plugin/gateway/stream-render.ts +578 -321
  23. package/telegram-plugin/registry/turns-schema.ts +21 -1
  24. package/telegram-plugin/retry-api-call.ts +46 -21
  25. package/telegram-plugin/session-tail.ts +13 -0
  26. package/telegram-plugin/shared/bot-runtime.ts +61 -17
  27. package/telegram-plugin/shared/gw-trace-gate.ts +18 -2
  28. package/telegram-plugin/tests/activity-card-wiring.test.ts +7 -7
  29. package/telegram-plugin/tests/activity-drain-fuse-drop-not-failure.test.ts +324 -0
  30. package/telegram-plugin/tests/agent-card-result-footer.test.ts +193 -0
  31. package/telegram-plugin/tests/boot-beacon.test.ts +462 -0
  32. package/telegram-plugin/tests/boot-pin-sweep-wiring.test.ts +6 -6
  33. package/telegram-plugin/tests/boot-sweep-gate.test.ts +42 -31
  34. package/telegram-plugin/tests/inbound-delivery-machine-dispatch.test.ts +2 -0
  35. package/telegram-plugin/tests/inbound-spool-progress.test.ts +2 -0
  36. package/telegram-plugin/tests/inbound-spool.test.ts +134 -5
  37. package/telegram-plugin/tests/narrative-lane-golden.test.ts +28 -0
  38. package/telegram-plugin/tests/obligation-determinism.test.ts +2 -0
  39. package/telegram-plugin/tests/obligation-store.test.ts +67 -1
  40. package/telegram-plugin/tests/status-pin-boot-recovery.test.ts +3 -3
  41. package/telegram-plugin/tests/status-pin-store.test.ts +26 -5
  42. package/telegram-plugin/tests/stream-render-golden.test.ts +20 -5
  43. package/telegram-plugin/tests/tg-post-logger-error-shape.test.ts +161 -0
  44. package/telegram-plugin/tests/turn-mint-defers-until-dequeue.test.ts +196 -0
  45. package/telegram-plugin/tests/turn-mint-harness.ts +155 -0
  46. package/telegram-plugin/tests/turn-supersede-finalizes-prior-card.test.ts +124 -0
  47. package/telegram-plugin/tests/worker-feed-pin-persistence.test.ts +30 -0
  48. package/telegram-plugin/tool-activity-summary.ts +104 -38
  49. package/telegram-plugin/worker-activity-feed.ts +33 -16
  50. package/telegram-plugin/gateway/dm-pin-sweep.test.ts +0 -251
  51. package/telegram-plugin/gateway/dm-pin-sweep.ts +0 -178
@@ -240,7 +240,27 @@ const PHASE4_MIGRATIONS = [
240
240
 
241
241
  function applySchema(db: SqliteDatabase): void {
242
242
  db.exec('PRAGMA journal_mode = WAL')
243
- db.exec('PRAGMA synchronous = NORMAL')
243
+ // FULL, not NORMAL. This registry is the crash-recovery ledger: the turn row
244
+ // is what tells the boot-time resume protocol that a turn was in flight when
245
+ // the process died. Under WAL, `synchronous = NORMAL` deliberately does NOT
246
+ // fsync the WAL on commit — it syncs only at checkpoint — so a container kill
247
+ // is survived but a HOST POWER CUT can lose the last commits, including the
248
+ // whole turn row. The interrupted turn then becomes invisible to resume:
249
+ // silently unrecoverable, which is exactly the case this ledger exists for.
250
+ //
251
+ // Cost is bounded and measured, not assumed: on this fleet's ext4/NVMe volume
252
+ // a FULL commit costs ~0.87ms mean (p99 1.7ms) vs ~0.011ms at NORMAL, i.e. a
253
+ // ~1150 commits/s ceiling. Steady state is ~330 commits/day, and the hot path
254
+ // is `bumpSubagentActivity` (subagents-schema.ts) at ~1Hz per running worker
255
+ // — ~4 commits/s during fan-out, a ~0.35% duty cycle.
256
+ //
257
+ // `synchronous` is per-connection and NOT persisted in the DB file (unlike
258
+ // `journal_mode`), so it binds only connections that set it. The PreToolUse
259
+ // subagent tracker (`hooks/subagent-tracker-pretool.mjs`) opens this same
260
+ // registry.db without the pragma and has therefore been running at SQLite's
261
+ // default FULL on the tool-call hot path since it shipped; the gateway's
262
+ // NORMAL was the outlier.
263
+ db.exec('PRAGMA synchronous = FULL')
244
264
  // Concurrency: multiple writers contend on this registry (the PreToolUse
245
265
  // subagent-tracker hook, the gateway's subagent-watcher backfill, the turns
246
266
  // writer) — especially when several sub-agents dispatch at once. Without a
@@ -92,13 +92,49 @@ export interface RetryCallOpts {
92
92
  priorityClass?: 'critical' | 'useful' | 'cosmetic'
93
93
  }
94
94
 
95
+ /**
96
+ * The three benign Telegram 400s this policy swallows (contract table above).
97
+ */
98
+ export type BenignTelegram400Kind = 'not_modified' | 'message_not_found' | 'delete_not_found'
99
+
100
+ /**
101
+ * Single source of truth for "is this Telegram 400 a benign no-op?".
102
+ *
103
+ * Benign here means exactly what the retry policy already treats as a
104
+ * non-event: the edit was a no-op because the content is identical, or the
105
+ * target message vanished before the edit/delete landed. Both are
106
+ * high-volume on a healthy agent (every coalesced card repaint can produce
107
+ * one) and neither means anything is wrong — so any surface that reports
108
+ * Telegram failures needs to tell them apart from a real rejection WITHOUT
109
+ * maintaining a second copy of the description list (#3927). The retry
110
+ * policy's own swallow branches call this, so there is one list.
111
+ *
112
+ * Returns the benign kind, or `null` for everything else — including 429,
113
+ * 403, and any 400 not on this deliberately narrow list. A new benign
114
+ * description belongs here, not in a caller.
115
+ */
116
+ export function classifyBenignTelegram400(
117
+ errorCode: number | undefined,
118
+ description: string | undefined,
119
+ ): BenignTelegram400Kind | null {
120
+ if (errorCode !== 400) return null
121
+ const desc = (description ?? '').toLowerCase()
122
+ // "message is not modified" / "message was not modified" — Telegram's
123
+ // no-op-on-equal-text.
124
+ if (desc.includes('not modified')) return 'not_modified'
125
+ // "message to edit/delete not found" — the target vanished.
126
+ if (desc.includes('message to edit not found')) return 'message_not_found'
127
+ if (desc.includes('message to delete not found')) return 'delete_not_found'
128
+ return null
129
+ }
130
+
95
131
  export interface RetryObserver {
96
132
  /** Fires just before sleeping for a retry. */
97
133
  onRetry?(info: { attempt: number; reason: 'flood_wait' | 'network'; delayMs: number }): void
98
134
  /** Fires when max retries is reached and the wrapper gives up. */
99
135
  onGiveUp?(info: { attempts: number; error: unknown }): void
100
136
  /** Fires for each benign error we swallowed (not-modified, not-found). */
101
- onBenign?(info: { kind: 'not_modified' | 'message_not_found' | 'delete_not_found' }): void
137
+ onBenign?(info: { kind: BenignTelegram400Kind }): void
102
138
  }
103
139
 
104
140
  export interface RetryApiCallConfig {
@@ -399,26 +435,15 @@ export function createRetryApiCall(
399
435
  continue
400
436
  }
401
437
 
402
- // Swallow "message is not modified" Telegram's no-op-on-equal-text.
403
- if (
404
- isGrammyErr &&
405
- (err as GrammyError).error_code === 400 &&
406
- desc.includes('not modified')
407
- ) {
408
- observer?.onBenign?.({ kind: 'not_modified' })
409
- return undefined as unknown as T
410
- }
411
-
412
- // Swallow "message to edit/delete not found" — target vanished.
413
- if (
414
- isGrammyErr &&
415
- (err as GrammyError).error_code === 400 &&
416
- (desc.includes('message to edit not found') ||
417
- desc.includes('message to delete not found'))
418
- ) {
419
- observer?.onBenign?.({
420
- kind: desc.includes('edit') ? 'message_not_found' : 'delete_not_found',
421
- })
438
+ // Swallow the benign 400s — "message is not modified" (Telegram's
439
+ // no-op-on-equal-text) and "message to edit/delete not found" (the
440
+ // target vanished). Classification lives in `classifyBenignTelegram400`
441
+ // so the tg-post logger reports the same family identically (#3927).
442
+ const benignKind = isGrammyErr
443
+ ? classifyBenignTelegram400((err as GrammyError).error_code, desc)
444
+ : null
445
+ if (benignKind !== null) {
446
+ observer?.onBenign?.({ kind: benignKind })
422
447
  return undefined as unknown as T
423
448
  }
424
449
 
@@ -93,6 +93,13 @@ export function findActiveSessionFile(projectsDir: string): string | null {
93
93
  export type SessionEvent =
94
94
  | { kind: 'enqueue'; chatId: string | null; messageId: string | null; threadId: string | null; rawContent: string; isSync?: boolean }
95
95
  | { kind: 'dequeue' }
96
+ // #3927 — `queue-operation` op `remove`: the queued item was folded into the
97
+ // ALREADY-RUNNING turn as a `queued_command` attachment rather than drained
98
+ // into a new turn. It is the OTHER terminal of an `enqueue` (the CLI writes
99
+ // exactly one of `dequeue` / `remove` per enqueue), and unlike `dequeue` it
100
+ // REPLAYS the enqueue's `content` byte-for-byte — which is what lets the
101
+ // gateway discard the right parked turn-start by identity.
102
+ | { kind: 'queue_remove'; rawContent: string }
96
103
  | { kind: 'thinking' }
97
104
  // Live model in use for the MAIN session, extracted from `message.model` on
98
105
  // each `type:"assistant"` transcript line (the exact model that served that
@@ -566,6 +573,12 @@ export function projectTranscriptLine(line: string): SessionEvent[] {
566
573
  if (op === 'dequeue') {
567
574
  return [{ kind: 'dequeue' }]
568
575
  }
576
+ // #3927 — the enqueue's OTHER terminal. Previously dropped, which left the
577
+ // gateway unable to tell "queued message folded into the running turn"
578
+ // (`remove`) from "queue drained into a new turn" (`dequeue`).
579
+ if (op === 'remove') {
580
+ return [{ kind: 'queue_remove', rawContent: (obj.content as string | undefined) ?? '' }]
581
+ }
569
582
  return []
570
583
  }
571
584
 
@@ -28,10 +28,10 @@ import { execFileSync, spawnSync } from 'child_process'
28
28
  import { createHash } from 'crypto'
29
29
  import { AsyncLocalStorage } from 'async_hooks'
30
30
  import { clearStaleTelegramPollingState } from '../startup-reset.js'
31
- import { createRetryApiCall } from '../retry-api-call.js'
31
+ import { createRetryApiCall, classifyBenignTelegram400 } from '../retry-api-call.js'
32
32
  import { makeFloodWaitRecorder, makeFloodWaitProbe } from '../flood-circuit-breaker.js'
33
33
  import { RICH_MESSAGE_MAX_CHARS } from '../format.js'
34
- import { shouldEmitTgPost } from './gw-trace-gate.js'
34
+ import { shouldEmitTgPost, type TgPostStatus } from './gw-trace-gate.js'
35
35
  import { guardAccidentalFormatting } from '../rich-send.js'
36
36
 
37
37
  // ─── tg-post tag plumbing ─────────────────────────────────────────────────
@@ -80,6 +80,17 @@ function formatTgPostTags(tags: TgPostTags | undefined): string {
80
80
 
81
81
  // ─── tg-post observability transformer ────────────────────────────────────
82
82
 
83
+ /**
84
+ * Sanitise a Telegram error description for single-line log output —
85
+ * collapse whitespace, strip newlines, cap at 80 chars, `-` when absent.
86
+ * PII-safe: Telegram error descriptions are server-generated and don't
87
+ * echo the request body.
88
+ */
89
+ function shortDesc(raw: string | undefined): string {
90
+ if (!raw) return '-'
91
+ return raw.replace(/\s+/g, ' ').slice(0, 80).replace(/[\r\n]/g, ' ') || '-'
92
+ }
93
+
83
94
  /**
84
95
  * Installs an API transformer on the bot that emits one stderr line per
85
96
  * outbound Telegram Bot API POST. This is the single catchment point for
@@ -91,7 +102,30 @@ function formatTgPostTags(tags: TgPostTags | undefined): string {
91
102
  *
92
103
  * Log shape (one line per POST, on both success and failure):
93
104
  *
94
- * tg-post method=<m> chat=<id> thread=<id|-> parse_mode=<HTML|MarkdownV2|none> bytes=<n> hash=<sha1-12> status=<ok|err> err=<class-or--> code=<http-or--> desc=<short|-->
105
+ * tg-post method=<m> chat=<id> thread=<id|-> parse_mode=<HTML|MarkdownV2|none> bytes=<n> hash=<sha1-12> status=<ok|benign|err> err=<class-or--> code=<http-or--> desc=<short|-->
106
+ *
107
+ * TRUTHFULNESS (#3927): grammY resolves the transformer chain with the raw
108
+ * `ApiResponse`, and only converts `{ok:false}` into a thrown `GrammyError`
109
+ * AFTER the chain returns — verified in the pinned grammy 1.44.0,
110
+ * `out/core/client.js:95-100`:
111
+ *
112
+ * const data = await this.call(method, payload, signal) // chain
113
+ * if (data.ok) return data.result
114
+ * else throw toGrammyError(data, method, payload) // OUTSIDE
115
+ *
116
+ * So EVERY Telegram-level rejection (429 flood-wait, 400, 403) arrives here
117
+ * as a RESOLVED response, and the `catch` below only ever sees transport
118
+ * (`HttpError`) failures. Before this was fixed the logger reported every
119
+ * one of them as `status=ok err=- code=- desc=-` — a rate-limited
120
+ * `unpinAllChatMessages` logged as SUCCESS 3ms before the retry policy
121
+ * logged `429 rate limited, waiting 3s`. We therefore inspect the resolved
122
+ * body and report `status=err err=telegram_<code>` from it.
123
+ *
124
+ * The narrow set of benign 400s the retry policy already swallows
125
+ * ("message is not modified", "message to edit/delete not found" —
126
+ * classified by the shared `classifyBenignTelegram400`) gets `status=benign`
127
+ * instead, so promoting real rejections out of `status=ok` doesn't bury
128
+ * `grep status=err` under high-volume card-repaint no-ops.
95
129
  *
96
130
  * Body content is never logged — only its length and a 12-char sha1 prefix
97
131
  * so we can recognise repeated identical sends without leaking PII. The
@@ -115,16 +149,34 @@ export function installTgPostLogger(bot: Bot): void {
115
149
  ? createHash('sha1').update(text).digest('hex').slice(0, 12)
116
150
  : '-'
117
151
  const tagSuffix = formatTgPostTags(_getTgPostTags())
118
- try {
119
- const res = await prev(method, payload, signal)
152
+ const emit = (status: TgPostStatus, errClass: string, code: string, desc: string) => {
120
153
  // #3025: suppress zero-signal per-poll heartbeats (getUpdates/getMe
121
154
  // status=ok, one line per ~30s long-poll tick) unless the operator
122
155
  // set SWITCHROOM_GW_TRACE. Errors and all other methods still log.
123
- if (shouldEmitTgPost(method, 'ok')) {
124
- process.stderr.write(
125
- `tg-post method=${method} chat=${chat} thread=${thread} parse_mode=${parseMode} bytes=${bytes} hash=${hash} status=ok err=- code=- desc=-${tagSuffix}\n`,
156
+ if (!shouldEmitTgPost(method, status)) return
157
+ process.stderr.write(
158
+ `tg-post method=${method} chat=${chat} thread=${thread} parse_mode=${parseMode} bytes=${bytes} hash=${hash} status=${status} err=${errClass} code=${code} desc=${desc}${tagSuffix}\n`,
159
+ )
160
+ }
161
+ try {
162
+ const res = await prev(method, payload, signal)
163
+ // A RESOLVED response can still be a Telegram-level rejection — grammy
164
+ // converts `{ok:false}` to a thrown GrammyError only after this chain
165
+ // returns (see the docblock). Same defensive shape as the edit fuse's
166
+ // `runObserved` (edit-flood-fuse.ts).
167
+ const r = res as unknown as { ok?: boolean; error_code?: number; description?: string }
168
+ if (r != null && typeof r === 'object' && r.ok === false) {
169
+ const code = r.error_code
170
+ const benign = classifyBenignTelegram400(code, r.description)
171
+ emit(
172
+ benign !== null ? 'benign' : 'err',
173
+ `telegram_${code ?? 'unknown'}`,
174
+ code != null ? String(code) : '-',
175
+ shortDesc(r.description),
126
176
  )
177
+ return res
127
178
  }
179
+ emit('ok', '-', '-', '-')
128
180
  return res
129
181
  } catch (err) {
130
182
  const errClass = err instanceof GrammyError
@@ -134,15 +186,7 @@ export function installTgPostLogger(bot: Bot): void {
134
186
  const rawDesc = err instanceof GrammyError
135
187
  ? (err as GrammyError).description
136
188
  : (err instanceof Error ? err.message : '')
137
- // Sanitise the description for single-line log output — collapse
138
- // whitespace, strip newlines, cap at 80 chars. PII-safe: Telegram
139
- // error descriptions are server-generated and don't echo body.
140
- const desc = rawDesc
141
- ? rawDesc.replace(/\s+/g, ' ').slice(0, 80).replace(/[\r\n]/g, ' ') || '-'
142
- : '-'
143
- process.stderr.write(
144
- `tg-post method=${method} chat=${chat} thread=${thread} parse_mode=${parseMode} bytes=${bytes} hash=${hash} status=err err=${errClass} code=${code} desc=${desc}${tagSuffix}\n`,
145
- )
189
+ emit('err', errClass, code, shortDesc(rawDesc))
146
190
  throw err
147
191
  }
148
192
  })
@@ -57,6 +57,20 @@ export const gwTraceVerbose: boolean = computeGwTraceVerbose(
57
57
  */
58
58
  const ZERO_SIGNAL_POLL_METHODS = new Set(['getUpdates', 'getMe'])
59
59
 
60
+ /**
61
+ * The status tiers a `tg-post` line can carry.
62
+ *
63
+ * `benign` (#3927) is a REJECTED call that is nonetheless a non-event —
64
+ * the narrow set of 400s the retry policy already swallows ("message is
65
+ * not modified", "message to edit/delete not found"), classified by
66
+ * `classifyBenignTelegram400`. It exists so that promoting resolved
67
+ * `ok:false` responses out of `status=ok` doesn't drown the newly-truthful
68
+ * `status=err` signal in high-volume card-repaint no-ops: `grep status=err`
69
+ * stays a list of real failures, and the benign lines are still there,
70
+ * honestly labelled, for anyone who wants them.
71
+ */
72
+ export type TgPostStatus = 'ok' | 'benign' | 'err'
73
+
60
74
  /**
61
75
  * Decide whether a `tg-post` line should be written.
62
76
  *
@@ -66,11 +80,13 @@ const ZERO_SIGNAL_POLL_METHODS = new Set(['getUpdates', 'getMe'])
66
80
  * Always emitted:
67
81
  * - any `status=err` (real failures — the whole point of the log),
68
82
  * - every other method's `ok` line (sendMessage/editMessageText/... are
69
- * genuine outbound observability, #656/#657).
83
+ * genuine outbound observability, #656/#657),
84
+ * - `status=benign` on any non-poll method — same volume as the
85
+ * `status=ok` line it replaces, just no longer mislabelled as success.
70
86
  */
71
87
  export function shouldEmitTgPost(
72
88
  method: string,
73
- status: 'ok' | 'err',
89
+ status: TgPostStatus,
74
90
  verbose: boolean = gwTraceVerbose,
75
91
  ): boolean {
76
92
  if (verbose) return true
@@ -119,17 +119,17 @@ describe('activity-card durability wiring', () => {
119
119
 
120
120
  it('the boot reaper runs ONLY after the startup mutex is won (never at import time)', () => {
121
121
  // #3026: the reaper is now invoked via the mutex-gated orchestrator
122
- // runBootPinCleanupAndDmSweep() (which runs it alongside
123
- // statusPinBootCleanup + queuedCardBootReaper, then runs the DM
124
- // stale-pin sweep). Invariant unchanged: reachable only post-lock.
122
+ // runBootPinCleanupAndStalePinSweep() (which runs it alongside
123
+ // statusPinBootCleanup + queuedCardBootReaper, then runs the stale-pin
124
+ // stack sweep). Invariant unchanged: reachable only post-lock.
125
125
  // (1) The orchestrator runs the reaper. Since the #3664 S2 salvage the
126
126
  // steps are INJECTED into runBootPinSweepSteps (each individually
127
- // absorbed, so a throwing reaper cannot strand the DM sweep behind it)
127
+ // absorbed, so a throwing reaper cannot strand the sweep behind it)
128
128
  // instead of being awaited inline — so the marker is the step binding.
129
129
  const orchestrator = between(
130
130
  gatewaySrc,
131
- 'function runBootPinCleanupAndDmSweep()',
132
- 'dmPinSweepEligible = true',
131
+ 'function runBootPinCleanupAndStalePinSweep()',
132
+ 'stalePinSweepEligible = true',
133
133
  )
134
134
  expect(orchestrator).toMatch(/activityCardReaper: activityCardBootReaper,/)
135
135
  // (2) Both orchestrator invocation sites (mutex-won + mutex-fallback)
@@ -146,7 +146,7 @@ describe('activity-card durability wiring', () => {
146
146
  // statusPinBootCleanup documents.
147
147
  const beforeMain = gatewaySrc.split('await acquireStartupLock({')[0] ?? ''
148
148
  expect(beforeMain).not.toMatch(/^\s*void activityCardBootReaper\(\)/m)
149
- expect(beforeMain).not.toMatch(/^\s*void runBootPinCleanupAndDmSweep\(\)/m)
149
+ expect(beforeMain).not.toMatch(/^\s*void runBootPinCleanupAndStalePinSweep\(\)/m)
150
150
  expect(beforeMain).not.toMatch(/^\s*bootPinSweepGate\.(arm|botReady)\(\)/m)
151
151
  })
152
152
 
@@ -0,0 +1,324 @@
1
+ /**
2
+ * The edit-flood fuse's DELIBERATE drops must not be reported as failures.
3
+ *
4
+ * ── The defect ────────────────────────────────────────────────────────────
5
+ * `edit-flood-fuse.ts` is installed as a grammY **transformer**
6
+ * (`bot.api.config.use`, `installEditFloodFuse`). A transformer's return
7
+ * contract is the raw `ApiResponse` ENVELOPE, and grammY's `Api.callApi`
8
+ * unwraps it:
9
+ *
10
+ * ```js
11
+ * const data = await this.call(method, payload, signal)
12
+ * if (data.ok) return data.result
13
+ * else throw toGrammyError(data, method, payload)
14
+ * ```
15
+ * (grammy `out/core/client.js`)
16
+ *
17
+ * The fuse resolved a dropped call with the bare `true` — the *result*, with no
18
+ * envelope. `(true).ok` is `undefined`, so EVERY intentional fuse drop took the
19
+ * `else` branch and was raised as
20
+ * `GrammyError: Call to 'editMessageText' failed! (undefined: undefined)`.
21
+ *
22
+ * Downstream, `narrative-lane.ts`'s activity-summary drain caught that, found
23
+ * nothing in its `isTransport` allowlist matching, incremented
24
+ * `turn.activityDrainFailures` and wrote a `activity-summary drain failed:` line
25
+ * to stderr. `activityDrainFailures` is what flags a turn DEGRADED at turn-end,
26
+ * so a healthy turn whose progress card was correctly rate-limited was reported
27
+ * as broken — observed reaching `failures=8` on a single turn.
28
+ *
29
+ * ── Why the fix is the envelope, not an `isTransport` entry ───────────────
30
+ * Classifying the string in `isTransport` would paper over a real contract
31
+ * violation that ALSO corrupts every other caller of a fused API method (each
32
+ * one sees a synthetic transport error for a benign no-op). Restoring the
33
+ * envelope is the durable fix; `isTransport` then needs no new entry at all,
34
+ * because nothing is thrown.
35
+ *
36
+ * ── R1 on the per-MESSAGE tier ───────────────────────────────────────────
37
+ * The second describe covers the other half: a card must never freeze mid-step
38
+ * because a frame was shed. `awaitRoom`'s `dropGuard` (R1: "only drop while
39
+ * something NEWER for this message is still in flight to repaint it") was wired
40
+ * into the two per-chat tiers but NOT the per-message tier, which passed
41
+ * `dropGuard: undefined` — read as "drop unconditionally at the deadline". A
42
+ * lone terminal `finalize`, which by definition has nothing newer coming, was
43
+ * therefore droppable there regardless of its priority class, freezing the card
44
+ * on "→ in-progress".
45
+ */
46
+ import { describe, it, expect, vi, afterEach } from 'vitest'
47
+ import { tmpdir } from 'node:os'
48
+ import { GrammyError } from 'grammy'
49
+
50
+ import { createEditFloodFuse, type EditFloodFuseConfig } from '../edit-flood-fuse.js'
51
+ import { withOutboundClass, type OutboundClass } from '../outbound-class.js'
52
+ import { createNarrativeLane } from '../gateway/narrative-lane.js'
53
+ import type { CurrentTurn, NarrativeLaneDeps } from '../gateway/gateway.js'
54
+
55
+ const CHAT = '1001'
56
+
57
+ /**
58
+ * grammY's `Api.callApi`, verbatim in behaviour: run the transformer chain,
59
+ * unwrap `ok:true`, throw `GrammyError` otherwise. This is the seam the defect
60
+ * lived in, so the harness must model it rather than assume the fuse's return
61
+ * value reaches the caller untouched (which is what the pre-existing fuse tests
62
+ * assumed — they hand `next` a bare `true` and never unwrap, which is exactly
63
+ * why the bug survived them).
64
+ */
65
+ function makeFusedApi(config: EditFloodFuseConfig) {
66
+ const fuse = createEditFloodFuse(config)
67
+ const landed: Array<{ method: string; message_id?: number; text: string | null }> = []
68
+ let nextId = 3000
69
+
70
+ const callApi = async (method: string, payload: Record<string, unknown>, run: () => unknown) => {
71
+ const data = (await fuse.apply(method, payload, async () => ({ ok: true, result: run() }))) as {
72
+ ok?: boolean
73
+ result?: unknown
74
+ }
75
+ if (data.ok === true) return data.result
76
+ throw new GrammyError(`Call to '${method}' failed!`, data as never, method, payload as never)
77
+ }
78
+
79
+ const api = {
80
+ sendRichMessage: (c: string, b: { markdown: string }) => {
81
+ const id = ++nextId
82
+ return callApi('sendRichMessage', { chat_id: c }, () => {
83
+ landed.push({ method: 'sendRichMessage', message_id: id, text: b.markdown })
84
+ return { message_id: id }
85
+ }) as Promise<{ message_id: number }>
86
+ },
87
+ editMessageText: (c: string, m: number, b: { markdown: string }) =>
88
+ callApi('editMessageText', { chat_id: c, message_id: m }, () => {
89
+ landed.push({ method: 'editMessageText', message_id: m, text: b.markdown })
90
+ return true
91
+ }),
92
+ deleteMessage: (c: string, m: number) =>
93
+ callApi('deleteMessage', { chat_id: c, message_id: m }, () => {
94
+ landed.push({ method: 'deleteMessage', message_id: m, text: null })
95
+ return true
96
+ }),
97
+ }
98
+ return { api, landed, fuse }
99
+ }
100
+
101
+ /**
102
+ * The real `createNarrativeLane`, wired to a bot whose API traverses the real
103
+ * fuse. `robustApiCall` stands in for the production chain
104
+ * `robustApiCall → sendGate.gate → withOutboundClass → fn`: the only part of it
105
+ * the fuse can observe is the published outbound class, so the stub publishes it
106
+ * for real via `withOutboundClass` and calls straight through.
107
+ */
108
+ function makeFusedLane(config: EditFloodFuseConfig) {
109
+ const { api, landed, fuse } = makeFusedApi(config)
110
+ const submitted: Array<{ verb?: string; priorityClass?: OutboundClass }> = []
111
+ const noop = () => {}
112
+ const fakeEA = {
113
+ mayDrain: () => true,
114
+ openOrEditCard: (_p: string, fn: () => void) => fn(),
115
+ finalizeCard: (fn: () => void) => fn(),
116
+ markSubstantiveFinalDelivered: (fn: () => void) => fn(),
117
+ }
118
+ const deps = {
119
+ ACTIVITY_CARD_STORE_PATH: `${tmpdir()}/lane-fuse-drop-activity-cards.json`,
120
+ CLEAR_STATUS_ON_COMPLETION: false,
121
+ FEED_HEARTBEAT_ENABLED: false,
122
+ FEED_HEARTBEAT_MIN_STALE_MS: 6000,
123
+ FEED_LIVENESS_OPEN_ENABLED: false,
124
+ FEED_LIVENESS_OPEN_MS: 5000,
125
+ PIN_STATUS_WHILE_WORKING: false,
126
+ POST_ANSWER_LIVENESS_STALE_MS: 90000,
127
+ STATIC: false,
128
+ activeDraftStreams: new Map(),
129
+ activityCardPersistEnabled: false,
130
+ activityCardStoreFs: {
131
+ readFileSync: () => '',
132
+ writeFileSync: noop,
133
+ mkdirSync: noop,
134
+ renameSync: noop,
135
+ unlinkSync: noop,
136
+ },
137
+ bot: { api },
138
+ cardDrainGate: (_t: unknown, _ea: unknown, run: () => void) => run(),
139
+ currentTurnMap: { get: () => null, byKey: new Map() },
140
+ earlyLivenessOpenTimers: new Map(),
141
+ emissionAuthorityFor: () => fakeEA,
142
+ feedOpenGateDeps: () => ({
143
+ hasOutboundDeliveredSince: () => false,
144
+ historyEnabled: false,
145
+ finalAnswerMinChars: 200,
146
+ }),
147
+ getCurrentTurn: () => null,
148
+ reconcileStatusPin: noop,
149
+ robustApiCall: (fn: () => Promise<unknown>, opts?: { verb?: string; priorityClass?: OutboundClass }) => {
150
+ submitted.push({ verb: opts?.verb, priorityClass: opts?.priorityClass })
151
+ return withOutboundClass(opts?.priorityClass ?? 'critical', fn)
152
+ },
153
+ statusKey: (c: string, t?: number | null) => `${c}:${t ?? 'main'}`,
154
+ } as unknown as NarrativeLaneDeps
155
+ const lane = createNarrativeLane(deps)
156
+ return { lane, landed, fuse, submitted }
157
+ }
158
+
159
+ function makeLaneTurn(lane: ReturnType<typeof createNarrativeLane>): CurrentTurn {
160
+ const turn = {
161
+ turnId: 'turn-fuse-drop-1',
162
+ sessionChatId: CHAT,
163
+ sessionThreadId: undefined,
164
+ sourceMessageId: null,
165
+ registryKey: null,
166
+ startedAt: Date.now() - 3000,
167
+ currentModel: null,
168
+ totalTokens: 0,
169
+ labeledToolCount: 0,
170
+ mirrorLines: [] as string[],
171
+ foregroundSubAgents: new Map<string, string[]>(),
172
+ activityPendingRender: null as string | null,
173
+ activityLastSentRender: null as string | null,
174
+ activityMessageId: null as number | null,
175
+ activityInFlight: null as Promise<void> | null,
176
+ activityEverOpened: false,
177
+ activityDrainFailures: 0,
178
+ finalAnswerEverDelivered: false,
179
+ finalAnswerDelivered: false,
180
+ replyCalled: false,
181
+ capturedText: [] as string[],
182
+ lastReplyText: '',
183
+ answerStream: null,
184
+ liveness: { recentlyStreaming: () => false, onStreamEvent: () => {}, note: () => {} },
185
+ } as unknown as CurrentTurn
186
+ ;(turn as { narrativeGate?: unknown }).narrativeGate = lane.makeNarrativeGate(turn)
187
+ return turn
188
+ }
189
+
190
+ const settle = () => new Promise((r) => setTimeout(r, 20))
191
+
192
+ /**
193
+ * A fuse tuned so a cosmetic repaint is genuinely DROPPED:
194
+ * - one cosmetic edit per message per window, so frame 2+ is over budget;
195
+ * - a short defer deadline so the wait ends inside the test;
196
+ * - `lateReleaseMaxPerWindow: 0`, i.e. the bounded R1 overshoot budget is
197
+ * already spent, which is the state in which a lone cosmetic frame is
198
+ * dropped rather than late-released. Without this the fix in the second
199
+ * describe would late-release the frame and no drop would occur at all.
200
+ */
201
+ const DROPPING_FUSE: EditFloodFuseConfig = {
202
+ perMessageMaxPerWindow: 1,
203
+ cosmeticPerMessageMaxPerWindow: 1,
204
+ perMessageWindowMs: 60_000,
205
+ maxDeferMs: 20,
206
+ lateReleaseMaxPerWindow: 0,
207
+ }
208
+
209
+ afterEach(() => {
210
+ vi.restoreAllMocks()
211
+ })
212
+
213
+ describe('edit-flood fuse — a deliberate drop is not a failure', () => {
214
+ it('a dropped cosmetic edit RESOLVES through grammY instead of throwing "(undefined: undefined)"', async () => {
215
+ const { api, fuse } = makeFusedApi(DROPPING_FUSE)
216
+ const results: unknown[] = []
217
+ const errors: string[] = []
218
+ for (let i = 0; i < 3; i++) {
219
+ await withOutboundClass('cosmetic', () => api.editMessageText(CHAT, 42, { markdown: `frame ${i}` }))
220
+ .then((r) => results.push(r))
221
+ .catch((e: unknown) => errors.push((e as Error).message))
222
+ }
223
+ // Pre-fix: two of the three surface as
224
+ // `Call to 'editMessageText' failed! (undefined: undefined)`.
225
+ expect(errors).toEqual([])
226
+ // The drop is real — this asserts the fuse actually shed, so the test cannot
227
+ // pass by simply never exercising the drop path.
228
+ expect(fuse.stats().dropped).toBeGreaterThan(0)
229
+ // grammY unwraps `{ok:true, result:true}` to the `true` an edit resolves to.
230
+ expect(results).toEqual([true, true, true])
231
+ })
232
+
233
+ it('the activity-summary drain records ZERO failures and logs no "drain failed" line', async () => {
234
+ const stderr = vi.spyOn(process.stderr, 'write').mockImplementation(() => true)
235
+ const { lane, fuse } = makeFusedLane(DROPPING_FUSE)
236
+ const turn = makeLaneTurn(lane)
237
+
238
+ // Open the card, then repaint it past the cosmetic ceiling so the fuse drops.
239
+ lane.showNarrativeStep(turn, 'Step one of the work')
240
+ await turn.activityInFlight
241
+ for (let i = 2; i <= 6; i++) {
242
+ lane.showNarrativeStep(turn, `Step ${i} of the work`)
243
+ await turn.activityInFlight
244
+ await settle()
245
+ }
246
+
247
+ expect(fuse.stats().dropped + fuse.stats().superseded).toBeGreaterThan(0)
248
+ // The counter that flags the turn DEGRADED at turn-end. Pre-fix this reached
249
+ // the number of dropped repaints (observed: 8 on a real turn).
250
+ expect(turn.activityDrainFailures).toBe(0)
251
+ const lines = stderr.mock.calls.map((c) => String(c[0]))
252
+ expect(lines.filter((l) => l.includes('drain failed'))).toEqual([])
253
+ })
254
+ })
255
+
256
+ describe('edit-flood fuse — a terminal card render is never shed', () => {
257
+ it('the lane submits the finalize above `cosmetic`, at a class the fuse will not shed', async () => {
258
+ // REGRESSION PIN. `useful` and `critical` are the only classes above
259
+ // `cosmetic` (`outbound-class.ts` / `retry-api-call.ts` `priorityClass`);
260
+ // the finalize takes `useful` — it may be DEFERRED, but the fuse's
261
+ // cosmetic-only per-message/per-chat ceilings and the R1 drop path do not
262
+ // apply to it. Live repaints staying `cosmetic` is correct and unchanged.
263
+ const { lane, submitted } = makeFusedLane({ enabled: false })
264
+ const turn = makeLaneTurn(lane)
265
+ lane.showNarrativeStep(turn, 'Doing the work now')
266
+ await turn.activityInFlight
267
+ lane.clearActivitySummary(turn)
268
+ await settle()
269
+
270
+ const drain = submitted.filter((s) => s.verb === 'activity-summary.edit')
271
+ const finalize = submitted.filter((s) => s.verb === 'activity-summary.finalize')
272
+ expect(finalize.length).toBeGreaterThanOrEqual(1)
273
+ for (const f of finalize) expect(f.priorityClass).toBe('useful')
274
+ // The live repaints are deliberately still sheddable.
275
+ for (const d of drain) expect(d.priorityClass).toBe('cosmetic')
276
+ })
277
+
278
+ it('the fuse LATE-RELEASES a lone terminal frame at the deadline instead of freezing the card', async () => {
279
+ // R1 on the per-MESSAGE tier. One `useful` edit fills the per-message
280
+ // window; the terminal frame that follows is over budget, is the ONLY frame
281
+ // in flight for its message (nothing newer will ever repaint it), and must
282
+ // therefore be released late rather than dropped.
283
+ const { api, landed, fuse } = makeFusedApi({
284
+ perMessageMaxPerWindow: 1,
285
+ perMessageWindowMs: 60_000,
286
+ maxDeferMs: 20,
287
+ })
288
+ await withOutboundClass('useful', () => api.editMessageText(CHAT, 77, { markdown: 'live frame' }))
289
+ // Nothing else is in flight for message 77 — this is the card's last frame.
290
+ const res = await withOutboundClass('useful', () =>
291
+ api.editMessageText(CHAT, 77, { markdown: 'TERMINAL ✅ done' }),
292
+ )
293
+
294
+ // Pre-fix this frame was dropped at the deadline (`dropGuard: undefined` on
295
+ // the per-message tier), so it never reached the API and the card stayed on
296
+ // "live frame" forever — and, via the envelope defect, the caller saw a
297
+ // synthetic transport error on top.
298
+ expect(landed.map((l) => l.text)).toEqual(['live frame', 'TERMINAL ✅ done'])
299
+ expect(res).toBe(true)
300
+ expect(fuse.stats().dropped).toBe(0)
301
+ })
302
+
303
+ it('a superseded frame is still dropped — R1 does not disable last-write-wins', async () => {
304
+ // The guard is a DEADLINE guard only. A genuinely newer frame for the same
305
+ // message must still kill the older waiter, or the fix would turn the
306
+ // per-message tier into an unbounded queue.
307
+ const { api, landed, fuse } = makeFusedApi({
308
+ perMessageMaxPerWindow: 1,
309
+ perMessageWindowMs: 60_000,
310
+ maxDeferMs: 5_000,
311
+ })
312
+ await withOutboundClass('cosmetic', () => api.editMessageText(CHAT, 88, { markdown: 'v1' }))
313
+ const stale = withOutboundClass('cosmetic', () => api.editMessageText(CHAT, 88, { markdown: 'v2' }))
314
+ await settle()
315
+ const newest = withOutboundClass('cosmetic', () => api.editMessageText(CHAT, 88, { markdown: 'v3' }))
316
+ // Both settle without throwing; the stale one simply never painted.
317
+ await expect(stale).resolves.toBe(true)
318
+ await settle()
319
+ expect(fuse.stats().superseded).toBeGreaterThan(0)
320
+ expect(landed.map((l) => l.text)).toEqual(['v1'])
321
+ // Release the still-waiting newest frame so the test does not leak a timer.
322
+ void newest.catch(() => {})
323
+ })
324
+ })