switchroom 0.18.6 → 0.18.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (116) hide show
  1. package/dist/agent-scheduler/index.js +1 -0
  2. package/dist/auth-broker/index.js +1 -0
  3. package/dist/cli/autoaccept-poll.js +140 -33
  4. package/dist/cli/notion-write-pretool.mjs +1 -0
  5. package/dist/cli/switchroom.js +1172 -812
  6. package/dist/host-control/main.js +2 -1
  7. package/dist/vault/approvals/kernel-server.js +1 -0
  8. package/dist/vault/broker/server.js +1 -0
  9. package/package.json +3 -3
  10. package/profiles/_base/cron-session.sh.hbs +55 -16
  11. package/profiles/_base/start.sh.hbs +146 -50
  12. package/profiles/default/CLAUDE.md.hbs +1 -1
  13. package/skills/switchroom-runtime/SKILL.md +2 -0
  14. package/telegram-plugin/dist/bridge/bridge.js +22 -0
  15. package/telegram-plugin/dist/gateway/gateway.js +2965 -862
  16. package/telegram-plugin/dist/server.js +24 -0
  17. package/telegram-plugin/flood-circuit-breaker.ts +123 -0
  18. package/telegram-plugin/gateway/activity-card-store.ts +63 -18
  19. package/telegram-plugin/gateway/always-allow-persist-queue.ts +438 -0
  20. package/telegram-plugin/gateway/approval-timeout-inbound-builders.ts +150 -0
  21. package/telegram-plugin/gateway/boot-card.ts +27 -0
  22. package/telegram-plugin/gateway/busy-ack.ts +106 -0
  23. package/telegram-plugin/gateway/clean-shutdown-marker.ts +68 -20
  24. package/telegram-plugin/gateway/gateway.ts +1618 -198
  25. package/telegram-plugin/gateway/inbound-spool.ts +2 -1
  26. package/telegram-plugin/gateway/inject-handler.test.ts +19 -0
  27. package/telegram-plugin/gateway/inject-handler.ts +17 -0
  28. package/telegram-plugin/gateway/ipc-protocol.ts +44 -2
  29. package/telegram-plugin/gateway/ipc-server.ts +40 -0
  30. package/telegram-plugin/gateway/mental-model-propose-diff.ts +61 -5
  31. package/telegram-plugin/gateway/model-command.ts +227 -54
  32. package/telegram-plugin/gateway/pending-card-expiry.ts +98 -0
  33. package/telegram-plugin/gateway/pending-card-store.ts +173 -0
  34. package/telegram-plugin/gateway/pending-inbound-buffer.ts +12 -2
  35. package/telegram-plugin/gateway/resume-inbound-builder.ts +240 -2
  36. package/telegram-plugin/gateway/session-model-file.ts +198 -0
  37. package/telegram-plugin/gateway/session-model-source.ts +73 -0
  38. package/telegram-plugin/gateway/status-pin-store.ts +82 -22
  39. package/telegram-plugin/gateway/worker-feed-dispatch.ts +24 -1
  40. package/telegram-plugin/gateway/worker-pin-reaper.ts +114 -0
  41. package/telegram-plugin/hooks/hooks.json +10 -10
  42. package/telegram-plugin/hooks/run-hook.sh +84 -0
  43. package/telegram-plugin/hooks/subagent-tracker-pretool.mjs +30 -7
  44. package/telegram-plugin/model-label.ts +69 -0
  45. package/telegram-plugin/model-unavailable.ts +26 -0
  46. package/telegram-plugin/operator-events.ts +24 -0
  47. package/telegram-plugin/permission-diff.ts +128 -0
  48. package/telegram-plugin/pty-partial-handler.ts +39 -0
  49. package/telegram-plugin/registry/subagents-schema.ts +80 -1
  50. package/telegram-plugin/registry/subagents.test.ts +90 -0
  51. package/telegram-plugin/render/rich-render.ts +79 -1
  52. package/telegram-plugin/retry-api-call.ts +62 -0
  53. package/telegram-plugin/session-tail.ts +28 -0
  54. package/telegram-plugin/shared/bot-runtime.ts +8 -1
  55. package/telegram-plugin/silence-poke.ts +14 -0
  56. package/telegram-plugin/silent-end.ts +49 -4
  57. package/telegram-plugin/stream-controller.ts +156 -38
  58. package/telegram-plugin/subagent-watcher.ts +222 -37
  59. package/telegram-plugin/tests/activity-card-store.test.ts +47 -2
  60. package/telegram-plugin/tests/always-allow-persist-queue.test.ts +529 -0
  61. package/telegram-plugin/tests/approval-card-restart-outcome.test.ts +218 -0
  62. package/telegram-plugin/tests/approval-timeout-inbound-builders.test.ts +94 -0
  63. package/telegram-plugin/tests/boot-card-flood-suppress.test.ts +111 -0
  64. package/telegram-plugin/tests/busy-ack-wiring.test.ts +118 -0
  65. package/telegram-plugin/tests/busy-ack.test.ts +121 -0
  66. package/telegram-plugin/tests/button-tap-turn-gated.test.ts +263 -0
  67. package/telegram-plugin/tests/flood-circuit-breaker.test.ts +74 -0
  68. package/telegram-plugin/tests/gateway-clean-shutdown-marker.test.ts +85 -27
  69. package/telegram-plugin/tests/gateway-session-model-relaunch.test.ts +179 -25
  70. package/telegram-plugin/tests/ipc-server-query-pending-permission.test.ts +157 -0
  71. package/telegram-plugin/tests/mental-model-name-entity-corruption.test.ts +119 -0
  72. package/telegram-plugin/tests/mental-model-propose-callback-gate.test.ts +8 -5
  73. package/telegram-plugin/tests/model-command.test.ts +203 -43
  74. package/telegram-plugin/tests/model-label.test.ts +64 -0
  75. package/telegram-plugin/tests/model-unavailable.test.ts +41 -0
  76. package/telegram-plugin/tests/operator-events.test.ts +1 -0
  77. package/telegram-plugin/tests/pending-card-durability-wiring.test.ts +202 -0
  78. package/telegram-plugin/tests/pending-card-expiry.test.ts +190 -0
  79. package/telegram-plugin/tests/pending-card-store.test.ts +173 -0
  80. package/telegram-plugin/tests/permission-diff.test.ts +111 -0
  81. package/telegram-plugin/tests/pty-partial-handler.test.ts +56 -0
  82. package/telegram-plugin/tests/render/render-outbound-chunks.test.ts +98 -0
  83. package/telegram-plugin/tests/resume-inbound-builder.test.ts +286 -0
  84. package/telegram-plugin/tests/retry-api-call.test.ts +59 -0
  85. package/telegram-plugin/tests/run-hook-wrapper.test.ts +132 -0
  86. package/telegram-plugin/tests/session-model-file.test.ts +132 -0
  87. package/telegram-plugin/tests/session-model-source.test.ts +67 -0
  88. package/telegram-plugin/tests/session-tail.test.ts +64 -0
  89. package/telegram-plugin/tests/silent-end.test.ts +46 -1
  90. package/telegram-plugin/tests/slot-banner-boot-recovery.test.ts +3 -3
  91. package/telegram-plugin/tests/status-pin-boot-recovery.test.ts +3 -3
  92. package/telegram-plugin/tests/status-pin-store.test.ts +62 -6
  93. package/telegram-plugin/tests/stream-controller-chunk-cap.test.ts +122 -0
  94. package/telegram-plugin/tests/subagent-tracker-hooks.test.ts +39 -0
  95. package/telegram-plugin/tests/subagent-watcher-boot-promotion-replay.test.ts +107 -4
  96. package/telegram-plugin/tests/subagent-watcher-handback-gaps.test.ts +42 -4
  97. package/telegram-plugin/tests/subagent-watcher-parent-turn-key.test.ts +47 -0
  98. package/telegram-plugin/tests/subagent-watcher-terminated-ids-cap.test.ts +150 -0
  99. package/telegram-plugin/tests/subagent-watcher.test.ts +54 -0
  100. package/telegram-plugin/tests/tool-activity-summary.test.ts +37 -0
  101. package/telegram-plugin/tests/typing-wrap.test.ts +23 -0
  102. package/telegram-plugin/tests/voice-send.test.ts +308 -0
  103. package/telegram-plugin/tests/worker-activity-feed.test.ts +11 -0
  104. package/telegram-plugin/tests/worker-feed-dispatch.test.ts +126 -0
  105. package/telegram-plugin/tests/worker-pin-reaper.test.ts +132 -0
  106. package/telegram-plugin/tool-activity-summary.ts +22 -2
  107. package/telegram-plugin/typing-wrap.ts +72 -25
  108. package/telegram-plugin/uat/scenarios/jtbd-deliberate-restart-resumes-dm.test.ts +118 -0
  109. package/telegram-plugin/uat/scenarios/jtbd-midflight-busy-ack-dm.test.ts +201 -0
  110. package/telegram-plugin/uat/scenarios/jtbd-worker-pin-lifecycle-dm.test.ts +208 -0
  111. package/telegram-plugin/uat/scenarios/vault-card-survives-gateway-restart-dm.test.ts +140 -0
  112. package/telegram-plugin/uat/scenarios/vault-deny-resumes-turn-dm.test.ts +84 -0
  113. package/telegram-plugin/uat/scenarios/vault-timeout-wakes-agent-dm.test.ts +91 -0
  114. package/telegram-plugin/voice-ondemand.ts +25 -1
  115. package/telegram-plugin/voice-send.ts +154 -0
  116. package/telegram-plugin/worker-activity-feed.ts +9 -0
@@ -25,7 +25,18 @@
25
25
 
26
26
  import { parse } from "./parse.js";
27
27
  import { renderSafe, type RenderResult } from "./render.js";
28
- import { RICH_MESSAGE_MAX_CHARS } from "../format.js";
28
+ import { RICH_MESSAGE_MAX_CHARS, splitMarkdownChunks } from "../format.js";
29
+
30
+ /**
31
+ * The legacy plain-text `sendMessage` / `editMessageText` wire cap (4096
32
+ * UTF-16 units). It does NOT apply to the rich path (`sendRichMessage`, up
33
+ * to `RICH_MESSAGE_MAX_CHARS`), but it DOES apply the moment `renderSafe`
34
+ * degrades a body to `mode: "plain"` — the send path then routes it through
35
+ * the plain `sendMessage` endpoint (see stream-controller `doSend`/`doEdit`).
36
+ * A degraded plain body larger than this is rejected by Telegram with
37
+ * `message is too long`, so `renderOutboundChunks` caps plain pieces here.
38
+ */
39
+ export const PLAIN_TEXT_MAX_CHARS = 4096;
29
40
 
30
41
  /** Parse the `SWITCHROOM_RICH_RENDER` flag value. Default OFF; accepts the
31
42
  * same truthy tokens as the other switchroom env flags. Pure so the default
@@ -70,3 +81,70 @@ export function maybeRenderOutbound(
70
81
  if (!richRenderEnabled(env)) return { text, mode: "markdown", degradations: [] };
71
82
  return renderOutbound(text, maxLen);
72
83
  }
84
+
85
+ /**
86
+ * Flag-gated, CAP-ENFORCING transform for the live send path.
87
+ *
88
+ * `maybeRenderOutbound` returns ONE `RenderResult` and can only ever fit a
89
+ * body into a single wire message. But `renderSafe`'s markdown re-escaping
90
+ * (GFM-special chars gain a leading `\`) GROWS a chunk: a raw body sized just
91
+ * under `maxLen` can escape PAST it. `renderSafe` handles that by degrading
92
+ * the WHOLE document to `mode: "plain"` (raw source, no rich wrapper) — it
93
+ * never re-splits a multi-block body. The send path then ships that plain
94
+ * body through the 4096-capped `sendMessage` endpoint, so a ~32k plain body
95
+ * is rejected by Telegram (`message is too long`) and the answer is dropped.
96
+ *
97
+ * This function closes that gap: it returns an ARRAY of pieces, EACH of which
98
+ * is guaranteed to fit its own wire cap —
99
+ * - a `markdown` piece is `<= maxLen` (`renderSafe`'s own guarantee), and
100
+ * - a `plain` piece is `<= PLAIN_TEXT_MAX_CHARS` (the plain endpoint's cap).
101
+ * Pieces are cut only at `splitMarkdownChunks`' safe boundaries, so a fenced
102
+ * code block or a table row is NEVER bisected. Re-splitting the RAW source and
103
+ * re-rendering each piece also RECOVERS rich formatting the whole-document
104
+ * plain degradation would have thrown away: the smaller pieces individually
105
+ * escape under `maxLen` and come back as `markdown`.
106
+ *
107
+ * - flag OFF (default): `[{ text, mode: "markdown", degradations: [] }]` —
108
+ * a single passthrough piece, identical to `maybeRenderOutbound`.
109
+ * - flag ON, body fits: `[renderSafe(...)]` — a single piece, identical to
110
+ * `maybeRenderOutbound` (byte-for-byte for the common case).
111
+ * - flag ON, body oversize: 2+ cap-respecting pieces in send order.
112
+ */
113
+ export function renderOutboundChunks(
114
+ text: string,
115
+ env: NodeJS.ProcessEnv = process.env,
116
+ maxLen: number = RICH_MESSAGE_MAX_CHARS,
117
+ plainMax: number = PLAIN_TEXT_MAX_CHARS,
118
+ ): RenderResult[] {
119
+ if (!richRenderEnabled(env)) {
120
+ return [{ text, mode: "markdown", degradations: [] }];
121
+ }
122
+
123
+ const whole = renderOutbound(text, maxLen);
124
+ // `markdown` mode from renderSafe is already `<= maxLen`; a `plain` result
125
+ // that fits the plain cap is a single deliverable piece too. Common case.
126
+ if (whole.mode === "markdown" && whole.text.length <= maxLen) return [whole];
127
+ if (whole.mode === "plain" && whole.text.length <= plainMax) return [whole];
128
+
129
+ // Oversize: re-split the RAW source at safe boundaries and re-render each
130
+ // piece. Each sub-piece is `<= maxLen` in raw form, so it usually escapes
131
+ // back under `maxLen` and renders as rich markdown; a piece whose escaped
132
+ // form STILL overflows (or an indivisible plain blob) is emitted as plain,
133
+ // further split to `plainMax` so it fits the plain endpoint.
134
+ const out: RenderResult[] = [];
135
+ for (const rawPiece of splitMarkdownChunks(text, maxLen)) {
136
+ const rendered = renderOutbound(rawPiece, maxLen);
137
+ if (rendered.mode === "markdown" && rendered.text.length <= maxLen) {
138
+ out.push(rendered);
139
+ continue;
140
+ }
141
+ // Plain piece — cap at the plain-endpoint limit. `renderSafe` returned the
142
+ // raw source for a plain result, so split that source (safe boundaries,
143
+ // never bisecting a fence/table) into `<= plainMax` slices.
144
+ for (const plainPiece of splitMarkdownChunks(rendered.text, plainMax)) {
145
+ out.push({ text: plainPiece, mode: "plain", degradations: rendered.degradations });
146
+ }
147
+ }
148
+ // Defensive: a degenerate input that split to nothing still yields one piece.
149
+ return out.length > 0 ? out : [whole];
150
+ }
@@ -63,8 +63,48 @@ export interface RetryApiCallConfig {
63
63
  observer?: RetryObserver
64
64
  /** Optional log sink for flood-wait / network lines. */
65
65
  log?: (line: string) => void
66
+ /**
67
+ * Fires whenever a Telegram 429 flood-wait is observed, BEFORE the sleep.
68
+ * The circuit-breaker (#2923) persists the flood-wait window here so that
69
+ * a container restart during an active ban can suppress non-essential
70
+ * sends (boot cards) instead of feeding the same per-bot-token flood
71
+ * counter and prolonging the ban. Best-effort; a throw here is swallowed.
72
+ */
73
+ onFloodWait?: (retryAfterSec: number) => void
74
+ }
75
+
76
+ /**
77
+ * True when the thrown error is a LOCAL resource-exhaustion failure —
78
+ * ENOSPC (disk/tmpfs full), EDQUOT (quota), EIO, or ENOMEM — rather than a
79
+ * remote Telegram API failure. Issue #2923: an agent's tmpfs filling up
80
+ * wedges the outbound send's local staging step; retrying that as if it
81
+ * were a transient REMOTE failure hammers the Bot API and trips a per-bot
82
+ * flood ban. A local disk error must NOT drive remote retries — surface it
83
+ * as a distinct, non-retryable degraded state instead.
84
+ */
85
+ export function isLocalResourceError(err: unknown): boolean {
86
+ const code = (err as { code?: unknown })?.code
87
+ if (typeof code === 'string' && ['ENOSPC', 'EDQUOT', 'EIO', 'ENOMEM'].includes(code)) {
88
+ return true
89
+ }
90
+ const msg = err instanceof Error ? err.message : String(err ?? '')
91
+ return (
92
+ // Word-boundaried so a substring can't false-match; covers the same set
93
+ // as the errno code list above (ENOSPC/EDQUOT/EIO/ENOMEM).
94
+ /\b(ENOSPC|EDQUOT|EIO|ENOMEM)\b/.test(msg) ||
95
+ /no space left on device/i.test(msg) ||
96
+ /disk quota exceeded/i.test(msg)
97
+ )
66
98
  }
67
99
 
100
+ /**
101
+ * Marker error thrown when `retryApiCall` refuses to retry a LOCAL
102
+ * resource-exhaustion failure (#2923). Callers can detect this to surface a
103
+ * "degraded: local disk full" state rather than treating it like a remote
104
+ * send failure worth retrying.
105
+ */
106
+ export const LOCAL_RESOURCE_EXHAUSTED = 'LOCAL_RESOURCE_EXHAUSTED'
107
+
68
108
  const DEFAULT_SLEEP = (ms: number) => new Promise<void>((r) => setTimeout(r, ms))
69
109
 
70
110
  /**
@@ -83,6 +123,7 @@ export function createRetryApiCall(
83
123
  const sleep = config.sleep ?? DEFAULT_SLEEP
84
124
  const observer = config.observer
85
125
  const log = config.log
126
+ const onFloodWait = config.onFloodWait
86
127
 
87
128
  return async function retryApiCall<T>(
88
129
  fn: () => Promise<T>,
@@ -96,12 +137,33 @@ export function createRetryApiCall(
96
137
  const msg = err instanceof Error ? err.message : String(err)
97
138
  const desc = isGrammyErr ? (err as GrammyError).description : msg
98
139
 
140
+ // LOCAL resource exhaustion (#2923) — ENOSPC/EDQUOT/EIO/ENOMEM. This
141
+ // is a LOCAL disk/memory failure, not a remote API failure: retrying
142
+ // it in a tight loop is exactly what tripped the per-bot flood ban.
143
+ // Do NOT retry. Throw a distinct, non-retryable marker so the caller
144
+ // surfaces a degraded "local disk full" state and backs off hard.
145
+ if (isLocalResourceError(err)) {
146
+ log?.(
147
+ `telegram gateway: LOCAL resource exhaustion (${(err as { code?: string }).code ?? 'disk/mem'}) — ` +
148
+ `not retrying the send (would feed a flood ban); surfacing degraded state\n`,
149
+ )
150
+ observer?.onGiveUp?.({ attempts: attempt + 1, error: err })
151
+ throw Object.assign(new Error(LOCAL_RESOURCE_EXHAUSTED), { original: err })
152
+ }
153
+
99
154
  // Flood-wait — sleep retry_after and try again.
100
155
  if (isGrammyErr && (err as GrammyError).error_code === 429) {
101
156
  const retryAfter = Number(
102
157
  (err as GrammyError).parameters?.retry_after ?? 5,
103
158
  )
104
159
  const delayMs = retryAfter * 1000
160
+ // Persist the flood window so a restart during the ban can suppress
161
+ // non-essential sends instead of extending it (#2923 circuit breaker).
162
+ try {
163
+ onFloodWait?.(retryAfter)
164
+ } catch {
165
+ /* best-effort — never let the breaker hook break the retry path */
166
+ }
105
167
  log?.(`telegram gateway: 429 rate limited, waiting ${retryAfter}s\n`)
106
168
  observer?.onRetry?.({ attempt, reason: 'flood_wait', delayMs })
107
169
  await sleep(delayMs)
@@ -42,6 +42,7 @@ function isMultiAgentEnabled(env: NodeJS.ProcessEnv = process.env): boolean {
42
42
  }
43
43
  import { classifyClaudeError, type OperatorEventKind } from './operator-events.js'
44
44
  import { createToolLabelSidecar, type ToolLabelSidecar } from './tool-label-sidecar.js'
45
+ import { isModelSentinel } from './model-label.js'
45
46
 
46
47
  /** Match Claude Code's cli.js VX() function. */
47
48
  export function sanitizeCwdToProjectName(cwd: string): string {
@@ -92,6 +93,13 @@ export type SessionEvent =
92
93
  | { kind: 'enqueue'; chatId: string | null; messageId: string | null; threadId: string | null; rawContent: string; isSync?: boolean }
93
94
  | { kind: 'dequeue' }
94
95
  | { kind: 'thinking' }
96
+ // Live model in use for the MAIN session, extracted from `message.model` on
97
+ // each `type:"assistant"` transcript line (the exact model that served that
98
+ // API call). Emitted FIRST among a line's events so a card rendered on the
99
+ // same batch already reflects the current model. Sentinels (`<synthetic>` on
100
+ // compaction lines, fixture junk) are filtered at projection — see
101
+ // isModelSentinel — so this only ever carries a real resolved model id.
102
+ | { kind: 'model'; model: string }
95
103
  | { kind: 'tool_use'; toolName: string; toolUseId?: string | null; input?: Record<string, unknown>; precomputedLabel?: string }
96
104
  // Real-time tool label from the PreToolUse-hook sidecar — fires when the
97
105
  // hook writes the label (synchronous at tool-call time), independent of
@@ -115,6 +123,11 @@ export type SessionEvent =
115
123
  // filename stem (e.g. "aac6f1…"). Routed through the same ingest path
116
124
  // as parent events; the reducer fans them out to per-sub-agent state.
117
125
  | { kind: 'sub_agent_started'; agentId: string; firstPromptText: string; subagentType?: string }
126
+ // Live model in use for a SUB-AGENT, extracted from `message.model` on each
127
+ // of its `type:"assistant"` transcript lines. Same contract as the main
128
+ // `model` kind (sentinel-filtered, emitted first) but agent-scoped so the
129
+ // watcher can track it per WorkerEntry and thread it onto the worker card.
130
+ | { kind: 'sub_agent_model'; agentId: string; model: string }
118
131
  | { kind: 'sub_agent_tool_use'; agentId: string; toolUseId: string | null; toolName: string; input?: Record<string, unknown>; precomputedLabel?: string }
119
132
  // Same shared contract as the main-agent `text` kind — see its doc above
120
133
  // (including the `lastInMessage` projection-artifact note). The wire-kind
@@ -320,6 +333,15 @@ export function projectTranscriptLine(line: string): SessionEvent[] {
320
333
  const content = message?.content as Array<Record<string, unknown>> | undefined
321
334
  if (!Array.isArray(content)) return []
322
335
  const events: SessionEvent[] = []
336
+ // Live model capture: `message.model` is the exact resolved model that
337
+ // served THIS assistant API call. Emit it FIRST (before the content events)
338
+ // so a card rendered on the same read batch already carries the current
339
+ // model. Sentinels (`<synthetic>` on compaction lines, fixture junk) are
340
+ // skipped — the reducer keeps the last real value.
341
+ const mainModel = message?.model
342
+ if (typeof mainModel === 'string' && !isModelSentinel(mainModel)) {
343
+ events.push({ kind: 'model', model: mainModel })
344
+ }
323
345
  // Text→narrative projection comes from the ONE shared kernel
324
346
  // (projectAssistantTextBlocks): it owns the empty-drop + blockIndex +
325
347
  // lastInMessage contract. We emit its events at their source positions
@@ -468,6 +490,12 @@ export function projectSubagentLine(
468
490
  const content = message?.content as Array<Record<string, unknown>> | undefined
469
491
  if (!Array.isArray(content)) return []
470
492
  const events: SessionEvent[] = []
493
+ // Live model capture (sub-agent tier): same contract as the main-agent
494
+ // branch — emit the sub-agent's resolved model first, sentinel-filtered.
495
+ const subModel = message?.model
496
+ if (typeof subModel === 'string' && !isModelSentinel(subModel)) {
497
+ events.push({ kind: 'sub_agent_model', agentId, model: subModel })
498
+ }
471
499
  // Text→narrative projection comes from the SAME shared kernel as the
472
500
  // main agent (projectAssistantTextBlocks): one source for the empty-drop
473
501
  // + blockIndex + lastInMessage contract. The `make` adapter only changes
@@ -29,6 +29,7 @@ import { createHash } from 'crypto'
29
29
  import { AsyncLocalStorage } from 'async_hooks'
30
30
  import { clearStaleTelegramPollingState } from '../startup-reset.js'
31
31
  import { createRetryApiCall } from '../retry-api-call.js'
32
+ import { makeFloodWaitRecorder } from '../flood-circuit-breaker.js'
32
33
  import { RICH_MESSAGE_MAX_CHARS } from '../format.js'
33
34
 
34
35
  // ─── tg-post tag plumbing ─────────────────────────────────────────────────
@@ -149,9 +150,15 @@ export function installTgPostLogger(bot: Bot): void {
149
150
  * Usage:
150
151
  * const robustApiCall = createRobustApiCall()
151
152
  */
152
- export function createRobustApiCall() {
153
+ export function createRobustApiCall(opts: { floodStatePath?: string } = {}) {
153
154
  return createRetryApiCall({
154
155
  log: (line) => process.stderr.write(line),
156
+ // #2923: persist every observed 429 flood-wait window so a restart during
157
+ // the ban can suppress non-essential sends (boot cards) instead of feeding
158
+ // the per-bot flood counter and prolonging the ban.
159
+ ...(opts.floodStatePath
160
+ ? { onFloodWait: makeFloodWaitRecorder(opts.floodStatePath) }
161
+ : {}),
155
162
  })
156
163
  }
157
164
 
@@ -423,6 +423,20 @@ export function formatFrameworkFallbackText(
423
423
  return null
424
424
  }
425
425
 
426
+ /**
427
+ * #2995 — the LONGEST-running in-flight tool for a turn key, or null when
428
+ * none is tracked. Read-only accessor over `inFlightTools` for the
429
+ * mid-flight busy-ack: the gateway needs the blocking step's name/label
430
+ * and its age to decide whether (and how) to ack a buffered mid-turn
431
+ * inbound. Does not touch the silence clock.
432
+ */
433
+ export function longestInFlightTool(key: string, now: number): ToolSnapshot | null {
434
+ const s = state.get(key)
435
+ if (s == null) return null
436
+ const snaps = snapshotInFlight(s, now)
437
+ return snaps.length > 0 ? snaps[0]! : null
438
+ }
439
+
426
440
  /** Snapshot in-flight tools sorted longest-running first — for the honest
427
441
  * floor/fallback message body. */
428
442
  function snapshotInFlight(s: SilencePokeState, now: number): ToolSnapshot[] {
@@ -60,6 +60,8 @@ export interface SilentEndDeps {
60
60
  * never suppress on doubt.
61
61
  */
62
62
  hasOutboundDeliveredSince?: (chatId: string, sinceMs: number, threadId?: number | null) => boolean
63
+ /** Wall-clock now, ms. Defaults to `Date.now`; injectable for tests. */
64
+ now?: () => number
63
65
  }
64
66
 
65
67
  /**
@@ -80,6 +82,31 @@ export interface SilentEndDeps {
80
82
  */
81
83
  export const SILENT_END_MAX_RETRIES = 2
82
84
 
85
+ /**
86
+ * Fix #8 (adversarial-validation progress-card fix pass): `turnKey` is the
87
+ * STABLE `statusKey(chatId, threadId)` — it never changes across turns on
88
+ * the same chat/thread — so `writeSilentEndState`'s "inherit retryCount IFF
89
+ * turnKey matches" rule has no per-turn discriminator on its own. Normally
90
+ * that's fine because `clearSilentEndState` always fires the moment a reply
91
+ * lands, wiping the exhausted record before the next turn starts. But if the
92
+ * gateway's own exhaust-read-and-clear is bypassed (crash/interrupt mid-turn,
93
+ * `gateway.ts` ~turn_end handler never runs), an exhausted
94
+ * (`retryCount >= SILENT_END_MAX_RETRIES`) record can survive into a LATER,
95
+ * unrelated turn. That later turn's first silent-end would then misread the
96
+ * old record as "this turn already exhausted its ladder" and skip its own
97
+ * re-prompt entirely.
98
+ *
99
+ * A record legitimately re-written within one turn's own retry ladder is
100
+ * always fresh (each Stop-hook block re-writes `timestamp`), so bounding by
101
+ * age is a safe, cheap staleness signal that doesn't require a new identity
102
+ * field: any record older than a turn's plausible lifetime could not still
103
+ * be "this turn's" in-flight retry state, and is therefore stale-carryover,
104
+ * not an active ladder. 30 minutes comfortably exceeds any real single-turn
105
+ * Stop-hook retry cycle (seconds, not minutes) while staying well inside the
106
+ * fleet's other staleness bounds (e.g. the 3h boot-resume window).
107
+ */
108
+ export const SILENT_END_STALE_RECORD_MAX_AGE_MS = 30 * 60_000
109
+
83
110
  /**
84
111
  * User-facing fallback text delivered when a user-message turn ends with no
85
112
  * final answer AND the deterministic Stop-hook re-prompt has already been
@@ -313,12 +340,30 @@ export function recordSilentTurnEnd(
313
340
  // If so, the record is satisfied-but-misdetected — drop it silently
314
341
  // and let this dark turn start its OWN fresh retry cycle instead of
315
342
  // inheriting someone else's spent budget.
316
- if (deps?.hasOutboundDeliveredSince?.(args.chatId, prev.timestamp, args.threadId)) {
343
+ // Fix #8: age-based staleness bound. `turnKey` has no per-turn nonce
344
+ // (see SILENT_END_STALE_RECORD_MAX_AGE_MS doc), so a record whose
345
+ // `timestamp` is older than a turn's plausible lifetime cannot belong to
346
+ // THIS dark turn's own retry ladder even absent any delivery evidence —
347
+ // it's carryover from a prior turn whose gateway-side clear was
348
+ // bypassed (crash/interrupt). Check this before (and independent of)
349
+ // the delivery-based check below so a bypassed-clear record is caught
350
+ // even when no reply was ever delivered on the chat/thread at all.
351
+ const now = deps?.now?.() ?? Date.now()
352
+ const staleByAge = now - prev.timestamp > SILENT_END_STALE_RECORD_MAX_AGE_MS
353
+ if (
354
+ staleByAge ||
355
+ deps?.hasOutboundDeliveredSince?.(args.chatId, prev.timestamp, args.threadId)
356
+ ) {
317
357
  emitLog(
318
358
  deps,
319
- `silent-end: stale exhausted record for turnKey=${args.turnKey} ` +
320
- `(retryCount=${prev.retryCount}) but a reply was delivered since ` +
321
- `${prev.timestamp} treating as satisfied-but-misdetected, not exhausted\n`,
359
+ staleByAge
360
+ ? `silent-end: stale exhausted record for turnKey=${args.turnKey} ` +
361
+ `(retryCount=${prev.retryCount}, age=${now - prev.timestamp}ms > ` +
362
+ `${SILENT_END_STALE_RECORD_MAX_AGE_MS}ms) — treating as carryover ` +
363
+ `from a prior turn, not exhausted\n`
364
+ : `silent-end: stale exhausted record for turnKey=${args.turnKey} ` +
365
+ `(retryCount=${prev.retryCount}) but a reply was delivered since ` +
366
+ `${prev.timestamp} — treating as satisfied-but-misdetected, not exhausted\n`,
322
367
  )
323
368
  // MUST clear before writing (adversarial review of #2892):
324
369
  // writeSilentEndState re-inherits retryCount whenever the on-disk
@@ -21,7 +21,7 @@
21
21
 
22
22
  import { createDraftStream, type DraftStreamHandle } from './draft-stream.js'
23
23
  import { richMessage, isParseEntitiesError } from './rich-send.js'
24
- import { maybeRenderOutbound } from './render/rich-render.js'
24
+ import { renderOutboundChunks } from './render/rich-render.js'
25
25
 
26
26
  /**
27
27
  * Minimal bot.api surface the controller needs. Real callers pass grammy's
@@ -223,81 +223,199 @@ export function createStreamController(cfg: StreamControllerConfig): DraftStream
223
223
  // `format:'text'` stream (plain sendMessage, no rich wrapper).
224
224
  // sendRichMessage does NOT accept link_preview_options (rich messages
225
225
  // control previews via entity detection), so strip it for the rich path.
226
- const doSend = (text: string, opts: StreamSendOpts) => {
227
- if (literalText) return bot.api.sendMessage(chatId, text, opts)
228
- // Flag-gated rich render (`SWITCHROOM_RICH_RENDER`, default OFF returns
229
- // the text untouched as markdown when off, so this is a no-op for every
230
- // agent until an operator opts in). A `plain` result (oversized/unsafe
231
- // content renderSafe declined to emit as rich) sends WITHOUT the wrapper.
232
- const rendered = maybeRenderOutbound(text)
233
- if (rendered.mode === 'plain') return bot.api.sendMessage(chatId, rendered.text, opts)
226
+ // Render the outbound body into 1+ wire-cap-respecting pieces. The common
227
+ // case is a SINGLE piece whose output is identical to the pre-existing
228
+ // `maybeRenderOutbound` path; a body whose markdown-escaping pushed the
229
+ // rendered form past the cap splits into several pieces, each of which fits
230
+ // its own wire cap and never bisects a fenced block / table row (see
231
+ // `renderOutboundChunks`). Flag OFF (default) and a literal `format:'text'`
232
+ // stream both yield a single passthrough piece — no behavioural change.
233
+ const renderPieces = (text: string): { text: string; rich: boolean }[] => {
234
+ if (literalText) return [{ text, rich: false }]
235
+ // A `plain`-mode piece (oversized/unsafe content renderSafe declined to
236
+ // emit as rich) sends WITHOUT the rich wrapper.
237
+ return renderOutboundChunks(text).map((r) => ({ text: r.text, rich: r.mode !== 'plain' }))
238
+ }
239
+ // Send ONE rendered piece. Rich pieces go through sendRichMessage (with
240
+ // link_preview_options stripped — rich messages control previews via entity
241
+ // detection); plain pieces (and literal streams) through sendMessage.
242
+ const sendPiece = (piece: { text: string; rich: boolean }, opts: StreamSendOpts) => {
243
+ if (!piece.rich) return bot.api.sendMessage(chatId, piece.text, opts)
234
244
  const richOpts = { ...opts }
235
245
  delete richOpts.link_preview_options
236
- return bot.api.sendRichMessage(chatId, richMessage(rendered.text), richOpts)
246
+ return bot.api.sendRichMessage(chatId, richMessage(piece.text), richOpts)
237
247
  }
238
- const doEdit = (id: number, text: string, opts: StreamSendOpts) => {
239
- if (literalText) return bot.api.editMessageText(chatId, id, text, opts)
240
- const rendered = maybeRenderOutbound(text)
241
- if (rendered.mode === 'plain') return bot.api.editMessageText(chatId, id, rendered.text, opts)
242
- return bot.api.editMessageText(chatId, id, richMessage(rendered.text), opts)
248
+ const editPiece = (id: number, piece: { text: string; rich: boolean }, opts: StreamSendOpts) => {
249
+ if (!piece.rich) return bot.api.editMessageText(chatId, id, piece.text, opts)
250
+ return bot.api.editMessageText(chatId, id, richMessage(piece.text), opts)
251
+ }
252
+
253
+ // Overflow-tail bookkeeping, shared across the send + edit closures for the
254
+ // whole stream lifetime. A body large enough to split into several
255
+ // wire-cap pieces anchors on piece[0] (edited in place by draft-stream) and
256
+ // parks pieces[1..n] as follow-up messages. The draft-stream edit callback
257
+ // fires on EVERY throttled flush as a streamed answer grows, so we MUST NOT
258
+ // re-send those tails each tick (that flooded the chat with duplicates — the
259
+ // blocker this fix closes). Instead we remember each tail's message_id the
260
+ // first time it is emitted and edit it in place on later flushes; a tail that
261
+ // did not exist on a prior flush (the piece count grew) is sent fresh once.
262
+ // End state after finalize: anchor + one message per tail piece, no dupes.
263
+ const tailIds: number[] = []
264
+ const tailLastText: string[] = []
265
+
266
+ // Emit or update a single tail piece (0-based index `ti` = piece index - 1).
267
+ // Sends a fresh message the first time; edits in place (skipping unchanged
268
+ // text) thereafter. A non-parse failure is logged as a partial-delivery
269
+ // warning and swallowed so the remaining tail pieces still get a chance to
270
+ // land — never a silent drop, never an abort of pieces K..N (concern C1).
271
+ const upsertTail = async (ti: number, piece: { text: string; rich: boolean }): Promise<void> => {
272
+ const existingId = tailIds[ti]
273
+ if (existingId != null) {
274
+ if (tailLastText[ti] === piece.text) return // unchanged — skip the API call
275
+ try {
276
+ await retry(() => editPiece(existingId, piece, baseOpts), { threadId, chat_id: chatId })
277
+ tailLastText[ti] = piece.text
278
+ onEdit?.(existingId, piece.text.length)
279
+ } catch (err) {
280
+ if (!literalText && piece.rich && isParseEntitiesError(err)) {
281
+ warn?.(
282
+ `stream-controller: tail-piece #${ti + 1} edit parse-entities rejected — retrying same id=${existingId} as plain text (${err instanceof Error ? err.message : String(err)})`,
283
+ )
284
+ await retry(
285
+ () => bot.api.editMessageText(chatId, existingId, piece.text, baseOpts),
286
+ { threadId, chat_id: chatId },
287
+ )
288
+ tailLastText[ti] = piece.text
289
+ onEdit?.(existingId, piece.text.length)
290
+ } else {
291
+ // Best-effort continue: leave tailLastText[ti] stale so the next
292
+ // flush retries this piece, and surface the partial delivery loudly.
293
+ warn?.(
294
+ `stream-controller: tail-piece #${ti + 1} edit FAILED (id=${existingId}) — partial delivery, this piece may be stale (${err instanceof Error ? err.message : String(err)})`,
295
+ )
296
+ }
297
+ }
298
+ return
299
+ }
300
+ // First emission of this tail piece → a fresh follow-up message.
301
+ try {
302
+ const sent = await retry(() => sendPiece(piece, sendOpts), { threadId, chat_id: chatId })
303
+ tailIds[ti] = sent.message_id
304
+ tailLastText[ti] = piece.text
305
+ onSend?.(sent.message_id, piece.text.length)
306
+ } catch (err) {
307
+ if (!literalText && piece.rich && isParseEntitiesError(err)) {
308
+ warn?.(
309
+ `stream-controller: tail-piece #${ti + 1} send parse-entities rejected — sending as plain text (${err instanceof Error ? err.message : String(err)})`,
310
+ )
311
+ const sent = await retry(
312
+ () => bot.api.sendMessage(chatId, piece.text, sendOpts),
313
+ { threadId, chat_id: chatId },
314
+ )
315
+ tailIds[ti] = sent.message_id
316
+ tailLastText[ti] = piece.text
317
+ onSend?.(sent.message_id, piece.text.length)
318
+ } else {
319
+ // Best-effort continue: no id recorded, so the next flush re-attempts
320
+ // this piece rather than silently dropping pieces K..N (concern C1).
321
+ warn?.(
322
+ `stream-controller: tail-piece #${ti + 1} send FAILED — partial delivery, this and later pieces may be missing this flush (${err instanceof Error ? err.message : String(err)})`,
323
+ )
324
+ }
325
+ }
243
326
  }
244
327
 
245
328
  return createDraftStream(
246
329
  async (text) => {
330
+ // Render → 1+ cap-respecting pieces. The FIRST piece's message_id anchors
331
+ // the stream (later edits target it); any overflow pieces are parked as
332
+ // follow-up messages via upsertTail. For the common single-piece case
333
+ // this is exactly one send.
334
+ const pieces = renderPieces(text)
335
+ const head = pieces[0]
336
+ let anchorId: number | undefined
247
337
  try {
248
338
  const sent = await retry(
249
- () => doSend(text, sendOpts),
339
+ () => sendPiece(head, sendOpts),
250
340
  { threadId, chat_id: chatId },
251
341
  )
252
- onSend?.(sent.message_id, text.length)
253
- return sent.message_id
342
+ anchorId = sent.message_id
254
343
  } catch (err) {
255
- if (!literalText && isParseEntitiesError(err)) {
256
- // First send rejected because the markdown couldn't be parsed.
257
- // There is no message_id to edit (the send 400'd before any
258
- // message was created), so a single fresh send as PLAIN text
259
- // (no rich wrapper, so the parser never runs) is the correct
260
- // recovery — see issue #657. The raw markdown source is itself
261
- // readable, so we send it verbatim.
344
+ if (!literalText && head.rich && isParseEntitiesError(err)) {
345
+ // Piece rejected because its markdown couldn't be parsed. There is
346
+ // no message_id to edit (the send 400'd before any message was
347
+ // created), so recover with a single fresh PLAIN send of the same
348
+ // body (no rich wrapper, so the parser never runs) see issue #657.
262
349
  warn?.(
263
350
  `stream-controller: send parse-entities rejected — retrying once as plain text (${err instanceof Error ? err.message : String(err)})`,
264
351
  )
352
+ // Resend the RAW source verbatim (readable, and the exact
353
+ // pre-existing #657 contract) rather than the escaped/rendered
354
+ // body. For a single-piece stream (the common case) this is the
355
+ // whole message; for a rare oversize split the head falls back to
356
+ // its own body and each tail to its own body below.
357
+ const fallbackBody = pieces.length === 1 ? text : head.text
265
358
  const sent = await retry(
266
- () => bot.api.sendMessage(chatId, text, sendOpts),
359
+ () => bot.api.sendMessage(chatId, fallbackBody, sendOpts),
267
360
  { threadId, chat_id: chatId },
268
361
  )
269
- onSend?.(sent.message_id, text.length)
270
- return sent.message_id
362
+ anchorId = sent.message_id
363
+ } else {
364
+ throw err
271
365
  }
272
- throw err
273
366
  }
367
+ // C2: report the ACTUAL emitted anchor-piece length, not the full body
368
+ // length — the anchor only holds the head piece, and the tails report
369
+ // their own lengths via upsertTail.
370
+ onSend?.(anchorId as number, head.text.length)
371
+ // Park overflow pieces (first-time send records their ids for reuse).
372
+ for (let pi = 1; pi < pieces.length; pi++) {
373
+ await upsertTail(pi - 1, pieces[pi])
374
+ }
375
+ return anchorId as number
274
376
  },
275
377
  async (id, text) => {
378
+ const pieces = renderPieces(text)
379
+ const head = pieces[0]
380
+ // Edit the anchor message in place with the FIRST piece.
276
381
  try {
277
382
  await retry(
278
- () => doEdit(id, text, baseOpts),
383
+ () => editPiece(id, head, baseOpts),
279
384
  { threadId, chat_id: chatId },
280
385
  )
281
- onEdit?.(id, text.length)
386
+ // C2: report the actual head-piece length, not the full body length.
387
+ onEdit?.(id, head.text.length)
282
388
  } catch (err) {
283
- if (!literalText && isParseEntitiesError(err)) {
389
+ if (!literalText && head.rich && isParseEntitiesError(err)) {
284
390
  // Edit rejected because the markdown couldn't be parsed — DO NOT
285
391
  // send a fresh message. The whole point of issue #657 is that the
286
392
  // previous implementation sent a duplicate message every time a
287
393
  // parse rejection fired. Retry the edit on the SAME message_id as
288
- // PLAIN text (no rich wrapper, so the parser never runs). The raw
289
- // markdown source is itself readable, so we send it verbatim.
394
+ // PLAIN text (no rich wrapper, so the parser never runs).
290
395
  warn?.(
291
396
  `stream-controller: edit parse-entities rejected — retrying same id=${id} as plain text (${err instanceof Error ? err.message : String(err)})`,
292
397
  )
398
+ // Re-edit the SAME id with the RAW source verbatim (the exact #657
399
+ // contract). For a single-piece stream (common case) this is the
400
+ // whole body; a rare oversize split edits the head piece's body.
401
+ const fallbackBody = pieces.length === 1 ? text : head.text
293
402
  await retry(
294
- () => bot.api.editMessageText(chatId, id, text, baseOpts),
403
+ () => bot.api.editMessageText(chatId, id, fallbackBody, baseOpts),
295
404
  { threadId, chat_id: chatId },
296
405
  )
297
- onEdit?.(id, text.length)
298
- return
406
+ onEdit?.(id, head.text.length)
407
+ } else {
408
+ throw err
299
409
  }
300
- throw err
410
+ }
411
+ // Oversize tail: the anchor message holds only the first piece. On EVERY
412
+ // edit flush we UPDATE the parked tail messages in place (or send a tail
413
+ // that only just came into existence) — we never re-send tails already
414
+ // emitted on a prior flush. This is the fix for the duplicate-flood
415
+ // blocker: the previous code re-sent pieces[1..n] as brand-new messages
416
+ // on each throttled edit tick.
417
+ for (let pi = 1; pi < pieces.length; pi++) {
418
+ await upsertTail(pi - 1, pieces[pi])
301
419
  }
302
420
  },
303
421
  {