switchroom 0.19.41 → 0.19.43

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (90) hide show
  1. package/dist/agent-scheduler/index.js +23 -3
  2. package/dist/auth-broker/index.js +54 -9
  3. package/dist/cli/notion-write-pretool.mjs +23 -3
  4. package/dist/cli/switchroom.js +1956 -900
  5. package/dist/host-control/main.js +227 -30
  6. package/dist/vault/approvals/kernel-server.js +54 -9
  7. package/dist/vault/broker/server.js +54 -9
  8. package/package.json +3 -3
  9. package/profiles/_base/start.sh.hbs +82 -4
  10. package/telegram-plugin/bridge/bridge.ts +1 -1
  11. package/telegram-plugin/card-layout.ts +63 -1
  12. package/telegram-plugin/dist/bridge/bridge.js +4 -1
  13. package/telegram-plugin/dist/gateway/gateway.js +1318 -693
  14. package/telegram-plugin/dist/server.js +5 -2
  15. package/telegram-plugin/flood-429-ledger.ts +5 -3
  16. package/telegram-plugin/flood-circuit-breaker.ts +11 -3
  17. package/telegram-plugin/format.ts +223 -13
  18. package/telegram-plugin/gateway/approval-callback-consume-record.test.ts +132 -0
  19. package/telegram-plugin/gateway/boot-card.ts +7 -2
  20. package/telegram-plugin/gateway/flood-reply-queue.ts +5 -0
  21. package/telegram-plugin/gateway/gateway.ts +56 -58
  22. package/telegram-plugin/gateway/handback-orphan-recovery.ts +69 -0
  23. package/telegram-plugin/gateway/handback-preturn-signal.ts +170 -32
  24. package/telegram-plugin/gateway/ipc-protocol.ts +31 -3
  25. package/telegram-plugin/gateway/ipc-server.ts +13 -4
  26. package/telegram-plugin/gateway/liveness-wiring.ts +12 -0
  27. package/telegram-plugin/gateway/model-command.ts +21 -109
  28. package/telegram-plugin/gateway/obligation-store.ts +55 -0
  29. package/telegram-plugin/gateway/outbound-send-path.ts +127 -11
  30. package/telegram-plugin/gateway/outbox-sweep.ts +371 -61
  31. package/telegram-plugin/gateway/rollout-status-edit.ts +175 -0
  32. package/telegram-plugin/gateway/stream-render.ts +62 -0
  33. package/telegram-plugin/gateway/update-announce.ts +11 -4
  34. package/telegram-plugin/hooks/audience-classify.d.mts +50 -0
  35. package/telegram-plugin/hooks/audience-classify.mjs +364 -0
  36. package/telegram-plugin/hooks/silent-end-interrupt-stop.mjs +86 -3
  37. package/telegram-plugin/hooks/silent-end-scan.mjs +80 -4
  38. package/telegram-plugin/outbox.ts +103 -2
  39. package/telegram-plugin/render/rich-render.ts +10 -2
  40. package/telegram-plugin/reply-quote.ts +150 -0
  41. package/telegram-plugin/retry-api-call.ts +254 -11
  42. package/telegram-plugin/shared/bot-runtime.ts +69 -34
  43. package/telegram-plugin/shared/gw-trace-gate.ts +14 -2
  44. package/telegram-plugin/silent-end.ts +34 -2
  45. package/telegram-plugin/status-no-truncate.ts +31 -16
  46. package/telegram-plugin/stream-controller.ts +44 -9
  47. package/telegram-plugin/stream-reply-handler.ts +5 -3
  48. package/telegram-plugin/tests/boot-card-render.test.ts +2 -1
  49. package/telegram-plugin/tests/card-budget-never-overflows.test.ts +165 -0
  50. package/telegram-plugin/tests/card-hard-truncate-entity-safe.test.ts +153 -0
  51. package/telegram-plugin/tests/card-variants.golden.txt +9 -9
  52. package/telegram-plugin/tests/flood-reply-queue.test.ts +89 -4
  53. package/telegram-plugin/tests/format-consistency.test.ts +160 -14
  54. package/telegram-plugin/tests/framework-fallback-drains-parked.test.ts +134 -0
  55. package/telegram-plugin/tests/handback-orphan-recovery.test.ts +131 -0
  56. package/telegram-plugin/tests/handback-preturn-signal.test.ts +212 -17
  57. package/telegram-plugin/tests/helpers/liveness-wiring-fixture.ts +3 -0
  58. package/telegram-plugin/tests/nested-indent-idiom.test.ts +162 -0
  59. package/telegram-plugin/tests/nested-prefix-indent.test.ts +115 -0
  60. package/telegram-plugin/tests/no-robust-api-call-factory.test.ts +62 -0
  61. package/telegram-plugin/tests/obligation-store.test.ts +51 -0
  62. package/telegram-plugin/tests/outbox-audience-3865.test.ts +579 -0
  63. package/telegram-plugin/tests/outbox-provenance-4141.test.ts +1126 -0
  64. package/telegram-plugin/tests/outbox-sweep-single-flight.test.ts +138 -0
  65. package/telegram-plugin/tests/pinned-card-collapse.test.ts +12 -10
  66. package/telegram-plugin/tests/plain-fallback-4096-cap.test.ts +509 -0
  67. package/telegram-plugin/tests/reply-quote-wire.test.ts +433 -0
  68. package/telegram-plugin/tests/retry-grammy-http-error.test.ts +404 -0
  69. package/telegram-plugin/tests/rollout-narration-edit-socket.test.ts +170 -0
  70. package/telegram-plugin/tests/rollout-status-edit-retry-policy.test.ts +366 -0
  71. package/telegram-plugin/tests/rollout-status-edit.test.ts +140 -0
  72. package/telegram-plugin/tests/rollout-status-wiring.test.ts +40 -7
  73. package/telegram-plugin/tests/tg-post-logger-error-shape.test.ts +4 -0
  74. package/telegram-plugin/tests/tg-post-retry-attribution.test.ts +196 -0
  75. package/telegram-plugin/tests/tool-activity-summary.test.ts +23 -17
  76. package/telegram-plugin/tool-activity-summary.ts +58 -30
  77. package/telegram-plugin/uat/scenarios/jtbd-foreground-subagent-activity-dm.test.ts +3 -2
  78. package/vendor/hindsight-memory/CHANGELOG.md +28 -0
  79. package/vendor/hindsight-memory/docs/measurements/subagent-volume-gate-3994.md +84 -0
  80. package/vendor/hindsight-memory/scripts/backfill_transcripts.py +254 -40
  81. package/vendor/hindsight-memory/scripts/lib/content.py +11 -2
  82. package/vendor/hindsight-memory/scripts/recall.py +18 -3
  83. package/vendor/hindsight-memory/scripts/subagent_retain.py +59 -38
  84. package/vendor/hindsight-memory/scripts/tests/data/replay_volume_gate_3994.py +295 -0
  85. package/vendor/hindsight-memory/scripts/tests/test_backfill_from_logs.py +109 -0
  86. package/vendor/hindsight-memory/scripts/tests/test_overlap_tokens.py +135 -0
  87. package/vendor/hindsight-memory/scripts/tests/test_recall_no_lexical_gate.py +20 -23
  88. package/vendor/hindsight-memory/scripts/tests/test_recall_query_shaping.py +48 -0
  89. package/vendor/hindsight-memory/scripts/tests/test_subagent_retain.py +94 -1
  90. package/vendor/hindsight-memory/scripts/tests/test_subagent_retain_learnings.py +98 -42
@@ -65,6 +65,19 @@ import {
65
65
  } from 'node:fs'
66
66
  import { join } from 'node:path'
67
67
  import { homedir } from 'node:os'
68
+ // W1-d (#3865). The audience vocabulary and its resolution rule live in ONE
69
+ // shared pure module because the unbundled Stop hooks (which stamp it) and this
70
+ // bundled gateway code (which persists and enforces it) must agree exactly. A
71
+ // hand-synced second copy here would be a leak waiting on a drift.
72
+ import type { Audience } from './hooks/audience-classify.mjs'
73
+ // #4141: the provenance-framing rule lives in the same shared pure module, for
74
+ // the same reason — the decision must be one implementation, not two.
75
+ import {
76
+ applyReplyThrowFraming,
77
+ shouldFrameReplyThrow,
78
+ } from './hooks/audience-classify.mjs'
79
+
80
+ export type { Audience }
68
81
 
69
82
  /** One captured, not-yet-delivered final message for a single main-session turn. */
70
83
  export interface OutboxRecord {
@@ -113,6 +126,42 @@ export interface OutboxRecord {
113
126
  * or in a sweep journal entry — is direct evidence of a regression.
114
127
  */
115
128
  replyAlreadyDeliveredThisTurn?: boolean
129
+ /**
130
+ * W1-d (#3865): WHO this text is addressed to, decided at capture time by
131
+ * `hooks/audience-classify.mjs` (`decideCaptureAudience`) — the only point in
132
+ * the pipeline that can still see the turn that produced it.
133
+ *
134
+ * `'internal'` means the text is the agent's own working notes (the #3861
135
+ * shape: the reply tool threw, nobody was waiting, and the structurally-
136
+ * selected terminal prose run is orchestration narration rather than an
137
+ * answer). Such a record is journaled and inspectable but MUST NEVER reach a
138
+ * chat — the sweep drops it at entry selection.
139
+ *
140
+ * OPTIONAL for legacy reasons: every record written before this field
141
+ * existed has no `audience`, and `resolveRecordAudience` maps missing (and
142
+ * any non-`'internal'` value) to `'user'`. Suppression requires an
143
+ * affirmative tag; absence is never evidence.
144
+ */
145
+ audience?: Audience
146
+ /**
147
+ * W1-d follow-up (#4141): did one of this turn's reply-tool calls come back
148
+ * `is_error: true`? Stamped at capture by `hooks/silent-end-scan.mjs` (which
149
+ * reads the `tool_result` blocks) and persisted here so the SWEEP — hours
150
+ * later, in another process, with the transcript long gone — can still state
151
+ * this text's provenance.
152
+ *
153
+ * This is the raw structural SIGNAL, not a verdict. `audience` already
154
+ * consumed it once (combined with the obligation state) to decide
155
+ * suppression; the sweep consumes it again, on its own, to decide FRAMING for
156
+ * the records suppression deliberately does NOT catch — the foreground case,
157
+ * where the reply threw but someone is provably still waiting, so the record
158
+ * is `'user'` and MUST deliver.
159
+ *
160
+ * OPTIONAL: absent on every pre-#4141 record and on gateway-authored records
161
+ * (`gateway/flood-reply-queue.ts`), both of which deliver unframed. Only an
162
+ * exact boolean `true` changes anything — see `shouldFrameReplyThrow`.
163
+ */
164
+ replyToolThrewThisTurn?: boolean
116
165
  }
117
166
 
118
167
  /** One line of the delivered-keys journal (`outbox/delivered.jsonl`). */
@@ -131,6 +180,30 @@ export interface DeliveredEntry {
131
180
  deliverySource?: 'sweep' | 'reply-tool' | 'flush'
132
181
  /** #3510 instrumentation: see `OutboxRecord.replyAlreadyDeliveredThisTurn`. */
133
182
  replyAlreadyDeliveredThisTurn?: boolean
183
+ /** W1-d (#3865): the record's audience, carried onto the journal line. */
184
+ audience?: Audience
185
+ /**
186
+ * W1-d (#3865): this journal line is a TERMINAL SUPPRESSION, not a delivery.
187
+ * The sweep withheld an `'internal'` record from the chat and wrote this line
188
+ * so the nonce is terminally claimed (a second tick sends nothing) and the
189
+ * decision is durably auditable.
190
+ *
191
+ * This is what keeps suppression honest: the outcome is recorded as an
192
+ * explicit, named state rather than as an absence. `tgMessageId` is absent on
193
+ * these lines because nothing was sent.
194
+ */
195
+ suppressedAudience?: Audience
196
+ /**
197
+ * W1-d follow-up (#4141): this delivery carried the reply-throw provenance
198
+ * banner (`REPLY_THROW_PROVENANCE_NOTICE`) in front of the captured prose.
199
+ *
200
+ * Durable and terminal, for the same reason `suppressedAudience` is: framing
201
+ * changes what a human sees, so the outcome is recorded as an explicit named
202
+ * state on the journal line rather than being inferable only from a log line
203
+ * that may have rotated away. `tgMessageId` IS present on these lines — a
204
+ * framed delivery is a delivery.
205
+ */
206
+ framedProvenance?: 'reply-throw'
134
207
  }
135
208
 
136
209
  export function sha256Hex(s: string): string {
@@ -439,6 +512,13 @@ export interface OutboxSweepDecision {
439
512
  action: OutboxSweepAction
440
513
  /** The text to deliver (with any "(delayed)"/"(from background task)" prefix). */
441
514
  text?: string
515
+ /**
516
+ * W1-d follow-up (#4141): `text` carries the reply-throw provenance banner.
517
+ * Surfaced on the decision (rather than left implicit in a string compare) so
518
+ * the caller can journal the outcome and emit telemetry without re-deriving
519
+ * the rule.
520
+ */
521
+ framedProvenance?: 'reply-throw'
442
522
  }
443
523
 
444
524
  /**
@@ -456,7 +536,10 @@ export interface OutboxSweepDecision {
456
536
  * send-delayed — older than max-age; deliver with a "(delayed)" prefix, never drop.
457
537
  */
458
538
  export function decideOutboxSweep(input: {
459
- record: Pick<OutboxRecord, 'turnNonce' | 'text' | 'createdAt'>
539
+ record: Pick<
540
+ OutboxRecord,
541
+ 'turnNonce' | 'text' | 'createdAt' | 'replyToolThrewThisTurn'
542
+ >
460
543
  now: number
461
544
  deliveredNonces: Set<string>
462
545
  textAlreadyDelivered: boolean
@@ -476,6 +559,13 @@ export function decideOutboxSweep(input: {
476
559
  * `normalizeOutboundBody` (it sends via `bot.api.sendMessage` directly).
477
560
  */
478
561
  shownLedgerHit?: boolean
562
+ /**
563
+ * W1-d follow-up (#4141) kill switch for the reply-throw provenance banner.
564
+ * `false` restores the pre-change body byte-for-byte; the default is ON.
565
+ * Passed in rather than read from env here so this core stays pure and the
566
+ * revert-check can flip it in-process.
567
+ */
568
+ provenanceFraming?: boolean
479
569
  }): OutboxSweepDecision {
480
570
  const {
481
571
  record,
@@ -487,6 +577,7 @@ export function decideOutboxSweep(input: {
487
577
  quietMs = OUTBOX_QUIET_MS,
488
578
  maxAgeMs = OUTBOX_MAX_AGE_MS,
489
579
  shownLedgerHit = false,
580
+ provenanceFraming = true,
490
581
  } = input
491
582
  if (deliveredNonces.has(record.turnNonce)) return { action: 'skip-journaled' }
492
583
  if (shownLedgerHit) return { action: 'skip-ephemeral-shown' }
@@ -496,7 +587,17 @@ export function decideOutboxSweep(input: {
496
587
  if (!routable) return { action: 'skip-unroutable' }
497
588
  const delayed = age > maxAgeMs
498
589
  const prefix = (delayed ? '(delayed) ' : '') + routePrefix
499
- return { action: delayed ? 'send-delayed' : 'send', text: prefix + record.text }
590
+ // #4141: the banner sits BETWEEN the delivery prefixes and the body — the
591
+ // prefixes describe the DELIVERY ("(delayed)", "(from background task)"),
592
+ // the banner describes the TEXT. It is additive only: it can never turn a
593
+ // send into a skip, so this branch cannot manufacture silence.
594
+ const framed = shouldFrameReplyThrow(record, { frameEnabled: provenanceFraming })
595
+ const body = framed ? applyReplyThrowFraming(record.text) : record.text
596
+ return {
597
+ action: delayed ? 'send-delayed' : 'send',
598
+ text: prefix + body,
599
+ ...(framed && body !== record.text ? { framedProvenance: 'reply-throw' as const } : {}),
600
+ }
500
601
  }
501
602
 
502
603
  export interface ResolvedChat {
@@ -28,7 +28,11 @@
28
28
 
29
29
  import { parse } from "./parse.js";
30
30
  import { renderSafe, type RenderResult } from "./render.js";
31
- import { RICH_MESSAGE_MAX_CHARS, splitMarkdownChunks } from "../format.js";
31
+ import {
32
+ PLAIN_TEXT_MAX_CHARS,
33
+ RICH_MESSAGE_MAX_CHARS,
34
+ splitMarkdownChunks,
35
+ } from "../format.js";
32
36
 
33
37
  /**
34
38
  * The legacy plain-text `sendMessage` / `editMessageText` wire cap (4096
@@ -38,8 +42,12 @@ import { RICH_MESSAGE_MAX_CHARS, splitMarkdownChunks } from "../format.js";
38
42
  * the plain `sendMessage` endpoint (see stream-controller `doSend`/`doEdit`).
39
43
  * A degraded plain body larger than this is rejected by Telegram with
40
44
  * `message is too long`, so `renderOutboundChunks` caps plain pieces here.
45
+ *
46
+ * Defined in `format.ts` (#4043) so the parse-reject plain FALLBACKS in the
47
+ * send path can share the one constant without importing the renderer;
48
+ * re-exported here for this module's existing importers.
41
49
  */
42
- export const PLAIN_TEXT_MAX_CHARS = 4096;
50
+ export { PLAIN_TEXT_MAX_CHARS };
43
51
 
44
52
  /** Parse the `SWITCHROOM_RICH_RENDER` kill-switch value. Default ON; disabled
45
53
  * only by an explicit falsey/off token (`0`/`false`/`off`/`no`,
@@ -0,0 +1,150 @@
1
+ /**
2
+ * Surgical quote-reply parameters (`reply_parameters.quote`).
3
+ *
4
+ * The `reply` / `stream_reply` tools document a `quote_text` argument: quote a
5
+ * specific fragment of the message being replied to, so the reply visibly
6
+ * anchors on that fragment instead of the whole message.
7
+ *
8
+ * WIRE SHAPE — the bug this module exists to make unrepeatable
9
+ * ------------------------------------------------------------
10
+ * Bot API `ReplyParameters` (https://core.telegram.org/bots/api#replyparameters):
11
+ *
12
+ * quote String Quoted part of the message to be replied to;
13
+ * 0-1024 characters after entities parsing.
14
+ * quote_parse_mode String Mode for parsing entities in the quote.
15
+ * quote_entities Array Entities in the quote.
16
+ * quote_position Integer Position of the quote in the original message in
17
+ * UTF-16 code units.
18
+ *
19
+ * `quote` is a **String** and `quote_position` is a **separate sibling
20
+ * Integer**. Both send sites used to emit `quote: { text, position: 0 }` — an
21
+ * object — so Telegram rejected every quoted reply with
22
+ * `400 Bad Request: field "quote" must be of type String`, the reply threw, and
23
+ * the agent re-attempted a payload that could never be accepted. A fleet sweep
24
+ * of `tg-post method=sendRichMessage … status=err` lines found 378 of 384 total
25
+ * send failures were this one error (agent `overlord`, 2026-07-29 → 07-30, in
26
+ * retry bursts). The feature had never worked.
27
+ *
28
+ * DELIBERATE OMISSIONS
29
+ * --------------------
30
+ * - **No `quote_position`.** Telegram locates the quote text on its own; a
31
+ * fabricated `0` is what encouraged the object shape in the first place, and
32
+ * neither send site has a real position source (the model supplies text, not
33
+ * an offset). A wrong position is worse than none: it either mislocates the
34
+ * highlight or 400s.
35
+ * - **No `quote_parse_mode` / `quote_entities`, and therefore NO HTML
36
+ * escaping.** With no parse mode the quote is matched as literal text, which
37
+ * is exactly what `quote_text` carries (plain text the model copied out of
38
+ * the user's message). Escaping it (`&` → `&amp;`) would make the string stop
39
+ * being an exact substring of the original and guarantee a rejection.
40
+ *
41
+ * REJECTION HANDLING
42
+ * ------------------
43
+ * The quote must be an exact substring of the target message; Telegram 400s
44
+ * when it isn't found. That is a *quote* failure, not an answer failure — the
45
+ * reply body is still correct and must still be delivered. {@link
46
+ * isQuoteRejectionError} classifies it and {@link dropQuoteFromSendOpts}
47
+ * produces the same options with the quote removed, so callers can retry once
48
+ * and land the reply as an ordinary (unquoted) reply.
49
+ */
50
+
51
+ import { GrammyError } from 'grammy'
52
+
53
+ import { isHtmlParseRejectError, isMessageTooLongError } from './retry-api-call.js'
54
+
55
+ /** Bot API cap: `quote` is 0-1024 characters after entities parsing. */
56
+ export const QUOTE_MAX_CHARS = 1024
57
+
58
+ /**
59
+ * Normalize a caller-supplied `quote_text` into the string Telegram accepts, or
60
+ * `null` when there is nothing worth quoting.
61
+ *
62
+ * - empty / whitespace-only ⇒ `null` (an empty quote highlights nothing and is
63
+ * a guaranteed rejection).
64
+ * - longer than {@link QUOTE_MAX_CHARS} ⇒ truncated. A PREFIX of an exact
65
+ * substring is still an exact substring, so truncation still matches the
66
+ * original message, whereas an over-length quote is a certain 400. The cut
67
+ * never splits a surrogate pair (a lone surrogate is not valid UTF-8 on the
68
+ * wire).
69
+ * - otherwise verbatim — never escaped, never trimmed (leading/trailing spaces
70
+ * are part of the substring the user may have selected).
71
+ */
72
+ export function normalizeQuoteText(quoteText: string | null | undefined): string | null {
73
+ if (typeof quoteText !== 'string') return null
74
+ if (quoteText.trim().length === 0) return null
75
+ if (quoteText.length <= QUOTE_MAX_CHARS) return quoteText
76
+ let end = QUOTE_MAX_CHARS
77
+ const code = quoteText.charCodeAt(end - 1)
78
+ // High surrogate at the cut boundary ⇒ drop it rather than emit half a pair.
79
+ if (code >= 0xd800 && code <= 0xdbff) end -= 1
80
+ return quoteText.slice(0, end)
81
+ }
82
+
83
+ /**
84
+ * Build the `reply_parameters` object for a reply to `messageId`, optionally
85
+ * carrying a surgical quote.
86
+ *
87
+ * This is the ONLY place the `quote` field is constructed. Both send sites (the
88
+ * `reply` chunk loop in `gateway/outbound-send-path.ts` and the `stream_reply`
89
+ * controller in `stream-controller.ts`) call it, so the wire shape cannot drift
90
+ * apart again.
91
+ */
92
+ export function buildReplyParameters(
93
+ messageId: number,
94
+ quoteText?: string | null,
95
+ ): { message_id: number; quote?: string } {
96
+ const quote = normalizeQuoteText(quoteText)
97
+ return { message_id: messageId, ...(quote != null ? { quote } : {}) }
98
+ }
99
+
100
+ /**
101
+ * True when Telegram rejected the send because of the QUOTE, not the body.
102
+ *
103
+ * Covers the "quote isn't in the original message" family (the documented
104
+ * failure mode) and any other 400 naming the quote — matched on the word rather
105
+ * than one exact phrase, because the Bot API's wording for this class has
106
+ * changed before (`QUOTE_TEXT_INVALID` vs the prose form) and a missed match
107
+ * costs the whole answer.
108
+ *
109
+ * A BODY error must never be claimed here, though, and the word alone is not
110
+ * enough to tell them apart: `<blockquote>` in a rich body produces
111
+ * `can't parse entities: Can't find end tag corresponding to start tag
112
+ * "blockquote"` / `Unsupported start tag "blockquote"` — 400s that contain the
113
+ * substring "quote" but are PARSE failures. Claiming one would drop the quote
114
+ * and resend the same unparseable body, hit the same 400, and throw straight
115
+ * out of the caller's fallback ladder — losing the answer that the plain-text
116
+ * rescue would otherwise have delivered. So defer to the body classifiers
117
+ * first, exactly as `isHtmlParseRejectError` defers to `isMessageTooLongError`
118
+ * (retry-api-call.ts) for the same reason. No false negative results: the real
119
+ * quote rejections (`QUOTE_TEXT_INVALID`, "message quote not found") contain
120
+ * neither a parse phrase nor a length phrase.
121
+ */
122
+ export function isQuoteRejectionError(err: unknown): boolean {
123
+ if (!(err instanceof GrammyError) || err.error_code !== 400) return false
124
+ if (isHtmlParseRejectError(err) || isMessageTooLongError(err)) return false
125
+ return (err.description || '').toLowerCase().includes('quote')
126
+ }
127
+
128
+ /** True when `opts` carries a `reply_parameters.quote`. Structural on purpose:
129
+ * the reply path builds untyped `Record<string, unknown>` send options and the
130
+ * stream path a typed `StreamSendOpts`, and both must be checkable. */
131
+ export function sendOptsHaveQuote(opts: object): boolean {
132
+ const rp = (opts as { reply_parameters?: { quote?: unknown } }).reply_parameters
133
+ return rp != null && typeof rp === 'object' && rp.quote != null
134
+ }
135
+
136
+ /**
137
+ * Same send options with the surgical quote removed — the retry payload after a
138
+ * {@link isQuoteRejectionError}. Everything else (reply target, thread, markup,
139
+ * notification) is preserved, so the answer still lands as a reply to the right
140
+ * message; only the highlight is sacrificed.
141
+ */
142
+ export function dropQuoteFromSendOpts<T extends object>(opts: T): T {
143
+ if (!sendOptsHaveQuote(opts)) return { ...opts }
144
+ const rp = { ...((opts as { reply_parameters: Record<string, unknown> }).reply_parameters) }
145
+ delete rp.quote
146
+ delete rp.quote_parse_mode
147
+ delete rp.quote_entities
148
+ delete rp.quote_position
149
+ return { ...opts, reply_parameters: rp }
150
+ }
@@ -22,11 +22,206 @@
22
22
  * | GrammyError 400 "message to edit not found" | swallow, return undefined |
23
23
  * | GrammyError 400 "message to delete not found" | swallow, return undefined |
24
24
  * | GrammyError 400 "thread not found" (w/ opts) | throw `THREAD_NOT_FOUND` wrapper |
25
- * | Network error (fetch failed / ECONN…) | exponential backoff retry, 3 attempts |
25
+ * | HttpError (grammy transport failure) | exponential backoff retry, 3 attempts |
26
+ * | Raw network error (fetch failed / ECONN…) | exponential backoff retry, 3 attempts |
27
+ * | ENOSPC/EDQUOT/EIO/ENOMEM (local exhaustion) | throw `LOCAL_RESOURCE_EXHAUSTED`, no retry|
26
28
  * | Anything else | rethrow immediately |
29
+ *
30
+ * The three "swallow" rows are the DEFAULT, not the law: a call site whose
31
+ * contract is "did the target message survive?" sets `rethrowBenign400: true`
32
+ * and gets the original `GrammyError` instead, so it can classify the failure
33
+ * itself. See that option's docblock for the #4065 inertness bug it exists for.
34
+ *
35
+ * The network row is classified by TYPE (`isTransientTransportError`), not by a
36
+ * substring of the message: grammy rewrites every transport failure's message
37
+ * to `Network request for '<method>' failed!`, so the substring form of this
38
+ * row never matched in production and the retry was dead code (#4123).
27
39
  */
28
40
 
29
- import { GrammyError } from 'grammy'
41
+ import { GrammyError, HttpError } from 'grammy'
42
+ import { AsyncLocalStorage } from 'async_hooks'
43
+
44
+ // ─── retry-attempt context (#3931) ────────────────────────────────────────
45
+
46
+ /**
47
+ * What the retry policy knows about the attempt currently on the wire.
48
+ *
49
+ * Published so the layer BELOW the policy — the `tg-post` observability
50
+ * transformer in `shared/bot-runtime.ts`, which runs inside `fn()` and
51
+ * therefore sees a failure *before* the policy decides what to do with it —
52
+ * can label a doomed-but-non-terminal attempt honestly instead of as a
53
+ * delivery failure. Without it, a 429 that the policy sleeps and retries
54
+ * SUCCESSFULLY still wrote `status=err` to the gateway log, and fleet-health's
55
+ * per-line `reply-delivery-failure` detector escalated a delivered reply
56
+ * (#3931).
57
+ */
58
+ export interface TgAttemptContext {
59
+ /** 0-based index of the attempt currently executing. */
60
+ attempt: number
61
+ /** Total attempts this policy will make. */
62
+ maxRetries: number
63
+ /** Longest 429 `retry_after` the policy will sleep in-process. */
64
+ maxFloodSleepMs: number
65
+ }
66
+
67
+ const attemptStore = new AsyncLocalStorage<TgAttemptContext>()
68
+
69
+ /** The attempt context of the enclosing `retryApiCall`, or undefined outside one. */
70
+ export function _getTgAttemptContext(): TgAttemptContext | undefined {
71
+ return attemptStore.getStore()
72
+ }
73
+
74
+ /** Run `fn` with `ctx` visible to `_getTgAttemptContext()` for its whole async chain. */
75
+ function withAttemptContext<T>(ctx: TgAttemptContext, fn: () => Promise<T>): Promise<T> {
76
+ return attemptStore.run(ctx, fn)
77
+ }
78
+
79
+ /**
80
+ * Substring classifier for a RAW transport error message (an undici/node error
81
+ * that has not been through grammy's wrapping).
82
+ *
83
+ * NOT sufficient on its own for anything that came out of `bot.api.*` — see
84
+ * `isTransientTransportError`, which is what the retry loop actually calls.
85
+ * grammy rewrites the message of every transport failure, so this predicate
86
+ * returns false for 100% of production send failures if used alone (#4123).
87
+ */
88
+ export function isTransientNetworkMessage(msg: string): boolean {
89
+ return (
90
+ msg.includes('ECONNRESET') ||
91
+ msg.includes('ETIMEDOUT') ||
92
+ msg.includes('fetch failed') ||
93
+ msg.includes('ENOTFOUND')
94
+ )
95
+ }
96
+
97
+ /**
98
+ * True for a cancellation: `AbortController.abort()` surfaces as a
99
+ * `DOMException`/`Error` with `name === 'AbortError'` (node also sets
100
+ * `code === 'ABORT_ERR'`). Checked structurally rather than by `instanceof`
101
+ * because the shape differs across the fetch implementations grammy can be
102
+ * handed (undici, node-fetch, a test double).
103
+ */
104
+ function isAbortError(err: unknown): boolean {
105
+ const e = err as { name?: unknown; code?: unknown } | null
106
+ return e?.name === 'AbortError' || e?.code === 'ABORT_ERR'
107
+ }
108
+
109
+ /**
110
+ * Is `err` a transient TRANSPORT failure the policy should back off and repeat?
111
+ *
112
+ * This is the single classifier the retry loop and `willRetryTelegramFailure`
113
+ * both call, so the policy and the logging tier cannot disagree about what
114
+ * counts as retryable (#3931).
115
+ *
116
+ * Why this is a TYPE test and not a message test (#4123): grammy funnels every
117
+ * fetch rejection through `toHttpError` (grammy 1.44.0, `out/core/client.js:58`
118
+ * → `out/core/error.js:76-83`), which throws away the original error text and
119
+ * renders `Network request for '<method>' failed!`. The underlying
120
+ * `fetch failed` / `ECONNRESET` / `ETIMEDOUT` / `ENOTFOUND` string is preserved
121
+ * only on `HttpError.error`, and is appended to the message only when
122
+ * `client.sensitiveLogs` is on — which we deliberately leave off, because it
123
+ * writes the bot-token URL into logs. So a substring match over the message
124
+ * matched NOTHING in production: the network-retry branch was dead code and a
125
+ * single connection blip terminally failed the send.
126
+ *
127
+ * The classes, and why each lands where it does:
128
+ *
129
+ * - `GrammyError` — an API-LEVEL rejection: Telegram answered, and said no.
130
+ * Never part of the network class. 429 has its own sleep-and-retry branch
131
+ * above; everything else (400/403/404/…) is terminal and must NOT be
132
+ * retried. This is the invariant the old `!isGrammyErr` guard at the
133
+ * callsite protected; it now lives here, in one place, next to the reason.
134
+ * - local resource exhaustion — ENOSPC/EDQUOT/EIO/ENOMEM is a LOCAL disk or
135
+ * memory failure. Retrying it in a tight loop is what tripped the per-bot
136
+ * flood ban (#2923). Excluded even when grammy wrapped it.
137
+ * - `HttpError` — transport by construction: it exists only on the
138
+ * "the fetch never produced a Telegram response" path (connection reset,
139
+ * DNS failure, request timeout, unparseable proxy 5xx). Retryable.
140
+ * - anything else — a raw error that never went through grammy (or a nested
141
+ * cause): fall back to the errno/message classifier.
142
+ */
143
+ export function isTransientTransportError(err: unknown): boolean {
144
+ if (err instanceof GrammyError) return false
145
+ if (isLocalResourceError(err)) return false
146
+ if (err instanceof HttpError) {
147
+ // grammy stashes the original rejection on `.error`.
148
+ const inner = (err as HttpError).error
149
+ // A local disk failure during a multipart upload can surface here; it
150
+ // stays non-retryable (#2923).
151
+ if (isLocalResourceError(inner)) return false
152
+ // A CALLER-initiated cancellation is not a transient failure — retrying it
153
+ // only re-aborts three times and adds backoff sleeps to shutdown. grammy's
154
+ // own request TIMEOUT is NOT this case: `createTimeout` rejects the race
155
+ // with a plain `Error("Request to '<m>' timed out after Ns")` BEFORE it
156
+ // aborts the controller (grammy 1.44.0, out/core/client.js:168-178), so a
157
+ // timeout arrives as that Error and remains retryable. An `AbortError`
158
+ // here means something aborted the signal we were handed (`bot.stop()`).
159
+ if (isAbortError(inner)) return false
160
+ return true
161
+ }
162
+ const msg = err instanceof Error ? err.message : String(err ?? '')
163
+ if (isTransientNetworkMessage(msg)) return true
164
+ // One level of standard cause-chaining (`new Error(x, { cause })`), so a
165
+ // wrapper that preserves the transport error is still classified correctly.
166
+ const cause = (err as { cause?: unknown } | null)?.cause
167
+ if (cause == null || cause === err) return false
168
+ if (isLocalResourceError(cause)) return false
169
+ return isTransientNetworkMessage(cause instanceof Error ? cause.message : String(cause))
170
+ }
171
+
172
+ /** The parts of a Telegram failure that decide retryability. */
173
+ export interface TgFailureShape {
174
+ /** Telegram `error_code`, or null/undefined for a transport-level error. */
175
+ errorCode?: number | null
176
+ /** Telegram `parameters.retry_after`, seconds. */
177
+ retryAfterSec?: number | null
178
+ /** Transport-level error message (only consulted when `errorCode` is absent). */
179
+ message?: string | null
180
+ /**
181
+ * The thrown error itself, when the caller has it.
182
+ *
183
+ * STRONGLY PREFERRED over `message` for anything that came out of `bot.api.*`:
184
+ * grammy rewrites the message of every transport failure, so a message-only
185
+ * shape cannot tell a retryable `HttpError` from a terminal one (#4123). Pass
186
+ * the error and the predicate classifies by type, exactly as the loop does.
187
+ */
188
+ cause?: unknown
189
+ }
190
+
191
+ /**
192
+ * Will the enclosing retry policy issue ANOTHER attempt after this failure?
193
+ *
194
+ * `true` means the failure is non-terminal: a later attempt may still deliver,
195
+ * so it must not be logged (or detected) as a delivery failure. `false` means
196
+ * either there is no retry policy above this call, or this attempt is the last
197
+ * one, or the error class is not retryable — i.e. the failure IS the outcome.
198
+ *
199
+ * Mirrors `createRetryApiCall`'s loop exactly:
200
+ * - no enclosing policy ⇒ terminal
201
+ * - last attempt (`attempt >= maxRetries-1`) ⇒ terminal (loop falls through)
202
+ * - 429 with `retry_after` over the sleep ceiling ⇒ terminal (FLOOD_WAIT_ACTIVE)
203
+ * - 429 under the ceiling ⇒ retried
204
+ * - transient transport error ⇒ retried
205
+ * - anything else ⇒ terminal (rethrown)
206
+ */
207
+ export function willRetryTelegramFailure(
208
+ failure: TgFailureShape,
209
+ ctx: TgAttemptContext | undefined = _getTgAttemptContext(),
210
+ ): boolean {
211
+ if (ctx == null) return false
212
+ if (ctx.attempt >= ctx.maxRetries - 1) return false
213
+ if (failure.errorCode === 429) {
214
+ const retryAfter = Number(failure.retryAfterSec ?? 5)
215
+ const delayMs = (Number.isFinite(retryAfter) ? retryAfter : 5) * 1000
216
+ return delayMs <= ctx.maxFloodSleepMs
217
+ }
218
+ if (failure.errorCode != null) return false
219
+ // Classify by TYPE when the caller handed us the error (the seam that
220
+ // matters — see `isTransientTransportError`); fall back to the message only
221
+ // for shapes reconstructed from a log line, which have nothing else.
222
+ if (failure.cause != null) return isTransientTransportError(failure.cause)
223
+ return isTransientTransportError(new Error(failure.message ?? ''))
224
+ }
30
225
 
31
226
  export interface RetryCallOpts {
32
227
  /**
@@ -90,6 +285,42 @@ export interface RetryCallOpts {
90
285
  * the authoritative statement.
91
286
  */
92
287
  priorityClass?: 'critical' | 'useful' | 'cosmetic'
288
+ /**
289
+ * Opt OUT of the benign-400 swallow for this one call (#4065 follow-up).
290
+ *
291
+ * By default `retryApiCall` treats the benign 400s
292
+ * (`classifyBenignTelegram400`: not-modified / message-to-edit-not-found /
293
+ * message-to-delete-not-found) as non-events and RESOLVES `undefined`. That
294
+ * is right for fire-and-forget card repaints, and wrong — silently, and in
295
+ * the worst direction — for any caller whose contract is "tell me whether
296
+ * the target message still exists".
297
+ *
298
+ * The concrete failure this exists for: the rollout narration card's edit
299
+ * relay (`gateway/rollout-status-edit.ts`) infers edit success from the
300
+ * ABSENCE of a throw and reports it to hostd. A seeded-resume narrator
301
+ * holding a carried `narration_message_id` for a card the operator deleted
302
+ * gets `400 "message to edit not found"` — which the default swallow turns
303
+ * into a resolved promise, so the handler replies `{ok:true}` and hostd
304
+ * records a frozen card as live and never re-posts. The `gone` signal — the
305
+ * whole reason that feature exists — is unreachable on the production path
306
+ * for its primary trigger. Classification has to happen where the error
307
+ * still exists, so the call site asks for the error instead of a
308
+ * lie-shaped success.
309
+ *
310
+ * When true, a benign 400 is RETHROWN unchanged (still `GrammyError`, with
311
+ * its `description` intact) after `observer.onBenign` fires, so the caller
312
+ * classifies it itself. Everything else — 429 handling, the flood breaker,
313
+ * network retries, `THREAD_NOT_FOUND` — is untouched. Default `false`:
314
+ * every existing call site keeps today's exact behaviour.
315
+ *
316
+ * Note this covers BOTH benign kinds deliberately. `not modified` also
317
+ * resolves `undefined` today, which coincidentally matches the intended
318
+ * semantics for an edit relay (the live card already carries this body =
319
+ * success) — but "coincidentally right" is not a contract. Rethrowing both
320
+ * puts ONE classifier in charge (the caller's), so the two kinds can never
321
+ * drift apart on the swallow boundary.
322
+ */
323
+ rethrowBenign400?: boolean
93
324
  }
94
325
 
95
326
  /**
@@ -133,7 +364,12 @@ export interface RetryObserver {
133
364
  onRetry?(info: { attempt: number; reason: 'flood_wait' | 'network'; delayMs: number }): void
134
365
  /** Fires when max retries is reached and the wrapper gives up. */
135
366
  onGiveUp?(info: { attempts: number; error: unknown }): void
136
- /** Fires for each benign error we swallowed (not-modified, not-found). */
367
+ /**
368
+ * Fires for each benign 400 we CLASSIFIED (not-modified, not-found) —
369
+ * whether it was then swallowed (the default) or rethrown because the call
370
+ * set `rethrowBenign400`. The Telegram-side event is the same either way, so
371
+ * counting it must not depend on the disposition.
372
+ */
137
373
  onBenign?(info: { kind: BenignTelegram400Kind }): void
138
374
  }
139
375
 
@@ -379,7 +615,10 @@ export function createRetryApiCall(
379
615
  throw gated
380
616
  }
381
617
  try {
382
- return await fn()
618
+ // #3931 — publish the attempt context for the duration of the wire
619
+ // call so the `tg-post` transformer underneath can tell a retried
620
+ // failure from a terminal one.
621
+ return await withAttemptContext({ attempt, maxRetries, maxFloodSleepMs }, fn)
383
622
  } catch (err) {
384
623
  const isGrammyErr = err instanceof GrammyError
385
624
  const msg = err instanceof Error ? err.message : String(err)
@@ -444,6 +683,11 @@ export function createRetryApiCall(
444
683
  : null
445
684
  if (benignKind !== null) {
446
685
  observer?.onBenign?.({ kind: benignKind })
686
+ // #4065 follow-up: a caller whose contract is "does the target
687
+ // message still exist?" opts out of the swallow, because a resolved
688
+ // promise is indistinguishable from a landed edit. Rethrow the
689
+ // ORIGINAL error so the caller classifies off `description`.
690
+ if (opts?.rethrowBenign400) throw err
447
691
  return undefined as unknown as T
448
692
  }
449
693
 
@@ -459,13 +703,12 @@ export function createRetryApiCall(
459
703
  }
460
704
 
461
705
  // Network-level transient errors — exponential backoff, bounded.
462
- if (
463
- !isGrammyErr &&
464
- (msg.includes('ECONNRESET') ||
465
- msg.includes('ETIMEDOUT') ||
466
- msg.includes('fetch failed') ||
467
- msg.includes('ENOTFOUND'))
468
- ) {
706
+ // Classified by TYPE, not by message substring: grammy rewrites the
707
+ // message of every transport failure, so the old substring test never
708
+ // fired in production and this whole branch was dead (#4123). The
709
+ // `!isGrammyErr` guard that used to sit here now lives inside
710
+ // `isTransientTransportError`, which excludes GrammyError explicitly.
711
+ if (isTransientTransportError(err)) {
469
712
  if (attempt < maxRetries - 1) {
470
713
  const delayMs = Math.pow(2, attempt) * 1000
471
714
  log?.(