switchroom 0.19.42 → 0.19.43

package/telegram-plugin/retry-api-call.ts

@@ -22,11 +22,206 @@
  *   | GrammyError 400 "message to edit not found"   | swallow, return undefined                 |
  *   | GrammyError 400 "message to delete not found" | swallow, return undefined                 |
  *   | GrammyError 400 "thread not found" (w/ opts)  | throw `THREAD_NOT_FOUND` wrapper          |
- *   | Network error (fetch failed / ECONN…)         | exponential backoff retry, 3 attempts     |
+ *   | HttpError (grammy transport failure)          | exponential backoff retry, 3 attempts     |
+ *   | Raw network error (fetch failed / ECONN…)     | exponential backoff retry, 3 attempts     |
+ *   | ENOSPC/EDQUOT/EIO/ENOMEM (local exhaustion)   | throw `LOCAL_RESOURCE_EXHAUSTED`, no retry|
  *   | Anything else                                 | rethrow immediately                       |
+ *
+ * The three "swallow" rows are the DEFAULT, not the law: a call site whose
+ * contract is "did the target message survive?" sets `rethrowBenign400: true`
+ * and gets the original `GrammyError` instead, so it can classify the failure
+ * itself. See that option's docblock for the #4065 inertness bug it exists for.
+ *
+ * The network row is classified by TYPE (`isTransientTransportError`), not by a
+ * substring of the message: grammy rewrites every transport failure's message
+ * to `Network request for '<method>' failed!`, so the substring form of this
+ * row never matched in production and the retry was dead code (#4123).
  */
 
-import { GrammyError } from 'grammy'
+import { GrammyError, HttpError } from 'grammy'
+import { AsyncLocalStorage } from 'async_hooks'
+
+// ─── retry-attempt context (#3931) ────────────────────────────────────────
+
+/**
+ * What the retry policy knows about the attempt currently on the wire.
+ *
+ * Published so the layer BELOW the policy — the `tg-post` observability
+ * transformer in `shared/bot-runtime.ts`, which runs inside `fn()` and
+ * therefore sees a failure *before* the policy decides what to do with it —
+ * can label a doomed-but-non-terminal attempt honestly instead of as a
+ * delivery failure. Without it, a 429 that the policy sleeps and retries
+ * SUCCESSFULLY still wrote `status=err` to the gateway log, and fleet-health's
+ * per-line `reply-delivery-failure` detector escalated a delivered reply
+ * (#3931).
+ */
+export interface TgAttemptContext {
+  /** 0-based index of the attempt currently executing. */
+  attempt: number
+  /** Total attempts this policy will make. */
+  maxRetries: number
+  /** Longest 429 `retry_after` the policy will sleep in-process. */
+  maxFloodSleepMs: number
+}
+
+const attemptStore = new AsyncLocalStorage<TgAttemptContext>()
+
+/** The attempt context of the enclosing `retryApiCall`, or undefined outside one. */
+export function _getTgAttemptContext(): TgAttemptContext | undefined {
+  return attemptStore.getStore()
+}
+
+/** Run `fn` with `ctx` visible to `_getTgAttemptContext()` for its whole async chain. */
+function withAttemptContext<T>(ctx: TgAttemptContext, fn: () => Promise<T>): Promise<T> {
+  return attemptStore.run(ctx, fn)
+}
+
+/**
+ * Substring classifier for a RAW transport error message (an undici/node error
+ * that has not been through grammy's wrapping).
+ *
+ * NOT sufficient on its own for anything that came out of `bot.api.*` — see
+ * `isTransientTransportError`, which is what the retry loop actually calls.
+ * grammy rewrites the message of every transport failure, so this predicate
+ * returns false for 100% of production send failures if used alone (#4123).
+ */
+export function isTransientNetworkMessage(msg: string): boolean {
+  return (
+    msg.includes('ECONNRESET') ||
+    msg.includes('ETIMEDOUT') ||
+    msg.includes('fetch failed') ||
+    msg.includes('ENOTFOUND')
+  )
+}
+
+/**
+ * True for a cancellation: `AbortController.abort()` surfaces as a
+ * `DOMException`/`Error` with `name === 'AbortError'` (node also sets
+ * `code === 'ABORT_ERR'`). Checked structurally rather than by `instanceof`
+ * because the shape differs across the fetch implementations grammy can be
+ * handed (undici, node-fetch, a test double).
+ */
+function isAbortError(err: unknown): boolean {
+  const e = err as { name?: unknown; code?: unknown } | null
+  return e?.name === 'AbortError' || e?.code === 'ABORT_ERR'
+}
+
+/**
+ * Is `err` a transient TRANSPORT failure the policy should back off and repeat?
+ *
+ * This is the single classifier the retry loop and `willRetryTelegramFailure`
+ * both call, so the policy and the logging tier cannot disagree about what
+ * counts as retryable (#3931).
+ *
+ * Why this is a TYPE test and not a message test (#4123): grammy funnels every
+ * fetch rejection through `toHttpError` (grammy 1.44.0, `out/core/client.js:58`
+ * → `out/core/error.js:76-83`), which throws away the original error text and
+ * renders `Network request for '<method>' failed!`. The underlying
+ * `fetch failed` / `ECONNRESET` / `ETIMEDOUT` / `ENOTFOUND` string is preserved
+ * only on `HttpError.error`, and is appended to the message only when
+ * `client.sensitiveLogs` is on — which we deliberately leave off, because it
+ * writes the bot-token URL into logs. So a substring match over the message
+ * matched NOTHING in production: the network-retry branch was dead code and a
+ * single connection blip terminally failed the send.
+ *
+ * The classes, and why each lands where it does:
+ *
+ *   - `GrammyError`  — an API-LEVEL rejection: Telegram answered, and said no.
+ *     Never part of the network class. 429 has its own sleep-and-retry branch
+ *     above; everything else (400/403/404/…) is terminal and must NOT be
+ *     retried. This is the invariant the old `!isGrammyErr` guard at the
+ *     callsite protected; it now lives here, in one place, next to the reason.
+ *   - local resource exhaustion — ENOSPC/EDQUOT/EIO/ENOMEM is a LOCAL disk or
+ *     memory failure. Retrying it in a tight loop is what tripped the per-bot
+ *     flood ban (#2923). Excluded even when grammy wrapped it.
+ *   - `HttpError`    — transport by construction: it exists only on the
+ *     "the fetch never produced a Telegram response" path (connection reset,
+ *     DNS failure, request timeout, unparseable proxy 5xx). Retryable.
+ *   - anything else  — a raw error that never went through grammy (or a nested
+ *     cause): fall back to the errno/message classifier.
+ */
+export function isTransientTransportError(err: unknown): boolean {
+  if (err instanceof GrammyError) return false
+  if (isLocalResourceError(err)) return false
+  if (err instanceof HttpError) {
+    // grammy stashes the original rejection on `.error`.
+    const inner = (err as HttpError).error
+    // A local disk failure during a multipart upload can surface here; it
+    // stays non-retryable (#2923).
+    if (isLocalResourceError(inner)) return false
+    // A CALLER-initiated cancellation is not a transient failure — retrying it
+    // only re-aborts three times and adds backoff sleeps to shutdown. grammy's
+    // own request TIMEOUT is NOT this case: `createTimeout` rejects the race
+    // with a plain `Error("Request to '<m>' timed out after Ns")` BEFORE it
+    // aborts the controller (grammy 1.44.0, out/core/client.js:168-178), so a
+    // timeout arrives as that Error and remains retryable. An `AbortError`
+    // here means something aborted the signal we were handed (`bot.stop()`).
+    if (isAbortError(inner)) return false
+    return true
+  }
+  const msg = err instanceof Error ? err.message : String(err ?? '')
+  if (isTransientNetworkMessage(msg)) return true
+  // One level of standard cause-chaining (`new Error(x, { cause })`), so a
+  // wrapper that preserves the transport error is still classified correctly.
+  const cause = (err as { cause?: unknown } | null)?.cause
+  if (cause == null || cause === err) return false
+  if (isLocalResourceError(cause)) return false
+  return isTransientNetworkMessage(cause instanceof Error ? cause.message : String(cause))
+}
+
+/** The parts of a Telegram failure that decide retryability. */
+export interface TgFailureShape {
+  /** Telegram `error_code`, or null/undefined for a transport-level error. */
+  errorCode?: number | null
+  /** Telegram `parameters.retry_after`, seconds. */
+  retryAfterSec?: number | null
+  /** Transport-level error message (only consulted when `errorCode` is absent). */
+  message?: string | null
+  /**
+   * The thrown error itself, when the caller has it.
+   *
+   * STRONGLY PREFERRED over `message` for anything that came out of `bot.api.*`:
+   * grammy rewrites the message of every transport failure, so a message-only
+   * shape cannot tell a retryable `HttpError` from a terminal one (#4123). Pass
+   * the error and the predicate classifies by type, exactly as the loop does.
+   */
+  cause?: unknown
+}
+
+/**
+ * Will the enclosing retry policy issue ANOTHER attempt after this failure?
+ *
+ * `true` means the failure is non-terminal: a later attempt may still deliver,
+ * so it must not be logged (or detected) as a delivery failure. `false` means
+ * either there is no retry policy above this call, or this attempt is the last
+ * one, or the error class is not retryable — i.e. the failure IS the outcome.
+ *
+ * Mirrors `createRetryApiCall`'s loop exactly:
+ *   - no enclosing policy                           ⇒ terminal
+ *   - last attempt (`attempt >= maxRetries-1`)      ⇒ terminal (loop falls through)
+ *   - 429 with `retry_after` over the sleep ceiling ⇒ terminal (FLOOD_WAIT_ACTIVE)
+ *   - 429 under the ceiling                         ⇒ retried
+ *   - transient transport error                     ⇒ retried
+ *   - anything else                                 ⇒ terminal (rethrown)
+ */
+export function willRetryTelegramFailure(
+  failure: TgFailureShape,
+  ctx: TgAttemptContext | undefined = _getTgAttemptContext(),
+): boolean {
+  if (ctx == null) return false
+  if (ctx.attempt >= ctx.maxRetries - 1) return false
+  if (failure.errorCode === 429) {
+    const retryAfter = Number(failure.retryAfterSec ?? 5)
+    const delayMs = (Number.isFinite(retryAfter) ? retryAfter : 5) * 1000
+    return delayMs <= ctx.maxFloodSleepMs
+  }
+  if (failure.errorCode != null) return false
+  // Classify by TYPE when the caller handed us the error (the seam that
+  // matters — see `isTransientTransportError`); fall back to the message only
+  // for shapes reconstructed from a log line, which have nothing else.
+  if (failure.cause != null) return isTransientTransportError(failure.cause)
+  return isTransientTransportError(new Error(failure.message ?? ''))
+}
 
 export interface RetryCallOpts {
   /**
@@ -90,6 +285,42 @@ export interface RetryCallOpts {
    * the authoritative statement.
    */
   priorityClass?: 'critical' | 'useful' | 'cosmetic'
+  /**
+   * Opt OUT of the benign-400 swallow for this one call (#4065 follow-up).
+   *
+   * By default `retryApiCall` treats the benign 400s
+   * (`classifyBenignTelegram400`: not-modified / message-to-edit-not-found /
+   * message-to-delete-not-found) as non-events and RESOLVES `undefined`. That
+   * is right for fire-and-forget card repaints, and wrong — silently, and in
+   * the worst direction — for any caller whose contract is "tell me whether
+   * the target message still exists".
+   *
+   * The concrete failure this exists for: the rollout narration card's edit
+   * relay (`gateway/rollout-status-edit.ts`) infers edit success from the
+   * ABSENCE of a throw and reports it to hostd. A seeded-resume narrator
+   * holding a carried `narration_message_id` for a card the operator deleted
+   * gets `400 "message to edit not found"` — which the default swallow turns
+   * into a resolved promise, so the handler replies `{ok:true}` and hostd
+   * records a frozen card as live and never re-posts. The `gone` signal — the
+   * whole reason that feature exists — is unreachable on the production path
+   * for its primary trigger. Classification has to happen where the error
+   * still exists, so the call site asks for the error instead of a
+   * lie-shaped success.
+   *
+   * When true, a benign 400 is RETHROWN unchanged (still `GrammyError`, with
+   * its `description` intact) after `observer.onBenign` fires, so the caller
+   * classifies it itself. Everything else — 429 handling, the flood breaker,
+   * network retries, `THREAD_NOT_FOUND` — is untouched. Default `false`:
+   * every existing call site keeps today's exact behaviour.
+   *
+   * Note this covers BOTH benign kinds deliberately. `not modified` also
+   * resolves `undefined` today, which coincidentally matches the intended
+   * semantics for an edit relay (the live card already carries this body =
+   * success) — but "coincidentally right" is not a contract. Rethrowing both
+   * puts ONE classifier in charge (the caller's), so the two kinds can never
+   * drift apart on the swallow boundary.
+   */
+  rethrowBenign400?: boolean
 }
 
 /**
@@ -133,7 +364,12 @@ export interface RetryObserver {
   onRetry?(info: { attempt: number; reason: 'flood_wait' | 'network'; delayMs: number }): void
   /** Fires when max retries is reached and the wrapper gives up. */
   onGiveUp?(info: { attempts: number; error: unknown }): void
-  /** Fires for each benign error we swallowed (not-modified, not-found). */
+  /**
+   * Fires for each benign 400 we CLASSIFIED (not-modified, not-found) —
+   * whether it was then swallowed (the default) or rethrown because the call
+   * set `rethrowBenign400`. The Telegram-side event is the same either way, so
+   * counting it must not depend on the disposition.
+   */
   onBenign?(info: { kind: BenignTelegram400Kind }): void
 }
 
@@ -379,7 +615,10 @@ export function createRetryApiCall(
         throw gated
       }
       try {
-        return await fn()
+        // #3931 — publish the attempt context for the duration of the wire
+        // call so the `tg-post` transformer underneath can tell a retried
+        // failure from a terminal one.
+        return await withAttemptContext({ attempt, maxRetries, maxFloodSleepMs }, fn)
       } catch (err) {
         const isGrammyErr = err instanceof GrammyError
         const msg = err instanceof Error ? err.message : String(err)
@@ -444,6 +683,11 @@ export function createRetryApiCall(
           : null
         if (benignKind !== null) {
           observer?.onBenign?.({ kind: benignKind })
+          // #4065 follow-up: a caller whose contract is "does the target
+          // message still exist?" opts out of the swallow, because a resolved
+          // promise is indistinguishable from a landed edit. Rethrow the
+          // ORIGINAL error so the caller classifies off `description`.
+          if (opts?.rethrowBenign400) throw err
           return undefined as unknown as T
         }
 
@@ -459,13 +703,12 @@ export function createRetryApiCall(
         }
 
         // Network-level transient errors — exponential backoff, bounded.
-        if (
-          !isGrammyErr &&
-          (msg.includes('ECONNRESET') ||
-            msg.includes('ETIMEDOUT') ||
-            msg.includes('fetch failed') ||
-            msg.includes('ENOTFOUND'))
-        ) {
+        // Classified by TYPE, not by message substring: grammy rewrites the
+        // message of every transport failure, so the old substring test never
+        // fired in production and this whole branch was dead (#4123). The
+        // `!isGrammyErr` guard that used to sit here now lives inside
+        // `isTransientTransportError`, which excludes GrammyError explicitly.
+        if (isTransientTransportError(err)) {
           if (attempt < maxRetries - 1) {
             const delayMs = Math.pow(2, attempt) * 1000
             log?.(

package/telegram-plugin/shared/bot-runtime.ts

@@ -5,8 +5,8 @@
  * standalone foreman bot before its retirement.
  *
  * What lives here:
- *   - `createRobustApiCall` — thin re-export of createRetryApiCall pre-wired
- *     with stderr logging (mirrors how gateway.ts constructs `robustApiCall`).
+ *   - `installTgPostLogger` / `installRichMarkdownGuard` — the grammy API
+ *     transformers every outbound call transits.
  *   - `makeSwitchroomExec` / `makeSwitchroomExecCombined` — factory fns for
  *     the switchroom CLI exec helpers (callers pass their own CLI path / config
  *     env so each process can be configured independently).
@@ -28,8 +28,7 @@ import { execFileSync, spawnSync } from 'child_process'
 import { createHash } from 'crypto'
 import { AsyncLocalStorage } from 'async_hooks'
 import { clearStaleTelegramPollingState } from '../startup-reset.js'
-import { createRetryApiCall, classifyBenignTelegram400 } from '../retry-api-call.js'
-import { makeFloodWaitRecorder, makeFloodWaitProbe } from '../flood-circuit-breaker.js'
+import { classifyBenignTelegram400, willRetryTelegramFailure } from '../retry-api-call.js'
 import { RICH_MESSAGE_MAX_CHARS } from '../format.js'
 import { shouldEmitTgPost, type TgPostStatus } from './gw-trace-gate.js'
 import { guardAccidentalFormatting } from '../rich-send.js'
@@ -102,7 +101,7 @@ function shortDesc(raw: string | undefined): string {
  *
  * Log shape (one line per POST, on both success and failure):
  *
- *   tg-post method=<m> chat=<id> thread=<id|-> parse_mode=<HTML|MarkdownV2|none> bytes=<n> hash=<sha1-12> status=<ok|benign|err> err=<class-or--> code=<http-or--> desc=<short|-->
+ *   tg-post method=<m> chat=<id> thread=<id|-> parse_mode=<HTML|MarkdownV2|none> bytes=<n> hash=<sha1-12> status=<ok|benign|retry|err> err=<class-or--> code=<http-or--> desc=<short|-->
  *
  * TRUTHFULNESS (#3927): grammY resolves the transformer chain with the raw
  * `ApiResponse`, and only converts `{ok:false}` into a thrown `GrammyError`
@@ -127,6 +126,17 @@ function shortDesc(raw: string | undefined): string {
  * instead, so promoting real rejections out of `status=ok` doesn't bury
  * `grep status=err` under high-volume card-repaint no-ops.
  *
+ * ATTEMPT vs OUTCOME (#3931): this transformer runs INSIDE `fn()`, i.e. below
+ * the retry policy, so it sees each attempt separately. A 429 that
+ * `createRetryApiCall` sleeps and then retries successfully is not a delivery
+ * failure — but it used to write `status=err`, and fleet-health's
+ * `reply-delivery-failure` detector (`src/fleet-health/detect.ts`) matches per
+ * LINE, so every retried-and-DELIVERED reply raised a severity-3 alert. The
+ * retry policy now publishes its attempt context (`_getTgAttemptContext`), and
+ * a failure the policy is about to retry is logged `status=retry` instead.
+ * `status=err` is therefore an OUTCOME: the logical send is over and nothing
+ * landed. Nothing is hidden — the retry lines are still emitted verbatim.
+ *
  * Body content is never logged — only its length and a 12-char sha1 prefix
  * so we can recognise repeated identical sends without leaking PII. The
  * `code` field carries the Telegram error_code (400/403/429/etc.) on
@@ -164,12 +174,27 @@ export function installTgPostLogger(bot: Bot): void {
       // converts `{ok:false}` to a thrown GrammyError only after this chain
       // returns (see the docblock). Same defensive shape as the edit fuse's
       // `runObserved` (edit-flood-fuse.ts).
-      const r = res as unknown as { ok?: boolean; error_code?: number; description?: string }
+      const r = res as unknown as {
+        ok?: boolean
+        error_code?: number
+        description?: string
+        parameters?: { retry_after?: number }
+      }
       if (r != null && typeof r === 'object' && r.ok === false) {
         const code = r.error_code
         const benign = classifyBenignTelegram400(code, r.description)
+        // #3931 — an attempt the enclosing retry policy is about to repeat is
+        // not an outcome; label it `retry` so it is never read as a delivery
+        // failure. Terminal rejections keep `status=err`.
+        const retrying =
+          benign === null &&
+          willRetryTelegramFailure({
+            errorCode: code ?? null,
+            retryAfterSec: r.parameters?.retry_after ?? null,
+            message: r.description ?? null,
+          })
         emit(
-          benign !== null ? 'benign' : 'err',
+          benign !== null ? 'benign' : retrying ? 'retry' : 'err',
           `telegram_${code ?? 'unknown'}`,
           code != null ? String(code) : '-',
           shortDesc(r.description),
@@ -186,7 +211,26 @@ export function installTgPostLogger(bot: Bot): void {
       const rawDesc = err instanceof GrammyError
         ? (err as GrammyError).description
         : (err instanceof Error ? err.message : '')
-      emit('err', errClass, code, shortDesc(rawDesc))
+      // #3931 — same attempt-vs-outcome rule as the resolved-rejection branch
+      // above. This branch carries the transport (`HttpError`) failures, whose
+      // transient members the retry policy backs off and repeats.
+      //
+      // #4123 — pass `cause`, not just `message`. grammy rewrites every
+      // transport failure's message to `Network request for '<method>' failed!`,
+      // so a message-only shape classified EVERY transport blip as terminal and
+      // wrote `status=err` — which fleet-health escalates as a
+      // `reply-delivery-failure` even when the next attempt delivers. The
+      // predicate needs the error itself to classify by type.
+      const retrying = willRetryTelegramFailure({
+        errorCode: err instanceof GrammyError ? (err as GrammyError).error_code : null,
+        retryAfterSec:
+          err instanceof GrammyError
+            ? ((err as GrammyError).parameters?.retry_after ?? null)
+            : null,
+        message: err instanceof Error ? err.message : String(err ?? ''),
+        cause: err,
+      })
+      emit(retrying ? 'retry' : 'err', errClass, code, shortDesc(rawDesc))
       throw err
     }
   })
@@ -248,32 +292,23 @@ export function installRichMarkdownGuard(bot: Bot): void {
   })
 }
 
-// ─── robustApiCall factory ────────────────────────────────────────────────
-
-/**
- * Creates a robust API call wrapper pre-wired with stderr logging.
- * This is exactly how gateway.ts constructs its `robustApiCall`.
- *
- * Usage:
- *   const robustApiCall = createRobustApiCall()
- */
-export function createRobustApiCall(opts: { floodStatePath?: string } = {}) {
-  return createRetryApiCall({
-    log: (line) => process.stderr.write(line),
-    // #2923: persist every observed 429 flood-wait window so a restart during
-    // the ban can suppress non-essential sends (boot cards) instead of feeding
-    // the per-bot flood counter and prolonging the ban.
-    // #3084: and refuse to issue a call while that window is still open —
-    // otherwise the card heartbeats re-drive a request into the ban every
-    // few seconds once the policy stops sleeping long waits.
-    ...(opts.floodStatePath
-      ? {
-          onFloodWait: makeFloodWaitRecorder(opts.floodStatePath),
-          floodWaitRemainingMs: makeFloodWaitProbe(opts.floodStatePath),
-        }
-      : {}),
-  })
-}
+// ─── robustApiCall factory: REMOVED (#3863) ───────────────────────────────
+//
+// `createRobustApiCall` used to live here as a second `createRetryApiCall`
+// wiring "pre-wired with stderr logging", kept for a standalone foreman bot
+// that was retired. It had ZERO production callers, and its breaker hooks were
+// OPTIONAL (`floodStatePath` defaulted to absent), so the one thing a second
+// wiring must never be — a send path blind to the shared flood window — was
+// its default. Docblocks in `flood-circuit-breaker.ts` and `flood-429-ledger.ts`
+// cited it as a live wiring, which is how a dead function became load-bearing
+// documentation for the breaker's completeness argument.
+//
+// There is one retry wiring per process, built at its own callsite with both
+// breaker hooks (`gateway.ts`'s `robustApiCall`, `outbox-sweep.ts`'s sweep
+// caller) and enforced by `scripts/check-retry-flood-hooks.mjs`. Do not
+// reintroduce a convenience factory here: it re-creates the hook-optional
+// default this deletion removed. `tests/no-robust-api-call-factory.test.ts`
+// guards that.
 
 // ─── Markdown escape helpers (#2669) ──────────────────────────────────────
 

package/telegram-plugin/shared/gw-trace-gate.ts

@@ -68,8 +68,18 @@ const ZERO_SIGNAL_POLL_METHODS = new Set(['getUpdates', 'getMe'])
  * `status=err` signal in high-volume card-repaint no-ops: `grep status=err`
  * stays a list of real failures, and the benign lines are still there,
  * honestly labelled, for anyone who wants them.
+ *
+ * `retry` (#3931) is a REJECTED call that the enclosing retry policy is about
+ * to attempt AGAIN — a 429 under the in-process sleep ceiling, or a transient
+ * transport error with attempts left. It is not an outcome: the logical send
+ * may still be delivered by the next attempt, and usually is. It exists
+ * because `status=err` is consumed as a delivery FAILURE by fleet-health's
+ * `reply-delivery-failure` detector (`src/fleet-health/detect.ts`), which
+ * matches per LINE — so labelling a survivable attempt `err` escalated
+ * successfully delivered replies. Post-#3931 a `status=err` line means the
+ * logical send is over and it did not land.
  */
-export type TgPostStatus = 'ok' | 'benign' | 'err'
+export type TgPostStatus = 'ok' | 'benign' | 'err' | 'retry'
 
 /**
  * Decide whether a `tg-post` line should be written.
@@ -79,6 +89,8 @@ export type TgPostStatus = 'ok' | 'benign' | 'err'
  *
  * Always emitted:
  *   - any `status=err` (real failures — the whole point of the log),
+ *   - any `status=retry` (the survivable attempts behind a flood episode —
+ *     suppressing them would hide the run-up to a ban, #3931),
  *   - every other method's `ok` line (sendMessage/editMessageText/... are
  *     genuine outbound observability, #656/#657),
  *   - `status=benign` on any non-poll method — same volume as the
@@ -90,7 +102,7 @@ export function shouldEmitTgPost(
   verbose: boolean = gwTraceVerbose,
 ): boolean {
   if (verbose) return true
-  if (status === 'err') return true
+  if (status === 'err' || status === 'retry') return true
   return !ZERO_SIGNAL_POLL_METHODS.has(method)
 }
 

package/telegram-plugin/silent-end.ts

@@ -66,6 +66,22 @@ export interface SilentEndState {
    * absent for the zero-prose (genuinely empty) silent-end.
    */
   pendingText?: string
+  /**
+   * W1-d follow-up (#4141) — one of THIS turn's reply-tool calls came back
+   * `is_error: true`, so `pendingText` is prose the model wrote *after* its
+   * delivery attempt failed, not text it chose to send as the answer.
+   *
+   * Stamped by the Stop hook's single-writer election
+   * (`silent-end-interrupt-stop.mjs` → `buildNextState`), the only component
+   * that can see the transcript; the gateway's own `writeSilentEndState` never
+   * sets it. Consumed by `deliverCapturedProse`, which prefixes a one-line
+   * provenance banner so the recipient is told the text was scraped off a
+   * failed turn rather than sent as an answer.
+   *
+   * Optional and only ever `true`: absent means "no positive evidence of a
+   * throw", which delivers exactly as before. It never suppresses.
+   */
+  replyToolThrewThisTurn?: boolean
 }
 
 export interface SilentEndDeps {
@@ -320,8 +336,16 @@ export const CAPTURED_PROSE_MIN_CHARS = 200
 export interface CapturedProseDecision {
   /** True → the gateway should deliver `text` directly on this silent-end. */
   deliver: boolean
-  /** The prose to deliver; present iff `deliver === true`. */
+  /** The prose to deliver; present iff `deliver === true`. RAW — never framed
+   *  here, so the caller's dedup and journal keys stay computed on the same
+   *  bytes the sweep path uses (#4141). */
   text?: string
+  /**
+   * #4141 — the persisted record says this turn's reply tool threw. Surfaced
+   * (not acted on) so `deliverCapturedProse` can state the prose's provenance.
+   * `true` only on positive evidence; never affects `deliver`.
+   */
+  replyToolThrewThisTurn?: boolean
   /** Machine-readable reason (for logs / tests). */
   reason:
     | 'captured-prose'
@@ -394,7 +418,15 @@ export function decideCapturedProseDelivery(
   if (deps?.backstopDeliveredNonceHit?.(state.turnId) === true) {
     return { deliver: false, reason: 'already-delivered' }
   }
-  return { deliver: true, text, reason: 'captured-prose' }
+  return {
+    deliver: true,
+    text,
+    reason: 'captured-prose',
+    // #4141: pass the hook's raw signal through untouched. Deliberately placed
+    // AFTER every `deliver:false` gate above — the banner is a labelling
+    // concern, and must never become an input to whether we deliver at all.
+    ...(state.replyToolThrewThisTurn === true ? { replyToolThrewThisTurn: true } : {}),
+  }
 }
 
 /**

package/telegram-plugin/status-no-truncate.ts

@@ -60,23 +60,38 @@ export const STATUS_LINE_MAX = 200
 export const STATUS_CARD_CHAR_BUDGET = RICH_MESSAGE_MAX_CHARS
 
 /**
- * Indent marker for a nested (foreground sub-agent) step line.
- *
- * NOTE: the three leading spaces here are ASCII, which Telegram's server-side
- * markdown parser DROPS — the visible nesting cue on this surface is the `↳`
- * glyph, not the indent. That is a known latent wart, deliberately left alone
- * — it needs its own live render check on the single-worker / agent card,
- * tracked in #3668. It is NOT broken by the #3662 mechanism (nothing here
- * depends on the ASCII run being visible), so it is not fixed here. If #3668
- * ever does make it a real indent, use U+2800 — NOT U+00A0, which was tried in
- * #3662 and renders flat. Do NOT copy this string as the idiom for a real
- * indent.
- *
- * @see WORKER_STEP_INDENT — the U+2800 indent used for actual left-nesting on
- * the combined (2+ worker) card, and the evidence for why ASCII (and U+00A0)
- * cannot work.
+ * Indent marker for a nested (foreground sub-agent) step line: three U+2800
+ * BRAILLE PATTERN BLANK, then the `↳` glyph.
+ *
+ * Until #3668 the indent was three ASCII spaces, which Telegram's server-side
+ * markdown parser LEFT-TRIMS off a content line — so the child block rendered
+ * FLAT against its parent steps and the only surviving nesting cue was the `↳`
+ * glyph. This constant now uses the SAME idiom as {@link WORKER_STEP_INDENT}
+ * for exactly the same reason, and there is only one indent idiom on the card
+ * surfaces again.
+ *
+ * The evidence is not an inference about Telegram's parser: U+2800 is general
+ * category So (Symbol, other), not Zs, so no whitespace-trimming rule — ASCII
+ * `\s`, Unicode `White_Space`, or `Zs` — can classify it as whitespace, yet it
+ * renders as blank width. It is ALSO live-verified: on 2026-07-26 four
+ * candidates were sent to a real phone in one message — three U+00A0 rendered
+ * FLAT (the #3662 failure), three U+2800 INDENTED CORRECTLY, a leading `↳ `
+ * rendered as visible ink, and a leading `· ` was promoted into a real list
+ * bullet. See {@link WORKER_STEP_INDENT}'s comment for the full record.
+ *
+ * The `↳ ` glyph is KEPT: it is what distinguishes a foreground sub-agent's
+ * step from the parent's own steps on a card where both appear, and the
+ * indent is what makes the block read as nested. The blanks lead so the
+ * glyph is not the thing being trimmed.
+ *
+ * Guard: `nested-prefix-indent.test.ts` asserts the PROPERTY (no character of
+ * the indent is whitespace under any of the three rules), not just the bytes —
+ * a byte-only assertion is what let #3662 ship green and inert.
+ *
+ * @see WORKER_STEP_INDENT — the same U+2800 idiom for the combined
+ * (2+ worker) card, and the full evidence for why ASCII and U+00A0 cannot work.
  */
-export const NESTED_PREFIX = '   ↳ '
+export const NESTED_PREFIX = '\u2800\u2800\u2800↳ '
 
 /**
  * Left indent for a step line rendered UNDER a worker header on the combined