switchroom 0.19.17 → 0.19.18

package/telegram-plugin/gateway/liveness-wiring.ts

@@ -38,10 +38,12 @@ export function buildSilencePokeOptions(deps: LivenessWiringDeps): Parameters<ty
     SILENCE_FALLBACK_HARD_MS,
     SILENCE_FLOOR_MS,
     SILENCE_DEFER_INFLIGHT_TOOLS,
+    isObligationOpenForTurn,
     TURN_PREVIEW_MAX,
     STATE_DIR,
     isLegitimatelyWorking,
     getCurrentTurn,
+    getCurrentTurnForKey,
     getInFlightUpdate,
     getTurnsDb,
     getInboundSpool,
@@ -68,6 +70,23 @@ export function buildSilencePokeOptions(deps: LivenessWiringDeps): Parameters<ty
   thresholdsMs: { fallback: SILENCE_FALLBACK_MS, fallbackHardCeiling: SILENCE_FALLBACK_HARD_MS, floor: SILENCE_FLOOR_MS },
   deferFallbackWhileToolInFlight: SILENCE_DEFER_INFLIGHT_TOOLS,
   isLegitimatelyWorking: (key) => isLegitimatelyWorking(key),
+  // #3552 — orphan-state reaper predicate. Deliberately the SAME condition the
+  // `onFrameworkFallback` late-fire guard below uses: no `activeTurnStartedAt`
+  // entry AND no current turn ⇒ the turn this state belongs to is over. Note
+  // `activeTurnStartedAt` alone is NOT sufficient — `releaseTurnBufferGate`
+  // clears it mid-turn on every final-answer reply while the turn keeps running
+  // (post-answer housekeeping), and silence-poke must keep watching that turn.
+  //
+  // #3580 — the second clause is read KEYED (`getCurrentTurnForKey(key)`, i.e.
+  // `currentTurnMap.get(key)`), never the bare `getCurrentTurn()` singleton.
+  // Under `SWITCHROOM_EMISSION_AUTHORITY=1` that singleton is a MOST-RECENT-SET
+  // MIRROR, so an unkeyed read answers about whichever topic started a turn last
+  // — not about `key`. Combined with the buffer-gate release above, topic A
+  // (live, gate cleared mid-turn) read DEAD as soon as topic B started and ended
+  // a turn, and the tick false-reaped A: permanent and silent, nothing re-arms
+  // until the next `startTurn`, so A loses both its mid-turn beat and its 300 s
+  // #1122 unwedge. Flag-OFF the keyed read IS the singleton, byte-for-byte.
+  isTurnLive: (key) => !(activeTurnStartedAt.get(key) == null && getCurrentTurnForKey(key) == null),
   emitMetric: (event) => {
     // Re-emit through the unified runtime-metrics fan-out (PostHog + JSONL).
     emitRuntimeMetric(event)
@@ -96,8 +115,10 @@ export function buildSilencePokeOptions(deps: LivenessWiringDeps): Parameters<ty
   // retired in #2667.
   onMidTurnFloor: async (ctx) => {
     // Late-fire guard, mirroring the fallback: a clean turn-end can race the
-    // tick. If the turn is gone, stay silent.
-    if (activeTurnStartedAt.get(ctx.key) == null && getCurrentTurn() == null) return
+    // tick. If the turn is gone, stay silent. #3580 — keyed read; an unkeyed
+    // `getCurrentTurn()` here answers about another topic's turn (see the
+    // `isTurnLive` note above) and silences a live turn's approval re-ping.
+    if (activeTurnStartedAt.get(ctx.key) == null && getCurrentTurnForKey(ctx.key) == null) return
     const blockedOnApproval = activeStatusReactions
       .get(statusKey(ctx.chatId, ctx.threadId))
       ?.isAwaiting() ?? false
@@ -142,7 +163,18 @@ export function buildSilencePokeOptions(deps: LivenessWiringDeps): Parameters<ty
     // events (124 of 138 `currentTurn_nulled=false` cases). Distinct
     // log line so observability still tracks the fact that the silence
     // crossed threshold; the wedge counter is no longer polluted.
-    if (activeTurnStartedAt.get(ctx.key) == null && getCurrentTurn() == null) {
+    //
+    // #3552: with the `isTurnLive` reaper wired above, this branch is now only
+    // the sub-poll-interval race (the turn ends between the tick's reap check
+    // and this handler running). The steady-state case it used to absorb —
+    // state armed for the full 300s against a turn that ended minutes earlier,
+    // 504 events in 14 days vs 110 real fires — is reaped at the tick instead,
+    // so this counter finally reads as the genuine race it names.
+    //
+    // #3580 — keyed read, same reason as `isTurnLive`: unkeyed, a sibling
+    // topic's turn flip made this guard skip the 300 s unwedge for a genuinely
+    // wedged turn (the #1122 permanent-wedge class) and log it as a clean race.
+    if (activeTurnStartedAt.get(ctx.key) == null && getCurrentTurnForKey(ctx.key) == null) {
       process.stderr.write(
         `telegram gateway: silence-poke framework-fallback late-fire skipped — ` +
         `turn ended cleanly during silence window ` +
@@ -259,6 +291,21 @@ export function buildSilencePokeOptions(deps: LivenessWiringDeps): Parameters<ty
     // stays null and we skip the send. The turn-teardown below (the unwedge —
     // the one job the live draft can't do, per the conversational-pacing RFC)
     // still runs unconditionally.
+    // #3551 L1/L2 — the teardown-notice suppression signal. It must be narrower
+    // than "some text went out", and it must mean DELIVERED.
+    //
+    // L1: `text` has two sources with opposite meanings. The approval re-ping is
+    // a LOUD, user-addressed message that explains why nothing is happening and
+    // asks the user to act — stacking a teardown notice on it is noise. The
+    // deterministic `formatUpdateStatusLine` is a SILENT status surface about an
+    // unrelated in-flight `update_apply`; it says nothing about this turn ending.
+    // A user who asked a question during an update and got killed anyway is
+    // EXACTLY the #3551 case, so that line must not suppress the notice.
+    //
+    // L2: computed from send SUCCESS, not from `text != null`. The send sits in a
+    // try/catch that only logs, so a throwing approval re-ping would otherwise
+    // suppress the notice while the user received nothing at all.
+    let userAddressedTextDelivered = false
     if (text != null) {
       try {
         // Conditional: when the turn is parked on an approval card, this
@@ -268,6 +315,7 @@ export function buildSilencePokeOptions(deps: LivenessWiringDeps): Parameters<ty
         // line stays SILENT (a status surface). Gate on `blockedOnApproval`.
         // The send stays in gateway.ts behind the bot-api retry policy.
         await sendSilenceText(ctx.chatId, ctx.threadId ?? null, text, blockedOnApproval ? false : true)
+        userAddressedTextDelivered = blockedOnApproval
       } catch (err) {
         process.stderr.write(
           `silence-poke fallback sendMessage failed chat=${ctx.chatId} thread=${ctx.threadId}: ${err}\n`,
@@ -299,6 +347,18 @@ export function buildSilencePokeOptions(deps: LivenessWiringDeps): Parameters<ty
       wedgedTurn != null &&
       wedgedTurn.sessionChatId === fbChatId &&
       wedgedTurn.sessionThreadId === fbThreadId
+    // #3580 L3 — the ONE liveness decision this fire makes about the wedged
+    // turn. It gates the teardown below AND is what `decideFallbackTeardownNotice`
+    // is told as `tearsDownLiveTurn`, so the notice can never claim a teardown
+    // the gate skipped (previously the notice got the looser bare
+    // `turnMatchesFallback`, and a user could be told "the framework ended that
+    // stalled turn" about a turn that never ended). Evaluated HERE — before the
+    // `purgeChatStale` self-heal and before `endCurrentTurnForKey`, both of
+    // which drop the very entry `turnLiveForItsTopic` reads — so it answers
+    // "was this turn live when the fallback fired", the question both the
+    // teardown and the notice actually mean.
+    const tearsDownLiveTurn =
+      turnMatchesFallback && wedgedTurn != null && turnLiveForItsTopic(wedgedTurn)
     const turnStartedAt = activeTurnStartedAt.get(fbKey)
     if (turnStartedAt != null) {
       const turnDurationMs = Date.now() - turnStartedAt
@@ -424,7 +484,14 @@ export function buildSilencePokeOptions(deps: LivenessWiringDeps): Parameters<ty
     // Flag-ON: `byKey.get(fbKey) === wedgedTurn`, so the keyed delete still
     // fires when the LIVE mirror has already flipped to another topic B (a bare
     // `currentTurn === wedgedTurn` would falsely skip and leak A's byKey entry).
-    if (turnMatchesFallback && wedgedTurn != null && turnLiveForItsTopic(wedgedTurn)) {
+    //
+    // #3580 L3 — the gate is the `tearsDownLiveTurn` const computed above, the
+    // SAME value handed to `decideFallbackTeardownNotice`. Do not re-inline the
+    // condition here: it must be evaluated before the `purgeChatStale` sweep
+    // above (which can drop this key's entry and so flip
+    // `turnLiveForItsTopic`), and the notice must speak for exactly what this
+    // gate did.
+    if (tearsDownLiveTurn && wedgedTurn != null) {
       // Status-surface observability: emit the lifecycle CLEAR for the
       // silence-poke teardown so a fallback-nulled turn has a turn-lifecycle
       // line like every other clear path (the framework-fallback line below is
@@ -439,6 +506,56 @@ export function buildSilencePokeOptions(deps: LivenessWiringDeps): Parameters<ty
       // the mirror iff it still points here — a live sibling topic is untouched.
       endCurrentTurnForKey(wedgedTurn, fbKey)
     }
+    // #3551 — the teardown NOTICE. Everything above just ENDED a live turn. On
+    // the non-approval branch `text` is null (`formatFrameworkFallbackText`
+    // returns a string only when parked on an approval card), so before this
+    // the user saw literally nothing: N minutes of silence, then the agent
+    // apparently starting the same request over. Killing a user's turn with no
+    // user-visible signal is a correctness bug, not a tuning knob — 110 fires /
+    // 14 days, ~9h of dead air (#3551). The teardown itself is NOT removed (it
+    // is the #1122 unwedge, the fallback's one load-bearing job); it is made
+    // observable. Sent AFTER the teardown so the notice cannot itself be
+    // clobbered by the state purge, and gated by `decideFallbackTeardownNotice`
+    // so it can only ever speak for a human-waiting, undelivered turn that this
+    // fire actually killed — at most once, since `endTurn(fbKey)` above dropped
+    // the key.
+    const teardownNotice = silencePoke.decideFallbackTeardownNotice({
+      tearsDownLiveTurn,
+      role: wedgedTurn?.role ?? null,
+      finalAnswerDelivered: wedgedTurn?.finalAnswerDelivered ?? false,
+      userAddressedTextDelivered,
+      // #3575 review B1 — this must be a question about THIS turn, not a static
+      // env flag. `OBLIGATION_LEDGER_ENABLED` is true for the whole process, so
+      // it promised "being re-asked now" in cases where no re-ask can ever
+      // happen: the obligation already hit OBLIGATION_REPRESENT_MAX (2) and
+      // escalated+closed, or it was closed silently by an outbound-since-open
+      // (obligation-wiring.ts:230/:294), or the inbound never opened one at all
+      // (synthetic / steering / interrupt). The user was then told to wait for a
+      // re-ask that never comes. Ask the ledger about this turn's own origin id
+      // instead, so the honest "please re-send it" branch fires when there is no
+      // open obligation. The lookup is deliberately made HERE, after the
+      // teardown above (which does not touch the ledger), so it reflects the
+      // ledger state at the moment we speak.
+      representWillFollow: isObligationOpenForTurn(wedgedTurn?.turnId ?? null),
+      silenceMs: ctx.silenceMs,
+    })
+    if (teardownNotice.send) {
+      try {
+        // LOUD (disable_notification: false). The user asked a question and is
+        // being told their attempt was killed — that is exactly the case where
+        // silence would train them to distrust the channel.
+        await sendSilenceText(fbChatId, ctx.threadId ?? null, teardownNotice.text, false)
+      } catch (err) {
+        process.stderr.write(
+          `silence-poke teardown notice sendMessage failed chat=${fbChatId} thread=${ctx.threadId}: ${err}\n`,
+        )
+      }
+    } else {
+      process.stderr.write(
+        `telegram gateway: silence-poke teardown notice skipped reason=${teardownNotice.reason} ` +
+        `chat=${fbChatId} thread=${ctx.threadId ?? '-'}\n`,
+      )
+    }
     // Best-effort: clear any pending silent-end marker so the Stop hook
     // doesn't double-block when claude eventually exits the wedged turn.
     try {
@@ -464,7 +581,10 @@ export function buildSilencePokeOptions(deps: LivenessWiringDeps): Parameters<ty
     process.stderr.write(
       `telegram gateway: silence-poke framework-fallback ended wedged turn ` +
       `chat=${fbChatId} thread=${ctx.threadId ?? '-'} silence_ms=${ctx.silenceMs} ` +
-      `currentTurn_nulled=${turnMatchesFallback} ` +
+      // #3580 L3 — report the gate that actually ran, not the looser
+      // chat/thread match: `turnMatchesFallback` was true in cases where the
+      // keyed liveness check skipped the teardown, so the field lied.
+      `currentTurn_nulled=${tearsDownLiveTurn} ` +
       `drained_buffered=${fbRedeliver.redelivered}/${fbRedeliver.drained}` +
       `${fbRedeliver.rebuffered > 0 ? ` rebuffered=${fbRedeliver.rebuffered}` : ''}` +
       `${fbExtraPurge.purged.length > 0 ? ` extra_keys_purged=${fbExtraPurge.purged.length}` : ''}\n`,

package/telegram-plugin/gateway/obligation-ledger.ts

@@ -242,6 +242,10 @@ export class ObligationLedger {
    * since `lastRepresentedAt`. Without this, the 5s sweep can fire again before
    * the re-presented turn even reaches the agent, burning the represent budget
    * immediately and producing back-to-back escalations on the same message.
+   * #3550: this window is retired early once a turn that started after the
+   * re-present has ENDED — at that point the trailing-answer grace above is the
+   * one covering the real risk, and holding the rest of the represent window is
+   * pure latency. See `representGraceStillProtecting`.
    */
   decideAtIdle(opts?: {
     now: number
@@ -267,6 +271,80 @@ export class ObligationLedger {
     return { action: 'represent', obligation: o }
   }
 
+  /**
+   * #3550 — is the per-represent grace still doing real work for this
+   * obligation? Pure so the rule is testable in isolation.
+   *
+   * The per-represent grace exists for ONE reason, stated at its definition:
+   * "the 5s sweep can fire again before the re-presented turn even reaches the
+   * agent". It is a proxy for "the re-present has not landed yet". Once a turn
+   * that started AFTER the re-present has ENDED, the proxy is spent — the
+   * re-present demonstrably reached the agent, the agent ran a whole turn on
+   * it, and that turn is over. Holding the remaining ~110s of a 120s window
+   * open at that point delays the next rung of a ladder that is already
+   * bounded by `maxRepresents`, for no protective benefit.
+   *
+   * The obligation is NOT released into the void when this returns false: the
+   * trailing-answer grace (`trailingGraceMs` from `lastTurnEndedAt`, 45s by
+   * default) is a separate, independently-evaluated window that covers exactly
+   * the risk that matters here — a slow answer still landing after its turn
+   * ended. So the early-out only applies when that grace is actually armed
+   * (`trailingGraceMs > 0`); with it disabled, the represent grace stays the
+   * full window rather than leaving the obligation ungated.
+   *
+   * A re-present with NO ended turn after it — the genuinely in-flight case
+   * the grace was written for — is untouched and keeps the full window.
+   *
+   * FLOOR (`MIN_REPRESENT_INTERVAL_MS`). Retiring the represent window makes the
+   * rung interval DERIVED — `trailingGraceMs + the represent-turn's duration`
+   * — where it used to be floored by the flat 120s window regardless of how the
+   * trailing grace was tuned. Without a floor,
+   * `SWITCHROOM_OBLIGATION_ESCALATE_GRACE_MS=1000` (a plausible "make it
+   * snappier" tune) collapses the whole ladder to ~1s+d per rung and escalates
+   * to the operator within ~15-20s of the original message. That is a config
+   * footgun the old code could not have, so it is closed by a MECHANISM, not a
+   * doc warning: the early-out is withheld until at least
+   * `min(representGraceMs, max(trailingGraceMs, MIN_REPRESENT_INTERVAL_MS))`
+   * has elapsed since the re-present. At the defaults (45s trailing / 120s
+   * represent) the floor is 45s and never binds — the trailing grace, measured
+   * from the strictly-later turn end, always expires after it — so this changes
+   * nothing in the shipped configuration.
+   */
+  static representGraceStillProtecting(
+    o: Pick<Obligation, 'lastRepresentedAt' | 'lastTurnEndedAt'>,
+    now: number,
+    representGraceMs: number,
+    trailingGraceMs: number,
+  ): boolean {
+    if (representGraceMs <= 0) return false
+    if (o.lastRepresentedAt == null) return false
+    const sinceRepresent = now - o.lastRepresentedAt
+    if (sinceRepresent >= representGraceMs) return false
+    // Inside the window. Is it still protecting anything?
+    const representTurnEnded =
+      o.lastTurnEndedAt != null && o.lastTurnEndedAt > o.lastRepresentedAt
+    if (representTurnEnded && trailingGraceMs > 0) {
+      // Spent — the trailing grace takes over, but never sooner than the floor
+      // (and never longer than the represent window the operator configured).
+      const floorMs = Math.min(
+        representGraceMs,
+        Math.max(trailingGraceMs, ObligationLedger.MIN_REPRESENT_INTERVAL_MS),
+      )
+      return sinceRepresent < floorMs
+    }
+    return true
+  }
+
+  /**
+   * The hard floor on the interval between two rungs of the represent ladder,
+   * used by `representGraceStillProtecting`. Independent of how the trailing
+   * grace is tuned, an obligation is never acted on again within this long of
+   * its own re-present. 30s comfortably exceeds the 5s sweep tick and the
+   * round-trip for a re-presented turn to reach the agent and answer, which is
+   * the only thing the represent window was ever debouncing.
+   */
+  static readonly MIN_REPRESENT_INTERVAL_MS = 30_000
+
   /** The oldest open obligation that is currently ELIGIBLE to act on — i.e. NOT
    *  within any grace window:
    *   - trailing-answer grace: its handling turn ended < `graceMs` ago (a queued
@@ -275,9 +353,11 @@ export class ObligationLedger {
    *   - background-work grace: when `backgroundWorkActive`, it was opened <
    *     `backgroundGraceMs` ago (genuine in-flight autonomous work — bounded by
    *     the ceiling so a stale/leaked worker can't suppress escalation forever);
-   *   - per-represent grace: it was re-presented < `representGraceMs` ago (prevents
-   *     a 5s sweep tick from immediately firing again on the same obligation before
-   *     the re-presented turn even reaches the agent). */
+   *   - per-represent grace: it was re-presented < `representGraceMs` ago AND that
+   *     re-present has not yet produced a completed turn (prevents a 5s sweep tick
+   *     from immediately firing again before the re-presented turn even reaches the
+   *     agent — see `representGraceStillProtecting` for why a turn that has already
+   *     ended retires this window early, #3550). */
   private oldestEligible(
     now: number,
     graceMs: number,
@@ -290,7 +370,7 @@ export class ObligationLedger {
       if (o.lastTurnEndedAt != null && now - o.lastTurnEndedAt < graceMs) continue // trailing-answer grace
       if (backgroundWorkActive && backgroundGraceMs > 0 && now - o.openedAt < backgroundGraceMs)
         continue // in-flight autonomous work, bounded by the ceiling
-      if (representGraceMs > 0 && o.lastRepresentedAt != null && now - o.lastRepresentedAt < representGraceMs)
+      if (ObligationLedger.representGraceStillProtecting(o, now, representGraceMs, graceMs))
         continue // per-represent grace: sweep fired before re-presented turn landed
       if (best === undefined || o.openedAt < best.openedAt) best = o
     }

package/telegram-plugin/gateway/resume-inbound-builder.ts

@@ -458,9 +458,13 @@ export function buildResumeDeferredReportInbound(
  * the gateway calls this with the classified `ended_via` so the
  * report-vs-resume policy lives in one testable place.
  *
- *   - 'timeout'                         → 'report'  (watchdog kill)
- *   - 'restart' | 'sigterm' | 'unknown' → 'resume'  (clean interrupt)
- *   - 'stop'                            → null      (finished; nothing to do)
+ *   - 'timeout'                                → 'report'  (watchdog kill)
+ *   - 'restart'|'reaped_stale'|'sigterm'|'unknown' → 'resume' (clean interrupt)
+ *   - 'stop'                                   → null      (finished; nothing to do)
+ *
+ * #3555: `'reaped_stale'` (mid-session stale-row sweep) is treated exactly
+ * like `'restart'` here — it is the same clean interrupt, just no longer
+ * mislabelled as a restart in the data.
  */
 export function selectResumeBuilder(
   endedVia: TurnEndedVia | null,
@@ -473,7 +477,12 @@ export function selectResumeBuilder(
 ): 'resume' | 'report' | null {
   let kind: 'resume' | 'report' | null
   if (endedVia === 'timeout') kind = 'report'
-  else if (endedVia === 'restart' || endedVia === 'sigterm' || endedVia === 'unknown') kind = 'resume'
+  else if (
+    endedVia === 'restart' ||
+    endedVia === 'reaped_stale' ||
+    endedVia === 'sigterm' ||
+    endedVia === 'unknown'
+  ) kind = 'resume'
   else if (endedVia == null) kind = 'resume' // still-open at boot = killed mid-flight
   else kind = null
   if (

package/telegram-plugin/gateway/stream-render.ts

@@ -372,10 +372,6 @@ export function handleSessionEvent(deps: StreamRenderDeps, ev: SessionEvent): vo
         // `activityMessageId` + `activityEverOpened` so `renderActivityFeed`
         // EDITS the existing card instead of opening a second one, and so the
         // turn's own end-of-turn `clearActivitySummary` finalizes it (lever 3).
-        // The handback turn also gets the turn-long typing loop it never had —
-        // whether or not a card was painted (the debounce may not have fired) —
-        // stopped by the canonical turn-end (`purgeReactionTracking →
-        // stopTurnTypingLoop`).
         if (HANDBACK_PRETURN_ENABLED) {
           const handbackAdoption = handbackPreturnSignal.tryAdopt(turnId)
           if (handbackAdoption != null) {
@@ -383,9 +379,32 @@ export function handleSessionEvent(deps: StreamRenderDeps, ev: SessionEvent): vo
               next.activityMessageId = handbackAdoption.activityMessageId
               next.activityEverOpened = true
             }
-            startTurnTypingLoop(ev.chatId, enqThreadIdNum ?? null)
+            // Observability (#3544): adoption is the rare, previously-silent
+            // branch — one line per adopted handback turn, not per turn.
+            process.stderr.write(
+              `telegram gateway: handback pre-turn adopted turnId=${turnId} ` +
+                `key=${handbackAdoption.statusKey} card=${handbackAdoption.activityMessageId ?? 'none'}\n`,
+            )
           }
         }
+        // #3544 — arm the turn-long `typing…` loop for EVERY minted turn,
+        // unconditionally. It used to hang off the handback ADOPTION above,
+        // which misses whenever the pre-turn entry was deduped (parallel
+        // workers on one topic), had no derivable turn id, or was already
+        // reaped — and the whole compose window went dark. Only the real-inbound
+        // path (`turn-start-surfaces.ts`) armed a loop, so a synthetic turn
+        // (handback / cron / wake) could have none at all. Unconditional is safe
+        // and costs nothing extra on the wire:
+        //   - `turnTypingLoop.start` is restart-safe (stops any prior loop on
+        //     the key first, so a real inbound's loop is replaced, not doubled);
+        //   - every send goes through the SHARED per-chat-key emitter floor
+        //     (`typing-emitter.ts`, TYPING_FLOOR_MS) so N arms on one chat still
+        //     cost at most one chat action per floor window — the 2026-07-11
+        //     flood-ban guard is what makes arming more loops free;
+        //   - `turn-end.ts` (`purgeReactionTracking → stopTurnTypingLoop`) is
+        //     already the single stop-owner for ALL turns, and a start is
+        //     self-healing anyway, so this cannot leak an interval.
+        startTurnTypingLoop(ev.chatId, enqThreadIdNum ?? null)
         // PR-4e — route the turn-SET through the keyed accessor: flag-OFF assigns
         // the singleton (byte-identical to `currentTurn = next`); flag-ON sets the
         // per-topic `byKey[statusKey]` entry AND the most-recent mirror. The key is