switchroom 0.19.11 → 0.19.13

package/telegram-plugin/gateway/stream-render.ts

@@ -55,7 +55,8 @@
 
 import { createAnswerStream } from '../answer-stream.js'
 import { LivenessTracker, isContextExhaustionText } from '../context-exhaustion.js'
-import { normalizeParagraphBreaks, normalizePunctuation, repairEscapedWhitespace, stripExcessBold } from '../format.js'
+import { normalizeOutboundBody } from './outbound-send-path.js'
+import { resolveEnvTimezone } from '../shared/local-time.js'
 import { hasOutboundDeliveredSince, recordOutbound } from '../history.js'
 import { isReplyTool } from '../narrative-dedup.js'
 import { NarrativeFlushController } from '../narrative-flush.js'
@@ -65,7 +66,6 @@ import { richMessage } from '../rich-send.js'
 import { emitRuntimeMetric } from '../runtime-metrics.js'
 import { CAPTURED_PROSE_MIN_CHARS, clearSilentEndState, decideCapturedProseDelivery, recordUndeliveredTurnEnd, silentEndFallbackText, writeSilentEndState } from '../silent-end.js'
 import { logStreamingEvent } from '../streaming-metrics.js'
-import { scrubVoice } from '../text-voice-scrub.js'
 import { appendActivityLabel } from '../tool-activity-summary.js'
 import { isTelegramReplyTool, isTelegramSurfaceTool } from '../tool-names.js'
 import { decideTurnFlush } from '../turn-flush-safety.js'
@@ -1523,19 +1523,6 @@ export function handleSessionEvent(deps: StreamRenderDeps, ev: SessionEvent): vo
       }
 
       if (turnEndDecision === 'flush' && flushDecision.kind === 'flush') {
-        let capturedText = flushDecision.text
-        // #2798 — turn-flush delivers the model's terminal prose when it
-        // skipped reply/stream_reply, but historically bypassed the reply
-        // path's markdown normalization entirely. Mirror executeReply's front
-        // of pipeline here so the backstop renders identically: repair LLM
-        // JSON-escape bungles (literal `\n`), then promote lone prose paragraph
-        // breaks into GFM hard breaks so the Bot API 10.1 rich path doesn't
-        // collapse them (lists/tables/code left untouched). Runs BEFORE the
-        // redact/scrub below, exactly as reply orders it (repair → normalize →
-        // redact → scrub), so masking sees the repaired text. Paragraph gaps
-        // are the plain `\n\n` normalizeParagraphBreaks guarantees — no spacer
-        // pass runs on the send side any more (removed in the #2669 follow-up).
-        capturedText = normalizeParagraphBreaks(repairEscapedWhitespace(capturedText))
         // Component 3 — origin-thread backstop. `chatId`/`threadId` are
         // captured from the turn atom (turn.sessionChatId/sessionThreadId)
         // at the top of this turn_end handler, NOT from the live
@@ -1547,41 +1534,30 @@ export function handleSessionEvent(deps: StreamRenderDeps, ev: SessionEvent): vo
         const backstopThreadId = threadId
         const backstopCtrl = ctrl
 
-        // Outbound secret scrub (#2044). Turn-flush delivers the model's
-        // terminal answer prose when it skipped reply/stream_reply — that
-        // is arbitrary agent free-text, sent via sendMessage/editMessageText
-        // and previewed to stderr below, so it needs the same mask as the
-        // three reply tools. Mirror the voice scrub: mask before the send,
-        // the preview, and recordOutbound.
-        capturedText = redactOutboundText(capturedText, 'turn_flush')
-
-        // #2798 reply-parity — normalize dashes/bullets and trip the over-bold
-        // guard deterministically, on code-masked text. This is the SAME chain
-        // the reply path applies (`stripExcessBold(normalizePunctuation(text))`)
-        // in the SAME order: after redact, before scrubVoice. Without it the
-        // turn-flush backstop rendered punctuation/bold differently from an
-        // identical reply. Kept inline to mirror reply exactly — a shared helper
-        // is a deliberate future refactor, not this change.
-        capturedText = stripExcessBold(normalizePunctuation(capturedText))
-
-        // Voice scrub (PR #1683 follow-up). Turn-flush is the path
-        // that fires when the model emits raw transcript text WITHOUT
-        // calling reply / stream_reply. That captured text bypasses
-        // PR #1683's executeReply scrub site entirely and is delivered
-        // via the rich-message path directly. Scrub the capturedText on the
-        // raw markdown so em-dashes never reach the wire. Kill switch:
-        // SWITCHROOM_DISABLE_VOICE_SCRUB.
-        {
-          const scrub = scrubVoice(capturedText)
-          if (scrub.replaced > 0) {
-            capturedText = scrub.scrubbed
-            emitRuntimeMetric({
-              kind: 'voice_scrub_applied',
-              chatKey: statusKey(backstopChatId, backstopThreadId),
-              replaced: scrub.replaced,
-              site: 'turn_flush',
-            })
-          }
+        // #3501: route through the single shared outbound seam. Turn-flush
+        // delivers the model's terminal prose when it skipped reply/stream_reply
+        // and used to hand-mirror the reply pipeline inline (repair → paragraph-
+        // break → redact → punctuation/bold → voice scrub). `normalizeOutboundBody`
+        // IS that pipeline with the same load-bearing order — the metric side
+        // effect (voice_scrub_applied on a non-zero replacement) is emitted here
+        // by the caller, exactly as the reply/edit sites do.
+        const _flushNorm = normalizeOutboundBody(
+          flushDecision.text,
+          'turn_flush',
+          redactOutboundText,
+          // #3501 temporal pass — a cron/mail-watcher notification that skipped
+          // reply is delivered here, so this is where "closed tomorrow (Thu 23
+          // Jul)" gets corrected against the agent's local date.
+          { tz: resolveEnvTimezone(), nowMs: Date.now() },
+        )
+        let capturedText = _flushNorm.text
+        if (_flushNorm.voiceReplaced > 0) {
+          emitRuntimeMetric({
+            kind: 'voice_scrub_applied',
+            chatKey: statusKey(backstopChatId, backstopThreadId),
+            replaced: _flushNorm.voiceReplaced,
+            site: 'turn_flush',
+          })
         }
 
         // #1664 — turn-flush only fires when !replyCalled (decideTurnFlush

package/telegram-plugin/hooks/silent-end-interrupt-stop.mjs

@@ -54,12 +54,16 @@
  * every session close.
  */
 
-import { readFileSync, writeFileSync, existsSync } from 'node:fs'
+import { readFileSync, writeFileSync, existsSync, mkdirSync } from 'node:fs'
 import { join } from 'node:path'
 import { homedir } from 'node:os'
 
+import { createHash } from 'node:crypto'
+import { renameSync } from 'node:fs'
+
 import {
   scanTurnForFinalReply,
+  scanForOutboxCapture,
   decideStopHookDisposition,
   isTurnFlushSafetyEnabledEnv,
   isCapturedProseDeliveryEnabledEnv,
@@ -131,6 +135,64 @@ function writeElectedState(statePath, base, decision) {
   }
 }
 
+const JOURNAL_FILE = 'delivered.jsonl'
+
+/** Is `nonce` already in the outbox delivered-keys journal? (best-effort) */
+function outboxAlreadyDelivered(outboxDir, nonce) {
+  const path = join(outboxDir, JOURNAL_FILE)
+  if (!existsSync(path)) return false
+  try {
+    for (const line of readFileSync(path, 'utf8').split('\n')) {
+      if (!line) continue
+      try {
+        if (JSON.parse(line)?.turnNonce === nonce) return true
+      } catch {
+        /* skip corrupt */
+      }
+    }
+  } catch {
+    /* best-effort */
+  }
+  return false
+}
+
+/**
+ * Write the durable outbox record for a captured undelivered final answer
+ * (atomic tmp+rename), mirroring `writeOutboxRecordAtomic` in `../outbox.ts`.
+ * The gateway heartbeat sweep is the single deliverer. Best-effort; never
+ * throws. Returns true when a record exists on disk for the nonce afterward.
+ */
+function writeOutboxRecord(stateDir, capture) {
+  const outboxDir = join(stateDir, 'outbox')
+  try {
+    mkdirSync(outboxDir, { recursive: true })
+    const finalPath = join(outboxDir, `${capture.turnNonce}.json`)
+    if (existsSync(finalPath)) return true
+    // Already delivered by another machine (reply/flush) under this nonce.
+    if (outboxAlreadyDelivered(outboxDir, capture.turnNonce)) return true
+    const record = {
+      turnNonce: capture.turnNonce,
+      chatId: capture.chatId,
+      threadId: capture.threadId,
+      text: capture.text,
+      textSha256: createHash('sha256').update(capture.text, 'utf8').digest('hex'),
+      createdAt: Date.now(),
+      source: capture.source,
+      anchorContent: capture.chatId == null ? capture.anchorContent : undefined,
+      // F2: per-session origin chat for envelope-less routing (fail-closed).
+      originChatId: capture.chatId == null ? (capture.originChatId ?? null) : undefined,
+      originThreadId: capture.chatId == null ? (capture.originThreadId ?? null) : undefined,
+    }
+    const tmpPath = join(outboxDir, `.${capture.turnNonce}.${process.pid}.tmp`)
+    writeFileSync(tmpPath, JSON.stringify(record), 'utf8')
+    renameSync(tmpPath, finalPath)
+    return true
+  } catch (err) {
+    process.stderr.write(`[silent-end-interrupt] failed to write outbox record: ${err.message}\n`)
+    return false
+  }
+}
+
 function main() {
   const raw = readStdin().trim()
   if (!raw) process.exit(0)
@@ -161,6 +223,37 @@ function main() {
     process.exit(0)
   }
 
+  const stateDir = getStateDir()
+
+  // ── Outbox capture (guaranteed final-message delivery) ────────────────
+  // Class-agnostic: fires on EVERY main-session turn end. When the turn ended
+  // with substantive undelivered trailing prose (Telegram inbound,
+  // task-notification handback, cron, or a future/unknown wake shape), write a
+  // durable outbox record. The gateway heartbeat sweep is the single deliverer
+  // — no CurrentTurn, no election, no re-prompt needed. This is the deterministic
+  // guarantee: the model calling `reply` is no longer required. Fail-open on any
+  // error (never loop the session).
+  //
+  // Kill switch symmetry: `SWITCHROOM_TG_OUTBOX_DELIVERY=0` disables the gateway
+  // sweep (the deliverer). Capture MUST honour the SAME gate — otherwise, with
+  // the flag off, capture would write records that nothing delivers AND
+  // short-circuit the legacy re-prompt path below = silent loss. Flag off ⇒ skip
+  // capture entirely and fall through to the legacy block/re-prompt behaviour.
+  if (process.env.SWITCHROOM_TG_OUTBOX_DELIVERY !== '0') try {
+    const capture = scanForOutboxCapture(jsonl)
+    if (capture.capture === true) {
+      writeOutboxRecord(stateDir, capture)
+      process.stderr.write(
+        `[silent-end-interrupt] captured undelivered final answer to outbox ` +
+          `(nonce=${capture.turnNonce} source=${capture.source} chars=${capture.text.length}) — ` +
+          `sweep will deliver; allowing stop\n`,
+      )
+      process.exit(0)
+    }
+  } catch (err) {
+    process.stderr.write(`[silent-end-interrupt] outbox capture error (fail-open): ${err.message}\n`)
+  }
+
   const decision = scanTurnForFinalReply(jsonl)
 
   // 'allow' (qualifying reply or silent marker) and 'unknown' (no
@@ -175,7 +268,6 @@ function main() {
   // transcript above. If a state file exists from a prior turn that
   // never got cleared (clean shutdown not perfect), this read still
   // works; if absent, retryCount defaults to 0.
-  const stateDir = getStateDir()
   const statePath = join(stateDir, 'silent-end-pending.json')
 
   let state = {}

package/telegram-plugin/hooks/silent-end-scan.mjs

@@ -57,6 +57,7 @@
 // re-verify only if a new outbound-delivery tool is added to bridge.ts.
 import { statSync } from 'node:fs'
 import { join } from 'node:path'
+import { createHash } from 'node:crypto'
 
 const REPLY_TOOLS = new Set([
   'mcp__switchroom-telegram__reply',
@@ -718,3 +719,281 @@ export function isGatewayHeartbeatFresh(stateDir, now = Date.now()) {
     return false
   }
 }
+
+// ── Outbox capture (guaranteed final-message delivery) ───────────────────────
+//
+// The Stop hook fires on EVERY main-session turn end, regardless of what woke
+// the turn (Telegram inbound, `<task-notification>` handback, cron, or a future
+// harness wake type). `scanForOutboxCapture` decides — class-agnostically —
+// whether this turn ended with substantive undelivered trailing prose that must
+// be captured into the durable outbox for the gateway sweep to deliver.
+//
+// Fail CLOSED (H4): capture keys on "a turn ended with unsent trailing prose",
+// never on recognising the wake shape. An anchor-less transcript tail still
+// captures (source='unknown') rather than silently allowing a lost answer.
+//
+// Cron (H5): NOT exempted — a cron turn that ends with prose is captured and
+// guaranteed-delivered, same as any other class. Only NO_REPLY stays silent.
+//
+// H6: a turn ending with real prose followed by a stray bare `NO_REPLY` still
+// captures the prose (the marker doesn't suppress a genuine answer), while a
+// legitimately-silent turn (pure NO_REPLY / narration-only) writes no record.
+
+/** Mirror of `deriveTurnNonce` in `../outbox.ts` — MUST stay in sync (a .mjs
+ * can't import the .ts). Test `outbox-nonce-parity` pins the equality. */
+export function deriveTurnNonce({ chatId, threadId, messageId, anchorTimestampMs, anchorContent }) {
+  if (chatId != null && messageId != null && messageId !== '' && String(messageId) !== '0') {
+    const key = `${chatId}:${threadId == null || threadId === 0 ? '_' : threadId}`
+    return `${key}#${messageId}`
+  }
+  return createHash('sha256').update(`${anchorTimestampMs}\n${anchorContent}`, 'utf8').digest('hex')
+}
+
+/**
+ * Strip a trailing bare silent-marker line (NO_REPLY / HEARTBEAT_OK) from a text
+ * block, returning the prose that precedes it (H6). If the whole block is the
+ * marker, prose is ''. If there is no trailing marker, `hadMarker` is false and
+ * prose is the block unchanged.
+ *
+ * @param {string} text
+ * @returns {{ prose: string, hadMarker: boolean }}
+ */
+export function stripTrailingSilentMarker(text) {
+  if (typeof text !== 'string') return { prose: '', hadMarker: false }
+  const rawLines = text.split('\n')
+  // Find the last non-empty line.
+  let lastIdx = -1
+  for (let i = rawLines.length - 1; i >= 0; i--) {
+    if (rawLines[i].trim().length > 0) {
+      lastIdx = i
+      break
+    }
+  }
+  if (lastIdx === -1) return { prose: '', hadMarker: false }
+  if (!SILENT_MARKER_RE.test(rawLines[lastIdx].trim())) {
+    return { prose: text.trim(), hadMarker: false }
+  }
+  const prose = rawLines.slice(0, lastIdx).join('\n').trim()
+  return { prose, hadMarker: true }
+}
+
+/**
+ * Resolve the turn-start anchor for capture. Primary: the most-recent
+ * `queue-operation`/`enqueue` line. H4 fallbacks (fail closed) when none is
+ * found: the last non-sidechain `user`-type line, else the last
+ * `queue-operation` of any operation, else the whole scanned range (startIdx=-1
+ * → scan from the top). `degraded` marks a non-primary anchor (source unknown).
+ *
+ * @param {string[]} lines
+ * @returns {{ startIdx: number, envelope: ReturnType<typeof parseChannelEnvelope>, anchorTimestampMs: number, anchorContent: string, degraded: boolean }}
+ */
+function resolveCaptureAnchor(lines) {
+  let enqueueIdx = -1
+  let anyQueueIdx = -1
+  let userIdx = -1
+  const parsed = new Array(lines.length)
+  for (let i = lines.length - 1; i >= 0; i--) {
+    const line = lines[i]
+    if (!line || line[0] !== '{') continue
+    let obj
+    try { obj = JSON.parse(line) } catch { continue }
+    parsed[i] = obj
+    if (obj?.type === 'queue-operation') {
+      if (obj.operation === 'enqueue' && enqueueIdx === -1) enqueueIdx = i
+      if (anyQueueIdx === -1) anyQueueIdx = i
+    }
+    if (obj?.type === 'user' && obj?.isSidechain !== true && userIdx === -1) userIdx = i
+  }
+  const anchorTs = (obj) => {
+    const t = obj?.timestamp
+    if (typeof t === 'number' && Number.isFinite(t)) return t
+    if (typeof t === 'string') {
+      const ms = Date.parse(t)
+      if (Number.isFinite(ms)) return ms
+    }
+    return Date.now()
+  }
+  if (enqueueIdx !== -1) {
+    const obj = parsed[enqueueIdx]
+    const content = typeof obj.content === 'string' ? obj.content : ''
+    return {
+      startIdx: enqueueIdx,
+      envelope: parseChannelEnvelope(content),
+      anchorTimestampMs: anchorTs(obj),
+      anchorContent: content,
+      degraded: false,
+    }
+  }
+  if (userIdx !== -1) {
+    const obj = parsed[userIdx]
+    const content = typeof obj?.message?.content === 'string' ? obj.message.content : JSON.stringify(obj?.message?.content ?? '')
+    return {
+      startIdx: userIdx,
+      envelope: { chatId: null, threadId: null, messageId: null, source: 'unknown' },
+      anchorTimestampMs: anchorTs(obj),
+      anchorContent: content,
+      degraded: true,
+    }
+  }
+  if (anyQueueIdx !== -1) {
+    const obj = parsed[anyQueueIdx]
+    const content = typeof obj.content === 'string' ? obj.content : ''
+    return {
+      startIdx: anyQueueIdx,
+      envelope: parseChannelEnvelope(content),
+      anchorTimestampMs: anchorTs(obj),
+      anchorContent: content,
+      degraded: true,
+    }
+  }
+  // No anchor at all — scan the whole range (fail closed on unknown shape).
+  return {
+    startIdx: -1,
+    envelope: { chatId: null, threadId: null, messageId: null, source: 'unknown' },
+    anchorTimestampMs: Date.now(),
+    anchorContent: '',
+    degraded: true,
+  }
+}
+
+/**
+ * Resolve THIS session's origin chat — the most-recent non-sidechain enqueue
+ * line whose content carries a real Telegram `<channel>` envelope with a
+ * chatId. Scoped to the session's OWN transcript, this is the conversation of
+ * record for an envelope-less handback turn (F2): the sweep routes an
+ * unresolved-registry record here instead of a gateway-global "last chat anyone
+ * messaged" fallback, so a DM-origin handback can never leak into an unrelated
+ * chat. Null when the session has no prior channel inbound → the record fails
+ * CLOSED (held, never delivered to an arbitrary chat).
+ *
+ * @param {string[]} lines
+ * @returns {{ chatId: string | null, threadId: number | null }}
+ */
+function resolveSessionOriginChat(lines) {
+  for (let i = lines.length - 1; i >= 0; i--) {
+    const line = lines[i]
+    if (!line || line[0] !== '{') continue
+    let obj
+    try { obj = JSON.parse(line) } catch { continue }
+    if (obj?.isSidechain === true) continue
+    if (obj?.type !== 'queue-operation') continue
+    const content = typeof obj.content === 'string' ? obj.content : ''
+    const env = parseChannelEnvelope(content)
+    if (env.chatId != null && env.chatId !== '') {
+      return { chatId: env.chatId, threadId: env.threadId ?? null }
+    }
+  }
+  return { chatId: null, threadId: null }
+}
+
+/**
+ * Class-agnostic capture scan. Walks the turn from its anchor, tracking the last
+ * delivery event (qualifying reply / silent marker) and the substantive prose
+ * blocks that trail it. Returns a capture descriptor when the turn ended with
+ * undelivered final-answer prose, else `{ capture: false }`.
+ *
+ * @param {string} jsonl
+ * @param {number} [now]
+ * @returns {{ capture: false, reason: string } | { capture: true, text: string, turnNonce: string, chatId: string|null, threadId: number|null, source: string, anchorContent: string }}
+ */
+export function scanForOutboxCapture(jsonl, now = Date.now()) {
+  const lines = jsonl.split('\n')
+  const anchor = resolveCaptureAnchor(lines)
+  const { envelope } = anchor
+
+  const blocks = []
+  for (let i = anchor.startIdx + 1; i < lines.length; i++) {
+    const line = lines[i]
+    if (!line || line[0] !== '{') continue
+    let obj
+    try { obj = JSON.parse(line) } catch { continue }
+    if (obj?.isSidechain === true) continue
+    if (obj?.type !== 'assistant') continue
+    const content = obj?.message?.content
+    if (!Array.isArray(content)) continue
+    for (const c of content) {
+      if (c?.type === 'text') {
+        const raw = String(c.text ?? '')
+        // H6: strip a trailing bare marker; if substantive non-narration prose
+        // remains, it is a genuine (undelivered) answer — capture it. If nothing
+        // but the marker (or narration) remains, the block is a silence event.
+        const { prose, hadMarker } = stripTrailingSilentMarker(raw)
+        if (hadMarker) {
+          // H6: a block ending "…real answer…\nNO_REPLY" carries a genuine
+          // answer — capture the prose and do NOT treat the block as a silence
+          // event (which would mask the answer). When the prose is empty or pure
+          // narration the marker is an intentional silence, but it must NOT emit
+          // a 'deliver' event: a *separate* bare-`NO_REPLY` block arriving AFTER
+          // a real (undelivered) prose block would otherwise move the
+          // last-delivery cursor past that prose and mask it (multi-block H6).
+          // A bare/narration marker is simply not a delivery — push nothing.
+          if (prose.length > 0 && !isNarrationBlock(prose)) {
+            blocks.push({ kind: 'text', chars: prose.length, text: prose })
+          }
+        } else if (prose.length > 0) {
+          blocks.push({ kind: 'text', chars: prose.length, text: prose })
+        }
+        continue
+      }
+      if (c?.type !== 'tool_use') continue
+      if (!REPLY_TOOLS.has(c.name)) continue
+      const input = c.input ?? {}
+      const text = String(input.text ?? '')
+      if (SILENT_MARKER_RE.test(text.trim()) || endsWithSilentMarker(text)) {
+        blocks.push({ kind: 'deliver', reason: 'silent-marker' })
+        continue
+      }
+      if (isFinalAnswerReply({
+        text,
+        disableNotification: input.disable_notification === true,
+        done: input.done === true,
+      })) {
+        blocks.push({ kind: 'deliver', reason: 'final-reply' })
+      }
+      // interim ack — neither delivery nor undelivered prose.
+    }
+  }
+
+  let lastDeliverIdx = -1
+  for (let i = 0; i < blocks.length; i++) {
+    if (blocks[i].kind === 'deliver') lastDeliverIdx = i
+  }
+  const trailing = blocks
+    .slice(lastDeliverIdx + 1)
+    .filter((b) => b.kind === 'text' && typeof b.text === 'string' && b.text.length > 0)
+    .map((b) => b.text)
+
+  const text = selectBridgePendingText(trailing)
+  if (text == null || text.trim().length < FINAL_ANSWER_MIN_CHARS) {
+    return { capture: false, reason: trailing.length === 0 ? 'no-trailing-prose' : 'below-floor' }
+  }
+
+  const turnNonce = deriveTurnNonce({
+    chatId: envelope.chatId,
+    threadId: envelope.threadId,
+    messageId: envelope.messageId,
+    anchorTimestampMs: anchor.anchorTimestampMs,
+    anchorContent: anchor.anchorContent,
+  })
+  let source = envelope.source
+  if (source == null) {
+    if (anchor.degraded) source = 'unknown'
+    else if (/task-notification|task-id/.test(anchor.anchorContent)) source = 'task-notification'
+    else source = 'channel'
+  }
+  // F2: for an envelope-less record, stamp THIS session's own origin chat so the
+  // sweep can route it without a gateway-global fallback. For an envelope-
+  // bearing record the anchor chatId already routes it (origin is redundant).
+  const origin = envelope.chatId != null ? { chatId: null, threadId: null } : resolveSessionOriginChat(lines)
+  return {
+    capture: true,
+    text: text.trim(),
+    turnNonce,
+    chatId: envelope.chatId,
+    threadId: envelope.threadId ?? null,
+    source,
+    anchorContent: anchor.anchorContent,
+    originChatId: origin.chatId,
+    originThreadId: origin.threadId,
+  }
+}