switchroom 0.18.24 → 0.18.26

package/telegram-plugin/subagent-watcher.ts

@@ -125,6 +125,19 @@ export interface WorkerEntry {
   lastActivityAt: number
   /** Number of tool calls seen so far. */
   toolCount: number
+  /**
+   * Running TOTAL tokens across every assistant message the worker has emitted
+   * (input + output + cache_creation, summed via sumUsageTokens; cache_read is
+   * excluded — replayed cached context, not new work).
+   * Accumulated from `sub_agent_usage` events, deduped by `seenUsageMessageIds`
+   * so the multi-line split-message shape (one logical message persisted as
+   * several JSONL lines sharing one `message.id` + identical `usage`) counts
+   * once. Rendered on the worker card's metrics line. 0 for a worker that never
+   * emitted a usage block (e.g. a non-Claude/litellm transcript).
+   */
+  totalTokens: number
+  /** message.id set already folded into `totalTokens` (usage dedup). */
+  seenUsageMessageIds: Set<string>
   /** True once a stall notification has been sent (suppresses repeat). */
   stallNotified: boolean
   /**
@@ -557,6 +570,9 @@ export interface SubagentWatcherConfig {
     state: WorkerState
     outcome: 'completed' | 'failed' | 'orphan'
     toolCount: number
+    /** Final running TOTAL tokens across the worker's whole life
+     *  (`WorkerEntry.totalTokens`). For the terminal worker-feed card. */
+    totalTokens: number
     durationMs: number
     /** Dispatch-time task description, for the handback envelope. */
     description: string
@@ -615,6 +631,10 @@ export interface SubagentWatcherConfig {
     lastTool: { name: string; sanitisedArg: string } | null
     /** Tool-use count observed so far. */
     toolCount: number
+    /** Running TOTAL tokens across the worker's assistant messages so far
+     *  (`WorkerEntry.totalTokens`). Threaded onto the worker/nested card's
+     *  metrics line. 0 for a worker that has emitted no usage yet / ever. */
+    totalTokens: number
     /** Friendly display line for THIS tick. Set on `sub_agent_tool_use`
      *  events to a `describeToolUse` label ("Reading X", "Running a
      *  command") so a foreground sub-agent that runs tools without
@@ -1089,6 +1109,8 @@ export function readSubTail(
     lastTool: { name: string; sanitisedArg: string } | null
     /** Tool-use count observed so far. */
     toolCount: number
+    /** Running total tokens so far (see SubagentWatcherConfig.onProgress). */
+    totalTokens: number
     /** Friendly display line for THIS tick (set on tool ticks; see the
      *  SubagentWatcherConfig.onProgress doc). */
     progressLine?: string
@@ -1179,6 +1201,7 @@ export function readSubTail(
               },
               lastTool: entry.lastTool,
               toolCount: entry.toolCount,
+              totalTokens: entry.totalTokens,
               model: entry.currentModel,
               skeleton: true,
             })
@@ -1320,6 +1343,7 @@ export function readSubTail(
               },
               lastTool: entry.lastTool,
               toolCount: entry.toolCount,
+              totalTokens: entry.totalTokens,
               model: entry.currentModel,
             })
             return true
@@ -1502,6 +1526,22 @@ export function readSubTail(
           }
           continue
         }
+        if (ev.kind === 'sub_agent_usage') {
+          // Accumulate the worker's running total tokens, deduped by
+          // message.id: the ≥2.1.x split-message shape stamps the SAME `usage`
+          // block on every JSONL line of one logical assistant message, so
+          // counting each line would 2-3x over-count. A null messageId is
+          // un-dedupable (older/edge shapes) — count it as-is (its usage is
+          // real). No card render here; the total rides the next onProgress
+          // tick's payload like the model does.
+          if (ev.messageId == null) {
+            entry.totalTokens += ev.totalTokens
+          } else if (!entry.seenUsageMessageIds.has(ev.messageId)) {
+            entry.seenUsageMessageIds.add(ev.messageId)
+            entry.totalTokens += ev.totalTokens
+          }
+          continue
+        }
         if (ev.kind === 'sub_agent_tool_use') {
           // Narrative-dedup gate step 2: a sub_agent_text block was pending;
           // this tool is the lookahead that decides it (SHOW unless it drafts
@@ -1567,6 +1607,7 @@ export function readSubTail(
                   },
                   lastTool: entry.lastTool,
                   toolCount: entry.toolCount,
+                  totalTokens: entry.totalTokens,
                   progressLine: toolLine,
                   model: entry.currentModel,
                 })
@@ -1892,6 +1933,8 @@ export function startSubagentWatcher(config: SubagentWatcherConfig): SubagentWat
       dispatchedAt: n,
       lastActivityAt: n,
       toolCount: 0,
+      totalTokens: 0,
+      seenUsageMessageIds: new Set<string>(),
       stallNotified: false,
       stalledAt: null,
       completionNotified: false,
@@ -2157,6 +2200,7 @@ export function startSubagentWatcher(config: SubagentWatcherConfig): SubagentWat
             // registerAgent's short-circuit + Gap 1 promotion).
             outcome: entry.errored ? 'failed' : entry.historical ? 'orphan' : 'completed',
             toolCount: entry.toolCount,
+            totalTokens: entry.totalTokens,
             durationMs: nowFn() - entry.dispatchedAt,
             description: entry.description,
             // For a failure, fall back to the error detail when the worker
@@ -2184,6 +2228,7 @@ export function startSubagentWatcher(config: SubagentWatcherConfig): SubagentWat
             state: entry.state,
             outcome: 'failed',
             toolCount: entry.toolCount,
+            totalTokens: entry.totalTokens,
             durationMs: nowFn() - entry.dispatchedAt,
             description: entry.description,
             resultText: entry.lastResultText,

package/telegram-plugin/tests/crash-redelivery-resume-exclusion.test.ts

@@ -0,0 +1,133 @@
+import { describe, it, expect } from 'vitest'
+import { decideRedeliverCapture } from '../gateway/redelivery-decision.js'
+import {
+  decideBootResumeKind,
+  RESUME_SYNTHETIC_PROMPT_PREFIX,
+} from '../gateway/resume-inbound-builder.js'
+import type { Turn, TurnEndedVia } from '../registry/turns-schema.js'
+
+/**
+ * Pins the MUTUAL EXCLUSION between crash-survival redelivery and the resume
+ * synthetic — the double-send guard. The gateway boot block composes exactly
+ * these two pure predicates: `decideBootResumeKind` classifies the interrupted
+ * turn, then `decideRedeliverCapture({ willBeResumed: kind === 'resume', ... })`
+ * decides whether to ALSO stage a redelivery.
+ *
+ * The load-bearing outcome: an interrupted turn that WILL be resumed (the model
+ * re-runs and emits a fresh answer) must NOT also redeliver its recovered draft
+ * — otherwise the same answer reaches the user twice. Conversely, a turn that
+ * will NOT be resumed (watchdog report, boot_resume:never suppression,
+ * resume-of-a-resume loop-guard) MUST redeliver, because nothing else re-answers.
+ *
+ * This mirrors the gateway wiring exactly, so it asserts the real composed
+ * outcome, not an isolated code path.
+ */
+
+const RESUME_MAX_AGE_MS = 10_800_000 // 3h, gateway default
+
+function makeTurn(over: Partial<Turn> = {}): Turn {
+  return {
+    turn_key: '900001:_#7',
+    chat_id: '900001',
+    thread_id: null,
+    started_at: Date.now() - 60_000, // 1 min ago — well within maxAge
+    ended_at: null,
+    ended_via: null,
+    last_assistant_msg_id: null,
+    last_assistant_done: null,
+    last_user_msg_id: null,
+    user_prompt_preview: 'deploy the staging stack',
+    assistant_reply_preview: null,
+    tool_call_count: 2,
+    interrupt_reason: null,
+    resumed_at: null,
+    session_id: 'sess-abcd', // durably pinned → redelivery is eligible on the floor
+    ...over,
+  } as Turn
+}
+
+/** Compose the two predicates exactly as the gateway boot block does. */
+function composeGate(turn: Turn, suppressed: boolean) {
+  const bootResumeKind = decideBootResumeKind({
+    pending: turn,
+    suppressed,
+    ageMs: Math.max(0, Date.now() - turn.started_at),
+    maxAgeMs: RESUME_MAX_AGE_MS,
+  })
+  const capture = decideRedeliverCapture({
+    willBeResumed: bootResumeKind === 'resume',
+    hasSessionId: Boolean(turn.session_id),
+  })
+  return { bootResumeKind, capture }
+}
+
+describe('crash-redelivery ↔ resume mutual exclusion (no double-send)', () => {
+  it('(a) an interrupted turn that WILL be resumed does NOT also redeliver', () => {
+    // ended_via 'restart' → decideBootResumeKind returns 'resume': the model
+    // re-runs and emits a fresh answer that supersedes any recovered draft.
+    const turn = makeTurn({ ended_via: 'restart' as TurnEndedVia })
+    const { bootResumeKind, capture } = composeGate(turn, /* suppressed */ false)
+    expect(bootResumeKind).toBe('resume')
+    expect(capture.capture).toBe(false)
+    expect(capture.skipReason).toBe('will-be-resumed')
+  })
+
+  it('(a2) a still-open (ended_via=null) killed-mid-flight turn is resumed, not redelivered', () => {
+    const turn = makeTurn({ ended_via: null })
+    const { bootResumeKind, capture } = composeGate(turn, false)
+    expect(bootResumeKind).toBe('resume')
+    expect(capture.capture).toBe(false)
+  })
+
+  it('(b) a watchdog-timeout turn (report, no auto re-answer) DOES redeliver', () => {
+    // ended_via 'timeout' → 'report': the synthetic only ASKS the user whether
+    // to retry — it does not auto-re-answer. Redelivery is the correct recovery.
+    const turn = makeTurn({ ended_via: 'timeout' as TurnEndedVia })
+    const { bootResumeKind, capture } = composeGate(turn, false)
+    expect(bootResumeKind).toBe('report')
+    expect(capture.capture).toBe(true)
+    expect(capture.skipReason).toBeUndefined()
+  })
+
+  it('(b2) a boot_resume:never-suppressed turn (defer-suppressed) DOES redeliver', () => {
+    // suppressed=true → 'defer-suppressed': no synthetic re-run, so redelivery
+    // is the ONLY recovery send.
+    const turn = makeTurn({ ended_via: 'restart' as TurnEndedVia })
+    const { bootResumeKind, capture } = composeGate(turn, /* suppressed */ true)
+    expect(bootResumeKind).toBe('defer-suppressed')
+    expect(capture.capture).toBe(true)
+  })
+
+  it('(b3) a resume-of-a-resume (loop-guard → defer-loop) turn DOES redeliver', () => {
+    // A turn whose prompt is itself a resume synthetic → 'defer-loop': the chain
+    // is capped, no re-run happens, so redelivery must still recover the answer.
+    const turn = makeTurn({
+      ended_via: 'restart' as TurnEndedVia,
+      user_prompt_preview: `${RESUME_SYNTHETIC_PROMPT_PREFIX} Continue the interrupted deploy.`,
+    })
+    const { bootResumeKind, capture } = composeGate(turn, false)
+    expect(bootResumeKind).toBe('defer-loop')
+    expect(capture.capture).toBe(true)
+  })
+
+  it('(b4) a STALE resume downgraded to report DOES redeliver (not suppressed)', () => {
+    // A 'restart' turn older than maxAge downgrades resume→report in
+    // selectResumeBuilder. Because the final kind is 'report' (no re-run),
+    // redelivery must fire — the gate keys on the FINAL kind, not the raw
+    // ended_via.
+    const turn = makeTurn({
+      ended_via: 'restart' as TurnEndedVia,
+      started_at: Date.now() - (RESUME_MAX_AGE_MS + 60_000),
+    })
+    const { bootResumeKind, capture } = composeGate(turn, false)
+    expect(bootResumeKind).toBe('report')
+    expect(capture.capture).toBe(true)
+  })
+
+  it('eligibility floor still applies: no session_id → no redelivery even when not resumed', () => {
+    const turn = makeTurn({ ended_via: 'timeout' as TurnEndedVia, session_id: null })
+    const { capture } = composeGate(turn, false)
+    expect(capture.capture).toBe(false)
+    expect(capture.skipReason).toBe('no-session-id')
+  })
+})

package/telegram-plugin/tests/crash-redelivery-wiring.test.ts

@@ -0,0 +1,72 @@
+import { describe, it, expect } from 'vitest'
+import { projectTrailingAnswerFromTranscript } from '../session-tail.js'
+import { decideRedeliver, REDELIVERY_PREFIX } from '../gateway/redelivery-decision.js'
+
+/**
+ * Pins the crash-survival redelivery WIRING GLUE — the composition
+ * `maybeRedeliverUndeliveredAnswer` performs in the boot send: project the
+ * interrupted turn's trailing answer from its transcript, then feed
+ * (`text`, `trailingIsText`) plus the durable oracle result into
+ * `decideRedeliver`. The oracle (`hasOutboundWithText`) and the projector are
+ * unit-tested in their own suites (history.test.ts / trailing-answer-projector
+ * .test.ts); this asserts the two ends plumb together — the projected answer is
+ * what gets framed, and a dangling mid-tool turn is refused before any send.
+ *
+ * The raw Telegram socket send and the post-`getMe()` connect sequencing are NOT
+ * unit-testable here (they require a connected grammy client); the ordering
+ * guarantee is documented on `maybeRedeliverUndeliveredAnswer` and enforced by
+ * its single call site inside the `didOneTimeSetup` block, which is unreachable
+ * until `bot.api.getMe()` resolves.
+ */
+
+const FINAL_ANSWER = 'The deploy finished — all three services are green and healthy.'
+
+function compose(transcript: string, hasDeliveredText: boolean) {
+  const projected = projectTrailingAnswerFromTranscript(transcript)
+  return decideRedeliver({
+    capturedText: projected.text,
+    trailingIsText: projected.trailingIsText,
+    hasDeliveredText,
+    alreadyRedelivered: false,
+    ageMs: 60_000,
+    maxAgeMs: 10_800_000,
+  })
+}
+
+function line(obj: unknown): string {
+  return JSON.stringify(obj)
+}
+const completedTurn = [
+  line({ type: 'user', message: { role: 'user', content: 'deploy status?' } }),
+  line({ type: 'assistant', message: { content: [{ type: 'text', text: 'Checking.' }] } }),
+  line({ type: 'assistant', message: { content: [{ type: 'tool_use', name: 'Bash', id: 'toolu_x', input: {} }] } }),
+  line({ type: 'assistant', message: { content: [{ type: 'text', text: FINAL_ANSWER }] } }),
+].join('\n')
+const danglingToolTurn = [
+  line({ type: 'user', message: { role: 'user', content: 'run the migration' } }),
+  line({ type: 'assistant', message: { content: [{ type: 'text', text: 'On it.' }] } }),
+  line({ type: 'assistant', message: { content: [{ type: 'tool_use', name: 'Bash', id: 'toolu_y', input: {} }] } }),
+].join('\n')
+
+describe('crash-redelivery wiring glue (project → decide)', () => {
+  it('frames the PROJECTED trailing answer when the oracle says it was not delivered', () => {
+    const d = compose(completedTurn, /* hasDeliveredText */ false)
+    expect(d.redeliver).toBe(true)
+    expect(d.framedText?.startsWith(REDELIVERY_PREFIX)).toBe(true)
+    expect(d.framedText).toContain(FINAL_ANSWER)
+  })
+
+  it('skips when the oracle reports the projected answer was already delivered', () => {
+    expect(compose(completedTurn, /* hasDeliveredText */ true).skipReason).toBe('already-delivered')
+  })
+
+  it('refuses a turn killed mid-tool before any send', () => {
+    // The projector RESETS the buffer on the trailing tool_use, so the recovered
+    // text is empty AND trailingIsText is false — either guard refuses. The
+    // decision reports 'empty-text' (checked first); the load-bearing property is
+    // simply that no send happens for a dangling mid-tool turn.
+    const d = compose(danglingToolTurn, false)
+    expect(d.redeliver).toBe(false)
+    expect(d.skipReason).toBe('empty-text')
+  })
+})

package/telegram-plugin/tests/history.test.ts

@@ -14,6 +14,8 @@ import {
   getRecentOutboundCount,
   getLatestInboundMessageId,
   hasOutboundDeliveredSince,
+  hasOutboundWithText,
+  normalizeDeliveryText,
   _resetForTests,
 } from '../history.js'
 
@@ -872,3 +874,92 @@ describe('forwarded-message origin columns', () => {
     expect(stored).toContain('Bob') // surrounding name preserved
   })
 })
+
+// ---------------------------------------------------------------------------
+// hasOutboundWithText — durable text-identity delivery oracle (crash-survival
+// redelivery). Keys on the ANSWER TEXT, not a chat+time window, so an interim
+// progress_update earlier in the same turn does NOT false-positive "delivered".
+// ---------------------------------------------------------------------------
+
+describe('hasOutboundWithText (durable text-identity oracle)', () => {
+  it('does NOT match an interim progress message against the (undelivered) final answer', () => {
+    initHistory(stateDir, 30)
+    // The turn sent only an interim progress_update; the real final answer was
+    // lost in the crash and never recorded. The time-windowed oracle would say
+    // "delivered" off the progress row — the text-identity oracle must not.
+    recordOutbound({
+      chat_id: '1', thread_id: null, message_ids: [10],
+      texts: ['on it — pulling yesterday’s GitHub activity'], ts: 200,
+    })
+    const finalAnswer = 'The deploy finished — all three services are green.'
+    expect(hasOutboundWithText('1', finalAnswer, null)).toBe(false)
+    // sanity: the coarse time-window oracle DOES false-positive here (this is
+    // exactly why we cannot use it as the redelivery gate).
+    expect(hasOutboundDeliveredSince('1', 100 * 1000, null, 1)).toBe(true)
+  })
+
+  it('matches when the final answer text was actually delivered', () => {
+    initHistory(stateDir, 30)
+    const finalAnswer = 'The deploy finished — all three services are green.'
+    recordOutbound({ chat_id: '1', thread_id: null, message_ids: [11], texts: [finalAnswer], ts: 300 })
+    expect(hasOutboundWithText('1', finalAnswer, null)).toBe(true)
+  })
+
+  it('matches a delivered chunk-1 against a longer projected answer (multi-chunk, no double-send)', () => {
+    initHistory(stateDir, 30)
+    const chunk1 = 'Part one of a long answer that was split across chunks.'
+    recordOutbound({ chat_id: '1', thread_id: null, message_ids: [12], texts: [chunk1], ts: 300 })
+    // The re-projected full answer starts with chunk-1 → treated as delivered.
+    expect(hasOutboundWithText('1', chunk1 + ' Part two continues here.', null)).toBe(true)
+  })
+
+  it('ignores whitespace/spacer differences via normalization', () => {
+    initHistory(stateDir, 30)
+    recordOutbound({ chat_id: '1', thread_id: null, message_ids: [13], texts: ['hello   world'], ts: 300 })
+    expect(hasOutboundWithText('1', 'hello world', null)).toBe(true)
+    expect(normalizeDeliveryText('hello   world')).toBe('hello world')
+  })
+
+  it('empty/whitespace text never matches', () => {
+    initHistory(stateDir, 30)
+    recordOutbound({ chat_id: '1', thread_id: null, message_ids: [14], texts: ['real'], ts: 300 })
+    expect(hasOutboundWithText('1', '   ', null)).toBe(false)
+  })
+
+  it('scopes by chat (a different chat does not satisfy the match)', () => {
+    initHistory(stateDir, 30)
+    recordOutbound({ chat_id: '1', thread_id: null, message_ids: [15], texts: ['scoped answer'], ts: 300 })
+    expect(hasOutboundWithText('2', 'scoped answer', null)).toBe(false)
+    expect(hasOutboundWithText('1', 'scoped answer', null)).toBe(true)
+  })
+
+  // diff-review defect #1 — a SHORT final answer must not false-positive-match an
+  // unrelated earlier row via the bidirectional-prefix rule (that would suppress a
+  // genuine redelivery = permanent silence). Short texts require full equality.
+  it('does NOT suppress a short answer that merely shares a prefix with an unrelated row', () => {
+    initHistory(stateDir, 30)
+    // An earlier turn delivered a longer line that starts with the short answer.
+    recordOutbound({ chat_id: '1', thread_id: null, message_ids: [20], texts: ['Done, deploying now.'], ts: 300 })
+    // The interrupted turn's real final answer was the short "Done." — never sent.
+    expect(hasOutboundWithText('1', 'Done.', null)).toBe(false)
+  })
+
+  it('still suppresses a short answer that was genuinely delivered (exact match)', () => {
+    initHistory(stateDir, 30)
+    recordOutbound({ chat_id: '1', thread_id: null, message_ids: [21], texts: ['Done.'], ts: 300 })
+    expect(hasOutboundWithText('1', 'Done.', null)).toBe(true)
+  })
+
+  // sinceMs scope: only rows delivered at/after the interrupted turn's started_at
+  // count, so an unrelated PRIOR turn's identical text can never suppress.
+  it('scopes by sinceMs (a prior-turn row before the floor does not match)', () => {
+    initHistory(stateDir, 30)
+    // Prior turn delivered this exact text at ts=200s.
+    recordOutbound({ chat_id: '1', thread_id: null, message_ids: [22], texts: ['repeated answer'], ts: 200 })
+    // Interrupted turn started at 250s (250_000 ms) — the prior row is out of scope.
+    expect(hasOutboundWithText('1', 'repeated answer', null, 250_000)).toBe(false)
+    // A row delivered within the turn window (ts=300s) does match.
+    recordOutbound({ chat_id: '1', thread_id: null, message_ids: [23], texts: ['repeated answer'], ts: 300 })
+    expect(hasOutboundWithText('1', 'repeated answer', null, 250_000)).toBe(true)
+  })
+})