switchroom 0.13.13 → 0.13.15

package/telegram-plugin/tests/pending-work-progress.test.ts

@@ -0,0 +1,354 @@
+/**
+ * Unit tests for cross-turn pending-async progress (#1445).
+ *
+ * Pins the deterministic state machine + edit cadence in isolation
+ * from the gateway. The integration with gateway hooks is exercised
+ * by the UAT scenario `silence-poke-debug-dm.test.ts`.
+ */
+
+import { afterEach, beforeEach, describe, expect, it } from 'vitest'
+
+import {
+  EDIT_INTERVAL_MS,
+  MAX_LIFETIME_MS,
+  TELEGRAM_MSG_CAP,
+  __getStateForTests,
+  __resetAllForTests,
+  __setDepsForTests,
+  __tickForTests,
+  clearPending,
+  noteAsyncDispatch,
+  noteOutbound,
+  noteTurnEnd,
+  startTurn,
+  type PendingProgressEditCtx,
+  type PendingProgressMetric,
+} from '../pending-work-progress.js'
+
+const KEY = '12345:_'
+
+interface Capture {
+  edits: PendingProgressEditCtx[]
+  metrics: PendingProgressMetric[]
+  now: number
+}
+
+function setup(): Capture {
+  const cap: Capture = { edits: [], metrics: [], now: 0 }
+  __resetAllForTests()
+  __setDepsForTests({
+    editMessage: async (ctx) => {
+      cap.edits.push(ctx)
+    },
+    emitMetric: (e) => {
+      cap.metrics.push(e)
+    },
+    nowMs: () => cap.now,
+  })
+  return cap
+}
+
+async function flush(): Promise<void> {
+  // Allow the fire-and-forget promise chain in tick() to settle.
+  await Promise.resolve()
+  await Promise.resolve()
+}
+
+describe('pending-work-progress', () => {
+  beforeEach(() => {
+    delete process.env.SWITCHROOM_DISABLE_PENDING_PROGRESS
+  })
+  afterEach(() => {
+    __resetAllForTests()
+  })
+
+  it('does nothing on turns without an async dispatch', () => {
+    const cap = setup()
+    startTurn(KEY)
+    noteOutbound(KEY, { messageId: 100, text: 'simple reply' })
+    noteTurnEnd(KEY)
+    expect(__getStateForTests(KEY)).toBeUndefined()
+    cap.now = 60_000
+    __tickForTests(cap.now)
+    expect(cap.edits).toHaveLength(0)
+    expect(cap.metrics).toHaveLength(0)
+  })
+
+  it('activates when turn ends with async dispatch + anchor', () => {
+    const cap = setup()
+    startTurn(KEY)
+    noteAsyncDispatch(KEY)
+    noteOutbound(KEY, { messageId: 100, text: 'worker dispatched' })
+    cap.now = 1_000
+    noteTurnEnd(KEY)
+    const s = __getStateForTests(KEY)
+    expect(s).toBeDefined()
+    expect(s?.activatedAt).toBe(1_000)
+    expect(s?.anchorMessageId).toBe(100)
+    expect(s?.anchorOriginalText).toBe('worker dispatched')
+    expect(cap.metrics).toContainEqual({
+      kind: 'pending_progress_started',
+      chatKey: KEY,
+    })
+  })
+
+  it('does not activate when async dispatch happened but no anchor was captured', () => {
+    const cap = setup()
+    startTurn(KEY)
+    noteAsyncDispatch(KEY)
+    // no noteOutbound — model never sent a reply (silent end)
+    noteTurnEnd(KEY)
+    expect(__getStateForTests(KEY)).toBeUndefined()
+    cap.now = 60_000
+    __tickForTests(cap.now)
+    expect(cap.edits).toHaveLength(0)
+  })
+
+  it('does not activate when an anchor exists but no async dispatch happened', () => {
+    const cap = setup()
+    startTurn(KEY)
+    noteOutbound(KEY, { messageId: 100, text: 'just chatting' })
+    noteTurnEnd(KEY)
+    expect(__getStateForTests(KEY)).toBeUndefined()
+    cap.now = 60_000
+    __tickForTests(cap.now)
+    expect(cap.edits).toHaveLength(0)
+  })
+
+  it('edits anchor with elapsed-time suffix at EDIT_INTERVAL_MS cadence', async () => {
+    const cap = setup()
+    startTurn(KEY)
+    noteAsyncDispatch(KEY)
+    noteOutbound(KEY, {
+      messageId: 100,
+      text: 'Background sleep running; awaiting completion.',
+    })
+    cap.now = 0
+    noteTurnEnd(KEY)
+
+    // Tick at half-interval — no edit yet.
+    cap.now = EDIT_INTERVAL_MS / 2
+    __tickForTests(cap.now)
+    await flush()
+    expect(cap.edits).toHaveLength(0)
+
+    // Tick at full interval — first edit fires, "1m" suffix.
+    cap.now = EDIT_INTERVAL_MS
+    __tickForTests(cap.now)
+    await flush()
+    expect(cap.edits).toHaveLength(1)
+    expect(cap.edits[0].messageId).toBe(100)
+    expect(cap.edits[0].newText).toBe(
+      'Background sleep running; awaiting completion.\n\n— still working (1m)',
+    )
+
+    // Tick at 3 intervals total — second edit, "3m".
+    cap.now = EDIT_INTERVAL_MS * 3
+    __tickForTests(cap.now)
+    await flush()
+    expect(cap.edits).toHaveLength(2)
+    expect(cap.edits[1].newText).toBe(
+      'Background sleep running; awaiting completion.\n\n— still working (3m)',
+    )
+  })
+
+  it('strips prior suffix before re-appending so anchor never accumulates', async () => {
+    const cap = setup()
+    startTurn(KEY)
+    noteAsyncDispatch(KEY)
+    // Simulate a noteOutbound for text that already carries a stale
+    // suffix from an earlier round (defence in depth).
+    noteOutbound(KEY, {
+      messageId: 100,
+      text: 'worker dispatched\n\n— still working (12m)',
+    })
+    noteTurnEnd(KEY)
+    cap.now = EDIT_INTERVAL_MS
+    __tickForTests(cap.now)
+    await flush()
+    // The new edit should be based on 'worker dispatched' alone.
+    expect(cap.edits[0].newText).toBe(
+      'worker dispatched\n\n— still working (1m)',
+    )
+  })
+
+  it("clears on 'inbound' reason — user re-engaged", () => {
+    const cap = setup()
+    startTurn(KEY)
+    noteAsyncDispatch(KEY)
+    noteOutbound(KEY, { messageId: 100, text: 'wd' })
+    noteTurnEnd(KEY)
+    cap.now = EDIT_INTERVAL_MS * 2
+    clearPending(KEY, 'inbound')
+    expect(__getStateForTests(KEY)).toBeUndefined()
+    expect(cap.metrics).toContainEqual({
+      kind: 'pending_progress_cleared',
+      chatKey: KEY,
+      elapsedMs: EDIT_INTERVAL_MS * 2,
+      reason: 'inbound',
+    })
+    // No further edits after clear.
+    cap.now = EDIT_INTERVAL_MS * 3
+    __tickForTests(cap.now)
+    expect(cap.edits).toHaveLength(0)
+  })
+
+  it("clears on 'handback' reason — model is about to re-engage", () => {
+    const cap = setup()
+    startTurn(KEY)
+    noteAsyncDispatch(KEY)
+    noteOutbound(KEY, { messageId: 100, text: 'wd' })
+    noteTurnEnd(KEY)
+    clearPending(KEY, 'handback')
+    expect(__getStateForTests(KEY)).toBeUndefined()
+    expect(cap.metrics.some((m) => m.kind === 'pending_progress_cleared' && m.reason === 'handback')).toBe(true)
+  })
+
+  it('times out at MAX_LIFETIME_MS', async () => {
+    const cap = setup()
+    startTurn(KEY)
+    noteAsyncDispatch(KEY)
+    noteOutbound(KEY, { messageId: 100, text: 'wd' })
+    cap.now = 0
+    noteTurnEnd(KEY)
+    // Halfway — still active.
+    cap.now = MAX_LIFETIME_MS / 2
+    __tickForTests(cap.now)
+    await flush()
+    expect(__getStateForTests(KEY)).toBeDefined()
+    // Past the budget — auto-cleared.
+    cap.now = MAX_LIFETIME_MS + 1
+    __tickForTests(cap.now)
+    await flush()
+    expect(__getStateForTests(KEY)).toBeUndefined()
+    expect(cap.metrics.some((m) => m.kind === 'pending_progress_cleared' && m.reason === 'timeout')).toBe(true)
+  })
+
+  it('skips edit (but advances cadence) if total would exceed Telegram message cap', async () => {
+    const cap = setup()
+    startTurn(KEY)
+    noteAsyncDispatch(KEY)
+    // Anchor text long enough that even the smallest suffix overflows.
+    const bigText = 'x'.repeat(TELEGRAM_MSG_CAP - 5)
+    noteOutbound(KEY, { messageId: 100, text: bigText })
+    cap.now = 0
+    noteTurnEnd(KEY)
+    cap.now = EDIT_INTERVAL_MS
+    __tickForTests(cap.now)
+    await flush()
+    expect(cap.edits).toHaveLength(0)
+    // lastEditAt still advanced — we won't spin retrying every tick.
+    const s = __getStateForTests(KEY)
+    expect(s?.lastEditAt).toBe(EDIT_INTERVAL_MS)
+  })
+
+  it('honors the kill switch — no state, no edits, no metrics', async () => {
+    const cap = setup()
+    process.env.SWITCHROOM_DISABLE_PENDING_PROGRESS = '1'
+    try {
+      startTurn(KEY)
+      noteAsyncDispatch(KEY)
+      noteOutbound(KEY, { messageId: 100, text: 'wd' })
+      noteTurnEnd(KEY)
+      expect(__getStateForTests(KEY)).toBeUndefined()
+      cap.now = EDIT_INTERVAL_MS * 3
+      __tickForTests(cap.now)
+      await flush()
+      expect(cap.edits).toHaveLength(0)
+      expect(cap.metrics).toHaveLength(0)
+    } finally {
+      delete process.env.SWITCHROOM_DISABLE_PENDING_PROGRESS
+    }
+  })
+
+  it('startTurn resets per-turn fields but NOT cross-turn activation', () => {
+    const cap = setup()
+    // Turn 1: dispatches async, ends, pending-progress active.
+    startTurn(KEY)
+    noteAsyncDispatch(KEY)
+    noteOutbound(KEY, { messageId: 100, text: 'wd' })
+    cap.now = 1_000
+    noteTurnEnd(KEY)
+    expect(__getStateForTests(KEY)?.activatedAt).toBe(1_000)
+    // Turn 2 starts (e.g. via the gateway's inbound path that already
+    // called clearPending). startTurn resets per-turn fields but the
+    // map entry has been deleted by clearPending, so this should
+    // simply do nothing dangerous if called against an absent key.
+    clearPending(KEY, 'inbound')
+    startTurn(KEY)
+    expect(__getStateForTests(KEY)).toBeUndefined()
+  })
+
+  it('no stale carryover: turn 1 activates, clearPending fires, turn 2 (no async) does not re-activate', async () => {
+    // Reproduces the reviewer's blocker #2 path: turn 1 with async
+    // dispatch activates pending-progress; an arriving turn 2 (real
+    // inbound OR synthesised wake) must clear state so a turn 2 that
+    // does NOT itself dispatch async never inherits the prior turn's
+    // `pending=true` and re-activates against turn 2's anchor.
+    const cap = setup()
+    // ── Turn 1: dispatch async, reply, end — activates.
+    noteAsyncDispatch(KEY)
+    noteOutbound(KEY, { messageId: 100, text: 'worker dispatched' })
+    cap.now = 1_000
+    noteTurnEnd(KEY)
+    expect(__getStateForTests(KEY)?.activatedAt).toBe(1_000)
+
+    // ── Inbound (or handback / cron / vault grant) for turn 2.
+    // Gateway clears state — exactly what the inbound/enqueue hooks
+    // wire up at handleInbound + handleSessionEvent.enqueue.
+    cap.now = 90_000
+    clearPending(KEY, 'inbound')
+    expect(__getStateForTests(KEY)).toBeUndefined()
+
+    // ── Turn 2: reply only, NO async dispatch this turn.
+    noteOutbound(KEY, { messageId: 200, text: 'just answering' })
+    cap.now = 91_000
+    noteTurnEnd(KEY)
+
+    // Turn 2 must NOT activate — no async was dispatched in this turn.
+    // Pre-fix this assertion would fail because the prior turn's
+    // `pending=true` was never reset and `noteTurnEnd` re-activated
+    // against turn 2's fresh anchor.
+    expect(__getStateForTests(KEY)).toBeUndefined()
+
+    // Confirm: no edits fire over the next several poll intervals.
+    cap.now = 91_000 + EDIT_INTERVAL_MS * 3
+    __tickForTests(cap.now)
+    await flush()
+    expect(cap.edits).toHaveLength(0)
+  })
+
+  it('multiple chats — independent state', async () => {
+    const cap = setup()
+    const KEY_A = 'A:_'
+    const KEY_B = 'B:42'
+    startTurn(KEY_A)
+    noteAsyncDispatch(KEY_A)
+    noteOutbound(KEY_A, { messageId: 10, text: 'wd-A' })
+    cap.now = 0
+    noteTurnEnd(KEY_A)
+
+    startTurn(KEY_B)
+    noteAsyncDispatch(KEY_B)
+    noteOutbound(KEY_B, { messageId: 20, text: 'wd-B' })
+    noteTurnEnd(KEY_B)
+
+    cap.now = EDIT_INTERVAL_MS
+    __tickForTests(cap.now)
+    await flush()
+    expect(cap.edits).toHaveLength(2)
+    const byMsg = new Map(cap.edits.map((e) => [e.messageId, e]))
+    expect(byMsg.get(10)?.chatId).toBe('A')
+    expect(byMsg.get(10)?.threadId).toBe(null)
+    expect(byMsg.get(20)?.chatId).toBe('B')
+    expect(byMsg.get(20)?.threadId).toBe(42)
+
+    // Clear A only; B should keep ticking.
+    clearPending(KEY_A, 'inbound')
+    cap.now = EDIT_INTERVAL_MS * 2
+    __tickForTests(cap.now)
+    await flush()
+    expect(cap.edits.filter((e) => e.messageId === 10)).toHaveLength(1)
+    expect(cap.edits.filter((e) => e.messageId === 20)).toHaveLength(2)
+  })
+})

package/telegram-plugin/uat/scenarios/cross-turn-pending-progress-dm.test.ts

@@ -0,0 +1,239 @@
+/**
+ * Cross-turn pending-async progress — UAT regression gate for #1445.
+ *
+ * Verifies the post-fix behaviour shipped in `pending-work-progress.ts`:
+ * when a turn ends with the model having dispatched async background
+ * work (here `Bash` with `run_in_background:true`) and the model has
+ * stopped speaking, the framework keeps editing the model's last reply
+ * *in place* at ~60s intervals so the user sees ambient liveness during
+ * the wait.
+ *
+ * ## Pre-fix behaviour (what the user complained about)
+ *
+ * 1. User sends a long-task prompt at t=0.
+ * 2. Model runs the bash command with `run_in_background:true` and
+ *    sends one PING reply at ~+20s ("Background sleep running…").
+ * 3. Turn ends.
+ * 4. Silence-poke ladder is per-turn — it stops the moment endTurn()
+ *    fires. There is no cross-turn ambient surface.
+ * 5. The user sees NOTHING for ~5 min until the framework's 300s
+ *    silence-poke fallback fires (or — as observed in the UAT that
+ *    drove the fix — does not fire at all, because the turn already
+ *    ended). Production data confirms: silence-poke succeeded/fired
+ *    rate is 0–7% across hundreds of fires.
+ *
+ * ## Post-fix behaviour (this scenario asserts)
+ *
+ * 1. Model sends one fresh reply at ~+20s — the anchor.
+ * 2. Turn ends.
+ * 3. Framework edits the anchor in place at ~+80s, ~+140s, ~+200s,
+ *    ~+260s, ~+320s with the suffix `\n\n— still working (Nm)`.
+ * 4. All edits are SILENT (`disable_notification: true` on the edit
+ *    or, equivalently, an edit which never pushes a notification).
+ *    The user sees ambient liveness without any added pings.
+ * 5. Sleep completes ~+350s; the model wakes (`BashOutput` /
+ *    background-task notification path), turn re-starts,
+ *    `clearPending` fires — no further edits.
+ *
+ * ## What this scenario asserts
+ *
+ *   1. At least one FRESH bot message lands (the initial anchor).
+ *   2. At least one EDIT to the anchor lands AFTER the initial reply,
+ *      whose text contains the framework suffix `— still working (\d+m)`.
+ *   3. Every observed EDIT message is `silent === true` (no push
+ *      notification fired by the in-place edit).
+ *   4. Edits are anchored to the SAME `messageId` as the initial
+ *      fresh reply (single in-place surface, not spammy new sends).
+ *
+ * The full per-message trail is dumped to console for forensic
+ * inspection regardless of pass / fail.
+ *
+ * Wall-clock budget: ~8 min.
+ */
+
+import { describe, expect, it } from "vitest";
+import { spinUp } from "../harness.js";
+import type { ObservedMessage } from "../driver.js";
+
+const SLEEP_SECONDS = 350;
+
+// Engineered to elicit the natural production pattern: the model
+// sends a quick ack reply ("on it — background sleep running"),
+// dispatches the sleep as a background Bash, ends its turn, then
+// returns with "done" once the sleep completes. The framework
+// fix-under-test owns the in-between ambient.
+const PROMPT =
+  `Please run \`sleep ${SLEEP_SECONDS}\` in the background using the ` +
+  `Bash tool with \`run_in_background: true\` — this is a stress ` +
+  `test of the cross-turn ambient progress surface, so the sleep ` +
+  `duration matters. Send a brief one-line acknowledgement that ` +
+  `you've dispatched it (your natural beat-1 ack is fine), then ` +
+  `wait for it to complete. When it finishes, reply with exactly ` +
+  `the single word "done".`;
+
+const OVERALL_DEADLINE_MS = (SLEEP_SECONDS + 240) * 1000;
+
+interface TrailEntry {
+  relMs: number;
+  kind: "fresh" | "edit";
+  silent: boolean;
+  messageId: number;
+  text: string;
+}
+
+const SUFFIX_RE = /\n\n— still working \(\d+m\)$/;
+
+function pad(s: string, n: number): string {
+  return s.length >= n ? s : s + " ".repeat(n - s.length);
+}
+
+describe("uat: cross-turn pending-async ambient progress (#1445)", () => {
+  it(
+    "framework edits the anchor in place during a 350s background bash",
+    async () => {
+      const sc = await spinUp({ agent: "test-harness" });
+      try {
+        const startedAt = Date.now();
+        await sc.sendDM(PROMPT);
+        console.log(`[cross-turn-pending] t=0 prompt sent`);
+
+        const trail: TrailEntry[] = [];
+
+        // Initial wait window — give the model 90s to send its first
+        // anchor reply. After that, we observe edits for the full
+        // sleep duration plus headroom; once the model's final fresh
+        // "done" lands we wind down within 10s.
+        let quiescenceDeadline = startedAt + 90_000;
+        const overallDeadline = startedAt + OVERALL_DEADLINE_MS;
+        let firstAnchorMsgId: number | null = null;
+        let sawDone = false;
+
+        while (Date.now() < overallDeadline) {
+          const remaining = Math.min(
+            quiescenceDeadline - Date.now(),
+            overallDeadline - Date.now(),
+          );
+          if (remaining <= 0) break;
+          try {
+            const msg = await sc.expectMessage(
+              (m: ObservedMessage) => m.fromBot,
+              { from: "bot", timeout: remaining },
+            );
+            const rel = Date.now() - startedAt;
+            const entry: TrailEntry = {
+              relMs: rel,
+              kind: msg.edited ? "edit" : "fresh",
+              silent: msg.silent,
+              messageId: msg.messageId,
+              text: msg.text,
+            };
+            trail.push(entry);
+            console.log(
+              `[cross-turn-pending] +${(rel / 1000).toFixed(1)}s ` +
+                `${entry.kind.toUpperCase()} msg=${entry.messageId} ` +
+                `silent=${entry.silent} text=${JSON.stringify(
+                  entry.text.slice(0, 120).replace(/\n/g, " ⏎ "),
+                )}`,
+            );
+            if (firstAnchorMsgId == null && entry.kind === "fresh") {
+              firstAnchorMsgId = entry.messageId;
+            }
+            const trimmedFinal = entry.text.trim().toLowerCase();
+            const looksLikeDone =
+              entry.kind === "fresh" &&
+              entry.messageId !== firstAnchorMsgId &&
+              (trimmedFinal === "done" || /\bdone\b/.test(trimmedFinal));
+            if (looksLikeDone) {
+              sawDone = true;
+              quiescenceDeadline = Date.now() + 10_000;
+            } else {
+              // Generous quiescence so we cover the whole sleep window
+              // plus a 90s headroom for the model's wake + final reply.
+              quiescenceDeadline = Date.now() + 120_000;
+            }
+          } catch {
+            // Timed out — quiescence reached.
+            break;
+          }
+        }
+
+        // Dump full trail.
+        console.log(
+          "\n========== CROSS-TURN PENDING-PROGRESS TRAIL ==========",
+        );
+        console.log(`prompt: ${SLEEP_SECONDS}s background bash`);
+        console.log(`total bot messages observed: ${trail.length}`);
+        console.log(`anchor messageId: ${firstAnchorMsgId}`);
+        console.log(`saw_done: ${sawDone}`);
+        console.log("");
+        console.log("  rel(s)   kind   silent  msg          text");
+        console.log("  -------  -----  ------  -----------  ----");
+        for (const e of trail) {
+          console.log(
+            `  ${pad((e.relMs / 1000).toFixed(1) + "s", 8)} ` +
+              `${pad(e.kind, 6)} ${pad(String(e.silent), 7)} ` +
+              `${pad(String(e.messageId), 12)} ` +
+              `${e.text.slice(0, 80).replace(/\n/g, " ⏎ ")}`,
+          );
+        }
+        console.log(
+          "=======================================================\n",
+        );
+
+        // ── Regression assertions ─────────────────────────────────
+
+        // (1) at least one fresh anchor reply landed
+        const fresh = trail.filter((e) => e.kind === "fresh");
+        expect(
+          fresh.length,
+          `no fresh bot replies observed — agent isn't responding`,
+        ).toBeGreaterThanOrEqual(1);
+        expect(firstAnchorMsgId).not.toBeNull();
+
+        // (2) at least one edit landed AFTER the initial anchor, and
+        // its text carries the framework's "— still working (Nm)"
+        // suffix. This is THE regression gate for the fix.
+        const edits = trail.filter((e) => e.kind === "edit");
+        const editsWithSuffix = edits.filter((e) => SUFFIX_RE.test(e.text));
+        expect(
+          editsWithSuffix.length,
+          `no in-place edits with the "— still working (Nm)" suffix ` +
+            `landed during the ${SLEEP_SECONDS}s background bash. ` +
+            `Total edits observed: ${edits.length}. The cross-turn ` +
+            `pending-progress fix is not active — see ` +
+            `\`pending-work-progress.ts\` and the gateway hooks at ` +
+            `\`noteAsyncDispatch\` / \`noteOutbound\` / \`noteTurnEnd\`. ` +
+            `Pre-fix this number is zero by construction.`,
+        ).toBeGreaterThanOrEqual(1);
+
+        // (3) every observed edit is silent (an edit never pings per
+        // Telegram semantics; we double-check via the receiving-side
+        // flag so any framework regression that switched to fresh
+        // sends fails loudly).
+        const loudEdits = edits.filter((e) => !e.silent);
+        expect(
+          loudEdits.length,
+          `${loudEdits.length} edit(s) pinged the device — edits ` +
+            `should never fire a notification.`,
+        ).toBe(0);
+
+        // (4) every edit is anchored to the same messageId as the
+        // initial fresh anchor — the framework is editing ONE
+        // surface, not spamming.
+        const offAnchorEdits = edits.filter(
+          (e) => e.messageId !== firstAnchorMsgId,
+        );
+        expect(
+          offAnchorEdits.length,
+          `${offAnchorEdits.length} edit(s) were anchored to a ` +
+            `different message id than the initial reply ` +
+            `(${firstAnchorMsgId}). The framework should edit a ` +
+            `single anchor in place, not a chain of messages.`,
+        ).toBe(0);
+      } finally {
+        await sc.tearDown();
+      }
+    },
+    OVERALL_DEADLINE_MS + 60_000,
+  );
+});