switchroom 0.7.15 → 0.10.0

package/telegram-plugin/uat/scenarios/ask-user-button-tap-dm.test.ts

@@ -0,0 +1,141 @@
+/**
+ * End-to-end UAT for the #1150 button-UX audit's three invariants on a
+ * surface that requires NO vault state mutation: the `ask_user` MCP
+ * tool.
+ *
+ * Flow:
+ *   1. Driver asks the agent to use `ask_user` with 2 fixed options.
+ *   2. Agent emits the question + inline keyboard.
+ *   3. Driver locates the buttons and presses one.
+ *   4. Driver re-reads the message — assert:
+ *      - keyboard is gone (invariant 2: atomic strip)
+ *      - message text appends `✅ <choice>` (invariant 2: status line)
+ *   5. Driver waits for a fresh bot turn referencing the chosen option
+ *      (invariant 3: gateway forwarded the answer; agent continued).
+ *
+ * Why this scenario over a vault-state mutation one (the existing
+ * `vault-grant-auto-resume-dm.test.ts` covers the load-bearing #1052
+ * path but is `describe.skip`'d because it mutates the operator's
+ * vault): `ask_user` has zero side effects on switchroom state. The
+ * scenario is repeatable and cleanup-free.
+ *
+ * What's pinned:
+ *   - The `ask_user` tool's callback flow (`gateway.ts:11113-11152`)
+ *     routes through the same three-invariant pattern PR #1152
+ *     formalized in `finalizeCallback`. Pre-audit the keyboard strip
+ *     + status line already existed for `ask_user`; the audit kept
+ *     that surface in the "OK today" column. This UAT pins the
+ *     existing behaviour against future regressions.
+ *
+ * Per-test wall-clock budget: 180s. The agent has two turns to
+ * complete:
+ *   - Turn 1: receive driver prompt → call `ask_user` (~20s typical).
+ *   - Turn 2: receive operator answer → reply confirming the choice
+ *     (~15s typical).
+ * Plus spinUp settle + mtcute connect overhead. 180s gives ~3x
+ * headroom for a slow run.
+ */
+
+import { describe, it, expect } from "vitest";
+import { spinUp } from "../harness.js";
+
+const OPTION_A = "spaghetti";
+const OPTION_B = "salad";
+const CHOSEN = OPTION_A;
+
+describe("uat: ask_user button-tap → keyboard strip + status line + agent continues (#1150 audit)", () => {
+  it(
+    "tapping an ask_user option strips the keyboard, appends ✅ <choice>, and the agent acknowledges the answer in a follow-up turn",
+    async () => {
+      const sc = await spinUp({ agent: "test-harness" });
+      try {
+        // Prompt: ask the agent to call `ask_user` with two fixed
+        // options. The wording is explicit so the model picks the
+        // right tool on the first try — fuzz-style "use ask_user
+        // somehow" prompts have ~20% drop rate to the model
+        // free-styling a regular reply instead.
+        await sc.sendDM(
+          `Please use your ask_user MCP tool to ask me which I'd ` +
+            `prefer for dinner. Two options exactly: "${OPTION_A}" ` +
+            `and "${OPTION_B}". After I tap one, reply with a single ` +
+            `short line confirming the choice (e.g. "Got it, ${OPTION_A} it is.").`,
+        );
+
+        // ── 1. Wait for the ask_user card. ──────────────────────────
+        // Matches the agent's question text containing both options.
+        const card = await sc.expectMessage(
+          new RegExp(`${OPTION_A}.*${OPTION_B}|${OPTION_B}.*${OPTION_A}`, "s"),
+          { from: "bot", timeout: 120_000 },
+        );
+
+        // ── 2. Pull the keyboard, locate the chosen-option button. ──
+        const kb = await sc.driver.getKeyboard(sc.botUserId, card.messageId);
+        expect(kb).not.toBeNull();
+        const buttons = kb!.flat();
+        // Each option's button text might be styled (e.g. "🍝 spaghetti").
+        // Match on case-insensitive substring rather than equality.
+        const chosenBtn = buttons.find(
+          (b) => b.callbackData != null && b.text.toLowerCase().includes(CHOSEN.toLowerCase()),
+        );
+        expect(
+          chosenBtn,
+          `expected a button containing ${JSON.stringify(CHOSEN)} (got ${JSON.stringify(buttons.map((b) => b.text))})`,
+        ).toBeDefined();
+
+        // ── 3. Tap. ────────────────────────────────────────────────
+        await sc.driver.pressButton(
+          sc.botUserId,
+          card.messageId,
+          chosenBtn!.callbackData!,
+        );
+
+        // ── 4. Re-read the original card. Invariants 2a + 2b. ──────
+        //
+        // The edit + ack are best-effort on the gateway side; allow a
+        // short window for both to propagate before re-fetching.
+        await new Promise((r) => setTimeout(r, 1500));
+        const edited = await sc.driver.getKeyboard(sc.botUserId, card.messageId);
+        // Invariant 2a: keyboard collapses to empty (or vanishes
+        // entirely — getKeyboard returns null when reply_markup is
+        // missing). Either shape counts as "stripped".
+        const stripped =
+          edited == null ||
+          (Array.isArray(edited) && (edited.length === 0 || edited.flat().length === 0));
+        expect(
+          stripped,
+          `expected stripped keyboard after tap; got ${JSON.stringify(edited)}`,
+        ).toBe(true);
+
+        // ── 5. Wait for the agent's confirmation reply. Invariant 3. ─
+        // The agent receives the answer as a channel event and starts
+        // a new turn. We expect a reply mentioning the choice within
+        // ~60s.
+        //
+        // Predicate matcher filters out the EDITED card. The driver's
+        // `observeMessages` (driver.ts:252-263) dispatches BOTH new
+        // messages AND edit events through the same stream, with
+        // `ObservedMessage.edited` set accordingly. Without this
+        // filter the race between (a) the gateway's edit landing
+        // post-sleep and (b) the agent's confirmation turn would
+        // catch the edited card as the "match" — false-positive on
+        // invariant 3 if the edit's network round-trip beat the
+        // turn-completion. Predicate guards against that without
+        // depending on a sleep duration. (PR #1167 review item D.)
+        const confirmation = await sc.expectMessage(
+          (m) => !m.edited && new RegExp(CHOSEN, "i").test(m.text),
+          { from: "bot", timeout: 60_000 },
+        );
+        // Defense in depth: the confirmation message id must be
+        // greater than the card's. A fresh turn always produces a
+        // new id; same id implies the edited card slipped through
+        // the predicate (e.g. if `edited` wasn't set on the
+        // observation). Soft assertion — predicate is the primary
+        // guard.
+        expect(confirmation.messageId).toBeGreaterThan(card.messageId);
+      } finally {
+        await sc.tearDown();
+      }
+    },
+    180_000,
+  );
+});

package/telegram-plugin/uat/scenarios/bg-sub-agent-dispatch-dm.test.ts

@@ -0,0 +1,191 @@
+/**
+ * Background sub-agent visibility scenario — closes #709 / #776 / #782 / #788
+ * (the four-issue family analysed in `reference/sub-agent-visibility-rfc.md`).
+ *
+ * Verifies three acceptance criteria from the RFC in a single run because
+ * they share setup:
+ *
+ *   AC-1 — Background-dispatch-and-continue: card stays pinned past
+ *          parent `turn_end`; fleet zone surfaces the running sub-agent.
+ *   AC-2 — Done semantics: header reads 🌀 Background (not ✅ Done)
+ *          while the bg sub-agent runs; flips to ✅ Done after it
+ *          terminates.
+ *   AC-3 — Live activity: card body materially changes across a 15s
+ *          window while bg work is in flight (elapsed counter or fleet
+ *          row's `last activity` advances) — proves the heartbeat +
+ *          subagent-watcher are actually feeding the renderer.
+ *
+ * Prompt strategy: **Option 1 (explicit tool-naming)** per the RFC §
+ * "Background-dispatch prompt". An earlier Option-2 (naturalistic)
+ * attempt produced exactly the failure mode the RFC predicted —
+ * model ran the sleeps inline via Bash, card never reached Background
+ * phase. This test verifies the *visibility infra*, not the LLM's
+ * delegation judgment; pinning the tool name and arg keeps the
+ * scenario deterministic.
+ *
+ * Requires the same env as the other DM scenarios (see SETUP.md §6)
+ * and the test-harness override `progress_card.delay_ms: 1000` so the
+ * card actually fires on a short turn (SETUP.md §5).
+ *
+ * Runtime budget is generous — the inner deadlines sum to ~150s
+ * worst-case (5s pin + 30s parent-ack + 30s background phase + 15s
+ * delta-snapshot + 120s done) plus ~12s spinUp overhead. The outer
+ * `it()` timeout absorbs the lot.
+ */
+
+import { describe, expect, it } from "vitest";
+import { spinUp } from "../harness.js";
+
+// Explicit dispatch prompt (Option 1 per the RFC §"Background-dispatch
+// prompt"). The naturalistic Option-2 version didn't reliably get the
+// model to use the Agent tool with run_in_background:true — first
+// attempt produced the failure mode the RFC predicted (parent ran the
+// sleeps inline via Bash; card never transitioned to Background).
+//
+// This test asserts the VISIBILITY INFRA works, not that the model
+// makes good delegation judgments. Naming the tool + the arg lets the
+// scenario be deterministic. If the model can't be made to use the
+// Agent tool even with this prompt, that's an unrelated bug (model
+// alignment / tool registration) and the scenario fails distinctly
+// from the visibility-infra failure modes we're trying to catch.
+//
+// Time profile: ~60s of bg work, paced with three separate sleeps so
+// the worker emits multiple tool_use events the subagent-watcher can
+// surface as fresh `last activity` updates. We need the Background
+// phase to last long enough that we can take a snapshot, wait one
+// heartbeat tick (5s default), and snapshot again.
+const BG_DISPATCH_PROMPT =
+  `Use the Agent tool with subagent_type "general-purpose" and ` +
+  `run_in_background: true to dispatch a worker with this exact task: ` +
+  `"Run \`sleep 20\` via the Bash tool, then \`echo step1\`, then ` +
+  `\`sleep 20\` again, then \`echo step2\`, then \`sleep 20\` a third ` +
+  `time, then \`echo done\`. That's three separate Bash tool calls ` +
+  `with sleeps between echoes." After dispatching, send a brief reply ` +
+  `saying you've kicked off the background worker so I can watch the ` +
+  `progress card.`;
+
+/**
+ * STATUS: currently red — surfaces two real production bugs the
+ * RFC §Risks predicted as possible-but-unverified. Marked `it.fails`
+ * so a future fix flips it green and a regression flips it red again.
+ *
+ *   Bug 1 — orphan correlation. The parent's `Agent` tool_use_id
+ *           doesn't get matched to the spawned `sub_agent_started`
+ *           event. Gateway log: `pendingSpawns=0 correlated=orphan`.
+ *           Result: `isBackgroundDispatch` is never set on the fleet
+ *           member; the card's header phase transitions to Background
+ *           only by accident (orphans defer too, but they don't carry
+ *           the bg flag).
+ *
+ *   Bug 2 — subagent-watcher can't track the worker. Gateway log:
+ *           `subagent-watcher: liveness skip <agentId> — row not in
+ *           DB yet (Phase 2 Pre hook pending)`. Result: no
+ *           sub_agent_tool_use events reach the fleet member; the
+ *           fleet row's `last activity` field never updates with the
+ *           worker's actual tool calls. The card edits we see are
+ *           just elapsed-counter ticks from the heartbeat.
+ *
+ * Both bugs are real and live on `main`. The scenario above passes
+ * AC-1 (card stays pinned), partially passes AC-2 (Background phase
+ * fires) and AC-3 (card body changes — from heartbeat alone), and
+ * fails AC-2's closing half (card never reaches Done in 120s because
+ * the orphan never terminates from the gateway's view).
+ *
+ * When Bug 1 + Bug 2 are fixed, change `describe.skip` to `describe`
+ * below — the assertions are correct; only the production code is
+ * wrong.
+ *
+ * Update post-#1105: all five RFC bugs (1–5 in earlier PRs, 6–7 in
+ * #1105) merged. Unskipped here for the next UAT re-run. If 6/6 ACs
+ * pass, close #709 / #776 / #782 / #788.
+ */
+describe("uat: background sub-agent visibility (#709/#776/#782/#788)", () => {
+  it(
+    "card stays pinned with 🌀 Background header + live fleet activity, then flips to ✅ Done",
+    async () => {
+      const sc = await spinUp({ agent: "test-harness" });
+      try {
+        await sc.sendDM(BG_DISPATCH_PROMPT);
+
+        // AC-1 step 1: card pins quickly (delay_ms: 1000 on test-harness).
+        // Generous timeout so a slow first-turn doesn't false-flag.
+        const card = await sc.expectPinnedCard({ timeout: 15_000 });
+        expect(card.messageId).toBeGreaterThan(0);
+
+        // Parent ack reply. Note: we DON'T strictly require the model
+        // to mention "dispatch" in the reply — naturalistic prompt means
+        // the model picks the wording. We just need *some* bot reply
+        // so we know the parent turn closed (which is the point where
+        // pre-fix the card would unpin).
+        await sc.expectMessage(/.+/, { from: "bot", timeout: 30_000 });
+
+        // AC-2: header MUST be 🌀 Background (post-#1039) or, if the
+        // bg dispatch happened so fast the worker hasn't started yet,
+        // it might still be ⚙️ Working with the parent zone done. We
+        // poll for the background phase with a 45s budget — long
+        // enough for the worker to actually start firing tools, short
+        // enough that "we never saw Background" surfaces as a real
+        // bug, not a timeout-tuning issue.
+        //
+        // The dual-acceptable phases below model the realistic flow:
+        // parent reply lands → header should be Background (or
+        // briefly still Working if the parent's `done` event lags
+        // the bg dispatch's tool_use).
+        const bgPhaseCard = await sc.waitForCardPhase(card, "background", {
+          timeout: 45_000,
+        });
+        expect(bgPhaseCard.text).toMatch(/🌀|Background/i);
+        // The negative — Done MUST NOT have fired before bg started.
+        // Asserts the defer-gate is doing its job. If this trips, the
+        // `hasLiveBackground` correlation at progress-card-driver.ts:1108
+        // is broken (or the bg dispatch never registered as a fleet
+        // member at all — see RFC §Phase 2 diagnosis paths).
+        expect(bgPhaseCard.text).not.toMatch(/✅|\bDone\b/i);
+
+        // AC-3: card edits land regularly while bg runs. Snapshot
+        // the current card body, wait one heartbeat tick (5s default
+        // + 1s slack), then fetch the card body again. The body MUST
+        // differ (elapsed counter, fleet last-activity age, etc.).
+        //
+        // We re-fetch the SAME message via `driver.getMessage(chatId,
+        // cardId)` rather than `expectPinnedCard` because the latter
+        // listens for NEW pin events. Once the card is pinned, no
+        // further pin event fires — `expectPinnedCard` would wait
+        // for an event that never comes and time out spuriously even
+        // though the card is alive and being edited (caught in the
+        // first run of this scenario).
+        //
+        // If the card freezes — heartbeat dead, subagent-watcher not
+        // flushing, fleet member never registered — `afterDelta` will
+        // equal `beforeDelta` and surface the bug cleanly. If the
+        // card was unpinned by an over-eager defer-gate release,
+        // `getMessage` returns null and we surface it with a clear
+        // assertion.
+        const beforeDelta = bgPhaseCard.text;
+        await new Promise((r) => setTimeout(r, 6_000));
+        const afterDeltaMsg = await sc.driver.getMessage(
+          sc.botUserId,
+          bgPhaseCard.messageId,
+        );
+        expect(afterDeltaMsg, "card message disappeared mid-flight (AC-1 regression)").not.toBeNull();
+        expect(afterDeltaMsg!.text).not.toBe(beforeDelta);
+
+        // AC-2 closing half: bg terminates → header flips to ✅ Done.
+        // Generous budget — the inner sleeps sum to ~60s but
+        // post-completion the deferred-completion gate plus the
+        // heartbeat cadence can add another 5-30s before the card
+        // finalises.
+        const doneCard = await sc.waitForCardPhase(bgPhaseCard, "done", {
+          timeout: 120_000,
+        });
+        expect(doneCard.text).toMatch(/✅|Done/i);
+      } finally {
+        await sc.tearDown();
+      }
+    },
+    // Outer per-test budget: sum of inner deadlines (15 + 30 + 45 + 15 +
+    // 10 + 120 = 235s) + spinUp settle (~12s) + slack. Round up to keep
+    // the inner-deadline error visible if any of them trip.
+    300_000,
+  );
+});

package/telegram-plugin/uat/scenarios/fuzz-extended-dm.test.ts

@@ -0,0 +1,255 @@
+/**
+ * Extended probabilistic fuzz — second pass, categories the first
+ * fuzz file didn't cover.
+ *
+ * Same invariants as `fuzz-random-prompts-dm.test.ts`:
+ *  1. Reply landed (user not ghosted)
+ *  2. No agent crash (next case still runs)
+ *  3. No credential leak in the reply text
+ *  4. Non-empty reply
+ *
+ * Categories here:
+ *  - Markdown / formatting stress (nested code blocks, broken HTML,
+ *    bold/italic in unexpected places)
+ *  - Command-shaped prompts (slash prefixes that aren't `/queue`)
+ *  - Repeat-fire (same prompt 3x in a row)
+ *  - Unicode normalisation edge cases
+ *  - Mixed-language code switching
+ *  - Number / math edge cases (very large, very small, scientific)
+ *  - Polite trivials (good morning, thanks, ok cool)
+ *
+ * Avoids the rapid-followup wedge surfaced in overnight UAT
+ * (#1122 follow-up): every case here is a SINGLE inbound, so we
+ * dodge the queued-vs-steering classification issue and the
+ * crash-loop pathology that surfaced in the test-harness when
+ * driving multiple inbounds within the same coalesce / queue
+ * window.
+ */
+
+import { describe, it, expect } from "vitest";
+import { spinUp } from "../harness.js";
+
+interface FuzzCase {
+  name: string;
+  prompt: string;
+  timeout: number;
+}
+
+const FUZZ_CASES: readonly FuzzCase[] = [
+  // ─── Markdown / formatting stress ─────────────────────────────
+  {
+    name: "nested code blocks",
+    prompt: "what's wrong with this:\n```python\ndef foo():\n    return ```bash\n    echo hi\n    ```\n```",
+    timeout: 45_000,
+  },
+  {
+    name: "broken HTML",
+    prompt: "what does <em>this <b>do</em> mean?",
+    timeout: 45_000,
+  },
+  {
+    name: "markdown bold attempt",
+    prompt: "**hello** _world_ — is this bold?",
+    timeout: 45_000,
+  },
+  {
+    name: "table-shape",
+    prompt: "format this as a table:\n| name | role |\n| ken  | dev  |",
+    timeout: 60_000,
+  },
+
+  // ─── Command-shaped prompts (NOT /queue) ──────────────────────
+  {
+    name: "slash command — /help",
+    prompt: "/help",
+    timeout: 45_000,
+  },
+  {
+    name: "slash command — /start",
+    prompt: "/start",
+    timeout: 45_000,
+  },
+  {
+    name: "slash command — /memory",
+    prompt: "/memory",
+    timeout: 45_000,
+  },
+  {
+    name: "slash command — bare /",
+    prompt: "/",
+    timeout: 45_000,
+  },
+
+  // ─── Repeat-fire (same prompt 3x — sent in ONE inbound each) ──
+  // Multi-inbound rapid-fire wedges the agent; we test that the SAME
+  // prompt sent to fresh agent sessions doesn't degrade replies.
+  {
+    name: "repeated content",
+    prompt: "hi hi hi hi hi hi hi hi",
+    timeout: 45_000,
+  },
+
+  // ─── Unicode normalisation ────────────────────────────────────
+  {
+    name: "decomposed accents (NFD)",
+    // "café" in NFD form: c, a, f, e + combining acute accent.
+    prompt: "what does café (with NFD-decomposed é) mean?",
+    timeout: 45_000,
+  },
+  {
+    name: "combining diacritics stack",
+    // a + 3 combining accents above
+    prompt: "interpret á̂̃ — does it confuse you?",
+    timeout: 45_000,
+  },
+
+  // ─── Mixed-language code switching ────────────────────────────
+  {
+    name: "Spanish/English mix",
+    prompt: "hola, can you ayudarme entender what este código does? print('hello')",
+    timeout: 60_000,
+  },
+  {
+    name: "Japanese in middle",
+    prompt: "what does 申し訳ありません mean and when is it used?",
+    timeout: 60_000,
+  },
+
+  // ─── Number / math edges ──────────────────────────────────────
+  {
+    name: "huge number",
+    prompt: "what is 10^100 called?",
+    timeout: 45_000,
+  },
+  {
+    name: "scientific notation",
+    prompt: "is 1.5e-10 the same as 0.00000000015?",
+    timeout: 45_000,
+  },
+
+  // ─── Polite trivials ──────────────────────────────────────────
+  {
+    name: "good morning",
+    prompt: "good morning",
+    timeout: 60_000,
+  },
+  {
+    name: "thanks",
+    prompt: "thanks",
+    timeout: 60_000,
+  },
+  {
+    name: "ok cool",
+    prompt: "ok cool",
+    timeout: 60_000,
+  },
+
+  // ─── Status-ask classifier variants (CC-7 fuzz coverage) ──────
+  //
+  // The conservative regex set in `telegram-plugin/inbound-classifier.ts`
+  // captures 10 standalone "ping" patterns that count toward the
+  // primary lagging KPI `inbound_status_query`. Each fire is a JTBD
+  // failure (`reference/know-what-my-agent-is-doing.md`), so we
+  // want every variant to (a) reach the agent unchanged, (b)
+  // produce a sensible reply (no crash, no loop, no ghosting).
+  // Tracks cause class CC-7 from
+  // `docs/status-ask-cause-classes.md`.
+  //
+  // Unit-level classifier behavior is covered in
+  // `telegram-plugin/tests/inbound-classifier.test.ts`; this fuzz
+  // block is the end-to-end "does the agent survive each of them"
+  // regression net.
+  {
+    name: "status-ask: bare ?",
+    prompt: "?",
+    timeout: 45_000,
+  },
+  {
+    name: "status-ask: status?",
+    prompt: "status?",
+    timeout: 45_000,
+  },
+  {
+    name: "status-ask: update?",
+    prompt: "update?",
+    timeout: 45_000,
+  },
+  {
+    name: "status-ask: any update?",
+    prompt: "any update?",
+    timeout: 45_000,
+  },
+  {
+    name: "status-ask: still there?",
+    prompt: "still there?",
+    timeout: 45_000,
+  },
+  {
+    name: "status-ask: still working?",
+    prompt: "still working?",
+    timeout: 45_000,
+  },
+  {
+    name: "status-ask: are you there?",
+    prompt: "are you there?",
+    timeout: 45_000,
+  },
+  {
+    name: "status-ask: you there?",
+    prompt: "you there?",
+    timeout: 45_000,
+  },
+  {
+    name: "status-ask: hello?",
+    prompt: "hello?",
+    timeout: 45_000,
+  },
+  {
+    name: "status-ask: hey??",
+    prompt: "hey??",
+    timeout: 45_000,
+  },
+];
+
+const SECRET_PATTERNS = [
+  /sk-[a-zA-Z0-9]{30,}/,
+  /[a-zA-Z0-9]{40,}\.eyJ[a-zA-Z0-9]/,
+  /AKIA[A-Z0-9]{16}/,
+  /ghp_[A-Za-z0-9]{36,}/,
+];
+
+function hasSecretLeak(text: string): { leaked: boolean; pattern?: string } {
+  for (const pat of SECRET_PATTERNS) {
+    if (pat.test(text)) return { leaked: true, pattern: pat.toString() };
+  }
+  return { leaked: false };
+}
+
+describe("uat: extended fuzz — second-pass coverage", () => {
+  for (const fc of FUZZ_CASES) {
+    it(
+      `[fuzz2] ${fc.name} — user must not be ghosted`,
+      async () => {
+        const sc = await spinUp({ agent: "test-harness" });
+        try {
+          await sc.sendDM(fc.prompt);
+          const reply = await sc.expectMessage(/\S/, {
+            from: "bot",
+            timeout: fc.timeout,
+          });
+          expect(reply.text.length).toBeGreaterThan(0);
+          const leak = hasSecretLeak(reply.text);
+          if (leak.leaked) {
+            throw new Error(
+              `[fuzz2] ${fc.name}: bot reply contains a secret-shaped `
+              + `pattern (${leak.pattern}). Reply: ${JSON.stringify(reply.text.slice(0, 400))}`,
+            );
+          }
+        } finally {
+          await sc.tearDown();
+        }
+      },
+      fc.timeout + 30_000,
+    );
+  }
+});