switchroom 0.7.15 → 0.10.0

package/telegram-plugin/uat/scenarios/jtbd-soft-commit-dm.test.ts

@@ -0,0 +1,67 @@
+/**
+ * JTBD scenario — soft-commit for slow turns.
+ *
+ * The new conversational-pacing prompt (#1122) instructs the agent
+ * to send a one-liner "let me check, back in a few" before slow
+ * work. This UAT exercises that behaviour: send a prompt that
+ * obviously needs >15s, expect the FIRST outbound to be a short
+ * soft-commit message, with the final answer landing later.
+ *
+ * Not strict — the agent's allowed to skip the soft-commit if it
+ * judges the work is fast enough. The assertion is "the user does
+ * NOT see a long silent gap before the first sign of life": either
+ * a soft-commit OR the actual reply lands within 20s.
+ */
+
+import { describe, it, expect } from "vitest";
+import { spinUp } from "../harness.js";
+
+// A prompt that needs real work (file reads / web search-ish / some
+// thinking) so the model is incentivised to soft-commit.
+const SLOW_PROMPT = (
+  "Read /etc/hostname and /etc/os-release, then summarise this "
+  + "machine in a single sentence (what OS family, what hostname). "
+  + "Take your time."
+);
+
+describe("uat: soft-commit pacing", () => {
+  it(
+    "user asks slow question → first reply lands within 20s",
+    async () => {
+      const sc = await spinUp({ agent: "test-harness" });
+      try {
+        const sendStart = Date.now();
+        await sc.sendDM(SLOW_PROMPT);
+
+        // 30s wall-clock budget gives mtcute polling jitter + the
+        // agent's first tool call enough headroom that a "near-miss
+        // soft commit" (model thinks for 25s then sends) still passes.
+        // Previous 25s/22s pair sat exactly in the model's natural
+        // think-then-respond window and produced flake unrelated to
+        // any real bug.
+        const firstReply = await sc.expectMessage(/\S/, {
+          from: "bot",
+          timeout: 30_000,
+        });
+        const ttfo = Date.now() - sendStart;
+
+        expect(firstReply.text.length).toBeGreaterThan(0);
+        expect(ttfo).toBeLessThan(30_000);
+
+        // If the first reply IS the final answer (short, complete),
+        // the model skipped soft-commit ceremony — fine, just note.
+        if (firstReply.text.length > 200) {
+          console.log(
+            `[soft-commit] model produced a long final answer as the `
+            + `first message (${firstReply.text.length} chars, ${ttfo}ms). `
+            + `Conversational pacing prompt would prefer a soft-commit `
+            + `first — but this is a soft preference, not a contract.`,
+          );
+        }
+      } finally {
+        await sc.tearDown();
+      }
+    },
+    50_000,
+  );
+});

package/telegram-plugin/uat/scenarios/jtbd-status-query-dm.test.ts

@@ -0,0 +1,49 @@
+/**
+ * JTBD scenario — `status?` inbound classifier.
+ *
+ * The conversational-pacing redesign (#1122 PR1) wired a primary
+ * lagging KPI: `inbound_status_query`, the count of users typing
+ * "status?", "still there?", "?", etc — every fire is a JTBD
+ * failure. We assert the classifier triggers AND the agent
+ * gracefully responds (i.e. doesn't crash, doesn't ignore, doesn't
+ * loop on it).
+ *
+ * Note: the classifier is fire-and-forget — it emits a runtime
+ * metric event but doesn't change routing. So all we can assert
+ * from the driver side is "the agent still replies sensibly" —
+ * the metric emission is verified by the unit tests in
+ * `tests/inbound-classifier.test.ts`. This UAT exists for
+ * end-to-end safety: "sending status? doesn't break anything."
+ */
+
+import { describe, it, expect } from "vitest";
+import { spinUp } from "../harness.js";
+
+const STATUS_QUERIES = ["status?", "still there?", "any update?", "?"];
+
+describe("uat: status-query inbound", () => {
+  for (const query of STATUS_QUERIES) {
+    it(
+      `user sends ${JSON.stringify(query)} → agent replies sensibly`,
+      async () => {
+        const sc = await spinUp({ agent: "test-harness" });
+        try {
+          await sc.sendDM(query);
+
+          // Any non-empty reply within 60s is acceptable. The
+          // interesting thing is the classifier metric fired —
+          // verified at the unit-test level. Here we just want
+          // "no crash, no silent-end, sensible reply."
+          const reply = await sc.expectMessage(/\S/, {
+            from: "bot",
+            timeout: 60_000,
+          });
+          expect(reply.text.length).toBeGreaterThan(0);
+        } finally {
+          await sc.tearDown();
+        }
+      },
+      90_000,
+    );
+  }
+});

package/telegram-plugin/uat/scenarios/location-inbound-dm.test.ts

@@ -0,0 +1,65 @@
+/**
+ * Location-inbound scenario — driver shares a geolocation with the
+ * test bot. Exercises the new `message:location` handler from #1077
+ * end-to-end: gateway parses the lat/lon, builds a `(location: …)`
+ * envelope, forwards to the agent, agent replies.
+ *
+ * Requires the same env as `smoke-dm-reply.test.ts` (see
+ * `uat/SETUP.md` §6).
+ *
+ * Coordinates are intentionally a well-known landmark (Sydney Opera
+ * House) so a failure trace makes "what was shared" obvious — and so
+ * a chatbot persona has something semantically grounded to respond to,
+ * which makes the bot's reply check more meaningful than asserting
+ * `.+`. We still tolerate ANY reply text — the goal is to prove the
+ * gateway forwarded the location, not to grade the agent's geography.
+ *
+ * Other 12 inbound types from #1077 are covered structurally in
+ * `tests/inbound-message-types.test.ts`. End-to-end UAT for them
+ * (contact, venue, dice, poll, web_app_data, users_shared,
+ * chat_shared, dice, game, story, paid_media, successful_payment,
+ * passport_data) is deferred — most require either a custom bot
+ * setup (mini-app, payments provider) or a Telegram client gesture
+ * (story share, dice roll) that the mtcute driver does not script
+ * cleanly enough to be worth the brittleness.
+ */
+
+import { describe, it, expect } from "vitest";
+import { spinUp } from "../harness.js";
+
+// Sydney Opera House — recognizable, non-sensitive, stable across runs.
+const SYDNEY_OPERA_HOUSE_LAT = -33.8568;
+const SYDNEY_OPERA_HOUSE_LON = 151.2153;
+
+describe("uat: location-inbound DM round-trip", () => {
+  it(
+    "driver shares a geolocation, bot replies within 90s",
+    async () => {
+      const sc = await spinUp({ agent: "test-harness" });
+
+      try {
+        await sc.driver.sendLocation(
+          sc.botUserId,
+          SYDNEY_OPERA_HOUSE_LAT,
+          SYDNEY_OPERA_HOUSE_LON,
+        );
+
+        // Same budget as smoke-dm-reply: 90s tolerates the gateway's
+        // coalescing window + one normal Claude turn. A healthy agent
+        // replies in <20s.
+        const reply = await sc.expectMessage(/.+/, {
+          from: "bot",
+          timeout: 90_000,
+        });
+
+        expect(reply.text.length).toBeGreaterThan(0);
+        expect(reply.senderUserId).toBe(sc.botUserId);
+      } finally {
+        await sc.tearDown();
+      }
+    },
+    // Mirrors smoke-dm-reply's 110s outer budget — must exceed the
+    // 90s inner deadline plus spinUp overhead.
+    110_000,
+  );
+});

package/telegram-plugin/uat/scenarios/midturn-silent-dm.test.ts

@@ -0,0 +1,175 @@
+/**
+ * Mid-turn `disable_notification` scenario.
+ *
+ * Goal context: cause class CC-2 in `docs/status-ask-cause-classes.md`
+ * — the L2 conversational layer. The conversational-pacing prompt
+ * (`profiles/_shared/telegram-style.md.hbs:10`) instructs the model to
+ * pass `disable_notification: true` on mid-turn `reply` calls so the
+ * user only gets a device ping on the FINAL answer. If that contract
+ * silently degrades — model regression, prompt drift, or a gateway
+ * code path that drops the flag — every mid-turn reply pings. Users
+ * mute the bot. They then can't tell working from done. They ask
+ * "are you alive?" — `inbound_status_query` ticks.
+ *
+ * The flag is observable on the receiving side via mtcute's
+ * `message.isSilent` getter (corresponds to Telegram's
+ * `message.silent` flag, set by sender's `disable_notification` Bot
+ * API param). The driver was extended in this PR to surface it on
+ * `ObservedMessage.silent`.
+ *
+ * ## What the scenario asserts
+ *
+ * 1. Send a prompt that should produce multiple bot outbounds (a
+ *    soft commit + mid-turn updates + a final answer). The prompt
+ *    is explicit about wanting paced updates so the model doesn't
+ *    optimize to a single reply.
+ * 2. Collect every bot message in the turn (waits for quiescence:
+ *    no fresh bot message for `QUIESCENCE_MS`).
+ * 3. Assert: every bot message EXCEPT THE LAST has `silent === true`.
+ * 4. Assert: the LAST bot message has `silent === false` (the final
+ *    answer should ping).
+ *
+ * ## Tolerances
+ *
+ * - If the turn has only one bot message (model judged the work fast
+ *   enough to skip pacing), the mid-turn assertion is vacuous and we
+ *   only check that the single final message is NOT silent. The
+ *   prompt is engineered to be slow enough that this is unlikely,
+ *   but we don't fail on it.
+ * - Quiescence window is 12s — long enough that a paused model isn't
+ *   mistaken for "done", short enough that test wall-clock stays
+ *   reasonable.
+ * - Edits don't count as fresh messages — we observe `edited === false`
+ *   only. This matches the production semantic: an edit doesn't push
+ *   a notification.
+ *
+ * ## Failure shapes
+ *
+ *   1. Mid-turn ping degrade — at least one non-last message has
+ *      `silent === false`. The error message names the offending
+ *      message index + text preview.
+ *   2. Final-answer silent — the last message has `silent === true`.
+ *      Means the final answer doesn't ping; user might miss the
+ *      reply landing.
+ *   3. No bot messages within timeout — distinct failure: agent
+ *      isn't responding at all.
+ *
+ * Requires the same env as `smoke-dm-reply.test.ts` (see
+ * `uat/SETUP.md` §6).
+ */
+
+import { describe, expect, it } from "vitest";
+import { spinUp } from "../harness.js";
+import type { ObservedMessage } from "../driver.js";
+
+const QUIESCENCE_MS = 12_000;
+const OVERALL_DEADLINE_MS = 120_000;
+
+// Multi-step prompt with explicit pacing expectations. Engineered so
+// a well-behaved model produces:
+//   1. soft commit ("on it" / "let me check")
+//   2. mid-turn update after each file (with disable_notification: true)
+//   3. final answer
+//
+// The work itself is two trivial file reads + a one-sentence
+// summary. If the model collapses this to a single reply, the test
+// still asserts the disable_notification contract on what it does
+// emit; the vacuous-mid-turn path is allowed.
+const PACED_PROMPT =
+  "Please follow this exact pacing protocol for this turn:\n" +
+  "  1. First send a brief 'on it' reply so I know you started.\n" +
+  "  2. Read /etc/hostname, then send a brief mid-turn update saying " +
+  "what the hostname is. Use disable_notification:true on that update.\n" +
+  "  3. Read /etc/os-release, then send a brief mid-turn update saying " +
+  "what the OS family is. Use disable_notification:true on that update.\n" +
+  "  4. Finally send a single-sentence summary as your final answer " +
+  "(no disable_notification flag — this one should ping me).\n" +
+  "Keep each message short.";
+
+describe("uat: mid-turn replies pass disable_notification (CC-2)", () => {
+  it(
+    "every mid-turn bot reply is silent; only the final answer pings",
+    async () => {
+      const sc = await spinUp({ agent: "test-harness" });
+      try {
+        await sc.sendDM(PACED_PROMPT);
+
+        const collected: ObservedMessage[] = [];
+        const overallDeadline = Date.now() + OVERALL_DEADLINE_MS;
+        let quiescenceDeadline = Date.now() + 30_000; // first message
+        // bigger budget
+
+        // Drain bot messages until QUIESCENCE_MS passes with no
+        // fresh non-edit observation, or the overall deadline hits.
+        while (Date.now() < overallDeadline) {
+          const remaining = Math.min(
+            quiescenceDeadline - Date.now(),
+            overallDeadline - Date.now(),
+          );
+          if (remaining <= 0) break;
+          try {
+            const msg = await sc.expectMessage(
+              (m: ObservedMessage) => m.fromBot && !m.edited,
+              { from: "bot", timeout: remaining },
+            );
+            collected.push(msg);
+            quiescenceDeadline = Date.now() + QUIESCENCE_MS;
+          } catch {
+            // Timed out — that's the quiescence signal we wanted.
+            break;
+          }
+        }
+
+        expect(
+          collected.length,
+          `no bot messages observed within ${OVERALL_DEADLINE_MS}ms — ` +
+            `agent isn't responding at all (distinct failure from CC-2).`,
+        ).toBeGreaterThan(0);
+
+        const trail = collected
+          .map(
+            (m, i) =>
+              `  [${i}] silent=${m.silent} text=${JSON.stringify(
+                m.text.slice(0, 80),
+              )}`,
+          )
+          .join("\n");
+
+        // Final answer should ping.
+        const last = collected[collected.length - 1];
+        expect(
+          last.silent,
+          `final answer (message ${collected.length - 1}) was marked ` +
+            `silent — the user won't get pinged when the turn finishes. ` +
+            `Trail:\n${trail}`,
+        ).toBe(false);
+
+        // Mid-turn updates should NOT ping. Vacuous when the model
+        // emitted only the final answer; meaningful when paced.
+        const midTurn = collected.slice(0, -1);
+        const loudMidTurn = midTurn.filter((m) => !m.silent);
+        expect(
+          loudMidTurn.length,
+          `${loudMidTurn.length} mid-turn message(s) were NOT silent — ` +
+            `each one pings the user's device. Conversational pacing ` +
+            `requires disable_notification:true on mid-turn replies. ` +
+            `Trail:\n${trail}`,
+        ).toBe(0);
+
+        if (midTurn.length === 0) {
+          console.warn(
+            `[midturn-silent] model produced only 1 bot reply — the ` +
+              `mid-turn assertion was vacuous. Prompt may not be ` +
+              `slow enough to force pacing, or the model is ignoring ` +
+              `the explicit step-by-step instructions. This is not a ` +
+              `failure of CC-2, but the scenario didn't cover its ` +
+              `intended ground.`,
+          );
+        }
+      } finally {
+        await sc.tearDown();
+      }
+    },
+    OVERALL_DEADLINE_MS + 30_000,
+  );
+});

package/telegram-plugin/uat/scenarios/reactions-dm.test.ts

@@ -0,0 +1,142 @@
+/**
+ * Reaction lifecycle scenario — driver DMs the test bot, bot reacts
+ * to the inbound message through the lifecycle and lands a terminal
+ * emoji once the reply ships.
+ *
+ * Part of: https://github.com/switchroom/switchroom/issues/866
+ * Goal context: cause class CC-1 / CC-6 in
+ * `docs/status-ask-cause-classes.md` (the L1 ambient layer should
+ * deliver a definitively-done terminal emoji within a few seconds
+ * of the bot's final reply — otherwise the user looks at their
+ * inbound, sees it still wearing 🤔, and asks "you done?").
+ *
+ * History: this scenario was previously `describe.skip` with a
+ * rationale that the pinned progress card "renders INSTEAD of
+ * reactions". The card was retired in #1126; the card-vs-reaction
+ * branch in the gateway is dead. We can now exercise the full
+ * lifecycle end-to-end without the two-agent split.
+ *
+ * What we assert (in priority order):
+ *
+ *  1. Within the turn, the driver sees AT LEAST ONE `+` reaction
+ *     op (the L1 "I'm alive" signal). Fast turns may collapse
+ *     intermediate states, so we only require *one* add, not a
+ *     specific emoji.
+ *  2. By the time the bot has sent a final reply (+ a short tail
+ *     for Telegram to deliver the terminal-emoji replace), the
+ *     LAST observed `+` op is in the `done` set (`👍 / 💯 / 🎉`).
+ *
+ * Why "last `+` op wins" rather than `expectReaction(['👍'])` with
+ * a literal sequence: `setMessageReaction` REPLACES the prior emoji
+ * atomically. mtcute's update stream can deliver the replace as a
+ * `-prev` followed by a `+next`, or as a single coalesced event,
+ * depending on server batching. The "last add wins" shape matches
+ * the production semantics — whatever's *currently* on the message
+ * is what the user actually sees.
+ *
+ * The observer must be attached BEFORE the reply lands so we
+ * capture the queued / working reactions, not just the terminal
+ * one. Pattern: `observeReactions` immediately after `sendDM`
+ * returns the messageId, drain into a trail array while we wait
+ * for the reply, then run a short tail to catch the terminal
+ * after the reply.
+ *
+ * Requires the same env as `smoke-dm-reply.test.ts` (see
+ * `uat/SETUP.md` §6).
+ */
+
+import { describe, expect, it } from "vitest";
+import { spinUp } from "../harness.js";
+
+const TERMINAL_DONE_EMOJI = new Set(["👍", "💯", "🎉"]);
+const TAIL_AFTER_REPLY_MS = 8_000;
+
+const INBOUND = (): string => `uat-reactions ${new Date().toISOString()}`;
+
+interface ObservedOp {
+  emoji: string;
+  op: "+" | "-";
+  at: number;
+}
+
+describe("uat: reaction lifecycle on driver DM", () => {
+  it(
+    "driver sees an alive reaction, then a terminal-done emoji by reply tail",
+    async () => {
+      const sc = await spinUp({ agent: "test-harness" });
+      try {
+        const sent = await sc.sendDM(INBOUND());
+
+        // Attach the observer immediately so the queued (👀) and
+        // working reactions don't fire before the listener exists.
+        const trail: ObservedOp[] = [];
+        const iter = sc.driver
+          .observeReactions(sc.botUserId, { messageId: sent.messageId })
+          [Symbol.asyncIterator]();
+        let pump: Promise<void> | null = null;
+        let stopPump = false;
+        pump = (async () => {
+          while (!stopPump) {
+            const next = await iter.next();
+            if (next.done === true) return;
+            trail.push({
+              emoji: next.value.emoji,
+              op: next.value.op,
+              at: Date.now(),
+            });
+          }
+        })();
+
+        try {
+          // Wait for the bot's reply (any content). Gives the L1
+          // lifecycle time to traverse queued → working → done.
+          const reply = await sc.expectMessage(/\S/, {
+            from: "bot",
+            timeout: 60_000,
+          });
+          expect(reply.text.length).toBeGreaterThan(0);
+
+          // Tail after the reply for Telegram to deliver the
+          // terminal-emoji replace. In practice <1s on a healthy bot;
+          // 8s ceiling absorbs server batching jitter.
+          await new Promise((resolve) =>
+            setTimeout(resolve, TAIL_AFTER_REPLY_MS),
+          );
+        } finally {
+          stopPump = true;
+          await iter.return?.();
+          if (pump) {
+            await pump.catch(() => {
+              /* generator return triggers rejection on pending iter.next() — ignore */
+            });
+          }
+        }
+
+        // L1 alive signal: at least one `+` op landed during the turn.
+        const adds = trail.filter((o) => o.op === "+");
+        expect(
+          adds.length,
+          `expected at least one reaction-add during the turn, got 0. ` +
+            `Full trail: ${trail.map((o) => `${o.op}${o.emoji}`).join(" ") || "(empty)"}`,
+        ).toBeGreaterThan(0);
+
+        // L1 terminal: the LAST `+` op should be a terminal-done emoji.
+        // Extra `-` ops after the final `+` are tolerated (Telegram
+        // sometimes emits a bare clean-up `-`); the last `+` is what
+        // the user actually sees.
+        const lastAdd = adds[adds.length - 1];
+        expect(
+          TERMINAL_DONE_EMOJI.has(lastAdd.emoji),
+          `expected last reaction-add to be one of ${[
+            ...TERMINAL_DONE_EMOJI,
+          ].join(", ")}, got ${lastAdd.emoji}. Full trail: ${trail
+            .map((o) => `${o.op}${o.emoji}`)
+            .join(" ")}`,
+        ).toBe(true);
+      } finally {
+        await sc.tearDown();
+      }
+    },
+    90_000,
+  );
+});

package/telegram-plugin/uat/scenarios/reactions-trigger-turn-dm.test.ts

@@ -0,0 +1,96 @@
+/**
+ * UAT scenario — driver reacts to a bot DM with a trigger emoji and
+ * observes the agent process a synthetic inbound turn (#1074).
+ *
+ * Flow:
+ *   1. Driver sends a DM that will provoke a bot reply.
+ *   2. Bot replies — driver observes the reply message id.
+ *   3. Driver places a 👎 reaction on the bot's reply.
+ *   4. Assert: the agent emits a subsequent action (another outbound
+ *      message). The reaction-trigger pipeline synthesizes a new
+ *      `<channel source="reaction">` inbound turn, which the agent's
+ *      Claude session treats as a normal turn and (per profile
+ *      guidance) acknowledges or course-corrects.
+ *
+ *   Negative:
+ *     - Driver also places a ❤️ reaction (not in the default
+ *       allowlist) on a separate bot message.
+ *     - Assert: NO new agent action within the negative-budget window.
+ *
+ * Requires the same env as `smoke-dm-reply.test.ts` — see
+ * `uat/SETUP.md` §6.
+ *
+ * NOTE: this scenario depends on the test-harness agent having the
+ * default `reactions:` config (allowlist includes 👎). If an operator
+ * has narrowed the allowlist this case will fail-with-message.
+ */
+
+import { describe, it, expect } from "vitest";
+import { spinUp } from "../harness.js";
+
+const TRIGGER_INBOUND = `uat-reaction-trigger ${new Date().toISOString()}`;
+const NEGATIVE_INBOUND = `uat-reaction-trigger-negative ${new Date().toISOString()}`;
+
+describe("uat: bot reaction triggers synthetic agent turn (#1074)", () => {
+  it(
+    "👎 on a bot reply dispatches a new agent turn; ❤️ does not",
+    async () => {
+      const sc = await spinUp({ agent: "test-harness" });
+      try {
+        // 1. Drive the first bot reply we'll react to.
+        await sc.sendDM(TRIGGER_INBOUND);
+        const firstReply = await sc.expectMessage(/.+/, {
+          from: "bot",
+          timeout: 90_000,
+        });
+        expect(firstReply.senderUserId).toBe(sc.botUserId);
+
+        // 2. React 👎 to the bot's reply. Default allowlist includes 👎,
+        //    so the gateway should dispatch a synthetic inbound after
+        //    the debounce window elapses.
+        await sc.driver.sendReaction(sc.botUserId, firstReply.messageId, "👎");
+
+        // 3. Wait for the agent to emit ANY subsequent message. The
+        //    debounce window is 30s by default, plus a Claude turn —
+        //    budget 120s to be safe.
+        const triggeredReply = await sc.expectMessage(/.+/, {
+          from: "bot",
+          timeout: 120_000,
+        });
+        expect(triggeredReply.messageId).not.toBe(firstReply.messageId);
+        expect(triggeredReply.senderUserId).toBe(sc.botUserId);
+
+        // ── Negative case ───────────────────────────────────────────────
+        await sc.sendDM(NEGATIVE_INBOUND);
+        const secondReply = await sc.expectMessage(/.+/, {
+          from: "bot",
+          timeout: 90_000,
+        });
+        await sc.driver.sendReaction(sc.botUserId, secondReply.messageId, "❤️");
+
+        // Wait the full debounce window + a generous Claude budget. If
+        // a new turn fires within this window, the negative case has
+        // failed (the allowlist leaked).
+        const NEGATIVE_BUDGET_MS = 45_000;
+        let leaked = false;
+        try {
+          await sc.expectMessage(/.+/, {
+            from: "bot",
+            timeout: NEGATIVE_BUDGET_MS,
+          });
+          leaked = true;
+        } catch {
+          // Expected — no new message within the negative window.
+        }
+        expect(leaked).toBe(false);
+      } finally {
+        await sc.tearDown();
+      }
+    },
+    // Per-test budget — must cover trigger turn + debounce + agent
+    // reply + negative-budget window + spinUp overhead. 5 minutes is
+    // generous but on the order of `progress-card-dm.test.ts` which
+    // also has multi-phase waits.
+    300_000,
+  );
+});