switchroom 0.7.15 → 0.10.0

package/telegram-plugin/turn-signal-tracker.ts

@@ -1,46 +1,82 @@
 /**
- * Per-turn silent-gap tracker for streaming observability.
+ * Per-turn signal + outbound tracker for streaming observability.
  *
- * Tracks the longest contiguous interval within a turn where no user-visible
- * signal was sent. Signals include: progress-card edits, status-reaction
- * transitions, answer-lane updates, and fresh sendMessage calls.
+ * Tracks TWO things, keyed by chatId+threadId:
+ *
+ *   1. **Signal gap** — longest contiguous interval where no user-visible
+ *      signal of ANY kind was sent (progress-card edits, status-reaction
+ *      transitions, answer-lane updates, fresh sendMessage calls). The
+ *      original use case from #203.
+ *
+ *   2. **Outbound messages** (added 2026-05 for the conversational-turn-
+ *      UX redesign, issue #1122) — strictly user-visible MESSAGES that
+ *      the agent sent: `reply`, `stream_reply` first-emits, progress
+ *      card flushes that produce a fresh sendMessage. Status reactions
+ *      and message edits don't count here — they don't ping the device
+ *      and aren't what "outbound silence" means for the KPI.
  *
  * Keyed by chatId+threadId so concurrent turns in different chats don't
- * collide. Designed to be fully standalone (no grammy/bot dependency) so
- * it's testable with deterministic time injection via vi.useFakeTimers().
+ * collide. Fully standalone — no grammy/bot dependency, deterministic
+ * time injection via vi.useFakeTimers().
  *
  * Usage:
- *   signalTracker.reset(key, now)       // at turn start
- *   signalTracker.noteSignal(key, now)  // on every user-visible signal
- *   signalTracker.getLongestGap(key)    // at turn_end
- *   signalTracker.clear(key)            // after emitting (cleanup)
+ *   signalTracker.reset(key, now)                  // at turn start
+ *   signalTracker.noteSignal(key, now)             // any signal (legacy)
+ *   signalTracker.noteOutbound(key, now)           // outbound message only
+ *   signalTracker.getLongestGap(key)               // at turn_end (signal)
+ *   signalTracker.getOutboundMetrics(key)          // at turn_end (KPIs)
+ *   signalTracker.clear(key)                       // after emitting
  */
 
 export interface TurnSignalState {
-  /** The time the current gap started (i.e., the last signal time). */
+  /** The time the turn began. Used to compute TTFO. */
+  turnStartedAt: number
+  /** Time the current signal gap started (last signal time). */
   lastSignalAt: number
-  /** The longest gap observed so far (ms). */
+  /** Longest signal-gap (any signal) observed so far (ms). */
   longestGapMs: number
+  /** First outbound message timestamp this turn, or null if none yet. */
+  firstOutboundAt: number | null
+  /** Most recent outbound message timestamp, or null. */
+  lastOutboundAt: number | null
+  /** Total outbound messages sent this turn. */
+  outboundCount: number
+  /** Longest gap between consecutive outbound messages (ms). */
+  longestOutboundGapMs: number
+}
+
+export interface OutboundMetrics {
+  /** ms between turn start and first outbound message; null if none sent. */
+  ttfoMs: number | null
+  /** Total outbound messages this turn. */
+  outboundCount: number
+  /** Longest gap between outbound messages — i.e. the "silent stretch"
+   *  metric for the conversational-pacing KPI. 0 if <2 messages. */
+  longestOutboundGapMs: number
 }
 
-/**
- * Module-scoped map: `"chatId:threadId"` → state. Using a module-level map
- * keeps the tracker lightweight and avoids passing state through every
- * call-site while remaining mockable in tests via the exported functions.
- */
 const state = new Map<string, TurnSignalState>()
 
 /**
  * Begin tracking a new turn. Records `now` as the initial signal time and
- * resets the gap accumulator. Call at the start of each fresh turn.
+ * resets the gap accumulator + outbound state. Call at the start of each
+ * fresh turn.
  */
 export function reset(key: string, now: number): void {
-  state.set(key, { lastSignalAt: now, longestGapMs: 0 })
+  state.set(key, {
+    turnStartedAt: now,
+    lastSignalAt: now,
+    longestGapMs: 0,
+    firstOutboundAt: null,
+    lastOutboundAt: null,
+    outboundCount: 0,
+    longestOutboundGapMs: 0,
+  })
 }
 
 /**
- * Record a user-visible signal. Measures the gap since the last signal and
- * updates `longestGapMs` if this gap is larger.
+ * Record a user-visible signal (any kind: reaction, edit, send). Measures
+ * the gap since the last signal and updates `longestGapMs` if larger.
  */
 export function noteSignal(key: string, now: number): void {
   const entry = state.get(key)
@@ -51,8 +87,31 @@ export function noteSignal(key: string, now: number): void {
 }
 
 /**
- * Returns the longest gap observed during the current turn (ms).
- * Returns 0 if no tracking state exists for this key.
+ * Record a fresh outbound MESSAGE (reply, stream_reply first-emit, or
+ * card flush that produced a new sendMessage). Updates the
+ * outbound-specific metrics: TTFO on first call, outbound-gap on
+ * subsequent calls.
+ *
+ * Does not double-update the signal-gap stream — callers that note an
+ * outbound message should ALSO call `noteSignal()` to keep the legacy
+ * signal-gap accurate.
+ */
+export function noteOutbound(key: string, now: number): void {
+  const entry = state.get(key)
+  if (entry == null) return
+  if (entry.firstOutboundAt == null) {
+    entry.firstOutboundAt = now
+  } else if (entry.lastOutboundAt != null) {
+    const gap = now - entry.lastOutboundAt
+    if (gap > entry.longestOutboundGapMs) entry.longestOutboundGapMs = gap
+  }
+  entry.lastOutboundAt = now
+  entry.outboundCount += 1
+}
+
+/**
+ * Returns the longest gap observed during the current turn (ms) — legacy
+ * "any signal" metric. Returns 0 if no tracking state exists for this key.
  */
 export function getLongestGap(key: string): number {
   return state.get(key)?.longestGapMs ?? 0
@@ -67,8 +126,25 @@ export function getLastSignalAt(key: string): number | undefined {
 }
 
 /**
- * Remove state for this key. Call after emitting the turn_signal_gap metric.
+ * Returns the outbound-message KPI bundle for the conversational-pacing
+ * redesign. Zeroed-out if no tracking state exists.
  */
+export function getOutboundMetrics(key: string): OutboundMetrics {
+  const entry = state.get(key)
+  if (entry == null) {
+    return { ttfoMs: null, outboundCount: 0, longestOutboundGapMs: 0 }
+  }
+  const ttfoMs = entry.firstOutboundAt != null
+    ? entry.firstOutboundAt - entry.turnStartedAt
+    : null
+  return {
+    ttfoMs,
+    outboundCount: entry.outboundCount,
+    longestOutboundGapMs: entry.longestOutboundGapMs,
+  }
+}
+
+/** Remove state for this key. Call after emitting the turn-end metrics. */
 export function clear(key: string): void {
   state.delete(key)
 }

package/telegram-plugin/uat/SETUP.md

@@ -112,49 +112,224 @@ If the driver account is locked entirely (e.g. SPAM_WAIT), only the
 account owner can resolve it via support@telegram.org. The harness has
 no recourse.
 
-## 5. Worktree-based agent install (NOT `switchroom agent add`)
+## 5. The `test-harness` agent (Phase 2a — DM focus)
 
-The UAT harness does **not** persistently install the test-harness
-agent through `switchroom agent add` (which writes a systemd unit + a
-persistent state dir — wrong shape for hermetic test runs). Instead,
-the harness `exec`s the agent as a child process per scenario with:
+Phase 2a tests run against a **persistent** `test-harness` agent
+created once via `switchroom agent add`. This pivots from the epic's
+original child-process-per-scenario plan (written before the Docker
+runtime landed) — the standard runtime now gets us most of the
+hermeticity we want without re-inventing the agent lifecycle. Forum
+topic + per-scenario STATE_DIR isolation rolls in with Phase 2b.
 
-- `STATE_DIR=$(mktemp -d)` — ephemeral; teardown rm-rfs it.
-- A unique `TELEGRAM_GATEWAY_PORT` (see port allocator note below).
-- `SWITCHROOM_AGENT_NAME=test-harness`.
-- The test bot token loaded from `telegram-test-bot-token`.
+### One-shot agent creation
 
-The Phase 1 scaffold stubs this out in `harness.ts`; Phase 2 wires it
-end-to-end.
+```bash
+# Resolve the driver's user_id once via mtcute (the helper prints
+# only the integer id to stdout; the session string never appears):
+cd ~/code/switchroom/telegram-plugin
+read -sp "Vault passphrase: " SWITCHROOM_VAULT_PASSPHRASE; echo
+export SWITCHROOM_VAULT_PASSPHRASE
+DRIVER_UID=$(bun uat/driver-info.ts)
+echo "Driver user_id: $DRIVER_UID"
+
+# Then create the agent. `--topology dm --allow-from $DRIVER_UID`
+# bypasses the @BotFather DM-pair flow and writes the driver's
+# user_id directly into allowFrom — so the bot will respond only
+# to DMs from the driver, never from arbitrary Telegram users
+# (important: the test bot's token is in vault scoped to
+# `test-harness` only, but the bot itself is publicly reachable
+# on Telegram).
+SWITCHROOM_BOT_TOKEN=$(switchroom vault get --no-broker telegram-test-bot-token) \
+  switchroom agent add test-harness \
+    --profile default \
+    --topology dm \
+    --bot-username meken_switchroom_test_bot \
+    --allow-from "$DRIVER_UID"
+unset SWITCHROOM_BOT_TOKEN SWITCHROOM_VAULT_PASSPHRASE
+
+# Verify the agent is up:
+switchroom agent status test-harness
+```
+
+`agent add` runs the n+1 wizard: scaffolds the per-agent dir under
+`~/.switchroom/agents/test-harness/`, refreshes the compose file,
+boots the container, runs a preflight. On success the agent is
+running and will reply to DMs from the driver user account.
+
+> **Hosts upgraded from before #1009.** If you set up the
+> `test-harness` agent on an older CLI build, its
+> `access.json` may carry the two pre-fix shapes — numeric
+> `allowFrom` (silently rejected by the gateway, #1001) and a
+> placeholder `groups: {"-100…"}` entry (404 boot-probe noise,
+> #1002). Both writers were corrected in #1009, but existing
+> scaffolds aren't auto-rewritten. To rebuild a clean access.json
+> on a host that hit the old shapes:
+>
+> ```bash
+> switchroom agent stop test-harness
+> rm ~/.switchroom/agents/test-harness/telegram/access.json
+> switchroom apply       # rewrites access.json via the fixed buildAccessJson
+> switchroom agent start test-harness
+> ```
+>
+> Fresh agent-add invocations on current main don't need this.
+
+### When this agent should be running
+
+- During UAT runs: yes. Scenarios fail with `expectMessage` timeouts
+  if the agent isn't responding.
+- Idle: harmless to leave running. It consumes one Claude turn only
+  when DMed by the driver — no scheduled work, no MCP polls.
+
+### Resetting state between runs
+
+Phase 2a accepts mild state pollution across scenarios (the agent's
+history accumulates). To reset hard:
 
-## 6. Port allocator vs unix sockets
+```bash
+switchroom agent stop test-harness
+rm -rf ~/.switchroom/agents/test-harness/state
+switchroom agent start test-harness
+```
+
+Phase 2b adds per-scenario state-dir scoping so this becomes
+automatic.
+
+### Optional: force progress-card on every turn (Phase 2c+ card scenarios)
+
+The gateway's `progress_card.delay_ms` defaults to 45 s, so short DM
+turns (most of UAT) never trigger the pinned card and the card-
+lifecycle scenarios (`progress-card-dm.test.ts`) skip themselves.
+To unskip — and validate `expectPinnedCard` / `waitForCardPhase`
+against real Telegram — override the delay on `test-harness` only:
+
+Edit `~/.switchroom/switchroom.yaml`, find the `test-harness:`
+block, and add the highlighted lines:
+
+```yaml
+  test-harness:
+    extends: default
+    topic_name: Test Harness
+    channels:
+      telegram:
+        progress_card:
+          delay_ms: 1000     # short — make every turn flash a card
+```
+
+Then apply + restart:
+
+```bash
+switchroom apply
+switchroom agent restart test-harness
+```
 
-Phase 1 commits to a **process-wide port allocator** (see
-`uat/port-allocator.ts`) rather than unix sockets. Rationale:
+Production agents keep the 45 s default; this override is test-only.
+Once configured, unskip the card scenario by changing
+`describe.skip(...)` → `describe(...)` in
+`scenarios/progress-card-dm.test.ts`.
 
-- The gateway already speaks IP loopback to the bridge; switching to
-  unix sockets is a code change in `gateway/` we don't want bundled
-  with the UAT scaffold work.
-- Tests only ever run from one harness process, so a node-local
-  monotonic counter starting at a high ephemeral port (default 47000)
-  is enough to avoid collisions with the system + with sibling
-  scenarios in the same run.
-- The allocator also `bind()`s a probe socket and releases it before
-  returning, which catches "port already in use by another process"
-  before the agent boots and produces a confusing crash.
+## 6. Running scenarios — env setup
 
-If we ever want concurrent harness runs from CI, swap to unix sockets;
-the harness API takes a `transport` shape so it's a one-line change.
+The harness reads four env vars at `spinUp()` time. The recommended
+workflow is to materialise them once into `.env`
+— the harness loads that file automatically on import (see
+`load-env.ts`). The file is gitignored repo-wide (`.env*` in
+`/.gitignore`); never commit a populated copy.
+
+Vault file perms (root:root 0600) mean the operator can't read
+`vault.enc` directly. Sourcing through the `test-harness` agent
+container — which already has these keys in its ACL — is the
+cleanest path:
+
+```bash
+cd ~/code/switchroom
+
+read -sp "Vault passphrase: " SWITCHROOM_VAULT_PASSPHRASE; echo
+export SWITCHROOM_VAULT_PASSPHRASE
+
+( umask 077 && {
+  echo "TELEGRAM_API_ID=$(docker exec switchroom-test-harness switchroom vault get telegram-uat-api-id)"
+  echo "TELEGRAM_API_HASH=$(docker exec switchroom-test-harness switchroom vault get telegram-uat-api-hash)"
+  echo "TELEGRAM_UAT_DRIVER_SESSION=$(docker exec switchroom-test-harness switchroom vault get telegram-uat-driver-session)"
+  echo "TELEGRAM_TEST_BOT_USERNAME=meken_switchroom_test_bot"
+} > .env )
+
+unset SWITCHROOM_VAULT_PASSPHRASE
+```
+
+> `umask 077` in the subshell guarantees the file is never
+> world-readable between creation and the redirection's implicit
+> chmod.
+
+> The `docker exec` path requires `test-harness` to have the three
+> `telegram-uat-*` keys in its `schedule[*].secrets` ACL (see
+> `~/.switchroom/switchroom.yaml`). If `vault get` returns
+> `VAULT-BROKER-DENIED`, add them and `switchroom apply`. The legacy
+> `vault get --no-broker` path no longer works for non-root operators
+> because the vault file is owned by the broker container's root user.
+
+After the `.env` is in place, just run the suite — no per-shell
+export dance:
+
+```bash
+bun test telegram-plugin/uat/scenarios/
+```
+
+To rotate or refresh the file, repeat the block above. The harness
+prefers existing `process.env` entries over `.env` values, so a
+one-off env override still works (`TELEGRAM_API_ID=99999 bun test ...`).
+
+The vault passphrase is unset before the test run so a misbehaving
+scenario can't smuggle it into a chat message. The session string in
+`.env` is bearer-equivalent to the driver account — treat the file
+as a long-lived secret.
 
 ## 7. Verification checklist before running scenarios
 
-- [ ] `switchroom vault get telegram-test-bot-token` returns a token.
-- [ ] `switchroom vault get telegram-uat-driver-session` returns a
-      session string (the command output may be redacted by the
-      vault — that's fine, you only need exit code 0).
-- [ ] `$SWITCHROOM_UAT_CHAT_ID` exported and is a negative int.
-- [ ] Test bot is admin in the supergroup.
-- [ ] Driver user is admin in the supergroup.
-- [ ] Topics enabled in the supergroup.
+- [ ] `switchroom vault list` shows `telegram-test-bot-token`,
+      `telegram-uat-api-id`, `telegram-uat-api-hash`,
+      `telegram-uat-driver-session` (and `telegram-uat-chat-id` for
+      Phase 2b).
+- [ ] `switchroom agent status test-harness` reports the agent active.
+- [ ] Driver user can DM `@meken_switchroom_test_bot` from Telegram
+      and get a reply (manual sanity check before automating).
+
+When all three are checked, the env block above + `bun run test:uat`
+is safe to run.
+
+## 8. CI gate — `:robot: UAT fuzz` Buildkite step
+
+Since the buildkite gate landed, the fuzz subset of scenarios
+(`fuzz-random-prompts-dm.test.ts`, `fuzz-extended-dm.test.ts`,
+`fuzz-human-style-dm.test.ts`) runs automatically on every PR that
+touches `telegram-plugin/`, `src/agents/`, or `telegram-plugin/uat/`.
+
+The step runs on a self-hosted Buildkite agent tagged
+`queue=uat-host` that lives on the same box as the `test-harness`
+agent. Secrets come from the Buildkite cluster secret store, not
+from local vault. See `.buildkite/README.md` § "UAT fuzz step" for
+agent setup + secret rotation.
+
+**Scope (CI):**
+
+| Scenario | In CI? | Why |
+|---|---|---|
+| `fuzz-random-prompts-dm` | ✅ gates PRs | JTBD-floor invariants; PR #1132. |
+| `fuzz-extended-dm` | ✅ gates PRs | Second-pass categories; PR #1134. |
+| `fuzz-human-style-dm` | ✅ gates PRs | Human-shape inbounds + meaningful-reply floor. |
+| `silent-end-recovery-dm` | ❌ local only | Passes, but the 5-min worst-case budget makes it costly to run every PR. Run nightly + ad-hoc. |
+| `jtbd-status-query-dm` | ❌ local only | Passes; defer to a follow-up that batches the cheap JTBD scenarios. |
+| `jtbd-soft-commit-dm` | ❌ local only | Already budget-tuned but real-Telegram timing flake risk; defer until we have flake telemetry. |
+| `jtbd-interrupt-marker-dm` | ❌ `describe.skip` | Suspected real bug per #1132 overnight. Investigate before unskipping. |
+| `jtbd-rapid-followup-dm` | ❌ `describe.skip` | Suspected real classification bug per #1132 overnight. Investigate before unskipping. |
+| vault / secret-redaction / voice / location / reactions / progress-card | ❌ local only | Need specific surfaces / config overrides not wired into the gate yet. |
+
+A local `bun run test:uat` runs the full include glob minus the two
+`describe.skip`'d JTBDs.
+
+## 9. Port allocator vs unix sockets (Phase 1 scaffold note)
 
-When all six are checked, `bun run test:uat` is safe to run.
+The Phase 1 `port-allocator.ts` is held in reserve for Phase 2b's
+child-process flow — Phase 2a (standard-runtime agent) doesn't need
+it. Kept rather than deleted because the allocator's bind-probe is
+the right shape for what 2b will need.