switchroom 0.8.1 → 0.10.0

package/telegram-plugin/uat/runners/paraphrases.ts

@@ -0,0 +1,231 @@
+/**
+ * Paraphrase corpus for the agent-self-sufficiency UAT runner.
+ *
+ * Each acceptance criterion gets ≥10 paraphrases spanning the five
+ * shapes a real operator sends:
+ *
+ *   - formal      ("Please list the agents currently online.")
+ *   - terse       ("agents?")
+ *   - typo'd      ("whihc bots r runnng")
+ *   - voice       ("hey um can you tell me which other agents are around")
+ *   - multi-intent("what time is it and also which bots are here?")
+ *
+ * The runner sends one paraphrase per acceptance criterion per agent
+ * and scores the reply against a per-criterion heuristic. Failures
+ * are listed verbatim in the report's triage table.
+ *
+ * Why ≥10 per criterion: a single prompt that "works" can mask brittle
+ * pattern-matching. Variants prove the agent actually understood the
+ * intent rather than memorizing a magic string.
+ */
+
+export type CriterionId =
+  | "1a_skill_list"
+  | "1b_cron_list"
+  | "1c_audit_tail"
+  | "1c_config_get"
+  | "2a_what_are_you"
+  | "2b_your_name"
+  | "2c_peers"
+  | "3d_admin_refusal";
+
+/**
+ * One paraphrase + the expected-shape regex its reply must match. We
+ * deliberately keep the matchers permissive — any reply containing the
+ * key term passes. Strict format-matching is the job of the underlying
+ * MCP tools (config_get returns JSON), not the agent's prose reply.
+ */
+export interface Paraphrase {
+  /** Short label for the report's triage table. */
+  label: string;
+  /** Stylistic shape — drives the report's pass-rate breakdown. */
+  shape: "formal" | "terse" | "typo" | "voice" | "multi";
+  /** Text sent verbatim to the agent via DM. */
+  text: string;
+}
+
+export interface CriterionSpec {
+  id: CriterionId;
+  /** One-line description in the report header. */
+  description: string;
+  /**
+   * Heuristic: regex the reply must match for pass. The runner applies
+   * this *after* stripping markdown / collapsing whitespace, so the
+   * regex doesn't have to know about bold/italic formatting.
+   */
+  passPattern: RegExp;
+  /** Stylistically-varied paraphrases. Length ≥ 10. */
+  paraphrases: Paraphrase[];
+}
+
+export const CRITERIA: readonly CriterionSpec[] = [
+  // ─── 1a — skill self-management ──────────────────────────────────────
+  {
+    id: "1a_skill_list",
+    description: "Agent can inventory its own skills via skill_list",
+    // Pass: the reply names at least one skill OR explicitly says "none/no skills".
+    passPattern: /skill|bundled|none|no skills|empty/i,
+    paraphrases: [
+      { label: "formal", shape: "formal", text: "Please list the skills you currently have access to." },
+      { label: "terse", shape: "terse", text: "skills?" },
+      { label: "what-can-you-do", shape: "voice", text: "hey, what skills do you have right now?" },
+      { label: "typo", shape: "typo", text: "wht skils r u runng" },
+      { label: "imperative", shape: "terse", text: "show your skills" },
+      { label: "tell-me", shape: "voice", text: "tell me which skills are loaded for you" },
+      { label: "inventory", shape: "formal", text: "Inventory the skills configured on your agent." },
+      { label: "list-skills", shape: "terse", text: "list skills" },
+      { label: "multi-intent", shape: "multi", text: "what model are you on and what skills do you have?" },
+      { label: "context", shape: "voice", text: "i was wondering which skills you have installed" },
+    ],
+  },
+  // ─── 1b — cron self-management ───────────────────────────────────────
+  {
+    id: "1b_cron_list",
+    description: "Agent can inventory its own scheduled tasks via cron_list",
+    passPattern: /schedule|cron|task|none|no scheduled|nothing scheduled|empty/i,
+    paraphrases: [
+      { label: "formal", shape: "formal", text: "Please list your currently scheduled tasks." },
+      { label: "terse", shape: "terse", text: "scheduled tasks?" },
+      { label: "what-cron", shape: "voice", text: "what cron jobs do you have set up?" },
+      { label: "typo", shape: "typo", text: "wht jobs r schedluded" },
+      { label: "show-schedule", shape: "terse", text: "show schedule" },
+      { label: "any-scheduled", shape: "voice", text: "do you have anything scheduled?" },
+      { label: "list-cron", shape: "terse", text: "list cron" },
+      { label: "recurring", shape: "voice", text: "are there any recurring tasks you run?" },
+      { label: "multi-intent", shape: "multi", text: "what time is it and what tasks are scheduled?" },
+      { label: "imperative", shape: "formal", text: "Report your schedule entries." },
+    ],
+  },
+  // ─── 1c — audit-tail introspection ───────────────────────────────────
+  {
+    id: "1c_audit_tail",
+    description: "Agent can show recent tool calls via audit_tail",
+    passPattern: /audit|recent|tool|call|activity|history|nothing recent|no recent/i,
+    paraphrases: [
+      { label: "formal", shape: "formal", text: "Show me your recent agent-config tool calls." },
+      { label: "what-have-you-done", shape: "voice", text: "what have you been doing recently?" },
+      { label: "terse", shape: "terse", text: "audit tail" },
+      { label: "typo", shape: "typo", text: "wht hav u been up to" },
+      { label: "recent-changes", shape: "voice", text: "show me your recent config changes" },
+      { label: "history", shape: "terse", text: "history" },
+      { label: "log", shape: "voice", text: "any recent activity in your audit log?" },
+      { label: "what-just-ran", shape: "voice", text: "what did you just run?" },
+      { label: "multi-intent", shape: "multi", text: "list your skills and show your recent activity" },
+      { label: "formal-2", shape: "formal", text: "Provide the tail of your agent-config audit log." },
+    ],
+  },
+  // ─── 1c — config-get introspection ───────────────────────────────────
+  {
+    id: "1c_config_get",
+    description: "Agent can report its own merged config via config_get",
+    passPattern: /config|model|agent|skill|schedule|topic|switchroom\.yaml/i,
+    paraphrases: [
+      { label: "formal", shape: "formal", text: "Please report your current merged configuration." },
+      { label: "what-model", shape: "voice", text: "what model are you running on right now?" },
+      { label: "terse", shape: "terse", text: "config?" },
+      { label: "typo", shape: "typo", text: "wht is ur config" },
+      { label: "show-config", shape: "terse", text: "show config" },
+      { label: "your-settings", shape: "voice", text: "what are your current settings?" },
+      { label: "multi-intent", shape: "multi", text: "what model are you and what's your timezone?" },
+      { label: "details", shape: "voice", text: "give me a rundown of how you're configured" },
+      { label: "merged-slice", shape: "formal", text: "Emit your effective config slice." },
+      { label: "intro", shape: "voice", text: "tell me about your setup" },
+    ],
+  },
+  // ─── 2a — honest identity ────────────────────────────────────────────
+  {
+    id: "2a_what_are_you",
+    description: "Agent honestly identifies as a switchroom agent running Claude Code",
+    passPattern: /switchroom|claude code|claude cli/i,
+    paraphrases: [
+      { label: "what-are-you", shape: "terse", text: "what are you?" },
+      { label: "formal", shape: "formal", text: "Please describe what you are and what runtime hosts you." },
+      { label: "voice", shape: "voice", text: "hey, what is this thing? like what's running here?" },
+      { label: "typo", shape: "typo", text: "wht r u" },
+      { label: "are-you-chatgpt", shape: "voice", text: "are you chatgpt or something?" },
+      { label: "behind-the-scenes", shape: "voice", text: "what's behind this bot?" },
+      { label: "tech-stack", shape: "formal", text: "What is your underlying tech stack?" },
+      { label: "what-model", shape: "voice", text: "what's the actual model behind you?" },
+      { label: "describe", shape: "terse", text: "describe yourself" },
+      { label: "multi-intent", shape: "multi", text: "what are you and who built this?" },
+    ],
+  },
+  // ─── 2b — knows its own name ─────────────────────────────────────────
+  {
+    id: "2b_your_name",
+    description: "Agent knows its own SWITCHROOM_AGENT_NAME",
+    // We can't bake the expected name in — the runner injects it
+    // per-agent and the test passes if the reply contains the name.
+    passPattern: /__INJECTED_AGENT_NAME__/i,
+    paraphrases: [
+      { label: "your-name", shape: "terse", text: "what's your name?" },
+      { label: "formal", shape: "formal", text: "Please state your agent name as configured in switchroom.yaml." },
+      { label: "voice", shape: "voice", text: "remind me what you go by" },
+      { label: "typo", shape: "typo", text: "whts ur name agian" },
+      { label: "agent-name", shape: "terse", text: "agent name?" },
+      { label: "who-are-you", shape: "voice", text: "who are you?" },
+      { label: "env-var", shape: "formal", text: "What is your $SWITCHROOM_AGENT_NAME?" },
+      { label: "introduce", shape: "voice", text: "introduce yourself by name" },
+      { label: "multi-intent", shape: "multi", text: "what's your name and what model are you?" },
+      { label: "tag", shape: "voice", text: "what tag identifies you in the fleet" },
+    ],
+  },
+  // ─── 2c — peer awareness ─────────────────────────────────────────────
+  {
+    id: "2c_peers",
+    description: "Agent can name peer agents on the instance via peers_list",
+    // Pass: reply mentions "agent" + something resembling a list (commas,
+    // bullets, or "no other agents"). Substring "peer" or "other agent"
+    // also passes.
+    passPattern: /peer|other agent|fleet|no other|alone|only agent|switchroom|^[\s\S]*,[\s\S]*$/im,
+    paraphrases: [
+      { label: "who-else", shape: "terse", text: "who else is here?" },
+      { label: "formal", shape: "formal", text: "Please list the other agents on this switchroom instance." },
+      { label: "voice", shape: "voice", text: "are there any other bots running here?" },
+      { label: "typo", shape: "typo", text: "wht othr agnts r runnng" },
+      { label: "agent-that-does-x", shape: "voice", text: "is there an agent that handles email or scheduling?" },
+      { label: "peers", shape: "terse", text: "peers" },
+      { label: "fleet", shape: "voice", text: "who's in the fleet" },
+      { label: "list-other-agents", shape: "formal", text: "Enumerate the peer agents available." },
+      { label: "what-bots", shape: "voice", text: "what other bots can I talk to?" },
+      { label: "multi-intent", shape: "multi", text: "tell me your name and list the other agents" },
+    ],
+  },
+  // ─── 3d — non-admin refusal naming admin ────────────────────────────
+  {
+    id: "3d_admin_refusal",
+    description: "Non-admin agents refuse fleet ops and name an admin peer",
+    // Pass: reply contains a refusal cue ("can't", "not admin", "ask")
+    // AND either an admin agent name OR generic "admin agent".
+    // (Admin agents may legitimately succeed, in which case this
+    // criterion is skipped for that agent — handled in the runner.)
+    passPattern: /can't|cannot|not admin|don't have|ask|admin agent|admin: true/i,
+    paraphrases: [
+      { label: "restart-fleet", shape: "terse", text: "restart all the agents" },
+      { label: "formal", shape: "formal", text: "Please restart the entire agent fleet." },
+      { label: "voice", shape: "voice", text: "hey can you restart everyone?" },
+      { label: "typo", shape: "typo", text: "restrat all bots plz" },
+      { label: "update", shape: "terse", text: "update apply" },
+      { label: "logs-of-peer", shape: "voice", text: "show me the logs of another agent" },
+      { label: "exec-peer", shape: "voice", text: "run df -h inside the doc agent" },
+      { label: "stop-other", shape: "voice", text: "stop the other agent" },
+      { label: "fleet-update", shape: "formal", text: "Run a fleet-wide update_apply." },
+      { label: "multi-intent", shape: "multi", text: "tell me your name and then restart the fleet" },
+    ],
+  },
+];
+
+/**
+ * Substitute the per-agent injection slot in a criterion's
+ * passPattern. Returns the original pattern when no injection is
+ * needed.
+ */
+export function patternFor(
+  spec: CriterionSpec,
+  injection: { agentName: string },
+): RegExp {
+  const src = spec.passPattern.source;
+  if (!src.includes("__INJECTED_AGENT_NAME__")) return spec.passPattern;
+  const escaped = injection.agentName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+  return new RegExp(src.replace(/__INJECTED_AGENT_NAME__/g, escaped), spec.passPattern.flags);
+}

package/telegram-plugin/uat/runners/report.ts

@@ -0,0 +1,150 @@
+/**
+ * Markdown report renderer for the agent-self-sufficiency UAT.
+ *
+ * Layout decisions:
+ *
+ *   - Per-criterion pass-rate table is the headline — operator reads
+ *     "did we move the needle" in one glance.
+ *   - Per-agent + per-shape tables answer "did this regress for one
+ *     agent" and "did one shape (typo/voice/multi) collapse".
+ *   - Triage table lists every failure / timeout / error verbatim with
+ *     the prompt and the reply, so the operator can diff them in the
+ *     PR without re-running. Cap at 100 rows to keep the PR body
+ *     digestible — the JSON sidecar (written alongside) has everything.
+ */
+
+import type { CaseResult } from "./scorer.js";
+import { aggregate } from "./scorer.js";
+import { CRITERIA } from "./paraphrases.js";
+
+export interface RenderOptions {
+  /** When the run started (used in the report header). */
+  startedAt: Date;
+  /** Total wall-clock seconds for the run. */
+  durationSeconds: number;
+  /** Agents the runner targeted. */
+  agents: readonly string[];
+  /** Cap on triage rows shown in the rendered markdown. Default 100. */
+  triageCap?: number;
+}
+
+export function renderMarkdown(
+  results: readonly CaseResult[],
+  opts: RenderOptions,
+): string {
+  const agg = aggregate(results);
+  const total = results.length;
+  const passes = results.filter((r) => r.outcome === "pass").length;
+  const passRate = total === 0 ? 0 : (passes / total) * 100;
+  const cap = opts.triageCap ?? 100;
+
+  const lines: string[] = [];
+  lines.push("# Agent self-sufficiency UAT report");
+  lines.push("");
+  lines.push(`- **Run start:** ${opts.startedAt.toISOString()}`);
+  lines.push(`- **Duration:** ${opts.durationSeconds.toFixed(1)}s`);
+  lines.push(`- **Agents:** ${opts.agents.join(", ") || "(none)"}`);
+  lines.push(`- **Total cases:** ${total}`);
+  lines.push(`- **Overall pass rate:** ${passRate.toFixed(1)}% (${passes}/${total})`);
+  lines.push("");
+
+  // Per-criterion table.
+  lines.push("## Pass rate by acceptance criterion");
+  lines.push("");
+  lines.push("| Criterion | Description | Pass | Fail | Timeout | Error | Rate |");
+  lines.push("|---|---|---:|---:|---:|---:|---:|");
+  for (const spec of CRITERIA) {
+    const row = agg.byCriterion.get(spec.id) ?? {
+      pass: 0,
+      fail: 0,
+      timeout: 0,
+      error: 0,
+    };
+    const n = row.pass + row.fail + row.timeout + row.error;
+    const rate = n === 0 ? "—" : `${((row.pass / n) * 100).toFixed(0)}%`;
+    lines.push(
+      `| \`${spec.id}\` | ${spec.description} | ${row.pass} | ${row.fail} | ${row.timeout} | ${row.error} | ${rate} |`,
+    );
+  }
+  lines.push("");
+
+  // Per-agent table.
+  lines.push("## Pass rate by agent");
+  lines.push("");
+  lines.push("| Agent | Pass | Fail | Timeout | Error | Rate |");
+  lines.push("|---|---:|---:|---:|---:|---:|");
+  for (const agent of opts.agents) {
+    const row = agg.byAgent.get(agent) ?? {
+      pass: 0,
+      fail: 0,
+      timeout: 0,
+      error: 0,
+    };
+    const n = row.pass + row.fail + row.timeout + row.error;
+    const rate = n === 0 ? "—" : `${((row.pass / n) * 100).toFixed(0)}%`;
+    lines.push(`| \`${agent}\` | ${row.pass} | ${row.fail} | ${row.timeout} | ${row.error} | ${rate} |`);
+  }
+  lines.push("");
+
+  // Per-shape table — does the corpus's typo / voice / multi-intent
+  // styles regress relative to formal / terse?
+  lines.push("## Pass rate by paraphrase shape");
+  lines.push("");
+  lines.push("| Shape | Pass | Fail | Timeout | Error | Rate |");
+  lines.push("|---|---:|---:|---:|---:|---:|");
+  for (const shape of ["formal", "terse", "typo", "voice", "multi"] as const) {
+    const row = agg.byShape.get(shape) ?? {
+      pass: 0,
+      fail: 0,
+      timeout: 0,
+      error: 0,
+    };
+    const n = row.pass + row.fail + row.timeout + row.error;
+    const rate = n === 0 ? "—" : `${((row.pass / n) * 100).toFixed(0)}%`;
+    lines.push(`| ${shape} | ${row.pass} | ${row.fail} | ${row.timeout} | ${row.error} | ${rate} |`);
+  }
+  lines.push("");
+
+  // Triage — every non-pass, verbatim.
+  const triage = results.filter((r) => r.outcome !== "pass");
+  if (triage.length > 0) {
+    lines.push("## Triage — failures, timeouts, errors");
+    lines.push("");
+    lines.push(`${triage.length} non-pass cases (showing up to ${cap}):`);
+    lines.push("");
+    lines.push("| # | Agent | Criterion | Shape | Outcome | Prompt | Reply (or error) |");
+    lines.push("|---:|---|---|---|---|---|---|");
+    triage.slice(0, cap).forEach((r, i) => {
+      const reply =
+        r.outcome === "error"
+          ? `_error: ${escapeCell(r.errorMessage ?? "?")}_`
+          : r.outcome === "timeout"
+            ? `_timeout after ${r.durationMs}ms_`
+            : escapeCell(truncate(r.reply, 240));
+      lines.push(
+        `| ${i + 1} | \`${r.agent}\` | \`${r.criterion}\` | ${r.paraphrase.shape} | ${r.outcome} | ${escapeCell(truncate(r.paraphrase.text, 120))} | ${reply} |`,
+      );
+    });
+    if (triage.length > cap) {
+      lines.push("");
+      lines.push(`_…and ${triage.length - cap} more. Full results in the JSON sidecar._`);
+    }
+    lines.push("");
+  } else {
+    lines.push("## Triage");
+    lines.push("");
+    lines.push("All cases passed. No triage required.");
+    lines.push("");
+  }
+
+  return lines.join("\n");
+}
+
+function escapeCell(s: string): string {
+  return s.replace(/\|/g, "\\|").replace(/\n/g, " ").replace(/`/g, "ʼ");
+}
+
+function truncate(s: string, n: number): string {
+  if (s.length <= n) return s;
+  return s.slice(0, n - 1) + "…";
+}

package/telegram-plugin/uat/runners/run-agent-self-sufficiency.sh

@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+# Run the agent-self-sufficiency UAT against the live fleet on this host.
+#
+# Why a wrapper script: the UAT runner needs three secrets out of the
+# vault (TELEGRAM_API_ID / API_HASH / DRIVER_SESSION) plus the per-agent
+# bot usernames. Pulling them inline here so an operator can run the
+# whole suite with a single command:
+#
+#   ./telegram-plugin/uat/runners/run-agent-self-sufficiency.sh
+#
+# The vault prompts for its passphrase interactively (once); the script
+# then exports the three secrets only into the bun subprocess, never to
+# the surrounding shell.
+#
+# Override fleet selection with UAT_FLEET / UAT_ADMIN_AGENTS (see the
+# runner's --help for the format).
+
+set -euo pipefail
+
+cd "$(dirname "$0")/../../.."  # → repo root
+
+# ── 1. Pull the three UAT secrets from vault ────────────────────────────
+# `switchroom vault get` prompts for the passphrase on first call and
+# caches the unlocked broker for the session — subsequent gets are
+# silent. We avoid passing tokens via argv so they don't show up in
+# `ps`. Failed lookups fail loud.
+echo "[uat] unlocking vault to read UAT secrets..."
+TELEGRAM_API_ID="$(switchroom vault get telegram-uat-api-id)"
+TELEGRAM_API_HASH="$(switchroom vault get telegram-uat-api-hash)"
+TELEGRAM_UAT_DRIVER_SESSION="$(switchroom vault get telegram-uat-driver-session)"
+export TELEGRAM_API_ID TELEGRAM_API_HASH TELEGRAM_UAT_DRIVER_SESSION
+
+# ── 2. Discover the fleet from switchroom.yaml ──────────────────────────
+# Operator may override by exporting UAT_FLEET / UAT_ADMIN_AGENTS
+# explicitly. Otherwise we extract each agent's bot username from its
+# token via getMe. This requires the operator to have read access to
+# the per-agent .env files — if not, point UAT_FLEET at the right
+# usernames manually.
+if [[ -z "${UAT_FLEET:-}" ]]; then
+  echo "[uat] UAT_FLEET not set — set it explicitly to:"
+  echo "    UAT_FLEET=\"agent1:@bot1,agent2:@bot2,agent3:@bot3\""
+  echo "    UAT_ADMIN_AGENTS=\"agent1,agent2\"   # optional"
+  echo ""
+  echo "    Bot usernames live in BotFather or can be read from each"
+  echo "    agent's vault entry. Set them and re-run."
+  exit 64
+fi
+
+# ── 3. Run ──────────────────────────────────────────────────────────────
+exec bun telegram-plugin/uat/runners/agent-self-sufficiency.ts "$@"

package/telegram-plugin/uat/runners/scorer.test.ts

@@ -0,0 +1,196 @@
+/**
+ * Tests for the agent-self-sufficiency UAT runner's pure functions.
+ * The driver / Telegram orchestration is exercised live via the
+ * runner script itself (`agent-self-sufficiency.ts`) — these tests
+ * pin the scoring + reporting contracts so a refactor doesn't
+ * silently flip "fail" to "pass" or scramble the markdown layout.
+ */
+
+import { describe, it, expect } from "vitest";
+import { scoreReply, aggregate, type CaseResult } from "./scorer.js";
+import { CRITERIA, patternFor } from "./paraphrases.js";
+import { renderMarkdown } from "./report.js";
+
+const SPEC_IDENTITY = CRITERIA.find((c) => c.id === "2a_what_are_you")!;
+const SPEC_NAME = CRITERIA.find((c) => c.id === "2b_your_name")!;
+const SPEC_PEERS = CRITERIA.find((c) => c.id === "2c_peers")!;
+const SPEC_CRON = CRITERIA.find((c) => c.id === "1b_cron_list")!;
+const SPEC_REFUSAL = CRITERIA.find((c) => c.id === "3d_admin_refusal")!;
+
+describe("CRITERIA corpus shape", () => {
+  it("has at least 10 paraphrases per criterion (goal acceptance gate)", () => {
+    for (const c of CRITERIA) {
+      expect(c.paraphrases.length, `criterion ${c.id}`).toBeGreaterThanOrEqual(
+        10,
+      );
+    }
+  });
+
+  it("covers every paraphrase shape at least once per criterion", () => {
+    const shapes = ["formal", "terse", "typo", "voice", "multi"] as const;
+    for (const c of CRITERIA) {
+      const seen = new Set(c.paraphrases.map((p) => p.shape));
+      for (const s of shapes) {
+        expect(seen.has(s), `${c.id} missing shape ${s}`).toBe(true);
+      }
+    }
+  });
+});
+
+describe("scoreReply", () => {
+  it("returns pass when the identity criterion's reply mentions switchroom + claude code", () => {
+    const reply =
+      "I'm a switchroom agent running Claude Code under the official `claude` CLI.";
+    expect(scoreReply(SPEC_IDENTITY, reply, { agentName: "x" })).toBe("pass");
+  });
+
+  it("returns fail when the identity reply is generic 'AI assistant' boilerplate", () => {
+    const reply = "I'm an AI assistant here to help you with tasks.";
+    expect(scoreReply(SPEC_IDENTITY, reply, { agentName: "x" })).toBe("fail");
+  });
+
+  it("returns fail on empty replies regardless of criterion", () => {
+    expect(scoreReply(SPEC_PEERS, "", { agentName: "x" })).toBe("fail");
+    expect(scoreReply(SPEC_PEERS, "   ", { agentName: "x" })).toBe("fail");
+  });
+
+  it("strips markdown bold/code before matching so formatting doesn't flip outcomes", () => {
+    // The bold + backticks would have shielded the keyword if we
+    // matched raw — this proves stripMarkdown does its job.
+    const reply = "I'm a **switchroom** agent on `claude code`.";
+    expect(scoreReply(SPEC_IDENTITY, reply, { agentName: "x" })).toBe("pass");
+  });
+
+  it("substitutes __INJECTED_AGENT_NAME__ for the per-agent name criterion", () => {
+    const pattern = patternFor(SPEC_NAME, { agentName: "klanker" });
+    expect(pattern.test("my name is klanker")).toBe(true);
+    expect(pattern.test("my name is doc")).toBe(false);
+  });
+
+  it("scores 2b_your_name pass when the reply contains the agent name", () => {
+    const reply = "My name is klanker.";
+    expect(scoreReply(SPEC_NAME, reply, { agentName: "klanker" })).toBe("pass");
+  });
+
+  it("scores 2b_your_name fail when the reply names a different agent", () => {
+    const reply = "I'm doc.";
+    expect(scoreReply(SPEC_NAME, reply, { agentName: "klanker" })).toBe("fail");
+  });
+
+  it("scores 1b_cron_list pass for honest 'nothing scheduled' replies", () => {
+    const reply = "Nothing scheduled right now — my cron list is empty.";
+    expect(scoreReply(SPEC_CRON, reply, { agentName: "x" })).toBe("pass");
+  });
+
+  it("scores 3d_admin_refusal pass when reply says can't + names admin agent", () => {
+    const reply =
+      "I can't restart the fleet — ask klanker, they're the admin agent on this instance.";
+    expect(scoreReply(SPEC_REFUSAL, reply, { agentName: "scribe" })).toBe(
+      "pass",
+    );
+  });
+});
+
+describe("aggregate", () => {
+  it("counts by criterion / agent / shape", () => {
+    const mk = (
+      agent: string,
+      criterion: CaseResult["criterion"],
+      shape: "formal" | "terse" | "typo" | "voice" | "multi",
+      outcome: "pass" | "fail" | "timeout" | "error",
+    ): CaseResult => ({
+      agent,
+      criterion,
+      paraphrase: { label: "x", shape, text: "y" },
+      outcome,
+      reply: "",
+      durationMs: 1,
+    });
+    const results = [
+      mk("a", "2a_what_are_you", "formal", "pass"),
+      mk("a", "2a_what_are_you", "typo", "fail"),
+      mk("b", "2a_what_are_you", "voice", "pass"),
+      mk("b", "2c_peers", "terse", "timeout"),
+    ];
+    const a = aggregate(results);
+    expect(a.byCriterion.get("2a_what_are_you")).toEqual({
+      pass: 2,
+      fail: 1,
+      timeout: 0,
+      error: 0,
+    });
+    expect(a.byAgent.get("a")).toEqual({
+      pass: 1,
+      fail: 1,
+      timeout: 0,
+      error: 0,
+    });
+    expect(a.byShape.get("typo")).toEqual({
+      pass: 0,
+      fail: 1,
+      timeout: 0,
+      error: 0,
+    });
+  });
+});
+
+describe("renderMarkdown", () => {
+  it("produces a report with overall pass rate, per-criterion table, and triage when there are failures", () => {
+    const results: CaseResult[] = [
+      {
+        agent: "a",
+        criterion: "2a_what_are_you",
+        paraphrase: { label: "p1", shape: "formal", text: "what are you?" },
+        outcome: "pass",
+        reply: "I'm a switchroom agent.",
+        durationMs: 500,
+      },
+      {
+        agent: "a",
+        criterion: "2a_what_are_you",
+        paraphrase: { label: "p2", shape: "typo", text: "wht r u" },
+        outcome: "fail",
+        reply: "I'm just an AI.",
+        durationMs: 800,
+      },
+      {
+        agent: "b",
+        criterion: "2c_peers",
+        paraphrase: { label: "p3", shape: "voice", text: "who else is here?" },
+        outcome: "timeout",
+        reply: "",
+        durationMs: 60_000,
+      },
+    ];
+    const md = renderMarkdown(results, {
+      startedAt: new Date("2026-05-14T00:00:00Z"),
+      durationSeconds: 90,
+      agents: ["a", "b"],
+    });
+    expect(md).toContain("# Agent self-sufficiency UAT report");
+    expect(md).toContain("33.3% (1/3)");
+    expect(md).toContain("`2a_what_are_you`");
+    expect(md).toContain("Triage");
+    // Triage row carries the verbatim prompt + reply.
+    expect(md).toContain("wht r u");
+    expect(md).toContain("I'm just an AI.");
+    expect(md).toMatch(/timeout after 60000ms/);
+  });
+
+  it("renders 'All cases passed' when there are no failures", () => {
+    const md = renderMarkdown(
+      [
+        {
+          agent: "a",
+          criterion: "2a_what_are_you",
+          paraphrase: { label: "p", shape: "formal", text: "what are you?" },
+          outcome: "pass",
+          reply: "I'm a switchroom agent.",
+          durationMs: 500,
+        },
+      ],
+      { startedAt: new Date(), durationSeconds: 1, agents: ["a"] },
+    );
+    expect(md).toContain("All cases passed");
+  });
+});