switchroom 0.7.15 → 0.10.0

package/telegram-plugin/uat/runners/report.ts

@@ -0,0 +1,150 @@
+/**
+ * Markdown report renderer for the agent-self-sufficiency UAT.
+ *
+ * Layout decisions:
+ *
+ *   - Per-criterion pass-rate table is the headline — operator reads
+ *     "did we move the needle" in one glance.
+ *   - Per-agent + per-shape tables answer "did this regress for one
+ *     agent" and "did one shape (typo/voice/multi) collapse".
+ *   - Triage table lists every failure / timeout / error verbatim with
+ *     the prompt and the reply, so the operator can diff them in the
+ *     PR without re-running. Cap at 100 rows to keep the PR body
+ *     digestible — the JSON sidecar (written alongside) has everything.
+ */
+
+import type { CaseResult } from "./scorer.js";
+import { aggregate } from "./scorer.js";
+import { CRITERIA } from "./paraphrases.js";
+
+export interface RenderOptions {
+  /** When the run started (used in the report header). */
+  startedAt: Date;
+  /** Total wall-clock seconds for the run. */
+  durationSeconds: number;
+  /** Agents the runner targeted. */
+  agents: readonly string[];
+  /** Cap on triage rows shown in the rendered markdown. Default 100. */
+  triageCap?: number;
+}
+
+export function renderMarkdown(
+  results: readonly CaseResult[],
+  opts: RenderOptions,
+): string {
+  const agg = aggregate(results);
+  const total = results.length;
+  const passes = results.filter((r) => r.outcome === "pass").length;
+  const passRate = total === 0 ? 0 : (passes / total) * 100;
+  const cap = opts.triageCap ?? 100;
+
+  const lines: string[] = [];
+  lines.push("# Agent self-sufficiency UAT report");
+  lines.push("");
+  lines.push(`- **Run start:** ${opts.startedAt.toISOString()}`);
+  lines.push(`- **Duration:** ${opts.durationSeconds.toFixed(1)}s`);
+  lines.push(`- **Agents:** ${opts.agents.join(", ") || "(none)"}`);
+  lines.push(`- **Total cases:** ${total}`);
+  lines.push(`- **Overall pass rate:** ${passRate.toFixed(1)}% (${passes}/${total})`);
+  lines.push("");
+
+  // Per-criterion table.
+  lines.push("## Pass rate by acceptance criterion");
+  lines.push("");
+  lines.push("| Criterion | Description | Pass | Fail | Timeout | Error | Rate |");
+  lines.push("|---|---|---:|---:|---:|---:|---:|");
+  for (const spec of CRITERIA) {
+    const row = agg.byCriterion.get(spec.id) ?? {
+      pass: 0,
+      fail: 0,
+      timeout: 0,
+      error: 0,
+    };
+    const n = row.pass + row.fail + row.timeout + row.error;
+    const rate = n === 0 ? "—" : `${((row.pass / n) * 100).toFixed(0)}%`;
+    lines.push(
+      `| \`${spec.id}\` | ${spec.description} | ${row.pass} | ${row.fail} | ${row.timeout} | ${row.error} | ${rate} |`,
+    );
+  }
+  lines.push("");
+
+  // Per-agent table.
+  lines.push("## Pass rate by agent");
+  lines.push("");
+  lines.push("| Agent | Pass | Fail | Timeout | Error | Rate |");
+  lines.push("|---|---:|---:|---:|---:|---:|");
+  for (const agent of opts.agents) {
+    const row = agg.byAgent.get(agent) ?? {
+      pass: 0,
+      fail: 0,
+      timeout: 0,
+      error: 0,
+    };
+    const n = row.pass + row.fail + row.timeout + row.error;
+    const rate = n === 0 ? "—" : `${((row.pass / n) * 100).toFixed(0)}%`;
+    lines.push(`| \`${agent}\` | ${row.pass} | ${row.fail} | ${row.timeout} | ${row.error} | ${rate} |`);
+  }
+  lines.push("");
+
+  // Per-shape table — does the corpus's typo / voice / multi-intent
+  // styles regress relative to formal / terse?
+  lines.push("## Pass rate by paraphrase shape");
+  lines.push("");
+  lines.push("| Shape | Pass | Fail | Timeout | Error | Rate |");
+  lines.push("|---|---:|---:|---:|---:|---:|");
+  for (const shape of ["formal", "terse", "typo", "voice", "multi"] as const) {
+    const row = agg.byShape.get(shape) ?? {
+      pass: 0,
+      fail: 0,
+      timeout: 0,
+      error: 0,
+    };
+    const n = row.pass + row.fail + row.timeout + row.error;
+    const rate = n === 0 ? "—" : `${((row.pass / n) * 100).toFixed(0)}%`;
+    lines.push(`| ${shape} | ${row.pass} | ${row.fail} | ${row.timeout} | ${row.error} | ${rate} |`);
+  }
+  lines.push("");
+
+  // Triage — every non-pass, verbatim.
+  const triage = results.filter((r) => r.outcome !== "pass");
+  if (triage.length > 0) {
+    lines.push("## Triage — failures, timeouts, errors");
+    lines.push("");
+    lines.push(`${triage.length} non-pass cases (showing up to ${cap}):`);
+    lines.push("");
+    lines.push("| # | Agent | Criterion | Shape | Outcome | Prompt | Reply (or error) |");
+    lines.push("|---:|---|---|---|---|---|---|");
+    triage.slice(0, cap).forEach((r, i) => {
+      const reply =
+        r.outcome === "error"
+          ? `_error: ${escapeCell(r.errorMessage ?? "?")}_`
+          : r.outcome === "timeout"
+            ? `_timeout after ${r.durationMs}ms_`
+            : escapeCell(truncate(r.reply, 240));
+      lines.push(
+        `| ${i + 1} | \`${r.agent}\` | \`${r.criterion}\` | ${r.paraphrase.shape} | ${r.outcome} | ${escapeCell(truncate(r.paraphrase.text, 120))} | ${reply} |`,
+      );
+    });
+    if (triage.length > cap) {
+      lines.push("");
+      lines.push(`_…and ${triage.length - cap} more. Full results in the JSON sidecar._`);
+    }
+    lines.push("");
+  } else {
+    lines.push("## Triage");
+    lines.push("");
+    lines.push("All cases passed. No triage required.");
+    lines.push("");
+  }
+
+  return lines.join("\n");
+}
+
+function escapeCell(s: string): string {
+  return s.replace(/\|/g, "\\|").replace(/\n/g, " ").replace(/`/g, "ʼ");
+}
+
+function truncate(s: string, n: number): string {
+  if (s.length <= n) return s;
+  return s.slice(0, n - 1) + "…";
+}

package/telegram-plugin/uat/runners/run-agent-self-sufficiency.sh

@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+# Run the agent-self-sufficiency UAT against the live fleet on this host.
+#
+# Why a wrapper script: the UAT runner needs three secrets out of the
+# vault (TELEGRAM_API_ID / API_HASH / DRIVER_SESSION) plus the per-agent
+# bot usernames. Pulling them inline here so an operator can run the
+# whole suite with a single command:
+#
+#   ./telegram-plugin/uat/runners/run-agent-self-sufficiency.sh
+#
+# The vault prompts for its passphrase interactively (once); the script
+# then exports the three secrets only into the bun subprocess, never to
+# the surrounding shell.
+#
+# Override fleet selection with UAT_FLEET / UAT_ADMIN_AGENTS (see the
+# runner's --help for the format).
+
+set -euo pipefail
+
+cd "$(dirname "$0")/../../.."  # → repo root
+
+# ── 1. Pull the three UAT secrets from vault ────────────────────────────
+# `switchroom vault get` prompts for the passphrase on first call and
+# caches the unlocked broker for the session — subsequent gets are
+# silent. We avoid passing tokens via argv so they don't show up in
+# `ps`. Failed lookups fail loud.
+echo "[uat] unlocking vault to read UAT secrets..."
+TELEGRAM_API_ID="$(switchroom vault get telegram-uat-api-id)"
+TELEGRAM_API_HASH="$(switchroom vault get telegram-uat-api-hash)"
+TELEGRAM_UAT_DRIVER_SESSION="$(switchroom vault get telegram-uat-driver-session)"
+export TELEGRAM_API_ID TELEGRAM_API_HASH TELEGRAM_UAT_DRIVER_SESSION
+
+# ── 2. Discover the fleet from switchroom.yaml ──────────────────────────
+# Operator may override by exporting UAT_FLEET / UAT_ADMIN_AGENTS
+# explicitly. Otherwise we extract each agent's bot username from its
+# token via getMe. This requires the operator to have read access to
+# the per-agent .env files — if not, point UAT_FLEET at the right
+# usernames manually.
+if [[ -z "${UAT_FLEET:-}" ]]; then
+  echo "[uat] UAT_FLEET not set — set it explicitly to:"
+  echo "    UAT_FLEET=\"agent1:@bot1,agent2:@bot2,agent3:@bot3\""
+  echo "    UAT_ADMIN_AGENTS=\"agent1,agent2\"   # optional"
+  echo ""
+  echo "    Bot usernames live in BotFather or can be read from each"
+  echo "    agent's vault entry. Set them and re-run."
+  exit 64
+fi
+
+# ── 3. Run ──────────────────────────────────────────────────────────────
+exec bun telegram-plugin/uat/runners/agent-self-sufficiency.ts "$@"

package/telegram-plugin/uat/runners/scorer.test.ts

@@ -0,0 +1,196 @@
+/**
+ * Tests for the agent-self-sufficiency UAT runner's pure functions.
+ * The driver / Telegram orchestration is exercised live via the
+ * runner script itself (`agent-self-sufficiency.ts`) — these tests
+ * pin the scoring + reporting contracts so a refactor doesn't
+ * silently flip "fail" to "pass" or scramble the markdown layout.
+ */
+
+import { describe, it, expect } from "vitest";
+import { scoreReply, aggregate, type CaseResult } from "./scorer.js";
+import { CRITERIA, patternFor } from "./paraphrases.js";
+import { renderMarkdown } from "./report.js";
+
+const SPEC_IDENTITY = CRITERIA.find((c) => c.id === "2a_what_are_you")!;
+const SPEC_NAME = CRITERIA.find((c) => c.id === "2b_your_name")!;
+const SPEC_PEERS = CRITERIA.find((c) => c.id === "2c_peers")!;
+const SPEC_CRON = CRITERIA.find((c) => c.id === "1b_cron_list")!;
+const SPEC_REFUSAL = CRITERIA.find((c) => c.id === "3d_admin_refusal")!;
+
+describe("CRITERIA corpus shape", () => {
+  it("has at least 10 paraphrases per criterion (goal acceptance gate)", () => {
+    for (const c of CRITERIA) {
+      expect(c.paraphrases.length, `criterion ${c.id}`).toBeGreaterThanOrEqual(
+        10,
+      );
+    }
+  });
+
+  it("covers every paraphrase shape at least once per criterion", () => {
+    const shapes = ["formal", "terse", "typo", "voice", "multi"] as const;
+    for (const c of CRITERIA) {
+      const seen = new Set(c.paraphrases.map((p) => p.shape));
+      for (const s of shapes) {
+        expect(seen.has(s), `${c.id} missing shape ${s}`).toBe(true);
+      }
+    }
+  });
+});
+
+describe("scoreReply", () => {
+  it("returns pass when the identity criterion's reply mentions switchroom + claude code", () => {
+    const reply =
+      "I'm a switchroom agent running Claude Code under the official `claude` CLI.";
+    expect(scoreReply(SPEC_IDENTITY, reply, { agentName: "x" })).toBe("pass");
+  });
+
+  it("returns fail when the identity reply is generic 'AI assistant' boilerplate", () => {
+    const reply = "I'm an AI assistant here to help you with tasks.";
+    expect(scoreReply(SPEC_IDENTITY, reply, { agentName: "x" })).toBe("fail");
+  });
+
+  it("returns fail on empty replies regardless of criterion", () => {
+    expect(scoreReply(SPEC_PEERS, "", { agentName: "x" })).toBe("fail");
+    expect(scoreReply(SPEC_PEERS, "   ", { agentName: "x" })).toBe("fail");
+  });
+
+  it("strips markdown bold/code before matching so formatting doesn't flip outcomes", () => {
+    // The bold + backticks would have shielded the keyword if we
+    // matched raw — this proves stripMarkdown does its job.
+    const reply = "I'm a **switchroom** agent on `claude code`.";
+    expect(scoreReply(SPEC_IDENTITY, reply, { agentName: "x" })).toBe("pass");
+  });
+
+  it("substitutes __INJECTED_AGENT_NAME__ for the per-agent name criterion", () => {
+    const pattern = patternFor(SPEC_NAME, { agentName: "klanker" });
+    expect(pattern.test("my name is klanker")).toBe(true);
+    expect(pattern.test("my name is doc")).toBe(false);
+  });
+
+  it("scores 2b_your_name pass when the reply contains the agent name", () => {
+    const reply = "My name is klanker.";
+    expect(scoreReply(SPEC_NAME, reply, { agentName: "klanker" })).toBe("pass");
+  });
+
+  it("scores 2b_your_name fail when the reply names a different agent", () => {
+    const reply = "I'm doc.";
+    expect(scoreReply(SPEC_NAME, reply, { agentName: "klanker" })).toBe("fail");
+  });
+
+  it("scores 1b_cron_list pass for honest 'nothing scheduled' replies", () => {
+    const reply = "Nothing scheduled right now — my cron list is empty.";
+    expect(scoreReply(SPEC_CRON, reply, { agentName: "x" })).toBe("pass");
+  });
+
+  it("scores 3d_admin_refusal pass when reply says can't + names admin agent", () => {
+    const reply =
+      "I can't restart the fleet — ask klanker, they're the admin agent on this instance.";
+    expect(scoreReply(SPEC_REFUSAL, reply, { agentName: "scribe" })).toBe(
+      "pass",
+    );
+  });
+});
+
+describe("aggregate", () => {
+  it("counts by criterion / agent / shape", () => {
+    const mk = (
+      agent: string,
+      criterion: CaseResult["criterion"],
+      shape: "formal" | "terse" | "typo" | "voice" | "multi",
+      outcome: "pass" | "fail" | "timeout" | "error",
+    ): CaseResult => ({
+      agent,
+      criterion,
+      paraphrase: { label: "x", shape, text: "y" },
+      outcome,
+      reply: "",
+      durationMs: 1,
+    });
+    const results = [
+      mk("a", "2a_what_are_you", "formal", "pass"),
+      mk("a", "2a_what_are_you", "typo", "fail"),
+      mk("b", "2a_what_are_you", "voice", "pass"),
+      mk("b", "2c_peers", "terse", "timeout"),
+    ];
+    const a = aggregate(results);
+    expect(a.byCriterion.get("2a_what_are_you")).toEqual({
+      pass: 2,
+      fail: 1,
+      timeout: 0,
+      error: 0,
+    });
+    expect(a.byAgent.get("a")).toEqual({
+      pass: 1,
+      fail: 1,
+      timeout: 0,
+      error: 0,
+    });
+    expect(a.byShape.get("typo")).toEqual({
+      pass: 0,
+      fail: 1,
+      timeout: 0,
+      error: 0,
+    });
+  });
+});
+
+describe("renderMarkdown", () => {
+  it("produces a report with overall pass rate, per-criterion table, and triage when there are failures", () => {
+    const results: CaseResult[] = [
+      {
+        agent: "a",
+        criterion: "2a_what_are_you",
+        paraphrase: { label: "p1", shape: "formal", text: "what are you?" },
+        outcome: "pass",
+        reply: "I'm a switchroom agent.",
+        durationMs: 500,
+      },
+      {
+        agent: "a",
+        criterion: "2a_what_are_you",
+        paraphrase: { label: "p2", shape: "typo", text: "wht r u" },
+        outcome: "fail",
+        reply: "I'm just an AI.",
+        durationMs: 800,
+      },
+      {
+        agent: "b",
+        criterion: "2c_peers",
+        paraphrase: { label: "p3", shape: "voice", text: "who else is here?" },
+        outcome: "timeout",
+        reply: "",
+        durationMs: 60_000,
+      },
+    ];
+    const md = renderMarkdown(results, {
+      startedAt: new Date("2026-05-14T00:00:00Z"),
+      durationSeconds: 90,
+      agents: ["a", "b"],
+    });
+    expect(md).toContain("# Agent self-sufficiency UAT report");
+    expect(md).toContain("33.3% (1/3)");
+    expect(md).toContain("`2a_what_are_you`");
+    expect(md).toContain("Triage");
+    // Triage row carries the verbatim prompt + reply.
+    expect(md).toContain("wht r u");
+    expect(md).toContain("I'm just an AI.");
+    expect(md).toMatch(/timeout after 60000ms/);
+  });
+
+  it("renders 'All cases passed' when there are no failures", () => {
+    const md = renderMarkdown(
+      [
+        {
+          agent: "a",
+          criterion: "2a_what_are_you",
+          paraphrase: { label: "p", shape: "formal", text: "what are you?" },
+          outcome: "pass",
+          reply: "I'm a switchroom agent.",
+          durationMs: 500,
+        },
+      ],
+      { startedAt: new Date(), durationSeconds: 1, agents: ["a"] },
+    );
+    expect(md).toContain("All cases passed");
+  });
+});

package/telegram-plugin/uat/runners/scorer.ts

@@ -0,0 +1,106 @@
+/**
+ * Heuristic pass/fail scoring for the agent-self-sufficiency UAT.
+ *
+ * Each result also carries the verbatim reply so the report's triage
+ * table can show the operator exactly what the agent said. Scoring is
+ * deliberately permissive — we're testing whether the agent
+ * understood the *intent* (and reached for the right tool), not
+ * whether the reply matches a specific phrasing.
+ *
+ * Failure modes the runner needs to distinguish from "wrong answer":
+ *
+ *   - timeout:     agent never replied within the budget. Could mean
+ *                  the agent is wedged, the bot token's wrong, or
+ *                  Telegram is throttling. Reported separately so the
+ *                  operator doesn't conflate "didn't reply" with
+ *                  "replied wrong".
+ *   - send_error:  driver couldn't even deliver the inbound (bot
+ *                  username missing, mtcute connection died, etc.).
+ *                  These bubble up as `error` results, not `fail`.
+ */
+
+import type { CriterionSpec, Paraphrase } from "./paraphrases.js";
+import { patternFor } from "./paraphrases.js";
+
+export type Outcome = "pass" | "fail" | "timeout" | "error";
+
+export interface CaseResult {
+  agent: string;
+  criterion: CriterionSpec["id"];
+  paraphrase: Paraphrase;
+  outcome: Outcome;
+  /** Verbatim reply text, empty for timeout/error. Trimmed; markdown
+   *  preserved so the report can show what the user actually saw. */
+  reply: string;
+  /** Wall-clock ms from sendDM to first reply (or to timeout). */
+  durationMs: number;
+  /** Optional error message for `error` outcomes. */
+  errorMessage?: string;
+}
+
+/**
+ * Score a single reply against a criterion. The runner does NOT call
+ * this on timeouts or errors — those outcomes are set directly. For
+ * `2b_your_name` and other criteria with `__INJECTED_AGENT_NAME__` in
+ * their passPattern, the caller passes the agent name so the matcher
+ * substitutes correctly.
+ */
+export function scoreReply(
+  spec: CriterionSpec,
+  reply: string,
+  injection: { agentName: string },
+): Outcome {
+  if (!reply.trim()) return "fail";
+  const normalized = stripMarkdown(reply).toLowerCase();
+  return patternFor(spec, injection).test(normalized) ? "pass" : "fail";
+}
+
+/**
+ * Strip markdown bold/italic/code-fence markers and collapse runs of
+ * whitespace. Permissive on purpose — the scorer's regex matches
+ * against words, not formatting.
+ */
+function stripMarkdown(s: string): string {
+  return s
+    .replace(/```[\s\S]*?```/g, " ")
+    .replace(/`([^`]+)`/g, "$1")
+    .replace(/\*\*([^*]+)\*\*/g, "$1")
+    .replace(/__([^_]+)__/g, "$1")
+    .replace(/\*([^*]+)\*/g, "$1")
+    .replace(/_([^_]+)_/g, "$1")
+    .replace(/\s+/g, " ")
+    .trim();
+}
+
+/**
+ * Aggregate per-criterion / per-agent / per-shape pass rates. Pure
+ * function — easy to test.
+ */
+export interface Aggregate {
+  byCriterion: Map<string, { pass: number; fail: number; timeout: number; error: number }>;
+  byAgent: Map<string, { pass: number; fail: number; timeout: number; error: number }>;
+  byShape: Map<string, { pass: number; fail: number; timeout: number; error: number }>;
+}
+
+export function aggregate(results: readonly CaseResult[]): Aggregate {
+  const acc: Aggregate = {
+    byCriterion: new Map(),
+    byAgent: new Map(),
+    byShape: new Map(),
+  };
+  const bump = (
+    m: Aggregate["byCriterion"],
+    k: string,
+    outcome: Outcome,
+  ): void => {
+    const row = m.get(k) ?? { pass: 0, fail: 0, timeout: 0, error: 0 };
+    row[outcome] += 1;
+    m.set(k, row);
+  };
+  for (const r of results) {
+    bump(acc.byCriterion, r.criterion, r.outcome);
+    bump(acc.byAgent, r.agent, r.outcome);
+    bump(acc.byShape, r.paraphrase.shape, r.outcome);
+  }
+  return acc;
+}

package/telegram-plugin/uat/runners/skill-coverage.test.ts

@@ -0,0 +1,100 @@
+/**
+ * Unit tests for the skill-coverage UAT runner's pure pieces:
+ * label extractor + sidecar JSONL reader. Live driver/network paths
+ * are validated by operator-driven runs (see runbook).
+ */
+
+import { describe, it, expect } from "vitest";
+import {
+  extractSkillFromLabel,
+  readSkillRowsSince,
+} from "./skill-coverage.js";
+
+describe("extractSkillFromLabel", () => {
+  it("pulls the slug from the hook's canonical label", () => {
+    expect(extractSkillFromLabel("Running skill switchroom-cli")).toBe(
+      "switchroom-cli",
+    );
+  });
+
+  it("is case-insensitive on the label but lowercases the slug", () => {
+    expect(extractSkillFromLabel("RUNNING SKILL BUILDKITE-API")).toBe(
+      "buildkite-api",
+    );
+  });
+
+  it("returns null for non-Skill labels", () => {
+    expect(extractSkillFromLabel("Reading scaffold.ts")).toBeNull();
+    expect(extractSkillFromLabel("Replying")).toBeNull();
+  });
+
+  it("returns null when the slug is missing or malformed", () => {
+    expect(extractSkillFromLabel("running skill")).toBeNull();
+    expect(extractSkillFromLabel("running skill (and)")).toBeNull();
+  });
+});
+
+describe("readSkillRowsSince", () => {
+  const files: Record<string, string> = {
+    "tool-labels-A.jsonl": [
+      // before sinceMs: ignored
+      JSON.stringify({ ts: 100, tool_use_id: "u1", agent_id: "ag", label: "Running skill docx", tool_name: "Skill" }),
+      // after sinceMs, Skill: kept
+      JSON.stringify({ ts: 1500, tool_use_id: "u2", agent_id: "ag", label: "Running skill switchroom-cli", tool_name: "Skill" }),
+      // after sinceMs, non-Skill: ignored
+      JSON.stringify({ ts: 1600, tool_use_id: "u3", agent_id: "ag", label: "Reading foo.ts", tool_name: "Read" }),
+    ].join("\n") + "\n",
+    "tool-labels-B.jsonl": [
+      JSON.stringify({ ts: 2000, tool_use_id: "u4", agent_id: "ag", label: "Running skill buildkite-cli", tool_name: "Skill" }),
+      // malformed line: ignored
+      "{not-json",
+      "",
+    ].join("\n") + "\n",
+    "other.jsonl": JSON.stringify({ ts: 2500, tool_name: "Skill", label: "Running skill x" }),
+  };
+
+  const fakeReaddir = (_p: string): string[] => Object.keys(files);
+  const fakeReadFile = (p: string): string => {
+    const name = p.split("/").pop()!;
+    if (files[name] === undefined) throw new Error("ENOENT");
+    return files[name]!;
+  };
+
+  it("returns only Skill rows from tool-labels-*.jsonl with ts >= sinceMs", () => {
+    const got = readSkillRowsSince("/fake", 1000, fakeReaddir, fakeReadFile);
+    const labels = got.map((r) => r.label).sort();
+    expect(labels).toEqual([
+      "Running skill buildkite-cli",
+      "Running skill switchroom-cli",
+    ]);
+  });
+
+  it("returns [] when the dir read throws", () => {
+    expect(
+      readSkillRowsSince("/fake", 0, () => { throw new Error("EACCES"); }, fakeReadFile),
+    ).toEqual([]);
+  });
+
+  it("skips files that fail to read but keeps siblings", () => {
+    const breakingRead = (p: string): string => {
+      if (p.endsWith("tool-labels-A.jsonl")) throw new Error("EACCES");
+      return fakeReadFile(p);
+    };
+    const got = readSkillRowsSince("/fake", 0, fakeReaddir, breakingRead);
+    expect(got.map((r) => r.label)).toEqual(["Running skill buildkite-cli"]);
+  });
+
+  it("ignores files that don't match the tool-labels-*.jsonl pattern", () => {
+    const files2: Record<string, string> = {
+      "other.jsonl": JSON.stringify({ ts: 100, tool_name: "Skill", label: "x" }),
+      "tool-labels-A.jsonl": "",
+    };
+    const got = readSkillRowsSince(
+      "/fake",
+      0,
+      () => Object.keys(files2),
+      (p) => files2[p.split("/").pop()!]!,
+    );
+    expect(got).toEqual([]);
+  });
+});