switchroom 0.8.1 → 0.11.0

package/telegram-plugin/uat/runners/scorer.ts

@@ -0,0 +1,106 @@
+/**
+ * Heuristic pass/fail scoring for the agent-self-sufficiency UAT.
+ *
+ * Each result also carries the verbatim reply so the report's triage
+ * table can show the operator exactly what the agent said. Scoring is
+ * deliberately permissive — we're testing whether the agent
+ * understood the *intent* (and reached for the right tool), not
+ * whether the reply matches a specific phrasing.
+ *
+ * Failure modes the runner needs to distinguish from "wrong answer":
+ *
+ *   - timeout:     agent never replied within the budget. Could mean
+ *                  the agent is wedged, the bot token's wrong, or
+ *                  Telegram is throttling. Reported separately so the
+ *                  operator doesn't conflate "didn't reply" with
+ *                  "replied wrong".
+ *   - send_error:  driver couldn't even deliver the inbound (bot
+ *                  username missing, mtcute connection died, etc.).
+ *                  These bubble up as `error` results, not `fail`.
+ */
+
+import type { CriterionSpec, Paraphrase } from "./paraphrases.js";
+import { patternFor } from "./paraphrases.js";
+
+export type Outcome = "pass" | "fail" | "timeout" | "error";
+
+export interface CaseResult {
+  agent: string;
+  criterion: CriterionSpec["id"];
+  paraphrase: Paraphrase;
+  outcome: Outcome;
+  /** Verbatim reply text, empty for timeout/error. Trimmed; markdown
+   *  preserved so the report can show what the user actually saw. */
+  reply: string;
+  /** Wall-clock ms from sendDM to first reply (or to timeout). */
+  durationMs: number;
+  /** Optional error message for `error` outcomes. */
+  errorMessage?: string;
+}
+
+/**
+ * Score a single reply against a criterion. The runner does NOT call
+ * this on timeouts or errors — those outcomes are set directly. For
+ * `2b_your_name` and other criteria with `__INJECTED_AGENT_NAME__` in
+ * their passPattern, the caller passes the agent name so the matcher
+ * substitutes correctly.
+ */
+export function scoreReply(
+  spec: CriterionSpec,
+  reply: string,
+  injection: { agentName: string },
+): Outcome {
+  if (!reply.trim()) return "fail";
+  const normalized = stripMarkdown(reply).toLowerCase();
+  return patternFor(spec, injection).test(normalized) ? "pass" : "fail";
+}
+
+/**
+ * Strip markdown bold/italic/code-fence markers and collapse runs of
+ * whitespace. Permissive on purpose — the scorer's regex matches
+ * against words, not formatting.
+ */
+function stripMarkdown(s: string): string {
+  return s
+    .replace(/```[\s\S]*?```/g, " ")
+    .replace(/`([^`]+)`/g, "$1")
+    .replace(/\*\*([^*]+)\*\*/g, "$1")
+    .replace(/__([^_]+)__/g, "$1")
+    .replace(/\*([^*]+)\*/g, "$1")
+    .replace(/_([^_]+)_/g, "$1")
+    .replace(/\s+/g, " ")
+    .trim();
+}
+
+/**
+ * Aggregate per-criterion / per-agent / per-shape pass rates. Pure
+ * function — easy to test.
+ */
+export interface Aggregate {
+  byCriterion: Map<string, { pass: number; fail: number; timeout: number; error: number }>;
+  byAgent: Map<string, { pass: number; fail: number; timeout: number; error: number }>;
+  byShape: Map<string, { pass: number; fail: number; timeout: number; error: number }>;
+}
+
+export function aggregate(results: readonly CaseResult[]): Aggregate {
+  const acc: Aggregate = {
+    byCriterion: new Map(),
+    byAgent: new Map(),
+    byShape: new Map(),
+  };
+  const bump = (
+    m: Aggregate["byCriterion"],
+    k: string,
+    outcome: Outcome,
+  ): void => {
+    const row = m.get(k) ?? { pass: 0, fail: 0, timeout: 0, error: 0 };
+    row[outcome] += 1;
+    m.set(k, row);
+  };
+  for (const r of results) {
+    bump(acc.byCriterion, r.criterion, r.outcome);
+    bump(acc.byAgent, r.agent, r.outcome);
+    bump(acc.byShape, r.paraphrase.shape, r.outcome);
+  }
+  return acc;
+}

package/telegram-plugin/uat/runners/skill-coverage.test.ts

@@ -0,0 +1,100 @@
+/**
+ * Unit tests for the skill-coverage UAT runner's pure pieces:
+ * label extractor + sidecar JSONL reader. Live driver/network paths
+ * are validated by operator-driven runs (see runbook).
+ */
+
+import { describe, it, expect } from "vitest";
+import {
+  extractSkillFromLabel,
+  readSkillRowsSince,
+} from "./skill-coverage.js";
+
+describe("extractSkillFromLabel", () => {
+  it("pulls the slug from the hook's canonical label", () => {
+    expect(extractSkillFromLabel("Running skill switchroom-cli")).toBe(
+      "switchroom-cli",
+    );
+  });
+
+  it("is case-insensitive on the label but lowercases the slug", () => {
+    expect(extractSkillFromLabel("RUNNING SKILL BUILDKITE-API")).toBe(
+      "buildkite-api",
+    );
+  });
+
+  it("returns null for non-Skill labels", () => {
+    expect(extractSkillFromLabel("Reading scaffold.ts")).toBeNull();
+    expect(extractSkillFromLabel("Replying")).toBeNull();
+  });
+
+  it("returns null when the slug is missing or malformed", () => {
+    expect(extractSkillFromLabel("running skill")).toBeNull();
+    expect(extractSkillFromLabel("running skill (and)")).toBeNull();
+  });
+});
+
+describe("readSkillRowsSince", () => {
+  const files: Record<string, string> = {
+    "tool-labels-A.jsonl": [
+      // before sinceMs: ignored
+      JSON.stringify({ ts: 100, tool_use_id: "u1", agent_id: "ag", label: "Running skill docx", tool_name: "Skill" }),
+      // after sinceMs, Skill: kept
+      JSON.stringify({ ts: 1500, tool_use_id: "u2", agent_id: "ag", label: "Running skill switchroom-cli", tool_name: "Skill" }),
+      // after sinceMs, non-Skill: ignored
+      JSON.stringify({ ts: 1600, tool_use_id: "u3", agent_id: "ag", label: "Reading foo.ts", tool_name: "Read" }),
+    ].join("\n") + "\n",
+    "tool-labels-B.jsonl": [
+      JSON.stringify({ ts: 2000, tool_use_id: "u4", agent_id: "ag", label: "Running skill buildkite-cli", tool_name: "Skill" }),
+      // malformed line: ignored
+      "{not-json",
+      "",
+    ].join("\n") + "\n",
+    "other.jsonl": JSON.stringify({ ts: 2500, tool_name: "Skill", label: "Running skill x" }),
+  };
+
+  const fakeReaddir = (_p: string): string[] => Object.keys(files);
+  const fakeReadFile = (p: string): string => {
+    const name = p.split("/").pop()!;
+    if (files[name] === undefined) throw new Error("ENOENT");
+    return files[name]!;
+  };
+
+  it("returns only Skill rows from tool-labels-*.jsonl with ts >= sinceMs", () => {
+    const got = readSkillRowsSince("/fake", 1000, fakeReaddir, fakeReadFile);
+    const labels = got.map((r) => r.label).sort();
+    expect(labels).toEqual([
+      "Running skill buildkite-cli",
+      "Running skill switchroom-cli",
+    ]);
+  });
+
+  it("returns [] when the dir read throws", () => {
+    expect(
+      readSkillRowsSince("/fake", 0, () => { throw new Error("EACCES"); }, fakeReadFile),
+    ).toEqual([]);
+  });
+
+  it("skips files that fail to read but keeps siblings", () => {
+    const breakingRead = (p: string): string => {
+      if (p.endsWith("tool-labels-A.jsonl")) throw new Error("EACCES");
+      return fakeReadFile(p);
+    };
+    const got = readSkillRowsSince("/fake", 0, fakeReaddir, breakingRead);
+    expect(got.map((r) => r.label)).toEqual(["Running skill buildkite-cli"]);
+  });
+
+  it("ignores files that don't match the tool-labels-*.jsonl pattern", () => {
+    const files2: Record<string, string> = {
+      "other.jsonl": JSON.stringify({ ts: 100, tool_name: "Skill", label: "x" }),
+      "tool-labels-A.jsonl": "",
+    };
+    const got = readSkillRowsSince(
+      "/fake",
+      0,
+      () => Object.keys(files2),
+      (p) => files2[p.split("/").pop()!]!,
+    );
+    expect(got).toEqual([]);
+  });
+});