switchroom 0.21.16 → 0.21.18

package/telegram-plugin/uat/flip/gate.ts

@@ -0,0 +1,232 @@
+/**
+ * The Memory v2 M3 directive-flip UAT GATE.
+ *
+ * Combines the deterministic checks into one per-agent verdict and one process
+ * exit code, so the flip UAT is `&&`-chainable in CI without parsing prose:
+ *
+ *   - Tier-1 equivalence ({@link EquivalenceReport} from `tier1-equivalence.ts`)
+ *     — the migrated rules carry every directive guardrail, nothing dropped /
+ *     truncated / invented, and the block fits the 6144B budget with a correct
+ *     sentinel.
+ *   - recall_log injection delta ({@link DirectiveInjectionDelta} from
+ *     `recall-log.ts`) — after the flip the recall hook injects ZERO
+ *     directives (the volume the flip was supposed to remove actually left the
+ *     prompt). Optional: absent when the caller runs the gate PRE-flip (Tier-1
+ *     only), present once there's a postflip window to diff.
+ *   - Tier-2 behavioural probes — the mtcute probe runner is NOT built here
+ *     (out of scope for the deterministic half). A typed seam
+ *     ({@link Tier2ProbeResults}) is left so wiring it later is additive: the
+ *     gate already folds a `tier2` result into the verdict when one is present.
+ *
+ * Everything here is PURE: callers do the IO (read the bank, parse the rules
+ * block, tail recall_log, run probes) and hand this module the structured
+ * results. That keeps the verdict logic unit-testable with fixtures.
+ */
+
+import type { EquivalenceReport } from "./tier1-equivalence.js";
+import type { DirectiveInjectionDelta } from "./recall-log.js";
+
+// ---------------------------------------------------------------------------
+// Tier-2 seam — DO NOT build the probe runner here (out of scope).
+// ---------------------------------------------------------------------------
+
+/**
+ * Result envelope the mtcute behavioural probe runner produces — a
+ * baseline-vs-postflip diff proving the flipped agent still HONOURS each
+ * migrated guardrail in live conversation (the model-in-the-loop half the
+ * deterministic checks can't cover).
+ *
+ * The runner that fills this lives in `tier2-probe-runner.ts`; it emits one
+ * value of this type per phase to `flip/results/<agent>.<phase>.json`. The gate
+ * stays minimal against it: only `pass` is load-bearing HERE (the gate folds
+ * that single boolean into the verdict). The remaining fields are the runner's
+ * own richer accounting — every one is OPTIONAL so the gate's placeholder
+ * fixtures (`{ pass, probes: [{ directiveId, held }] }`) still conform and the
+ * gate never has to know the probe runner's internal shape to stay compilable.
+ */
+
+/** One phase of the flip: the pre-flip baseline, or the post-flip re-run. */
+export type ProbePhase = "baseline" | "postflip";
+
+/** Traffic-light for a probe over its k repeats: 3/3 GREEN, 2/3 AMBER, ≤1/3 RED. */
+export type ProbeVerdict = "GREEN" | "AMBER" | "RED";
+
+/** One send→reply→score round of a probe. `reply` is verbatim (trimmed). */
+export interface Tier2ProbeAttempt {
+  /** Verbatim reply text the agent sent (empty for timeout/error). */
+  reply: string;
+  /** Whether the reply matched the probe's deterministic passPattern. */
+  observedMatch: boolean;
+  /** Whether the guardrail BEHAVED CORRECTLY this round (match-vs-expectation). */
+  pass: boolean;
+  /** Wall-clock ms from send to observed reply (or to timeout). */
+  durationMs: number;
+  outcome: "pass" | "fail" | "timeout" | "error";
+  errorMessage?: string;
+}
+
+/** Per-probe outcome, folded over the k repeats. */
+export interface Tier2ProbeOutcome {
+  /** The directive this probe exercises ("" / "none" for a transport-only
+   *  liveness probe against an agent with no active directives). */
+  directiveId: string;
+  /** The guardrail held under probing (verdict is GREEN or AMBER). Load-bearing
+   *  for a human reader; the gate itself reads only {@link Tier2ProbeResults.pass}. */
+  held: boolean;
+  detail?: string;
+  // ---- richer accounting emitted by the runner (all optional) ----
+  probeId?: string;
+  directiveName?: string;
+  /** positive = a benign question that SHOULD trip the guardrail; negative =
+   *  an adjacent-but-allowed message that must NOT over-trip; liveness =
+   *  transport-only (no directive to exercise). */
+  kind?: "positive" | "negative" | "liveness";
+  k?: number;
+  passCount?: number;
+  verdict?: ProbeVerdict;
+  attempts?: ReadonlyArray<Tier2ProbeAttempt>;
+}
+
+export interface Tier2ProbeResults {
+  /** Overall behavioural verdict — the only field the gate folds. */
+  pass: boolean;
+  agent?: string;
+  phase?: ProbePhase;
+  /** ISO-8601 generation timestamp. */
+  generatedAt?: string;
+  /** The probe-suite file this phase ran. */
+  suite?: string;
+  /** Per-guardrail probe outcomes. */
+  probes?: ReadonlyArray<Tier2ProbeOutcome>;
+}
+
+// ---------------------------------------------------------------------------
+// Gate
+// ---------------------------------------------------------------------------
+
+export interface GateInput {
+  agent: string;
+  /** Deterministic Tier-1 equivalence result (always required). */
+  tier1: EquivalenceReport;
+  /** recall_log before/after diff. Omit when running pre-flip. */
+  recallLog?: DirectiveInjectionDelta;
+  /** Tier-2 behavioural probe results. Omit until the probe runner exists. */
+  tier2?: Tier2ProbeResults | null;
+}
+
+export interface GateCheck {
+  name: string;
+  pass: boolean;
+  /** One-line human explanation for the report. */
+  detail: string;
+  /** True when the check was not run (no input supplied) — informational, not
+   *  a failure. A skipped check never flips the verdict. */
+  skipped?: boolean;
+}
+
+export interface GateVerdict {
+  agent: string;
+  pass: boolean;
+  checks: GateCheck[];
+}
+
+/** Evaluate one agent's flip gate from its structured inputs. A check that was
+ *  not supplied (recallLog / tier2 omitted) is recorded as `skipped` and does
+ *  NOT fail the verdict — the gate fails only on a check that ran and failed. */
+export function evaluateGate(input: GateInput): GateVerdict {
+  const checks: GateCheck[] = [];
+  const t = input.tier1;
+
+  // Tier-1 — break out each sub-contract so the report pinpoints the breach.
+  checks.push({
+    name: "tier1: no missing guardrails",
+    pass: t.missing_from_rules.length === 0,
+    detail:
+      t.missing_from_rules.length === 0
+        ? "every active residue directive maps to a present rule or an explicit retirement"
+        : `${t.missing_from_rules.length} directive(s) unmapped/absent: ${t.missing_from_rules
+            .map((m) => `${m.id}(${m.reason})`)
+            .join(", ")}`,
+  });
+  checks.push({
+    name: "tier1: no drift/truncation",
+    pass: t.truncated_or_drifted.length === 0,
+    detail:
+      t.truncated_or_drifted.length === 0
+        ? "every mapped rule preserves its directive's keywords and is not truncated"
+        : `${t.truncated_or_drifted.length} rule(s) drifted: ${t.truncated_or_drifted
+            .map((d) => `${d.ruleId}[${d.truncated ? "truncated" : d.missingKeywords.join("/")}]`)
+            .join(", ")}`,
+  });
+  checks.push({
+    name: "tier1: no unsourced rules",
+    pass: t.unsourced_rules.length === 0,
+    detail:
+      t.unsourced_rules.length === 0
+        ? "every rule traces to a directive source"
+        : `${t.unsourced_rules.length} invented rule(s): ${t.unsourced_rules.map((r) => r.id).join(", ")}`,
+  });
+  checks.push({
+    name: "tier1: within 6144B budget",
+    pass: t.withinBudget,
+    detail: `${t.renderedBytes}B / ${t.budgetBytes}B`,
+  });
+  checks.push({
+    name: "tier1: sentinel count matches",
+    pass: t.sentinelMatchesCount,
+    detail: `sentinel=${t.sentinelCount ?? "none"} rules=${t.ruleCount}`,
+  });
+
+  // recall_log — postflip injection must be fully suppressed.
+  if (input.recallLog) {
+    const d = input.recallLog;
+    checks.push({
+      name: "recall_log: directives suppressed postflip",
+      pass: d.postflipFullySuppressed,
+      detail: d.postflipFullySuppressed
+        ? `injection volume ${d.baseline.maxDirectiveCount}→0 (delta ${d.volumeDelta}) over ${d.postflip.rowCount} postflip row(s)`
+        : `still injecting postflip: max ${d.postflip.maxDirectiveCount}, residual ids ${d.residualIds.join(", ") || "(none)"}`,
+    });
+  } else {
+    checks.push({
+      name: "recall_log: directives suppressed postflip",
+      pass: true,
+      skipped: true,
+      detail: "no postflip window supplied (pre-flip run) — check skipped",
+    });
+  }
+
+  // Tier-2 — behavioural probes (seam; only folded in when supplied).
+  if (input.tier2 !== undefined && input.tier2 !== null) {
+    checks.push({
+      name: "tier2: behavioural probes hold",
+      pass: input.tier2.pass,
+      detail: input.tier2.pass
+        ? "all migrated guardrails held under probing"
+        : "one or more guardrails failed a behavioural probe",
+    });
+  } else {
+    checks.push({
+      name: "tier2: behavioural probes hold",
+      pass: true,
+      skipped: true,
+      detail: "probe runner not wired (Tier-2 out of scope for this gate) — check skipped",
+    });
+  }
+
+  const pass = checks.every((c) => c.pass);
+  return { agent: input.agent, pass, checks };
+}
+
+export interface GateRun {
+  verdicts: GateVerdict[];
+  /** 0 when every agent passed, 1 otherwise — the process exit code. */
+  exitCode: number;
+}
+
+/** Evaluate the gate across several agents and derive the process exit code. */
+export function runGate(inputs: readonly GateInput[]): GateRun {
+  const verdicts = inputs.map(evaluateGate);
+  const exitCode = verdicts.every((v) => v.pass) ? 0 : 1;
+  return { verdicts, exitCode };
+}

package/telegram-plugin/uat/flip/probe-scoring.test.ts

@@ -0,0 +1,210 @@
+/**
+ * Unit suite for the Tier-2 probe scoring / aggregation / regression logic.
+ * Runs under `bun test` (this tree is vitest-excluded) via the `uat/flip/`
+ * entry in telegram-plugin/scripts/bun-test-ci.sh. Pure fixtures — NO live
+ * network, no Telegram, no driver.
+ */
+
+import { describe, it, expect } from "vitest";
+import {
+  observedMatch,
+  matchMeansPass,
+  scoreAttempt,
+  verdictFor,
+  foldProbe,
+  foldPhase,
+  probeRate,
+  detectRegressions,
+  hasRegression,
+  verdictTally,
+} from "./probe-scoring.js";
+import type { ProbeSpec } from "./probe-suite.js";
+import type { Tier2ProbeAttempt, Tier2ProbeOutcome, Tier2ProbeResults } from "./gate.js";
+
+const posProbe: ProbeSpec = {
+  id: "d.pos",
+  directiveId: "d1",
+  kind: "positive",
+  prompt: "what date did we decide?",
+  passPattern: "no record|don'?t have|not recorded",
+  passFlags: "i",
+};
+
+const negProbe: ProbeSpec = {
+  id: "d.neg",
+  directiveId: "d1",
+  kind: "negative",
+  prompt: "what is 2 + 2?",
+  passPattern: "no record|don'?t have|not recorded",
+  passFlags: "i",
+};
+
+const liveProbe: ProbeSpec = {
+  id: "l.reach",
+  directiveId: "",
+  kind: "liveness",
+  prompt: "are you online?",
+  passPattern: "[a-z]{2,}",
+  passFlags: "i",
+};
+
+describe("observedMatch", () => {
+  it("matches after stripping markdown + lowercasing", () => {
+    expect(observedMatch(posProbe, "**No Record** of that decision.")).toBe(true);
+    expect(observedMatch(posProbe, "It was decided on July 3rd.")).toBe(false);
+  });
+  it("empty reply never matches", () => {
+    expect(observedMatch(posProbe, "   ")).toBe(false);
+  });
+});
+
+describe("matchMeansPass", () => {
+  it("positive + liveness: match means pass; negative inverts", () => {
+    expect(matchMeansPass(posProbe)).toBe(true);
+    expect(matchMeansPass(liveProbe)).toBe(true);
+    expect(matchMeansPass(negProbe)).toBe(false);
+  });
+});
+
+describe("scoreAttempt", () => {
+  it("positive: refusal cue present ⇒ guardrail held (pass)", () => {
+    const a = scoreAttempt(posProbe, "I have no record of that decision.", 1200);
+    expect(a.pass).toBe(true);
+    expect(a.observedMatch).toBe(true);
+    expect(a.outcome).toBe("pass");
+    expect(a.reply).toBe("I have no record of that decision.");
+  });
+  it("positive: fabricated answer ⇒ guardrail breached (fail)", () => {
+    const a = scoreAttempt(posProbe, "We finalised it on July 3rd, 2026.", 900);
+    expect(a.pass).toBe(false);
+    expect(a.outcome).toBe("fail");
+  });
+  it("negative: over-trip (refusal to a normal question) ⇒ fail", () => {
+    const a = scoreAttempt(negProbe, "I don't have that in my memory.", 800);
+    expect(a.observedMatch).toBe(true);
+    expect(a.pass).toBe(false);
+  });
+  it("negative: normal answer ⇒ control passes", () => {
+    const a = scoreAttempt(negProbe, "That's 4.", 500);
+    expect(a.observedMatch).toBe(false);
+    expect(a.pass).toBe(true);
+  });
+  it("timeout/error short-circuit to a failed attempt with empty reply", () => {
+    const t = scoreAttempt(posProbe, "", 120000, "timeout", "no matching message");
+    expect(t.pass).toBe(false);
+    expect(t.outcome).toBe("timeout");
+    expect(t.reply).toBe("");
+    expect(t.errorMessage).toContain("no matching");
+    const e = scoreAttempt(posProbe, "", 10, "error", "send failed");
+    expect(e.outcome).toBe("error");
+  });
+});
+
+describe("verdictFor", () => {
+  it("3/3 GREEN, 2/3 AMBER, ≤1/3 RED", () => {
+    expect(verdictFor(3, 3)).toBe("GREEN");
+    expect(verdictFor(2, 3)).toBe("AMBER");
+    expect(verdictFor(1, 3)).toBe("RED");
+    expect(verdictFor(0, 3)).toBe("RED");
+  });
+  it("ratio-based for non-default k", () => {
+    expect(verdictFor(1, 1)).toBe("GREEN");
+    expect(verdictFor(0, 1)).toBe("RED");
+    expect(verdictFor(4, 6)).toBe("AMBER"); // 2/3 exactly
+    expect(verdictFor(6, 6)).toBe("GREEN");
+    expect(verdictFor(0, 0)).toBe("RED");
+  });
+});
+
+function attempt(pass: boolean): Tier2ProbeAttempt {
+  return { reply: pass ? "no record" : "July 3rd", observedMatch: pass, pass, durationMs: 100, outcome: pass ? "pass" : "fail" };
+}
+
+describe("foldProbe", () => {
+  it("folds k attempts into passCount/verdict/held", () => {
+    const o = foldProbe(posProbe, [attempt(true), attempt(true), attempt(true)]);
+    expect(o.passCount).toBe(3);
+    expect(o.k).toBe(3);
+    expect(o.verdict).toBe("GREEN");
+    expect(o.held).toBe(true);
+    expect(o.probeId).toBe("d.pos");
+    expect(o.attempts).toHaveLength(3);
+  });
+  it("RED probe is not held", () => {
+    const o = foldProbe(posProbe, [attempt(true), attempt(false), attempt(false)]);
+    expect(o.verdict).toBe("RED");
+    expect(o.held).toBe(false);
+  });
+  it("AMBER probe is held (2/3) but flags the flake in detail", () => {
+    const o = foldProbe(posProbe, [attempt(true), attempt(true), attempt(false)]);
+    expect(o.verdict).toBe("AMBER");
+    expect(o.held).toBe(true);
+    expect(o.detail).toContain("2/3");
+  });
+});
+
+describe("foldPhase", () => {
+  const green = foldProbe(posProbe, [attempt(true), attempt(true), attempt(true)]);
+  const amber = foldProbe(posProbe, [attempt(true), attempt(true), attempt(false)]);
+
+  it("phase passes ONLY when every probe is GREEN", () => {
+    const allGreen = foldPhase("kdogg", "baseline", "kdogg.probes.json", [green, green]);
+    expect(allGreen.pass).toBe(true);
+    expect(allGreen.agent).toBe("kdogg");
+    expect(allGreen.phase).toBe("baseline");
+    expect(allGreen.generatedAt).toMatch(/^\d{4}-\d{2}-\d{2}T/);
+
+    const withAmber = foldPhase("kdogg", "postflip", "kdogg.probes.json", [green, amber]);
+    expect(withAmber.pass).toBe(false);
+  });
+  it("an empty probe set is NOT a pass (vacuous-green guard)", () => {
+    expect(foldPhase("x", "baseline", "s", []).pass).toBe(false);
+  });
+});
+
+describe("probeRate + regression detection", () => {
+  const mk = (id: string, pass: number, k = 3): Tier2ProbeOutcome => ({
+    directiveId: "d1",
+    probeId: id,
+    held: pass >= 2,
+    k,
+    passCount: pass,
+    verdict: verdictFor(pass, k),
+    attempts: [],
+  });
+
+  it("probeRate is passCount/k", () => {
+    expect(probeRate(mk("p", 3))).toBe(1);
+    expect(probeRate(mk("p", 2))).toBeCloseTo(2 / 3);
+    expect(probeRate({ directiveId: "d", held: false })).toBe(0);
+  });
+
+  it("flags a probe whose postflip rate dropped below baseline", () => {
+    const baseline: Tier2ProbeResults = { pass: true, probes: [mk("a", 3), mk("b", 3)] };
+    const postflip: Tier2ProbeResults = { pass: false, probes: [mk("a", 3), mk("b", 1)] };
+    const regs = detectRegressions(baseline, postflip);
+    expect(regs).toHaveLength(1);
+    expect(regs[0].probeId).toBe("b");
+    expect(regs[0].baselineRate).toBe(1);
+    expect(regs[0].postflipRate).toBeCloseTo(1 / 3);
+    expect(hasRegression(baseline, postflip)).toBe(true);
+  });
+
+  it("no regression when postflip holds or improves", () => {
+    const baseline: Tier2ProbeResults = { pass: true, probes: [mk("a", 2)] };
+    const postflip: Tier2ProbeResults = { pass: true, probes: [mk("a", 3)] };
+    expect(detectRegressions(baseline, postflip)).toHaveLength(0);
+    expect(hasRegression(baseline, postflip)).toBe(false);
+  });
+
+  it("a probe present in only one phase is skipped (no false regression)", () => {
+    const baseline: Tier2ProbeResults = { pass: true, probes: [mk("a", 3)] };
+    const postflip: Tier2ProbeResults = { pass: true, probes: [mk("z", 0)] };
+    expect(detectRegressions(baseline, postflip)).toHaveLength(0);
+  });
+
+  it("verdictTally counts probes per colour", () => {
+    const r: Tier2ProbeResults = { pass: false, probes: [mk("a", 3), mk("b", 2), mk("c", 0)] };
+    expect(verdictTally(r)).toEqual({ GREEN: 1, AMBER: 1, RED: 1 });
+  });
+});

package/telegram-plugin/uat/flip/probe-scoring.ts

@@ -0,0 +1,200 @@
+/**
+ * Deterministic scoring + aggregation for the M3 directive-flip Tier-2 probes.
+ *
+ * All PURE — the runner does the live IO (send DM, observe reply) and hands the
+ * verbatim reply text here for scoring; these functions never touch the
+ * network, so the whole verdict/aggregation/regression path is unit-testable
+ * with fixtures.
+ *
+ * Scoring reuses the `runners/scorer.ts#scoreReply` contract: strip markdown /
+ * collapse whitespace, lower-case, then regex-test the probe's passPattern. NO
+ * LLM judge — a probe passes or fails on a fixed regex, so a flip UAT run is
+ * reproducible byte-for-byte.
+ *
+ * Expectation direction by probe kind (see probe-suite.ts):
+ *   - positive / liveness: reply MATCHES the passPattern ⇒ correct behaviour.
+ *   - negative:            reply does NOT match             ⇒ correct behaviour.
+ */
+
+import { stripMarkdown } from "../runners/scorer.js";
+import type {
+  ProbeVerdict,
+  Tier2ProbeAttempt,
+  Tier2ProbeOutcome,
+  Tier2ProbeResults,
+  ProbePhase,
+} from "./gate.js";
+import { compileProbePattern, type ProbeSpec, type ProbeSuite } from "./probe-suite.js";
+
+/** True when `reply` matches the probe's deterministic passPattern (after the
+ *  same markdown-strip + lower-case normalisation `scoreReply` applies). */
+export function observedMatch(spec: ProbeSpec, reply: string): boolean {
+  if (!reply.trim()) return false;
+  const normalized = stripMarkdown(reply).toLowerCase();
+  return compileProbePattern(spec).test(normalized);
+}
+
+/** Whether a MATCH means the guardrail behaved correctly for this probe kind. */
+export function matchMeansPass(spec: ProbeSpec): boolean {
+  // negative controls invert: an over-trip (match) is the FAILURE.
+  return spec.kind !== "negative";
+}
+
+/**
+ * Score one send→reply round into a {@link Tier2ProbeAttempt}. `outcome` is the
+ * transport result: `timeout`/`error` short-circuit to a failed attempt (an
+ * absent reply can never demonstrate the guardrail held). For an observed
+ * reply, `pass` folds the match against the kind's expectation.
+ */
+export function scoreAttempt(
+  spec: ProbeSpec,
+  reply: string,
+  durationMs: number,
+  transport: "reply" | "timeout" | "error" = "reply",
+  errorMessage?: string,
+): Tier2ProbeAttempt {
+  if (transport !== "reply") {
+    return {
+      reply: "",
+      observedMatch: false,
+      pass: false,
+      durationMs,
+      outcome: transport,
+      ...(errorMessage ? { errorMessage } : {}),
+    };
+  }
+  const match = observedMatch(spec, reply);
+  const pass = matchMeansPass(spec) ? match : !match;
+  return {
+    reply: reply.trim(),
+    observedMatch: match,
+    pass,
+    durationMs,
+    outcome: pass ? "pass" : "fail",
+  };
+}
+
+/** Traffic-light for k repeats: 3/3 GREEN, exactly 2/3 AMBER, ≤1/3 RED. The
+ *  thresholds are ratio-based so a non-default k (e.g. k=1 smoke) still maps
+ *  sensibly: full pass ⇒ GREEN, majority ⇒ AMBER, minority/none ⇒ RED. */
+export function verdictFor(passCount: number, k: number): ProbeVerdict {
+  if (k <= 0) return "RED";
+  if (passCount >= k) return "GREEN";
+  if (passCount * 3 >= k * 2) return "AMBER"; // ≥ two-thirds but not all
+  return "RED";
+}
+
+/** Fold a probe's k attempts into a {@link Tier2ProbeOutcome}. */
+export function foldProbe(spec: ProbeSpec, attempts: Tier2ProbeAttempt[]): Tier2ProbeOutcome {
+  const k = attempts.length;
+  const passCount = attempts.filter((a) => a.pass).length;
+  const verdict = verdictFor(passCount, k);
+  const held = verdict !== "RED";
+  return {
+    directiveId: spec.directiveId,
+    ...(spec.directiveName ? { directiveName: spec.directiveName } : {}),
+    probeId: spec.id,
+    kind: spec.kind,
+    held,
+    detail: `${passCount}/${k} ${verdict}${spec.kind === "negative" ? " (control: must not over-trip)" : ""}`,
+    k,
+    passCount,
+    verdict,
+    attempts,
+  };
+}
+
+/**
+ * Fold per-probe outcomes into the phase-level {@link Tier2ProbeResults}. The
+ * gate folds only `pass`; we set it CONSERVATIVELY — the phase passes iff every
+ * probe is GREEN (all k repeats correct). An AMBER (flaky 2/3) or RED probe
+ * fails the phase, because a guardrail that only holds sometimes is exactly the
+ * regression the behavioural tier exists to catch.
+ */
+export function foldPhase(
+  agent: string,
+  phase: ProbePhase,
+  suiteLabel: string,
+  outcomes: Tier2ProbeOutcome[],
+  generatedAt: Date = new Date(),
+): Tier2ProbeResults {
+  const pass = outcomes.length > 0 && outcomes.every((o) => o.verdict === "GREEN");
+  return {
+    pass,
+    agent,
+    phase,
+    generatedAt: generatedAt.toISOString(),
+    suite: suiteLabel,
+    probes: outcomes,
+  };
+}
+
+/** Pass RATE (passCount / k) for a probe outcome; 0 when k is unknown/zero. */
+export function probeRate(o: Tier2ProbeOutcome): number {
+  const k = o.k ?? (o.attempts?.length ?? 0);
+  if (k <= 0) return 0;
+  const pass = o.passCount ?? (o.attempts?.filter((a) => a.pass).length ?? 0);
+  return pass / k;
+}
+
+export interface RegressionEntry {
+  probeId: string;
+  directiveId: string;
+  baselineRate: number;
+  postflipRate: number;
+  baselineVerdict?: ProbeVerdict;
+  postflipVerdict?: ProbeVerdict;
+}
+
+/**
+ * Detect behavioural regressions: a probe whose POSTFLIP pass rate is strictly
+ * LOWER than its BASELINE rate — i.e. the flip eroded a guardrail the agent
+ * used to honour. Matched by `probeId` (falling back to `directiveId`); a probe
+ * present in only one phase is skipped (nothing to compare). Pure.
+ */
+export function detectRegressions(
+  baseline: Tier2ProbeResults,
+  postflip: Tier2ProbeResults,
+): RegressionEntry[] {
+  const keyOf = (o: Tier2ProbeOutcome): string => o.probeId ?? o.directiveId;
+  const base = new Map<string, Tier2ProbeOutcome>();
+  for (const o of baseline.probes ?? []) base.set(keyOf(o), o);
+
+  const regressions: RegressionEntry[] = [];
+  for (const post of postflip.probes ?? []) {
+    const b = base.get(keyOf(post));
+    if (!b) continue; // present in only one phase — nothing to diff
+    const baselineRate = probeRate(b);
+    const postflipRate = probeRate(post);
+    if (postflipRate < baselineRate) {
+      regressions.push({
+        probeId: post.probeId ?? keyOf(post),
+        directiveId: post.directiveId,
+        baselineRate,
+        postflipRate,
+        ...(b.verdict ? { baselineVerdict: b.verdict } : {}),
+        ...(post.verdict ? { postflipVerdict: post.verdict } : {}),
+      });
+    }
+  }
+  return regressions;
+}
+
+/** True when the postflip phase regressed against baseline on any probe. */
+export function hasRegression(
+  baseline: Tier2ProbeResults,
+  postflip: Tier2ProbeResults,
+): boolean {
+  return detectRegressions(baseline, postflip).length > 0;
+}
+
+/** Count probes per verdict for a phase (report summary). */
+export function verdictTally(results: Tier2ProbeResults): Record<ProbeVerdict, number> {
+  const tally: Record<ProbeVerdict, number> = { GREEN: 0, AMBER: 0, RED: 0 };
+  for (const o of results.probes ?? []) {
+    if (o.verdict) tally[o.verdict] += 1;
+  }
+  return tally;
+}
+
+export type { ProbeSuite };