psyclaw 0.27.22 → 0.28.0

package/vendor/ars/scripts/test_claim_audit_calibration.py

@@ -0,0 +1,882 @@
+"""Calibration-mode test for v3.8 claim_ref_alignment_audit_agent (T-C1..T-C3).
+
+Per spec §7.7 in
+docs/design/2026-05-15-issue-103-claim-alignment-audit-spec.md.
+
+Three-tier acceptance:
+
+- T-C1: FNR < 0.15 AND FPR < 0.10 against the synthetic 25-tuple gold set.
+  Threshold failures are CI gates, not advisory.
+- T-C2: per-class FNR/FPR (SUPPORTED / UNSUPPORTED / AMBIGUOUS /
+  violated-constraint) appear in the calibration report.
+- T-C3: gold-set shape integrity — each tuple has tuple_kind ∈ {alignment,
+  constraint}, alignment tuples carry expected_judgment ∈ {SUPPORTED,
+  UNSUPPORTED, AMBIGUOUS, RETRIEVAL_FAILED} with no constraint fields,
+  constraint tuples carry expected_judgment ∈ {VIOLATED, NOT_VIOLATED} +
+  constraint_under_test_id + (constraint_under_test_rule_text OR
+  manifest_fixture_path), and the gold set has ≥3 NOT_VIOLATED constraint
+  tuples (else constraint FPR is unmeasurable).
+
+Why three tiers:
+- T-C2 catches calibration tooling regressions (script doesn't compute or
+  doesn't write report) distinct from gold-set degradation.
+- T-C3 catches gold-set authoring bugs (missing required rule text /
+  insufficient NOT_VIOLATED count) before they silently bypass T-C1.
+- T-C1 catches model/judge quality regression.
+
+Spec § 7 writes `tests/test_claim_audit_calibration.py`; repo convention
+is `scripts/test_*.py` per spec §13 step 9 path-mapping rule. CI uses
+`python -m unittest scripts.test_*`.
+
+Run:
+    python -m unittest scripts.test_claim_audit_calibration -v
+"""
+from __future__ import annotations
+
+import json
+import unittest
+from pathlib import Path
+from typing import Any, Callable
+
+from scripts.claim_audit_calibration import (
+    GoldSetValidationError,
+    run_calibration,
+    validate_gold_set,
+)
+
+REPO_ROOT = Path(__file__).resolve().parent.parent
+GOLD_SET_PATH = REPO_ROOT / "scripts" / "fixtures" / "claim_audit_calibration" / "gold_set.json"
+
+
+def _load_gold_set() -> list[dict[str, Any]]:
+    with GOLD_SET_PATH.open(encoding="utf-8") as fh:
+        return json.load(fh)
+
+
+def _tuple_lookup_key(tup: dict[str, Any]) -> tuple[str, str, str | None]:
+    """Composite lookup key so identical claim_text on different tuple_kind /
+    constraint_id pairs disambiguate (round-1 review closure).
+
+    The canonical gold set currently has unique claim_text per tuple, but
+    future gold sets may evaluate the same claim under different MNC ids
+    (e.g. one alignment tuple + one constraint tuple sharing prose). A
+    claim_text-only key would silently overwrite earlier entries and
+    cause _perfect_judge to return the wrong stub response, falsely
+    triggering T-C1 failures.
+    """
+    return (
+        tup["tuple_kind"],
+        tup["claim_text"],
+        tup.get("constraint_under_test_id"),
+    )
+
+
+def _perfect_judge() -> Callable[..., dict[str, Any]]:
+    """Stub judge that returns the gold-set expected_judgment verbatim.
+
+    Used by T-C1 to verify the calibration script's FNR / FPR computation
+    on a path where the judge perfectly matches the gold set. The stub
+    looks up the tuple by composite key (tuple_kind, claim_text,
+    constraint_under_test_id) so duplicate claim_text under different
+    constraint scopes does not collapse. The dispatch infers tuple kind
+    from kwargs: a non-empty `active_constraints` list means the runner
+    is on the constraint code path. A real LLM judge wires here at
+    operational deployment per the protocol doc.
+    """
+    tuples_by_key: dict[tuple[str, str, str | None], dict[str, Any]] = {
+        _tuple_lookup_key(t): t for t in _load_gold_set()
+    }
+
+    def fn(**kwargs: Any) -> dict[str, Any]:
+        claim_text = kwargs.get("claim_text", "")
+        active = kwargs.get("active_constraints") or []
+        constraint_id = active[0]["constraint_id"] if active else None
+        kind = "constraint" if active else "alignment"
+        tup = tuples_by_key.get((kind, claim_text, constraint_id))
+        if tup is None:
+            raise AssertionError(
+                f"perfect_judge: no gold tuple for kind={kind!r} "
+                f"claim_text={claim_text!r} constraint_id={constraint_id!r}"
+            )
+        expected = tup["expected_judgment"]
+        if kind == "alignment":
+            resp: dict[str, Any] = {"judgment": expected, "rationale": "perfect-judge stub"}
+            # A genuinely-perfect judge on a partial fixture (#213) emits the
+            # normalized UNSUPPORTED verdict AND a breakdown that actually
+            # decomposes THIS claim — it echoes the fixture's expected_sub_claims
+            # (key tokens + verdict) so it passes both the shape gate and the
+            # content-match (#213 P1-3). A dummy two-line breakdown would now miss.
+            if tup.get("expected_prompt_verdict") == "PARTIAL":
+                resp["sub_claim_breakdown"] = [
+                    {
+                        "sub_claim_text": " ".join(esc.get("key_tokens", [])) or "sub-claim",
+                        "sub_verdict": esc.get("sub_verdict", "UNSUPPORTED"),
+                    }
+                    for esc in tup.get("expected_sub_claims", [])
+                ]
+            return resp
+        # constraint tuple
+        if expected == "VIOLATED":
+            return {
+                "judgment": "VIOLATED",
+                "violated_constraint_id": tup["constraint_under_test_id"],
+                "rationale": "perfect-judge stub",
+            }
+        return {
+            "judgment": "NOT_VIOLATED",
+            "rationale": "perfect-judge stub",
+        }
+
+    return fn
+
+
+_BAD_JUDGE_FLIP_ALIGNMENT: dict[str, str] = {
+    "SUPPORTED": "UNSUPPORTED",
+    "UNSUPPORTED": "SUPPORTED",
+    "AMBIGUOUS": "UNSUPPORTED",
+    "RETRIEVAL_FAILED": "SUPPORTED",
+}
+
+
+def _bad_judge() -> Callable[..., dict[str, Any]]:
+    """Stub judge that intentionally returns wrong labels to drive non-zero
+    FNR/FPR (round-1 review closure on tautological T-C1).
+
+    Used by `TC1ThresholdEnforcementBadJudge` to verify that
+    `run_calibration` accurately COMPUTES FNR/FPR and that the threshold
+    gate WOULD fire when the judge degrades. Without this companion test
+    T-C1 only proves tooling correctness on a perfect-mirror stub;
+    pairing both proves tooling correctness AND threshold-gate
+    enforceability.
+
+    Strategy: flip every alignment expected label to its opposite class
+    and flip every VIOLATED expected to NOT_VIOLATED. Drives every tuple
+    into FN/FP territory so aggregate FNR + FPR both massively exceed
+    the thresholds.
+
+    round-2 review closure: lookup tables are built ONCE at factory time
+    (not per fn() call) so the gold set isn't re-read from disk for every
+    judge invocation. Both alignment and constraint paths use the
+    composite key from `_tuple_lookup_key` so duplicate claim_text under
+    different (tuple_kind, constraint_id) never silently collapses
+    — same hardening _perfect_judge already carries.
+    """
+    tuples_by_key: dict[tuple[str, str, str | None], dict[str, Any]] = {
+        _tuple_lookup_key(t): t for t in _load_gold_set()
+    }
+
+    def fn(**kwargs: Any) -> dict[str, Any]:
+        claim_text = kwargs.get("claim_text", "")
+        active = kwargs.get("active_constraints") or []
+        constraint_id = active[0]["constraint_id"] if active else None
+        kind = "constraint" if active else "alignment"
+        tup = tuples_by_key.get((kind, claim_text, constraint_id))
+        if tup is None:
+            raise AssertionError(
+                f"bad_judge: no gold tuple for kind={kind!r} "
+                f"claim_text={claim_text!r} constraint_id={constraint_id!r}"
+            )
+        if kind == "alignment":
+            return {
+                "judgment": _BAD_JUDGE_FLIP_ALIGNMENT[tup["expected_judgment"]],
+                "rationale": "bad-judge stub (flipped)",
+            }
+        # Constraint path — flip VIOLATED ↔ NOT_VIOLATED.
+        if tup["expected_judgment"] == "VIOLATED":
+            return {"judgment": "NOT_VIOLATED", "rationale": "bad-judge flip"}
+        return {
+            "judgment": "VIOLATED",
+            "violated_constraint_id": constraint_id,
+            "rationale": "bad-judge flip",
+        }
+
+    return fn
+
+
+# ---------------------------------------------------------------------------
+# T-C3 — Gold-set shape integrity.
+# ---------------------------------------------------------------------------
+
+
+class TC3GoldSetShape(unittest.TestCase):
+    """T-C3 catches gold-set authoring bugs before T-C1 can silently bypass them.
+
+    Spec §7.7 list (a)-(d):
+      (a) every tuple has tuple_kind ∈ {alignment, constraint}
+      (b) alignment tuples → expected_judgment in 4-alignment-judgment set,
+          NO constraint fields
+      (c) constraint tuples → expected_judgment ∈ {VIOLATED, NOT_VIOLATED}
+          AND constraint_under_test_id AND
+          (constraint_under_test_rule_text OR manifest_fixture_path)
+      (d) ≥3 NOT_VIOLATED constraint tuples (else FPR is unmeasurable)
+
+    Any violation is rejected at calibration ingestion with a diagnostic
+    naming the rule that failed.
+    """
+
+    def setUp(self) -> None:
+        self.gold_set = _load_gold_set()
+
+    def test_a_every_tuple_has_valid_tuple_kind(self) -> None:
+        # All tuples must declare tuple_kind ∈ {alignment, constraint}.
+        # Spec §7.7 rule (a).
+        for idx, tup in enumerate(self.gold_set):
+            self.assertIn(
+                tup.get("tuple_kind"),
+                {"alignment", "constraint"},
+                f"tuple {idx} has invalid tuple_kind {tup.get('tuple_kind')!r}",
+            )
+
+    def test_b_alignment_tuples_shape_pinned(self) -> None:
+        # Alignment tuples carry expected_judgment in 4-judgment set AND
+        # MUST NOT carry constraint fields. Spec §7.7 rule (b).
+        alignment_set = {"SUPPORTED", "UNSUPPORTED", "AMBIGUOUS", "RETRIEVAL_FAILED"}
+        constraint_fields = (
+            "constraint_under_test_id",
+            "constraint_under_test_rule_text",
+            "manifest_fixture_path",
+        )
+        for idx, tup in enumerate(self.gold_set):
+            if tup.get("tuple_kind") != "alignment":
+                continue
+            self.assertIn(
+                tup.get("expected_judgment"),
+                alignment_set,
+                f"alignment tuple {idx} carries invalid expected_judgment "
+                f"{tup.get('expected_judgment')!r}",
+            )
+            for field in constraint_fields:
+                self.assertNotIn(
+                    field,
+                    tup,
+                    f"alignment tuple {idx} must not carry constraint field {field!r}",
+                )
+
+    def test_c_constraint_tuples_have_required_fields(self) -> None:
+        # Constraint tuples carry expected_judgment ∈ {VIOLATED, NOT_VIOLATED}
+        # AND constraint_under_test_id AND
+        # (constraint_under_test_rule_text OR manifest_fixture_path). Spec §7.7 rule (c).
+        for idx, tup in enumerate(self.gold_set):
+            if tup.get("tuple_kind") != "constraint":
+                continue
+            self.assertIn(
+                tup.get("expected_judgment"),
+                {"VIOLATED", "NOT_VIOLATED"},
+                f"constraint tuple {idx} carries invalid expected_judgment "
+                f"{tup.get('expected_judgment')!r}",
+            )
+            self.assertIn(
+                "constraint_under_test_id",
+                tup,
+                f"constraint tuple {idx} missing constraint_under_test_id",
+            )
+            has_rule_text = bool(tup.get("constraint_under_test_rule_text"))
+            has_manifest_path = bool(tup.get("manifest_fixture_path"))
+            self.assertTrue(
+                has_rule_text or has_manifest_path,
+                f"constraint tuple {idx} missing both "
+                f"constraint_under_test_rule_text and manifest_fixture_path",
+            )
+
+    def test_d_at_least_three_not_violated_constraint_tuples(self) -> None:
+        # ≥3 NOT_VIOLATED constraint tuples — without them constraint FPR
+        # is unmeasurable and T-C1 cannot fail-on-threshold on the
+        # constraint line. Spec §7.7 rule (d).
+        not_violated = [
+            t for t in self.gold_set
+            if t.get("tuple_kind") == "constraint"
+            and t.get("expected_judgment") == "NOT_VIOLATED"
+        ]
+        self.assertGreaterEqual(
+            len(not_violated),
+            3,
+            f"gold set must include ≥3 NOT_VIOLATED constraint tuples; "
+            f"got {len(not_violated)}",
+        )
+
+    def test_validate_gold_set_rejects_invalid_tuple_kind(self) -> None:
+        # Rule (a) negative ingestion test — round-1 review
+        # closure on T-C3 rule coverage. Diagnostic must name rule (a).
+        broken = [
+            {
+                "tuple_kind": "comparison",  # not in {alignment, constraint}
+                "claim_text": "broken",
+                "expected_judgment": "SUPPORTED",
+            }
+        ]
+        with self.assertRaises(GoldSetValidationError) as ctx:
+            validate_gold_set(broken)
+        msg = str(ctx.exception)
+        self.assertIn("rule (a)", msg, f"diagnostic must name rule (a); got {msg!r}")
+        self.assertIn("tuple 0", msg, f"diagnostic must name offending index; got {msg!r}")
+
+    def test_validate_gold_set_rejects_alignment_with_constraint_field(self) -> None:
+        # validate_gold_set is the production entrypoint that the
+        # calibration runner calls at ingestion time. It MUST raise the
+        # documented error on rule-(b) violation. round-1 review closure:
+        # also assert rule (b) name in diagnostic.
+        broken = [
+            {
+                "tuple_kind": "alignment",
+                "claim_text": "broken",
+                "expected_judgment": "SUPPORTED",
+                "constraint_under_test_id": "MNC-1",  # forbidden on alignment
+            }
+        ]
+        with self.assertRaises(GoldSetValidationError) as ctx:
+            validate_gold_set(broken)
+        msg = str(ctx.exception)
+        self.assertIn("rule (b)", msg, f"diagnostic must name rule (b); got {msg!r}")
+        self.assertIn("alignment", msg.lower())
+
+    def test_validate_gold_set_rejects_constraint_without_rule_text(self) -> None:
+        # Rule (c) — constraint tuple missing both rule_text AND
+        # manifest_fixture_path. Most common silent-skip authoring bug.
+        broken = [
+            {
+                "tuple_kind": "constraint",
+                "claim_text": "broken",
+                "expected_judgment": "VIOLATED",
+                "constraint_under_test_id": "MNC-1",
+                # neither rule_text nor manifest_fixture_path
+            }
+        ]
+        with self.assertRaises(GoldSetValidationError) as ctx:
+            validate_gold_set(broken)
+        msg = str(ctx.exception)
+        self.assertIn("rule (c)", msg, f"diagnostic must name rule (c); got {msg!r}")
+        msg_lower = msg.lower()
+        self.assertTrue("rule_text" in msg_lower or "manifest_fixture_path" in msg_lower)
+
+    def test_validate_gold_set_rejects_under_three_not_violated(self) -> None:
+        # Rule (d) — fewer than 3 NOT_VIOLATED constraint tuples.
+        broken: list[dict[str, Any]] = [
+            {
+                "tuple_kind": "constraint",
+                "claim_text": f"v{i}",
+                "expected_judgment": "VIOLATED",
+                "constraint_under_test_id": "MNC-1",
+                "constraint_under_test_rule_text": "no causal language",
+            }
+            for i in range(5)
+        ] + [
+            {
+                "tuple_kind": "constraint",
+                "claim_text": "nv1",
+                "expected_judgment": "NOT_VIOLATED",
+                "constraint_under_test_id": "MNC-1",
+                "constraint_under_test_rule_text": "no causal language",
+            }
+        ]  # only 1 NOT_VIOLATED, need ≥3
+        with self.assertRaises(GoldSetValidationError) as ctx:
+            validate_gold_set(broken)
+        msg = str(ctx.exception)
+        self.assertIn("rule (d)", msg, f"diagnostic must name rule (d); got {msg!r}")
+        self.assertIn("NOT_VIOLATED", msg)
+
+    def test_validate_gold_set_accepts_canonical_gold_set(self) -> None:
+        # Positive path — the shipped gold set MUST validate cleanly.
+        self.assertIsNone(validate_gold_set(self.gold_set))
+
+    def test_validate_gold_set_rejects_partial_without_expected_sub_claims(self) -> None:
+        # #355 P2#4 — rule (e): a PARTIAL fixture is the ONLY thing exercising
+        # the atomic-decomposition subset metric. If it omits expected_sub_claims,
+        # `_breakdown_covers_expected` early-returns True and the metric is
+        # silently skipped — any generic two-line true-partial breakdown reports
+        # miss_rate=0, defeating the whole point of the #213 calibration. Ingestion
+        # MUST reject a PARTIAL tuple with missing/empty expected_sub_claims.
+        for missing in ({}, {"expected_sub_claims": []}, {"expected_sub_claims": None}):
+            with self.subTest(missing=missing):
+                broken = [
+                    {
+                        "tuple_kind": "alignment",
+                        "claim_text": "compound claim with two parts",
+                        "ref_text_excerpt": "excerpt",
+                        "anchor": {"kind": "page", "value": "1"},
+                        "expected_judgment": "UNSUPPORTED",
+                        "expected_prompt_verdict": "PARTIAL",
+                        **missing,
+                    }
+                ]
+                with self.assertRaises(GoldSetValidationError) as ctx:
+                    validate_gold_set(broken)
+                msg = str(ctx.exception)
+                self.assertIn("rule (e)", msg, f"diagnostic must name rule (e); got {msg!r}")
+                self.assertIn("expected_sub_claims", msg)
+
+    def test_validate_gold_set_accepts_partial_with_expected_sub_claims(self) -> None:
+        # Positive: a PARTIAL fixture that declares expected_sub_claims passes
+        # rule (e). Guards the rule from over-firing on well-formed fixtures.
+        # Padded with 3 NOT_VIOLATED constraint tuples to satisfy rule (d).
+        ok = [
+            {
+                "tuple_kind": "alignment",
+                "claim_text": "compound claim with two parts",
+                "ref_text_excerpt": "excerpt",
+                "anchor": {"kind": "page", "value": "1"},
+                "expected_judgment": "UNSUPPORTED",
+                "expected_prompt_verdict": "PARTIAL",
+                "expected_sub_claims": [
+                    {"key_tokens": ["first"], "sub_verdict": "SUPPORTED"},
+                    {"key_tokens": ["second"], "sub_verdict": "UNSUPPORTED"},
+                ],
+            },
+            *[
+                {
+                    "tuple_kind": "constraint",
+                    "claim_text": f"nv-filler-{i}",
+                    "expected_judgment": "NOT_VIOLATED",
+                    "constraint_under_test_id": "MNC-1",
+                    "constraint_under_test_rule_text": "filler rule",
+                }
+                for i in range(3)
+            ],
+        ]
+        self.assertIsNone(validate_gold_set(ok))
+
+    def test_run_calibration_rejects_manifest_only_constraint_tuple(self) -> None:
+        # round-2 review closure: validate_gold_set accepts EITHER inline
+        # rule_text OR manifest_fixture_path per spec §7.7 rule (c), but
+        # the v3.8.0 runner only supports the inline form. A manifest-only
+        # tuple validating clean but reaching the judge with rule="" is
+        # exactly the silent-skip authoring bug T-C3 is supposed to
+        # prevent. The runner MUST raise NotImplementedError at run time
+        # rather than pass the empty rule through to the judge. We pin
+        # this against a minimal synthetic set (no stub lookup needed
+        # because run_calibration raises before any judge invocation
+        # for the constraint tuples).
+        def _never_called_judge(**kwargs: Any) -> dict[str, Any]:
+            raise AssertionError("judge MUST NOT be called when manifest_only tuple raises")
+
+        manifest_only = [
+            # constraint tuple using manifest_fixture_path only (rule (c)
+            # second branch). Validates clean but runner must refuse.
+            {
+                "tuple_kind": "constraint",
+                "claim_text": "manifest-only constraint",
+                "ref_text_excerpt": None,
+                "anchor": {"kind": "page", "value": "1"},
+                "expected_judgment": "VIOLATED",
+                "constraint_under_test_id": "MNC-3",
+                "manifest_fixture_path": "scripts/fixtures/claim_audit_calibration/nonexistent.json",
+            },
+            # Three NOT_VIOLATED constraint tuples to satisfy rule (d).
+            # Use inline rule_text so validate_gold_set passes; the
+            # NotImplementedError fires on the FIRST manifest-only
+            # tuple it encounters, before any of these run.
+            *[
+                {
+                    "tuple_kind": "constraint",
+                    "claim_text": f"nv-filler-{i}",
+                    "ref_text_excerpt": None,
+                    "anchor": {"kind": "page", "value": "1"},
+                    "expected_judgment": "NOT_VIOLATED",
+                    "constraint_under_test_id": "MNC-3",
+                    "constraint_under_test_rule_text": "filler rule",
+                }
+                for i in range(3)
+            ],
+        ]
+        # Step 1: validate_gold_set MUST accept the manifest-only tuple
+        # (spec §7.7 rule (c) second branch).
+        self.assertIsNone(validate_gold_set(manifest_only))
+        # Step 2: run_calibration MUST refuse it at run time.
+        with self.assertRaises(NotImplementedError) as ctx:
+            run_calibration(manifest_only, judge_fn=_never_called_judge)
+        msg = str(ctx.exception)
+        self.assertIn("manifest_fixture_path", msg)
+        self.assertIn("post-v3.8", msg.lower())
+
+
+# ---------------------------------------------------------------------------
+# T-C2 — Per-class FNR/FPR reporting.
+# ---------------------------------------------------------------------------
+
+
+class TC2PerClassReport(unittest.TestCase):
+    """T-C2 catches calibration tooling regressions (script doesn't compute /
+    doesn't write report) distinct from gold-set or model degradation.
+
+    Spec §7.7: 'FNR/FPR are computed AND surfaced per judgment-class
+    (SUPPORTED vs UNSUPPORTED, AMBIGUOUS, violated-constraint) in the
+    calibration report output.'
+    """
+
+    def setUp(self) -> None:
+        self.gold_set = _load_gold_set()
+        self.report = run_calibration(self.gold_set, judge_fn=_perfect_judge())
+
+    def test_report_has_per_class_block(self) -> None:
+        # Report must surface a per_class section keyed by judgment-class.
+        self.assertIn("per_class", self.report)
+        per_class = self.report["per_class"]
+        self.assertIsInstance(per_class, dict)
+
+    def test_per_class_includes_four_judgment_classes(self) -> None:
+        # Spec §7.7 enumerates SUPPORTED, UNSUPPORTED, AMBIGUOUS, and
+        # violated-constraint as the four classes that must appear.
+        per_class = self.report["per_class"]
+        for cls in ("SUPPORTED", "UNSUPPORTED", "AMBIGUOUS", "violated_constraint"):
+            self.assertIn(cls, per_class, f"per_class missing class {cls!r}")
+
+    def test_each_class_exposes_fnr_and_fpr(self) -> None:
+        # Per-class reporting must include both FNR and FPR (not just one).
+        # Reviewer asymmetry — missing FPR on UNSUPPORTED was the
+        # historical hole reviewer-calibration_mode_protocol referenced.
+        per_class = self.report["per_class"]
+        for cls, payload in per_class.items():
+            self.assertIn("FNR", payload, f"class {cls!r} missing FNR")
+            self.assertIn("FPR", payload, f"class {cls!r} missing FPR")
+
+    def test_each_class_exposes_denominators(self) -> None:
+        # Protocol doc Phase 4 contract: each class entry carries
+        # n_positive + n_negative so 0.0 FNR on 0 positives is
+        # distinguishable from 0.0 FNR on N positives. round-1 review
+        # closure — earlier test only asserted FNR/FPR keys.
+        per_class = self.report["per_class"]
+        for cls, payload in per_class.items():
+            self.assertIn("n_positive", payload, f"class {cls!r} missing n_positive")
+            self.assertIn("n_negative", payload, f"class {cls!r} missing n_negative")
+
+    def test_canonical_denominators_match_gold_set(self) -> None:
+        # Pin the expected one-vs-rest denominators against the canonical
+        # 17-alignment + 8-constraint gold set (12 base alignment + 5 #213
+        # partial fixtures). Contract pinning (round-1 review) — without this a
+        # future gold-set rebalance could silently shift class distributions
+        # away from the documented Phase 4 example.
+        per_class = self.report["per_class"]
+        # Alignment gold set: 5 SUPPORTED + 8 UNSUPPORTED (3 base + 5 partial)
+        # + 3 AMBIGUOUS + 1 RETRIEVAL_FAILED = 17; one-vs-rest counts the
+        # positive class against the other 16 alignment tuples.
+        self.assertEqual(per_class["SUPPORTED"]["n_positive"], 5)
+        self.assertEqual(per_class["SUPPORTED"]["n_negative"], 12)
+        self.assertEqual(per_class["UNSUPPORTED"]["n_positive"], 8)
+        self.assertEqual(per_class["UNSUPPORTED"]["n_negative"], 9)
+        self.assertEqual(per_class["AMBIGUOUS"]["n_positive"], 3)
+        self.assertEqual(per_class["AMBIGUOUS"]["n_negative"], 14)
+        # Constraint gold set: 5 VIOLATED + 3 NOT_VIOLATED (unchanged by #213).
+        self.assertEqual(per_class["violated_constraint"]["n_positive"], 5)
+        self.assertEqual(per_class["violated_constraint"]["n_negative"], 3)
+
+
+# ---------------------------------------------------------------------------
+# T-C1 — Threshold enforcement.
+# ---------------------------------------------------------------------------
+
+
+class TC1ThresholdEnforcement(unittest.TestCase):
+    """T-C1 catches model/judge quality regression.
+
+    Spec §7.7: 'FNR < 0.15 AND FPR < 0.10 against the synthetic gold set.
+    Test FAILS when either threshold is exceeded. CI fails — author must
+    either curate a better gold set, tighten judge prompts, or update
+    judge_model.'
+
+    The synthetic gold set is paired with a perfect-judge stub so the
+    test exercises the threshold gate AND the calibration tooling
+    end-to-end without requiring a live LLM call. Production deployment
+    plugs a real judge_fn in place of the stub.
+    """
+
+    def setUp(self) -> None:
+        self.gold_set = _load_gold_set()
+        self.report = run_calibration(self.gold_set, judge_fn=_perfect_judge())
+
+    def test_fnr_below_threshold(self) -> None:
+        # Spec §7.7 gate: FNR < 0.15. Perfect-judge stub yields FNR = 0.
+        self.assertLess(
+            self.report["FNR"],
+            0.15,
+            f"FNR threshold violation: {self.report['FNR']!r} ≥ 0.15",
+        )
+
+    def test_fpr_below_threshold(self) -> None:
+        # Spec §7.7 gate: FPR < 0.10. Perfect-judge stub yields FPR = 0.
+        self.assertLess(
+            self.report["FPR"],
+            0.10,
+            f"FPR threshold violation: {self.report['FPR']!r} ≥ 0.10",
+        )
+
+    def test_report_records_thresholds_used(self) -> None:
+        # Operational concern: when CI fails on T-C1, the report MUST
+        # surface the threshold values it was checked against so the
+        # operator can distinguish a regression (judge degraded) from a
+        # threshold tightening (spec bump). Spec §7.7 + §9 acceptance.
+        self.assertEqual(self.report["thresholds"]["FNR"], 0.15)
+        self.assertEqual(self.report["thresholds"]["FPR"], 0.10)
+
+
+class TC1ThresholdEnforcementBadJudge(unittest.TestCase):
+    """T-C1 companion — proves the threshold gate accurately fires on a
+    degraded judge.
+
+    round-1 dual-track review closure: the canonical T-C1
+    perfect-judge path only proves tooling CORRECTNESS (FNR/FPR
+    computation works), not the threshold's ENFORCEABILITY (the gate
+    fires when the judge degrades). Without this companion, T-C1 could
+    silently pass on a future regression where the perfect-judge stub
+    decoupled from the real judge_fn signature and the gate became
+    structurally unreachable.
+
+    The bad-judge stub flips every gold-set expected_label to its
+    wrong-class counterpart so FNR + FPR both vastly exceed the spec
+    §7.7 thresholds (0.15 / 0.10). The test asserts run_calibration's
+    output is what an operator running CI on a degraded judge would
+    see: FNR ≥ 0.15 AND FPR ≥ 0.10. Operational deployment swaps
+    _bad_judge for a real judge_fn; the gate semantics are identical.
+    """
+
+    def setUp(self) -> None:
+        self.gold_set = _load_gold_set()
+        self.report = run_calibration(self.gold_set, judge_fn=_bad_judge())
+
+    def test_bad_judge_drives_fnr_over_threshold(self) -> None:
+        # Bad judge flips every alignment expected → opposite class +
+        # every VIOLATED → NOT_VIOLATED. Aggregate FNR must be >= 0.15.
+        self.assertGreaterEqual(
+            self.report["FNR"],
+            0.15,
+            f"bad_judge should drive FNR over threshold; got {self.report['FNR']!r}",
+        )
+
+    def test_bad_judge_drives_fpr_over_threshold(self) -> None:
+        # Same logic for FPR. The mirror-flip on alignment + the
+        # NOT_VIOLATED → VIOLATED flip on constraints both contribute.
+        self.assertGreaterEqual(
+            self.report["FPR"],
+            0.10,
+            f"bad_judge should drive FPR over threshold; got {self.report['FPR']!r}",
+        )
+
+    def test_bad_judge_per_class_fnr_non_zero(self) -> None:
+        # Per-class reporting must surface non-zero FNR on the flipped
+        # classes — confirms the per_class accumulator is wired to the
+        # judge output (not hardcoded zero) and would expose which
+        # class drove a real-judge regression.
+        per_class = self.report["per_class"]
+        for cls in ("SUPPORTED", "UNSUPPORTED", "violated_constraint"):
+            self.assertGreater(
+                per_class[cls]["FNR"],
+                0.0,
+                f"bad-judge flip should drive non-zero FNR on {cls!r}; "
+                f"got {per_class[cls]!r}",
+            )
+
+
+# ---------------------------------------------------------------------------
+# Partial-support subset metric (#213).
+# ---------------------------------------------------------------------------
+
+
+def _bare_unsupported_judge() -> Callable[..., dict[str, Any]]:
+    """Regressed judge: on a partial fixture it stops decomposing and returns a
+    bare UNSUPPORTED with no sub_claim_breakdown.
+
+    For NON-partial alignment tuples it mirrors the gold label (so it does not
+    blow the aggregate FNR/FPR gate) — the whole point of the subset metric is
+    that this judge can pass the aggregate while still missing the partial subset.
+    """
+    tuples_by_key: dict[tuple[str, str, str | None], dict[str, Any]] = {
+        _tuple_lookup_key(t): t for t in _load_gold_set()
+    }
+
+    def fn(**kwargs: Any) -> dict[str, Any]:
+        claim_text = kwargs.get("claim_text", "")
+        active = kwargs.get("active_constraints") or []
+        constraint_id = active[0]["constraint_id"] if active else None
+        kind = "constraint" if active else "alignment"
+        tup = tuples_by_key.get((kind, claim_text, constraint_id))
+        if tup is None:
+            raise AssertionError(
+                f"bare_unsupported_judge: no gold tuple for kind={kind!r} "
+                f"claim_text={claim_text!r} constraint_id={constraint_id!r}"
+            )
+        if kind == "constraint":
+            if tup["expected_judgment"] == "VIOLATED":
+                return {"judgment": "VIOLATED", "violated_constraint_id": constraint_id}
+            return {"judgment": "NOT_VIOLATED"}
+        # alignment: mirror the gold label, but NEVER emit a breakdown — even on
+        # partial fixtures (whose gold expected_judgment is UNSUPPORTED). This is
+        # the regression #213 closes: correct aggregate label, no decomposition.
+        return {"judgment": tup["expected_judgment"], "rationale": "bare-unsupported stub"}
+
+    return fn
+
+
+def _dummy_breakdown_judge() -> Callable[..., dict[str, Any]]:
+    """Cheating judge (#213 P1-3): emits a well-formed-SHAPED but generic
+    two-line breakdown on EVERY partial fixture without decomposing the actual
+    claim. It passes the shape gate (is_true_partial_breakdown) but must fail the
+    content-match against each fixture's expected_sub_claims.
+
+    Mirrors the gold label on every alignment tuple so the aggregate stays green.
+    """
+    tuples_by_key: dict[tuple[str, str, str | None], dict[str, Any]] = {
+        _tuple_lookup_key(t): t for t in _load_gold_set()
+    }
+
+    def fn(**kwargs: Any) -> dict[str, Any]:
+        claim_text = kwargs.get("claim_text", "")
+        active = kwargs.get("active_constraints") or []
+        constraint_id = active[0]["constraint_id"] if active else None
+        kind = "constraint" if active else "alignment"
+        tup = tuples_by_key.get((kind, claim_text, constraint_id))
+        if tup is None:
+            raise AssertionError(f"dummy_breakdown_judge: no gold tuple for {claim_text!r}")
+        if kind == "constraint":
+            if tup["expected_judgment"] == "VIOLATED":
+                return {"judgment": "VIOLATED", "violated_constraint_id": constraint_id}
+            return {"judgment": "NOT_VIOLATED"}
+        resp: dict[str, Any] = {"judgment": tup["expected_judgment"], "rationale": "dummy"}
+        if tup.get("expected_prompt_verdict") == "PARTIAL":
+            # Generic, claim-agnostic — same two lines for every partial fixture.
+            resp["sub_claim_breakdown"] = [
+                {"sub_claim_text": "first sub-claim", "sub_verdict": "SUPPORTED"},
+                {"sub_claim_text": "second sub-claim", "sub_verdict": "UNSUPPORTED"},
+            ]
+        return resp
+
+    return fn
+
+
+def _full_claim_text_judge() -> Callable[..., dict[str, Any]]:
+    """Cheating judge (ship-gate round-2): on a partial fixture it returns two
+    items whose sub_claim_text is the FULL claim text (one SUPPORTED, one
+    UNSUPPORTED). Each item contains every expected key token, so a naive
+    coverage check would pass — but this is NOT an atomic decomposition. The
+    atomicity + distinct-item rules in _breakdown_covers_expected must reject it.
+    """
+    tuples_by_key: dict[tuple[str, str, str | None], dict[str, Any]] = {
+        _tuple_lookup_key(t): t for t in _load_gold_set()
+    }
+
+    def fn(**kwargs: Any) -> dict[str, Any]:
+        claim_text = kwargs.get("claim_text", "")
+        active = kwargs.get("active_constraints") or []
+        constraint_id = active[0]["constraint_id"] if active else None
+        kind = "constraint" if active else "alignment"
+        tup = tuples_by_key.get((kind, claim_text, constraint_id))
+        if tup is None:
+            raise AssertionError(f"full_claim_text_judge: no gold tuple for {claim_text!r}")
+        if kind == "constraint":
+            if tup["expected_judgment"] == "VIOLATED":
+                return {"judgment": "VIOLATED", "violated_constraint_id": constraint_id}
+            return {"judgment": "NOT_VIOLATED"}
+        resp: dict[str, Any] = {"judgment": tup["expected_judgment"], "rationale": "full-text"}
+        if tup.get("expected_prompt_verdict") == "PARTIAL":
+            resp["sub_claim_breakdown"] = [
+                {"sub_claim_text": claim_text, "sub_verdict": "SUPPORTED"},
+                {"sub_claim_text": claim_text, "sub_verdict": "UNSUPPORTED"},
+            ]
+        return resp
+
+    return fn
+
+
+class PartialSupportSubsetMetric(unittest.TestCase):
+    """#213: the partial-support subset metric catches a judge that regresses to
+    bare UNSUPPORTED on compound claims even though its aggregate FNR stays green.
+
+    A partial fixture carries expected_judgment=UNSUPPORTED (B1) + the
+    expected_prompt_verdict=PARTIAL discriminator. The aggregate gate matches on
+    expected_judgment, so a bare-UNSUPPORTED judge passes it. Only the subset
+    metric — which requires UNSUPPORTED AND a well-formed true-partial breakdown
+    — exposes the regression.
+    """
+
+    def setUp(self) -> None:
+        self.gold_set = _load_gold_set()
+        self.partial_tuples = [
+            t for t in self.gold_set if t.get("expected_prompt_verdict") == "PARTIAL"
+        ]
+
+    def test_gold_set_has_at_least_five_partial_fixtures(self) -> None:
+        self.assertGreaterEqual(
+            len(self.partial_tuples),
+            5,
+            f"gold set must carry >=5 partial fixtures; got {len(self.partial_tuples)}",
+        )
+
+    def test_report_has_partial_support_block(self) -> None:
+        report = run_calibration(self.gold_set, judge_fn=_perfect_judge())
+        self.assertIn("partial_support", report)
+        self.assertIn("miss_rate", report["partial_support"])
+        self.assertEqual(report["partial_support"]["n_partial"], len(self.partial_tuples))
+
+    def test_perfect_judge_passes_partial_subset(self) -> None:
+        # A judge that emits UNSUPPORTED + a well-formed breakdown misses nothing.
+        # Proves the metric is not trivially always-failing.
+        report = run_calibration(self.gold_set, judge_fn=_perfect_judge())
+        self.assertEqual(
+            report["partial_support"]["miss_rate"],
+            0.0,
+            f"perfect judge should miss no partial fixtures; got {report['partial_support']!r}",
+        )
+
+    def test_bare_unsupported_judge_misses_partial_subset(self) -> None:
+        # The regression case: aggregate FNR stays under threshold (label matches),
+        # but the subset miss_rate is non-zero because no breakdown was emitted.
+        report = run_calibration(self.gold_set, judge_fn=_bare_unsupported_judge())
+        self.assertLess(
+            report["FNR"],
+            0.15,
+            f"bare-unsupported judge should still pass the aggregate FNR gate; "
+            f"got {report['FNR']!r} (if this fails the subset metric is redundant "
+            f"with the aggregate and proves nothing)",
+        )
+        self.assertEqual(
+            report["partial_support"]["miss_rate"],
+            1.0,
+            f"bare-unsupported judge must miss EVERY partial fixture; "
+            f"got {report['partial_support']!r}",
+        )
+
+    def test_partial_fixtures_declare_expected_sub_claims(self) -> None:
+        # #213 P1-3: each partial fixture must declare expected_sub_claims so the
+        # subset metric can verify the judge decomposed THIS claim, not just
+        # emitted any two-line breakdown.
+        for tup in self.partial_tuples:
+            esc = tup.get("expected_sub_claims")
+            self.assertIsInstance(
+                esc, list, f"partial fixture {tup['claim_text']!r} missing expected_sub_claims"
+            )
+            self.assertGreaterEqual(len(esc), 2, "expected_sub_claims must have >=2 entries")
+            for entry in esc:
+                self.assertIn("key_tokens", entry)
+                self.assertIn("sub_verdict", entry)
+
+    def test_full_claim_text_judge_misses_partial_subset(self) -> None:
+        # Ship-gate round-2: a judge returning two items each = the FULL claim text
+        # (one SUPPORTED, one UNSUPPORTED) passes the verdict-mix gate and contains
+        # every expected token, but is not an atomic decomposition. The atomicity +
+        # distinct-item rules must catch it -> miss_rate = 1.0.
+        report = run_calibration(self.gold_set, judge_fn=_full_claim_text_judge())
+        self.assertLess(report["FNR"], 0.15)
+        self.assertEqual(
+            report["partial_support"]["miss_rate"],
+            1.0,
+            f"full-claim-text (non-atomic) judge must miss every partial fixture; "
+            f"got {report['partial_support']!r}",
+        )
+
+    def test_dummy_breakdown_judge_misses_partial_subset(self) -> None:
+        # #213 P1-3: a judge that emits a well-formed-SHAPED but generic two-line
+        # breakdown on every partial fixture passes the aggregate AND the shape
+        # gate, but must FAIL the subset content-match (it never decomposed the
+        # actual claim). This is the gap a later review round flagged: shape-only
+        # verification was gameable.
+        report = run_calibration(self.gold_set, judge_fn=_dummy_breakdown_judge())
+        self.assertLess(
+            report["FNR"], 0.15, f"dummy-breakdown judge should pass aggregate; got {report['FNR']!r}"
+        )
+        self.assertEqual(
+            report["partial_support"]["miss_rate"],
+            1.0,
+            f"dummy-breakdown judge must miss every partial fixture on content-match; "
+            f"got {report['partial_support']!r}",
+        )
+
+
+if __name__ == "__main__":  # pragma: no cover
+    unittest.main()