oracle-gate 0.1.0__py3-none-any.whl

oracle_gate/__init__.py

@@ -0,0 +1,2 @@
+"""oracle-gate — the reference implementation of The Oracle Gate framework."""
+__version__ = "0.1.0"

oracle_gate/check.py

@@ -0,0 +1,346 @@
+"""Scoring and evidence emission for `oracle-gate check` — pure, no IO.
+
+Two rules govern this module, and both exist because the framework accuses other tools
+of breaking them:
+
+1. **Percentages are reported, never gated.** `mutation_score` and its flattering cousin
+   `mutation_score_covered` are printed together, each with its formula, so nobody has to
+   trust which denominator we chose. The gate is `survivors.triage`.
+
+2. **`check` fills G1 and G2 and nothing else.** It cannot certify human accountability,
+   an independent oracle, or a trustworthy eval. Those gates are emitted `not_met` with a
+   prompt, every time, even when everything it *can* measure passes.
+
+Metrics it produces are stamped `source = "measured"`, distinguishing them from the
+human-typed `declared` values the conformance checker also accepts. That is the third rung
+of the ladder — attached, declared, measured — and the fourth rung, *correct*, is not
+reachable by this or any tool.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from .checklist import PROFILE_KEY_BY_TIER, gates_for_tier
+from .survivors import UNCLASSIFIED, UnclassifiedStatus
+
+__all__ = [
+    "NoMutantsError",
+    "CountsDisagree",
+    "Verdict",
+    "scores",
+    "verdict",
+    "evidence",
+    "counts_from_mutants",
+    "cross_check",
+    "format_report",
+]
+
+# The mutmut status strings that count toward a score, mapped to this tool's count keys.
+# "no tests" -> "no_tests" is the one rename; everything else that reaches here is
+# already classified by `undetected`/`counts_from_mutants` raising on anything else.
+_STATUS_TO_COUNT_KEY = {"killed": "killed", "timeout": "timeout", "survived": "survived", "no tests": "no_tests"}
+
+_FORMULA = {
+    "mutation_score": "detected / valid   (detected = killed + timeout; valid = detected + survived + no_tests)",
+    "mutation_score_covered": "detected / covered   (covered = detected + survived)  — reported, not gated",
+}
+
+# Gates `check` is allowed to fill. Everything else needs a human.
+_MACHINE_GATES = ("G1", "G2")
+
+
+class NoMutantsError(ValueError):
+    """No valid mutants were produced, so there is no score to report."""
+
+
+class CountsDisagree(ValueError):
+    """Two independently-derived count dicts disagree on a field.
+
+    `run_mutation` gives us counts two ways: mutmut's own `mutmut-cicd-stats.json`, and
+    whatever `counts_from_mutants` tallies by parsing `mutmut results --all true`. They
+    should always agree -- they describe the same run. If they do not, something is
+    wrong with the run or with how one of the two was read, and printing a score built
+    from either one would be trusting a source of truth that just proved itself
+    unreliable. Fail closed instead.
+    """
+
+
+@dataclass(frozen=True)
+class Verdict:
+    passed: bool
+    unexplained: int
+
+    @property
+    def exit_code(self) -> int:
+        return 0 if self.passed else 1
+
+
+def scores(counts) -> dict:
+    """Compute both mutation scores, with their formulas attached.
+
+    Fails closed if mutmut reported a status we cannot place on Stryker's map: silently
+    dropping those counts would choose a denominator by accident.
+    """
+    for name in sorted(UNCLASSIFIED):
+        # no default: absent and zero are both falsy, and spelling one invites an
+        # equivalent mutant that no test can ever kill.
+        if counts.get(name):
+            raise UnclassifiedStatus(
+                f"mutmut reported {counts[name]} {name!r} mutant(s). These do not map onto "
+                f"a known mutant state, and guessing would silently pick a denominator. "
+                f"Investigate or exclude them from the run."
+            )
+
+    killed = counts.get("killed", 0)
+    timeout = counts.get("timeout", 0)
+    survived = counts.get("survived", 0)
+    no_tests = counts.get("no_tests", 0)
+
+    detected = killed + timeout
+    valid = detected + survived + no_tests
+    covered = detected + survived
+
+    if valid == 0:
+        raise NoMutantsError("mutmut produced no valid mutants; there is nothing to score")
+
+    return {
+        "mutation_score": detected / valid,
+        "mutation_score_covered": (detected / covered) if covered else None,
+        "counts": {
+            "killed": killed,
+            "timeout": timeout,
+            "survived": survived,
+            "no_tests": no_tests,
+            "detected": detected,
+            "valid": valid,
+            "covered": covered,
+        },
+        "formula": dict(_FORMULA),
+    }
+
+
+def verdict(triage) -> Verdict:
+    """The gate: every undetected mutant is killed or explained."""
+    return Verdict(passed=triage.passed, unexplained=len(triage.unexplained))
+
+
+def counts_from_mutants(mutants) -> dict:
+    """Tally {"killed", "timeout", "survived", "no_tests"} straight from parsed mutants.
+
+    This is a second, independent path to the same counts `run_mutation` reads out of
+    mutmut's stats file -- see `cross_check`. Any status other than the four this tool
+    knows how to place (including "suspicious"/"segfault"/"skipped") is refused rather
+    than silently dropped or lumped in: naming the status and the mutant lets a human
+    decide, instead of this tool guessing a denominator.
+    """
+    counts = {"killed": 0, "timeout": 0, "survived": 0, "no_tests": 0}
+    for m in mutants:
+        key = _STATUS_TO_COUNT_KEY.get(m.status)
+        if key is None:
+            raise UnclassifiedStatus(
+                f"mutmut reported status {m.status!r} for {m.id}, which does not map "
+                f"onto a known mutant state. Refusing to guess: classify it or exclude "
+                f"it from the run."
+            )
+        counts[key] += 1
+    return counts
+
+
+def cross_check(counts_from_stats, counts_from_results) -> None:
+    """Refuse to score unless mutmut's two outputs agree on every count.
+
+    `counts_from_stats` comes from `mutmut-cicd-stats.json`; `counts_from_results`
+    comes from parsing `mutmut results --all true` (see `counts_from_mutants`). They
+    describe the same run and must match. A disagreement means one of the two readings
+    is wrong, and printing a score anyway would silently pick which one to trust.
+    """
+    for field in ("killed", "survived", "no_tests", "timeout"):
+        a = counts_from_stats.get(field, 0)
+        b = counts_from_results.get(field, 0)
+        if a != b:
+            raise CountsDisagree(
+                f"counts disagree on {field!r}: mutmut-cicd-stats.json says {a}, "
+                f"mutmut results --all true says {b}"
+            )
+
+
+def _fmt(value) -> str:
+    return f"{value:.6g}"
+
+
+def evidence(
+    *,
+    tier: int,
+    coverage: float,
+    scores: dict,
+    triage,
+    undetected_total: int,
+    commit_sha: str,
+    generated_at: str,
+    coverage_report: str,
+    coverage_sha256: str,
+    mutation_report: str,
+    mutation_sha256: str,
+) -> str:
+    """Emit a TOML evidence package with G1/G2 measured and every other gate left to a human."""
+    profile = PROFILE_KEY_BY_TIER[tier]
+    g2_passed = triage.passed
+
+    lines = [
+        f"# Oracle Gate evidence package — Tier {tier} ({profile})",
+        "# G1/G2 were MEASURED by `oracle-gate check`. Every other gate needs a human.",
+        "# A pass attests completeness, declared floors, and provenance. Never correctness.",
+        "",
+        f'profile = "{profile}"',
+        "has_model_component = false",
+        f'commit_sha = "{commit_sha}"',
+        f'generated_at = "{generated_at}"',
+        "",
+    ]
+
+    for gate in gates_for_tier(tier):
+        level = gate.levels[profile]
+        lines.append(f"[gates.{gate.id}]  # {gate.title}  ({level})")
+
+        if gate.id == "G1":
+            lines += [
+                'status = "met"',
+                f"value = {_fmt(coverage)}   # coverage — reported, NOT gated",
+                'source = "measured"',
+                f'report = "{coverage_report}"',
+                f'sha256 = "{coverage_sha256}"',
+            ]
+        elif gate.id == "G2":
+            lines += [
+                f'status = "{"met" if g2_passed else "not_met"}"',
+                f"value = {_fmt(scores['mutation_score'])}   # mutation_score — reported, NOT gated",
+                f"survivors_total = {undetected_total}",
+                f"survivors_explained = {len(triage.explained)}",
+                'source = "measured"',
+                f'report = "{mutation_report}"',
+                f'sha256 = "{mutation_sha256}"',
+            ]
+            if not g2_passed:
+                lines.append(
+                    f"# {len(triage.unexplained)} undetected mutant(s) are neither killed nor "
+                    f"explained. Kill them, or sign an entry in survivors.toml."
+                )
+        else:
+            lines.append('status = "not_met"')
+            lines.append(f"# {_prompt(gate)}")
+
+        lines.append("")
+
+    return "\n".join(lines)
+
+
+def _prompt(gate) -> str:
+    if gate.automatable:
+        return "attach a report + its sha256, then set status."
+    return "a named human must approve: set approver, artifact, date, sha256."
+
+
+def should_emit_evidence(*, scope_applied: bool) -> bool:
+    """An evidence package attests a REVISION. A diff-scoped run only gates a change.
+
+    Emitting a package that records `G2 status = "met"` after examining a subset of the
+    repository would claim more than was verified — the exact failure this framework
+    names. So a run that actually filtered by diff writes no package. A `--scope diff`
+    run whose base could not be resolved fell back to the whole repo, really did examine
+    it, and therefore still attests.
+    """
+    return not scope_applied
+
+
+def _pct(value) -> str:
+    return "  n/a " if value is None else f"{value * 100:5.1f}%"
+
+
+def format_report(
+    coverage,
+    scores,
+    triage,
+    *,
+    base_warning: bool = False,
+    no_mutated_files: bool = False,
+    filtered: tuple | None = None,
+) -> str:
+    """The human-facing report. Both scores appear; neither is a gate.
+
+    The score section always reflects the whole configured mutmut scope -- only the
+    gate below it can be narrowed to a diff. Three optional, mutually exclusive flags
+    describe how `--scope diff` went:
+
+      `base_warning`     -- no diff base could be resolved; the run fell back to the
+                             full repo scope.
+      `no_mutated_files`  -- a base resolved, but none of the files mutmut mutates
+                             appear in the diff, so there is nothing to gate.
+      `filtered`          -- `(filtered_out, total)` undetected mutants outside the
+                             diff were dropped from the gate below.
+    """
+    counts = scores["counts"]
+    covered = scores["mutation_score_covered"]
+
+    lines = [""]
+    if base_warning:
+        lines.append("  WARNING: could not resolve a diff base; falling back to repo scope.")
+    lines += [
+        f"  coverage             {_pct(coverage)}   reported, not gated",
+        f"  mutation score       {_pct(scores['mutation_score'])}   reported, not gated"
+        "   detected / valid",
+        f"    on covered code    {_pct(covered)}   reported, not gated"
+        "   detected / covered",
+        "",
+        f"  {counts['valid']} mutants: {counts['killed']} killed, "
+        f"{counts['survived']} survived, "
+        f"{counts['no_tests']} never reached by any test.",
+    ]
+    if filtered is not None:
+        filtered_out, total = filtered
+        lines.append(
+            f"  mutmut mutated the full configured scope; {filtered_out} of {total} "
+            f"undetected mutants were filtered out as untouched by this diff."
+        )
+    lines.append("")
+
+    undetected_n = len(triage.undetected)
+    if no_mutated_files:
+        lines.append("  no mutated files changed — nothing to gate.")
+    elif triage.passed and filtered is not None:
+        # Mutants outside the diff were FILTERED, not killed and not explained. Saying
+        # "all killed or explained" here would be false.
+        lines.append(f"  G2 PASS — {undetected_n} undetected in the changed files.")
+        filtered_out, _total = filtered
+        if filtered_out:
+            lines.append(f"  {filtered_out} undetected mutants remain elsewhere in this repo.")
+    elif triage.passed:
+        lines.append(f"  G2 PASS — {undetected_n} undetected, all killed or explained.")
+    else:
+        lines.append(
+            f"  G2 FAIL — {undetected_n} undetected, "
+            f"{len(triage.explained)} explained, {len(triage.unexplained)} unexplained."
+        )
+        for mutant in triage.unexplained[:10]:
+            lines.append(f"    {mutant.id}  ({mutant.status})")
+        if len(triage.unexplained) > 10:
+            lines.append(f"    … and {len(triage.unexplained) - 10} more")
+        lines.append("")
+        lines.append("  Kill each one, or have a named human explain it in survivors.toml.")
+
+    if triage.absorbed:
+        lines.append("")
+        lines.append("  Explanations applied:")
+        for match, count in sorted(triage.absorbed.items()):
+            lines.append(f"    {match:40} absorbed {count} mutant(s)")
+
+    if triage.stale:
+        lines.append("")
+        lines.append("  STALE explanations (their file changed; they absorb nothing):")
+        for match in triage.stale:
+            lines.append(f"    {match}")
+
+    lines.append("")
+    lines.append("  A pass attests completeness, declared floors, and provenance. Never correctness.")
+    lines.append("")
+    return "\n".join(lines)

oracle_gate/checklist.py

@@ -0,0 +1,131 @@
+"""The gate model, the per-tier checklist, and the machine-readable schema.
+
+Each gate carries, per conformance profile, a requirement level: "must", "should", or "na"
+(not applicable). This is the normative core the framework's prose gestured at, made precise so a
+conformance claim can be checked (see conformance.py) instead of self-declared.
+
+Pure logic (no network, no scanners) — fully testable and mutation-tested.
+
+Honesty rule: gates that a tool cannot verify (requirements, human accountability, evals) are marked
+`automatable = False` and are NEVER auto-passed — they require attached human evidence.
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+# Profiles, and the tier -> profile mapping (framework Part 3/4).
+PROFILE_KEY_BY_TIER = {1: "baseline", 2: "baseline", 3: "regulated", 4: "safety_critical"}
+PROFILE_NAME = {
+    "baseline": "Baseline",
+    "regulated": "Regulated",
+    "safety_critical": "Safety-or-Security-Critical",
+}
+PROFILE_ORDER = ("baseline", "regulated", "safety_critical")
+
+# Back-compat display map (tier -> profile display name).
+PROFILE_BY_TIER = {t: PROFILE_NAME[PROFILE_KEY_BY_TIER[t]] for t in PROFILE_KEY_BY_TIER}
+
+
+@dataclass(frozen=True)
+class Gate:
+    id: str
+    title: str
+    automatable: bool          # can a tool ever verify this, or is it human/process?
+    levels: dict               # profile_key -> "must" | "should" | "na"
+    metric: dict | None = None  # optional {name, floors: {profile_key -> float}} for content-checking
+
+
+def _lv(baseline, regulated, safety):
+    return {"baseline": baseline, "regulated": regulated, "safety_critical": safety}
+
+
+def _metric(name, **floors):
+    """A reported metric, and optionally per-profile floors.
+
+    THIS FRAMEWORK SHIPS NO DEFAULT FLOORS. No published mutation- or coverage-score threshold has an
+    evidentiary basis (EVIDENCE.md), and a threshold invites Goodhart's law: a score rises either by
+    writing tests or by arguing that survivors are equivalent, and only one of those is free. The gate
+    is surviving mutants (see `survivors.py`), not a percentage.
+
+    Adopters MAY declare a local floor in their evidence package; `conformance._check_metric` enforces
+    whatever they declare. With no floor, a metric gate requires only that the value be REPORTED —
+    disclosure, not a target.
+    """
+    return {"name": name, "floors": dict(floors)}
+
+
+# The twelve gates. `automatable` = whether a tool can ever check it. `levels` = the requirement per
+# profile. G8 (evals) applies only when the deliverable has an AI/model component (checker handles
+# that); G11 (waivers) is validated structurally by the checker when a waiver is present, not required
+# as standalone evidence.
+GATES = (
+    Gate("G0",  "Approved requirements & traceability",                    False, _lv("na",     "must",   "must")),
+    Gate("G1",  "Coverage reported (a disclosure, never a target)",        True,  _lv("must",   "must",   "must"),
+         metric=_metric("coverage")),
+    Gate("G2",  "Fault-detection shown (no unexplained surviving mutants)", True,  _lv("should", "must",   "must"),
+         metric=_metric("mutation_score")),
+    Gate("G3",  "Independent oracle (independence established)",           False, _lv("should", "must",   "must")),
+    Gate("G4",  "Real supply-chain control (SCA, SBOM, pinning)",          True,  _lv("should", "must",   "must")),
+    Gate("G4b", "Security gates by tier (static + secret scan; more at T3+)", True,  _lv("must",   "must",   "must")),
+    Gate("G5",  "Cross-model review (advisory, human-triaged)",            True,  _lv("should", "should", "should")),
+    Gate("G6",  "Human accountability / code review",                      False, _lv("should", "must",   "must")),
+    Gate("G7",  "Tests are protected (no silent weakening)",               True,  _lv("should", "must",   "must")),
+    Gate("G8",  "Trustworthy evals (AI/model components)",                 False, _lv("should", "must",   "must")),
+    Gate("G9",  "AI-use provenance recorded",                             False, _lv("should", "must",   "must")),
+    Gate("G10", "Evidence package assembled",                              False, _lv("should", "must",   "must")),
+    Gate("G11", "Waivers explicit and time-boxed",                         False, _lv("should", "should", "must")),
+)
+
+GATE_BY_ID = {g.id: g for g in GATES}
+
+
+def gates_for_tier(tier: int) -> list[Gate]:
+    """The gates that apply at a given tier (1-4): those whose level for the tier's profile != 'na'."""
+    if tier not in PROFILE_KEY_BY_TIER:
+        raise ValueError(f"tier must be 1-4, got {tier}")
+    profile = PROFILE_KEY_BY_TIER[tier]
+    return [g for g in GATES if g.levels[profile] != "na"]
+
+
+def render_checklist(tier: int) -> str:
+    """A Markdown evidence-package skeleton for the tier: what applies, and how each is satisfied."""
+    applicable = gates_for_tier(tier)   # validates the tier first (clean ValueError on bad input)
+    profile = PROFILE_BY_TIER[tier]
+    auto = [g for g in applicable if g.automatable]
+    human = [g for g in applicable if not g.automatable]
+
+    lines = [
+        f"# Oracle Gate — evidence package (Tier {tier} · {profile} profile)",
+        "",
+        f"{len(applicable)} gates apply at this tier: {len(auto)} machine-checkable, "
+        f"{len(human)} require human evidence (never auto-passed).",
+        "",
+        "## Machine-checkable gates",
+    ]
+    for g in auto:
+        lines.append(f"- [ ] **{g.id}** — {g.title}  _(tool can run; attach the report)_")
+    lines += ["", "## Human-evidence gates (sign-off required — a tool cannot verify these)"]
+    for g in human:
+        lines.append(f"- [ ] **{g.id}** — {g.title}  _(attach the artifact + approver)_")
+    lines += [
+        "",
+        "## Conformance statement",
+        f"> Meets The Oracle Gate, **{profile}** profile, when every box above is checked with "
+        "attached evidence. Anything short ships under an explicit, time-boxed waiver (G11).",
+    ]
+    return "\n".join(lines)
+
+
+def schema() -> dict:
+    """The machine-readable gate/profile schema — so other tools can build against it."""
+    return {
+        "framework": "The Oracle Gate",
+        "profiles": {k: PROFILE_NAME[k] for k in PROFILE_ORDER},
+        "tier_to_profile": {str(t): PROFILE_KEY_BY_TIER[t] for t in PROFILE_KEY_BY_TIER},
+        "levels": ["must", "should", "na"],
+        "gates": [
+            {"id": g.id, "title": g.title, "automatable": g.automatable, "levels": g.levels,
+             "metric": g.metric}
+            for g in GATES
+        ],
+    }