oracle-gate 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- oracle_gate-0.1.0/LICENSE +21 -0
- oracle_gate-0.1.0/PKG-INFO +94 -0
- oracle_gate-0.1.0/README.md +63 -0
- oracle_gate-0.1.0/oracle_gate/__init__.py +2 -0
- oracle_gate-0.1.0/oracle_gate/check.py +346 -0
- oracle_gate-0.1.0/oracle_gate/checklist.py +131 -0
- oracle_gate-0.1.0/oracle_gate/cli.py +280 -0
- oracle_gate-0.1.0/oracle_gate/conformance.py +260 -0
- oracle_gate-0.1.0/oracle_gate/provenance.py +58 -0
- oracle_gate-0.1.0/oracle_gate/providers.py +162 -0
- oracle_gate-0.1.0/oracle_gate/review.py +41 -0
- oracle_gate-0.1.0/oracle_gate/runner.py +188 -0
- oracle_gate-0.1.0/oracle_gate/survivors.py +246 -0
- oracle_gate-0.1.0/oracle_gate.egg-info/PKG-INFO +94 -0
- oracle_gate-0.1.0/oracle_gate.egg-info/SOURCES.txt +28 -0
- oracle_gate-0.1.0/oracle_gate.egg-info/dependency_links.txt +1 -0
- oracle_gate-0.1.0/oracle_gate.egg-info/entry_points.txt +2 -0
- oracle_gate-0.1.0/oracle_gate.egg-info/requires.txt +9 -0
- oracle_gate-0.1.0/oracle_gate.egg-info/top_level.txt +1 -0
- oracle_gate-0.1.0/pyproject.toml +57 -0
- oracle_gate-0.1.0/setup.cfg +4 -0
- oracle_gate-0.1.0/tests/test_check.py +605 -0
- oracle_gate-0.1.0/tests/test_checklist.py +77 -0
- oracle_gate-0.1.0/tests/test_checklist_golden.py +25 -0
- oracle_gate-0.1.0/tests/test_cli.py +184 -0
- oracle_gate-0.1.0/tests/test_conformance.py +491 -0
- oracle_gate-0.1.0/tests/test_provenance.py +79 -0
- oracle_gate-0.1.0/tests/test_providers.py +89 -0
- oracle_gate-0.1.0/tests/test_runner.py +297 -0
- oracle_gate-0.1.0/tests/test_survivors.py +471 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Jeff Otterson
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: oracle-gate
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Reference CLI for The Oracle Gate — a framework for testing AI-built code. Model-agnostic cross-model review + per-tier evidence checklist.
|
|
5
|
+
Author: Jeff Otterson
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/Jott2121/oracle-gate
|
|
8
|
+
Project-URL: Repository, https://github.com/Jott2121/oracle-gate
|
|
9
|
+
Project-URL: Issues, https://github.com/Jott2121/oracle-gate/issues
|
|
10
|
+
Keywords: ai,testing,code-quality,mutation-testing,llm,test-quality
|
|
11
|
+
Classifier: Development Status :: 4 - Beta
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: Programming Language :: Python :: 3
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
18
|
+
Classifier: Topic :: Software Development :: Quality Assurance
|
|
19
|
+
Classifier: Topic :: Software Development :: Testing
|
|
20
|
+
Requires-Python: >=3.10
|
|
21
|
+
Description-Content-Type: text/markdown
|
|
22
|
+
License-File: LICENSE
|
|
23
|
+
Provides-Extra: dev
|
|
24
|
+
Requires-Dist: pytest>=8; extra == "dev"
|
|
25
|
+
Requires-Dist: hypothesis>=6; extra == "dev"
|
|
26
|
+
Requires-Dist: mutmut>=3; extra == "dev"
|
|
27
|
+
Provides-Extra: check
|
|
28
|
+
Requires-Dist: mutmut<4,>=3; extra == "check"
|
|
29
|
+
Requires-Dist: pytest-cov>=4; extra == "check"
|
|
30
|
+
Dynamic: license-file
|
|
31
|
+
|
|
32
|
+
# oracle-gate (CLI)
|
|
33
|
+
|
|
34
|
+
The reference implementation of [The Oracle Gate](https://github.com/Jott2121/oracle-gate/blob/main/README.md) — a framework for testing AI-built
|
|
35
|
+
code. This tool does the parts a machine *can* do honestly, and refuses to fake the parts it can't.
|
|
36
|
+
|
|
37
|
+
## What it does today
|
|
38
|
+
|
|
39
|
+
- **`oracle-gate review <file>`** — the cross-model review gate (**G5**). Sends an artifact to a model
|
|
40
|
+
for adversarial review. **Model-agnostic:** pick any provider; adding a model is one small adapter.
|
|
41
|
+
Framework-faithful: pass `--builder-lineage` and it warns if your reviewer shares the builder's
|
|
42
|
+
lineage (which would defeat the point).
|
|
43
|
+
- **`oracle-gate checklist --tier N`** — emits the per-tier **evidence-package skeleton**: which gates
|
|
44
|
+
apply, which the tool can check, and which require human sign-off. Human-judgment gates
|
|
45
|
+
(requirements, accountability, evals) are **never auto-passed** — they render as "evidence required."
|
|
46
|
+
- **`oracle-gate schema`** — the machine-readable gate/profile schema (JSON), so other tools can build
|
|
47
|
+
against the framework.
|
|
48
|
+
- **`oracle-gate evidence --tier N`** — a fillable **TOML evidence-package template** for a tier.
|
|
49
|
+
- **`oracle-gate conformance <evidence.toml> [--repo PATH]`** — checks a filled evidence package
|
|
50
|
+
against the profile it claims. It enforces three rising tiers of rigor — **none of which attests the
|
|
51
|
+
software is correct:** (1) **completeness** — a machine gate marked "met" must attach a report; a
|
|
52
|
+
human gate must name an approver + artifact + date; anything short needs a valid time-boxed waiver;
|
|
53
|
+
(2) **declared floor** — metric gates (coverage, mutation score) must declare a `value` that meets
|
|
54
|
+
the profile floor, so a bad-score report no longer reads as conformant; (3) **version binding** — the
|
|
55
|
+
package must carry a `commit_sha` (an ancestor of `HEAD` in `--repo`, default cwd), a `generated_at`,
|
|
56
|
+
and a `sha256` per attached artifact that matches the file on disk, so stale or borrowed evidence
|
|
57
|
+
can't pass. Whether the declared numbers are *true*, and whether the code is *correct*, stay a
|
|
58
|
+
human's job. Exit: 0 = pass, 1 = fail, 2 = error (CI-friendly).
|
|
59
|
+
- **`oracle-gate providers`** / **`oracle-gate ping`** — list model backends / check auth.
|
|
60
|
+
|
|
61
|
+
Deterministic scanner gates (running mutation testing, SCA/dependency checks, secret scanning behind
|
|
62
|
+
an `oracle-gate check` command) are a planned later phase.
|
|
63
|
+
|
|
64
|
+
## Install
|
|
65
|
+
|
|
66
|
+
```
|
|
67
|
+
pip install -e ".[dev]" # from this tool/ directory
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
## Keys (for the review gate)
|
|
71
|
+
|
|
72
|
+
Store a provider key at `~/.config/oracle-gate/<provider>.key` (chmod 600) or set its env var
|
|
73
|
+
(`OPENAI_API_KEY`, `ANTHROPIC_API_KEY`).
|
|
74
|
+
|
|
75
|
+
```
|
|
76
|
+
oracle-gate ping --provider openai
|
|
77
|
+
oracle-gate review my-notes.md --provider openai --builder-lineage anthropic
|
|
78
|
+
oracle-gate checklist --tier 3 > EVIDENCE.md
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
**Guardrail:** `review` sends the artifact to an external model. **Non-sensitive artifacts only** —
|
|
82
|
+
never route sensitive, controlled, or proprietary data to an unapproved commercial model.
|
|
83
|
+
|
|
84
|
+
## Dogfooding
|
|
85
|
+
|
|
86
|
+
This tool is tested with the framework it implements: its pure decision logic is **mutation-tested and
|
|
87
|
+
mutation-clean** (see [MUTATION.md](https://github.com/Jott2121/oracle-gate/blob/main/tool/MUTATION.md) — 411 killed; every pass/fail branch has no surviving
|
|
88
|
+
killable mutant; residual survivors are documented prose/equivalents). Its checklist and evidence
|
|
89
|
+
templates are pinned with golden-approval tests, and the version-binding IO is covered by a live git
|
|
90
|
+
integration test. It passes its own G2 gate.
|
|
91
|
+
|
|
92
|
+
## License
|
|
93
|
+
|
|
94
|
+
MIT (the code). The framework prose is CC BY 4.0. See the repository root.
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
# oracle-gate (CLI)
|
|
2
|
+
|
|
3
|
+
The reference implementation of [The Oracle Gate](https://github.com/Jott2121/oracle-gate/blob/main/README.md) — a framework for testing AI-built
|
|
4
|
+
code. This tool does the parts a machine *can* do honestly, and refuses to fake the parts it can't.
|
|
5
|
+
|
|
6
|
+
## What it does today
|
|
7
|
+
|
|
8
|
+
- **`oracle-gate review <file>`** — the cross-model review gate (**G5**). Sends an artifact to a model
|
|
9
|
+
for adversarial review. **Model-agnostic:** pick any provider; adding a model is one small adapter.
|
|
10
|
+
Framework-faithful: pass `--builder-lineage` and it warns if your reviewer shares the builder's
|
|
11
|
+
lineage (which would defeat the point).
|
|
12
|
+
- **`oracle-gate checklist --tier N`** — emits the per-tier **evidence-package skeleton**: which gates
|
|
13
|
+
apply, which the tool can check, and which require human sign-off. Human-judgment gates
|
|
14
|
+
(requirements, accountability, evals) are **never auto-passed** — they render as "evidence required."
|
|
15
|
+
- **`oracle-gate schema`** — the machine-readable gate/profile schema (JSON), so other tools can build
|
|
16
|
+
against the framework.
|
|
17
|
+
- **`oracle-gate evidence --tier N`** — a fillable **TOML evidence-package template** for a tier.
|
|
18
|
+
- **`oracle-gate conformance <evidence.toml> [--repo PATH]`** — checks a filled evidence package
|
|
19
|
+
against the profile it claims. It enforces three rising tiers of rigor — **none of which attests the
|
|
20
|
+
software is correct:** (1) **completeness** — a machine gate marked "met" must attach a report; a
|
|
21
|
+
human gate must name an approver + artifact + date; anything short needs a valid time-boxed waiver;
|
|
22
|
+
(2) **declared floor** — metric gates (coverage, mutation score) must declare a `value` that meets
|
|
23
|
+
the profile floor, so a bad-score report no longer reads as conformant; (3) **version binding** — the
|
|
24
|
+
package must carry a `commit_sha` (an ancestor of `HEAD` in `--repo`, default cwd), a `generated_at`,
|
|
25
|
+
and a `sha256` per attached artifact that matches the file on disk, so stale or borrowed evidence
|
|
26
|
+
can't pass. Whether the declared numbers are *true*, and whether the code is *correct*, stay a
|
|
27
|
+
human's job. Exit: 0 = pass, 1 = fail, 2 = error (CI-friendly).
|
|
28
|
+
- **`oracle-gate providers`** / **`oracle-gate ping`** — list model backends / check auth.
|
|
29
|
+
|
|
30
|
+
Deterministic scanner gates (running mutation testing, SCA/dependency checks, secret scanning behind
|
|
31
|
+
an `oracle-gate check` command) are a planned later phase.
|
|
32
|
+
|
|
33
|
+
## Install
|
|
34
|
+
|
|
35
|
+
```
|
|
36
|
+
pip install -e ".[dev]" # from this tool/ directory
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
## Keys (for the review gate)
|
|
40
|
+
|
|
41
|
+
Store a provider key at `~/.config/oracle-gate/<provider>.key` (chmod 600) or set its env var
|
|
42
|
+
(`OPENAI_API_KEY`, `ANTHROPIC_API_KEY`).
|
|
43
|
+
|
|
44
|
+
```
|
|
45
|
+
oracle-gate ping --provider openai
|
|
46
|
+
oracle-gate review my-notes.md --provider openai --builder-lineage anthropic
|
|
47
|
+
oracle-gate checklist --tier 3 > EVIDENCE.md
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
**Guardrail:** `review` sends the artifact to an external model. **Non-sensitive artifacts only** —
|
|
51
|
+
never route sensitive, controlled, or proprietary data to an unapproved commercial model.
|
|
52
|
+
|
|
53
|
+
## Dogfooding
|
|
54
|
+
|
|
55
|
+
This tool is tested with the framework it implements: its pure decision logic is **mutation-tested and
|
|
56
|
+
mutation-clean** (see [MUTATION.md](https://github.com/Jott2121/oracle-gate/blob/main/tool/MUTATION.md) — 411 killed; every pass/fail branch has no surviving
|
|
57
|
+
killable mutant; residual survivors are documented prose/equivalents). Its checklist and evidence
|
|
58
|
+
templates are pinned with golden-approval tests, and the version-binding IO is covered by a live git
|
|
59
|
+
integration test. It passes its own G2 gate.
|
|
60
|
+
|
|
61
|
+
## License
|
|
62
|
+
|
|
63
|
+
MIT (the code). The framework prose is CC BY 4.0. See the repository root.
|
|
@@ -0,0 +1,346 @@
|
|
|
1
|
+
"""Scoring and evidence emission for `oracle-gate check` — pure, no IO.
|
|
2
|
+
|
|
3
|
+
Two rules govern this module, and both exist because the framework accuses other tools
|
|
4
|
+
of breaking them:
|
|
5
|
+
|
|
6
|
+
1. **Percentages are reported, never gated.** `mutation_score` and its flattering cousin
|
|
7
|
+
`mutation_score_covered` are printed together, each with its formula, so nobody has to
|
|
8
|
+
trust which denominator we chose. The gate is `survivors.triage`.
|
|
9
|
+
|
|
10
|
+
2. **`check` fills G1 and G2 and nothing else.** It cannot certify human accountability,
|
|
11
|
+
an independent oracle, or a trustworthy eval. Those gates are emitted `not_met` with a
|
|
12
|
+
prompt, every time, even when everything it *can* measure passes.
|
|
13
|
+
|
|
14
|
+
Metrics it produces are stamped `source = "measured"`, distinguishing them from the
|
|
15
|
+
human-typed `declared` values the conformance checker also accepts. That is the third rung
|
|
16
|
+
of the ladder — attached, declared, measured — and the fourth rung, *correct*, is not
|
|
17
|
+
reachable by this or any tool.
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
from __future__ import annotations
|
|
21
|
+
|
|
22
|
+
from dataclasses import dataclass
|
|
23
|
+
|
|
24
|
+
from .checklist import PROFILE_KEY_BY_TIER, gates_for_tier
|
|
25
|
+
from .survivors import UNCLASSIFIED, UnclassifiedStatus
|
|
26
|
+
|
|
27
|
+
__all__ = [
|
|
28
|
+
"NoMutantsError",
|
|
29
|
+
"CountsDisagree",
|
|
30
|
+
"Verdict",
|
|
31
|
+
"scores",
|
|
32
|
+
"verdict",
|
|
33
|
+
"evidence",
|
|
34
|
+
"counts_from_mutants",
|
|
35
|
+
"cross_check",
|
|
36
|
+
"format_report",
|
|
37
|
+
]
|
|
38
|
+
|
|
39
|
+
# The mutmut status strings that count toward a score, mapped to this tool's count keys.
|
|
40
|
+
# "no tests" -> "no_tests" is the one rename; everything else that reaches here is
|
|
41
|
+
# already classified by `undetected`/`counts_from_mutants` raising on anything else.
|
|
42
|
+
_STATUS_TO_COUNT_KEY = {"killed": "killed", "timeout": "timeout", "survived": "survived", "no tests": "no_tests"}
|
|
43
|
+
|
|
44
|
+
_FORMULA = {
|
|
45
|
+
"mutation_score": "detected / valid (detected = killed + timeout; valid = detected + survived + no_tests)",
|
|
46
|
+
"mutation_score_covered": "detected / covered (covered = detected + survived) — reported, not gated",
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
# Gates `check` is allowed to fill. Everything else needs a human.
|
|
50
|
+
_MACHINE_GATES = ("G1", "G2")
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
class NoMutantsError(ValueError):
|
|
54
|
+
"""No valid mutants were produced, so there is no score to report."""
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
class CountsDisagree(ValueError):
|
|
58
|
+
"""Two independently-derived count dicts disagree on a field.
|
|
59
|
+
|
|
60
|
+
`run_mutation` gives us counts two ways: mutmut's own `mutmut-cicd-stats.json`, and
|
|
61
|
+
whatever `counts_from_mutants` tallies by parsing `mutmut results --all true`. They
|
|
62
|
+
should always agree -- they describe the same run. If they do not, something is
|
|
63
|
+
wrong with the run or with how one of the two was read, and printing a score built
|
|
64
|
+
from either one would be trusting a source of truth that just proved itself
|
|
65
|
+
unreliable. Fail closed instead.
|
|
66
|
+
"""
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
@dataclass(frozen=True)
|
|
70
|
+
class Verdict:
|
|
71
|
+
passed: bool
|
|
72
|
+
unexplained: int
|
|
73
|
+
|
|
74
|
+
@property
|
|
75
|
+
def exit_code(self) -> int:
|
|
76
|
+
return 0 if self.passed else 1
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def scores(counts) -> dict:
|
|
80
|
+
"""Compute both mutation scores, with their formulas attached.
|
|
81
|
+
|
|
82
|
+
Fails closed if mutmut reported a status we cannot place on Stryker's map: silently
|
|
83
|
+
dropping those counts would choose a denominator by accident.
|
|
84
|
+
"""
|
|
85
|
+
for name in sorted(UNCLASSIFIED):
|
|
86
|
+
# no default: absent and zero are both falsy, and spelling one invites an
|
|
87
|
+
# equivalent mutant that no test can ever kill.
|
|
88
|
+
if counts.get(name):
|
|
89
|
+
raise UnclassifiedStatus(
|
|
90
|
+
f"mutmut reported {counts[name]} {name!r} mutant(s). These do not map onto "
|
|
91
|
+
f"a known mutant state, and guessing would silently pick a denominator. "
|
|
92
|
+
f"Investigate or exclude them from the run."
|
|
93
|
+
)
|
|
94
|
+
|
|
95
|
+
killed = counts.get("killed", 0)
|
|
96
|
+
timeout = counts.get("timeout", 0)
|
|
97
|
+
survived = counts.get("survived", 0)
|
|
98
|
+
no_tests = counts.get("no_tests", 0)
|
|
99
|
+
|
|
100
|
+
detected = killed + timeout
|
|
101
|
+
valid = detected + survived + no_tests
|
|
102
|
+
covered = detected + survived
|
|
103
|
+
|
|
104
|
+
if valid == 0:
|
|
105
|
+
raise NoMutantsError("mutmut produced no valid mutants; there is nothing to score")
|
|
106
|
+
|
|
107
|
+
return {
|
|
108
|
+
"mutation_score": detected / valid,
|
|
109
|
+
"mutation_score_covered": (detected / covered) if covered else None,
|
|
110
|
+
"counts": {
|
|
111
|
+
"killed": killed,
|
|
112
|
+
"timeout": timeout,
|
|
113
|
+
"survived": survived,
|
|
114
|
+
"no_tests": no_tests,
|
|
115
|
+
"detected": detected,
|
|
116
|
+
"valid": valid,
|
|
117
|
+
"covered": covered,
|
|
118
|
+
},
|
|
119
|
+
"formula": dict(_FORMULA),
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def verdict(triage) -> Verdict:
|
|
124
|
+
"""The gate: every undetected mutant is killed or explained."""
|
|
125
|
+
return Verdict(passed=triage.passed, unexplained=len(triage.unexplained))
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def counts_from_mutants(mutants) -> dict:
|
|
129
|
+
"""Tally {"killed", "timeout", "survived", "no_tests"} straight from parsed mutants.
|
|
130
|
+
|
|
131
|
+
This is a second, independent path to the same counts `run_mutation` reads out of
|
|
132
|
+
mutmut's stats file -- see `cross_check`. Any status other than the four this tool
|
|
133
|
+
knows how to place (including "suspicious"/"segfault"/"skipped") is refused rather
|
|
134
|
+
than silently dropped or lumped in: naming the status and the mutant lets a human
|
|
135
|
+
decide, instead of this tool guessing a denominator.
|
|
136
|
+
"""
|
|
137
|
+
counts = {"killed": 0, "timeout": 0, "survived": 0, "no_tests": 0}
|
|
138
|
+
for m in mutants:
|
|
139
|
+
key = _STATUS_TO_COUNT_KEY.get(m.status)
|
|
140
|
+
if key is None:
|
|
141
|
+
raise UnclassifiedStatus(
|
|
142
|
+
f"mutmut reported status {m.status!r} for {m.id}, which does not map "
|
|
143
|
+
f"onto a known mutant state. Refusing to guess: classify it or exclude "
|
|
144
|
+
f"it from the run."
|
|
145
|
+
)
|
|
146
|
+
counts[key] += 1
|
|
147
|
+
return counts
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
def cross_check(counts_from_stats, counts_from_results) -> None:
|
|
151
|
+
"""Refuse to score unless mutmut's two outputs agree on every count.
|
|
152
|
+
|
|
153
|
+
`counts_from_stats` comes from `mutmut-cicd-stats.json`; `counts_from_results`
|
|
154
|
+
comes from parsing `mutmut results --all true` (see `counts_from_mutants`). They
|
|
155
|
+
describe the same run and must match. A disagreement means one of the two readings
|
|
156
|
+
is wrong, and printing a score anyway would silently pick which one to trust.
|
|
157
|
+
"""
|
|
158
|
+
for field in ("killed", "survived", "no_tests", "timeout"):
|
|
159
|
+
a = counts_from_stats.get(field, 0)
|
|
160
|
+
b = counts_from_results.get(field, 0)
|
|
161
|
+
if a != b:
|
|
162
|
+
raise CountsDisagree(
|
|
163
|
+
f"counts disagree on {field!r}: mutmut-cicd-stats.json says {a}, "
|
|
164
|
+
f"mutmut results --all true says {b}"
|
|
165
|
+
)
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
def _fmt(value) -> str:
|
|
169
|
+
return f"{value:.6g}"
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def evidence(
|
|
173
|
+
*,
|
|
174
|
+
tier: int,
|
|
175
|
+
coverage: float,
|
|
176
|
+
scores: dict,
|
|
177
|
+
triage,
|
|
178
|
+
undetected_total: int,
|
|
179
|
+
commit_sha: str,
|
|
180
|
+
generated_at: str,
|
|
181
|
+
coverage_report: str,
|
|
182
|
+
coverage_sha256: str,
|
|
183
|
+
mutation_report: str,
|
|
184
|
+
mutation_sha256: str,
|
|
185
|
+
) -> str:
|
|
186
|
+
"""Emit a TOML evidence package with G1/G2 measured and every other gate left to a human."""
|
|
187
|
+
profile = PROFILE_KEY_BY_TIER[tier]
|
|
188
|
+
g2_passed = triage.passed
|
|
189
|
+
|
|
190
|
+
lines = [
|
|
191
|
+
f"# Oracle Gate evidence package — Tier {tier} ({profile})",
|
|
192
|
+
"# G1/G2 were MEASURED by `oracle-gate check`. Every other gate needs a human.",
|
|
193
|
+
"# A pass attests completeness, declared floors, and provenance. Never correctness.",
|
|
194
|
+
"",
|
|
195
|
+
f'profile = "{profile}"',
|
|
196
|
+
"has_model_component = false",
|
|
197
|
+
f'commit_sha = "{commit_sha}"',
|
|
198
|
+
f'generated_at = "{generated_at}"',
|
|
199
|
+
"",
|
|
200
|
+
]
|
|
201
|
+
|
|
202
|
+
for gate in gates_for_tier(tier):
|
|
203
|
+
level = gate.levels[profile]
|
|
204
|
+
lines.append(f"[gates.{gate.id}] # {gate.title} ({level})")
|
|
205
|
+
|
|
206
|
+
if gate.id == "G1":
|
|
207
|
+
lines += [
|
|
208
|
+
'status = "met"',
|
|
209
|
+
f"value = {_fmt(coverage)} # coverage — reported, NOT gated",
|
|
210
|
+
'source = "measured"',
|
|
211
|
+
f'report = "{coverage_report}"',
|
|
212
|
+
f'sha256 = "{coverage_sha256}"',
|
|
213
|
+
]
|
|
214
|
+
elif gate.id == "G2":
|
|
215
|
+
lines += [
|
|
216
|
+
f'status = "{"met" if g2_passed else "not_met"}"',
|
|
217
|
+
f"value = {_fmt(scores['mutation_score'])} # mutation_score — reported, NOT gated",
|
|
218
|
+
f"survivors_total = {undetected_total}",
|
|
219
|
+
f"survivors_explained = {len(triage.explained)}",
|
|
220
|
+
'source = "measured"',
|
|
221
|
+
f'report = "{mutation_report}"',
|
|
222
|
+
f'sha256 = "{mutation_sha256}"',
|
|
223
|
+
]
|
|
224
|
+
if not g2_passed:
|
|
225
|
+
lines.append(
|
|
226
|
+
f"# {len(triage.unexplained)} undetected mutant(s) are neither killed nor "
|
|
227
|
+
f"explained. Kill them, or sign an entry in survivors.toml."
|
|
228
|
+
)
|
|
229
|
+
else:
|
|
230
|
+
lines.append('status = "not_met"')
|
|
231
|
+
lines.append(f"# {_prompt(gate)}")
|
|
232
|
+
|
|
233
|
+
lines.append("")
|
|
234
|
+
|
|
235
|
+
return "\n".join(lines)
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
def _prompt(gate) -> str:
|
|
239
|
+
if gate.automatable:
|
|
240
|
+
return "attach a report + its sha256, then set status."
|
|
241
|
+
return "a named human must approve: set approver, artifact, date, sha256."
|
|
242
|
+
|
|
243
|
+
|
|
244
|
+
def should_emit_evidence(*, scope_applied: bool) -> bool:
|
|
245
|
+
"""An evidence package attests a REVISION. A diff-scoped run only gates a change.
|
|
246
|
+
|
|
247
|
+
Emitting a package that records `G2 status = "met"` after examining a subset of the
|
|
248
|
+
repository would claim more than was verified — the exact failure this framework
|
|
249
|
+
names. So a run that actually filtered by diff writes no package. A `--scope diff`
|
|
250
|
+
run whose base could not be resolved fell back to the whole repo, really did examine
|
|
251
|
+
it, and therefore still attests.
|
|
252
|
+
"""
|
|
253
|
+
return not scope_applied
|
|
254
|
+
|
|
255
|
+
|
|
256
|
+
def _pct(value) -> str:
|
|
257
|
+
return " n/a " if value is None else f"{value * 100:5.1f}%"
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
def format_report(
|
|
261
|
+
coverage,
|
|
262
|
+
scores,
|
|
263
|
+
triage,
|
|
264
|
+
*,
|
|
265
|
+
base_warning: bool = False,
|
|
266
|
+
no_mutated_files: bool = False,
|
|
267
|
+
filtered: tuple | None = None,
|
|
268
|
+
) -> str:
|
|
269
|
+
"""The human-facing report. Both scores appear; neither is a gate.
|
|
270
|
+
|
|
271
|
+
The score section always reflects the whole configured mutmut scope -- only the
|
|
272
|
+
gate below it can be narrowed to a diff. Three optional, mutually exclusive flags
|
|
273
|
+
describe how `--scope diff` went:
|
|
274
|
+
|
|
275
|
+
`base_warning` -- no diff base could be resolved; the run fell back to the
|
|
276
|
+
full repo scope.
|
|
277
|
+
`no_mutated_files` -- a base resolved, but none of the files mutmut mutates
|
|
278
|
+
appear in the diff, so there is nothing to gate.
|
|
279
|
+
`filtered` -- `(filtered_out, total)` undetected mutants outside the
|
|
280
|
+
diff were dropped from the gate below.
|
|
281
|
+
"""
|
|
282
|
+
counts = scores["counts"]
|
|
283
|
+
covered = scores["mutation_score_covered"]
|
|
284
|
+
|
|
285
|
+
lines = [""]
|
|
286
|
+
if base_warning:
|
|
287
|
+
lines.append(" WARNING: could not resolve a diff base; falling back to repo scope.")
|
|
288
|
+
lines += [
|
|
289
|
+
f" coverage {_pct(coverage)} reported, not gated",
|
|
290
|
+
f" mutation score {_pct(scores['mutation_score'])} reported, not gated"
|
|
291
|
+
" detected / valid",
|
|
292
|
+
f" on covered code {_pct(covered)} reported, not gated"
|
|
293
|
+
" detected / covered",
|
|
294
|
+
"",
|
|
295
|
+
f" {counts['valid']} mutants: {counts['killed']} killed, "
|
|
296
|
+
f"{counts['survived']} survived, "
|
|
297
|
+
f"{counts['no_tests']} never reached by any test.",
|
|
298
|
+
]
|
|
299
|
+
if filtered is not None:
|
|
300
|
+
filtered_out, total = filtered
|
|
301
|
+
lines.append(
|
|
302
|
+
f" mutmut mutated the full configured scope; {filtered_out} of {total} "
|
|
303
|
+
f"undetected mutants were filtered out as untouched by this diff."
|
|
304
|
+
)
|
|
305
|
+
lines.append("")
|
|
306
|
+
|
|
307
|
+
undetected_n = len(triage.undetected)
|
|
308
|
+
if no_mutated_files:
|
|
309
|
+
lines.append(" no mutated files changed — nothing to gate.")
|
|
310
|
+
elif triage.passed and filtered is not None:
|
|
311
|
+
# Mutants outside the diff were FILTERED, not killed and not explained. Saying
|
|
312
|
+
# "all killed or explained" here would be false.
|
|
313
|
+
lines.append(f" G2 PASS — {undetected_n} undetected in the changed files.")
|
|
314
|
+
filtered_out, _total = filtered
|
|
315
|
+
if filtered_out:
|
|
316
|
+
lines.append(f" {filtered_out} undetected mutants remain elsewhere in this repo.")
|
|
317
|
+
elif triage.passed:
|
|
318
|
+
lines.append(f" G2 PASS — {undetected_n} undetected, all killed or explained.")
|
|
319
|
+
else:
|
|
320
|
+
lines.append(
|
|
321
|
+
f" G2 FAIL — {undetected_n} undetected, "
|
|
322
|
+
f"{len(triage.explained)} explained, {len(triage.unexplained)} unexplained."
|
|
323
|
+
)
|
|
324
|
+
for mutant in triage.unexplained[:10]:
|
|
325
|
+
lines.append(f" {mutant.id} ({mutant.status})")
|
|
326
|
+
if len(triage.unexplained) > 10:
|
|
327
|
+
lines.append(f" … and {len(triage.unexplained) - 10} more")
|
|
328
|
+
lines.append("")
|
|
329
|
+
lines.append(" Kill each one, or have a named human explain it in survivors.toml.")
|
|
330
|
+
|
|
331
|
+
if triage.absorbed:
|
|
332
|
+
lines.append("")
|
|
333
|
+
lines.append(" Explanations applied:")
|
|
334
|
+
for match, count in sorted(triage.absorbed.items()):
|
|
335
|
+
lines.append(f" {match:40} absorbed {count} mutant(s)")
|
|
336
|
+
|
|
337
|
+
if triage.stale:
|
|
338
|
+
lines.append("")
|
|
339
|
+
lines.append(" STALE explanations (their file changed; they absorb nothing):")
|
|
340
|
+
for match in triage.stale:
|
|
341
|
+
lines.append(f" {match}")
|
|
342
|
+
|
|
343
|
+
lines.append("")
|
|
344
|
+
lines.append(" A pass attests completeness, declared floors, and provenance. Never correctness.")
|
|
345
|
+
lines.append("")
|
|
346
|
+
return "\n".join(lines)
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
"""The gate model, the per-tier checklist, and the machine-readable schema.
|
|
2
|
+
|
|
3
|
+
Each gate carries, per conformance profile, a requirement level: "must", "should", or "na"
|
|
4
|
+
(not applicable). This is the normative core the framework's prose gestured at, made precise so a
|
|
5
|
+
conformance claim can be checked (see conformance.py) instead of self-declared.
|
|
6
|
+
|
|
7
|
+
Pure logic (no network, no scanners) — fully testable and mutation-tested.
|
|
8
|
+
|
|
9
|
+
Honesty rule: gates that a tool cannot verify (requirements, human accountability, evals) are marked
|
|
10
|
+
`automatable = False` and are NEVER auto-passed — they require attached human evidence.
|
|
11
|
+
"""
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
from dataclasses import dataclass
|
|
15
|
+
|
|
16
|
+
# Profiles, and the tier -> profile mapping (framework Part 3/4).
|
|
17
|
+
PROFILE_KEY_BY_TIER = {1: "baseline", 2: "baseline", 3: "regulated", 4: "safety_critical"}
|
|
18
|
+
PROFILE_NAME = {
|
|
19
|
+
"baseline": "Baseline",
|
|
20
|
+
"regulated": "Regulated",
|
|
21
|
+
"safety_critical": "Safety-or-Security-Critical",
|
|
22
|
+
}
|
|
23
|
+
PROFILE_ORDER = ("baseline", "regulated", "safety_critical")
|
|
24
|
+
|
|
25
|
+
# Back-compat display map (tier -> profile display name).
|
|
26
|
+
PROFILE_BY_TIER = {t: PROFILE_NAME[PROFILE_KEY_BY_TIER[t]] for t in PROFILE_KEY_BY_TIER}
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
@dataclass(frozen=True)
|
|
30
|
+
class Gate:
|
|
31
|
+
id: str
|
|
32
|
+
title: str
|
|
33
|
+
automatable: bool # can a tool ever verify this, or is it human/process?
|
|
34
|
+
levels: dict # profile_key -> "must" | "should" | "na"
|
|
35
|
+
metric: dict | None = None # optional {name, floors: {profile_key -> float}} for content-checking
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _lv(baseline, regulated, safety):
|
|
39
|
+
return {"baseline": baseline, "regulated": regulated, "safety_critical": safety}
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def _metric(name, **floors):
|
|
43
|
+
"""A reported metric, and optionally per-profile floors.
|
|
44
|
+
|
|
45
|
+
THIS FRAMEWORK SHIPS NO DEFAULT FLOORS. No published mutation- or coverage-score threshold has an
|
|
46
|
+
evidentiary basis (EVIDENCE.md), and a threshold invites Goodhart's law: a score rises either by
|
|
47
|
+
writing tests or by arguing that survivors are equivalent, and only one of those is free. The gate
|
|
48
|
+
is surviving mutants (see `survivors.py`), not a percentage.
|
|
49
|
+
|
|
50
|
+
Adopters MAY declare a local floor in their evidence package; `conformance._check_metric` enforces
|
|
51
|
+
whatever they declare. With no floor, a metric gate requires only that the value be REPORTED —
|
|
52
|
+
disclosure, not a target.
|
|
53
|
+
"""
|
|
54
|
+
return {"name": name, "floors": dict(floors)}
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
# The twelve gates. `automatable` = whether a tool can ever check it. `levels` = the requirement per
|
|
58
|
+
# profile. G8 (evals) applies only when the deliverable has an AI/model component (checker handles
|
|
59
|
+
# that); G11 (waivers) is validated structurally by the checker when a waiver is present, not required
|
|
60
|
+
# as standalone evidence.
|
|
61
|
+
GATES = (
|
|
62
|
+
Gate("G0", "Approved requirements & traceability", False, _lv("na", "must", "must")),
|
|
63
|
+
Gate("G1", "Coverage reported (a disclosure, never a target)", True, _lv("must", "must", "must"),
|
|
64
|
+
metric=_metric("coverage")),
|
|
65
|
+
Gate("G2", "Fault-detection shown (no unexplained surviving mutants)", True, _lv("should", "must", "must"),
|
|
66
|
+
metric=_metric("mutation_score")),
|
|
67
|
+
Gate("G3", "Independent oracle (independence established)", False, _lv("should", "must", "must")),
|
|
68
|
+
Gate("G4", "Real supply-chain control (SCA, SBOM, pinning)", True, _lv("should", "must", "must")),
|
|
69
|
+
Gate("G4b", "Security gates by tier (static + secret scan; more at T3+)", True, _lv("must", "must", "must")),
|
|
70
|
+
Gate("G5", "Cross-model review (advisory, human-triaged)", True, _lv("should", "should", "should")),
|
|
71
|
+
Gate("G6", "Human accountability / code review", False, _lv("should", "must", "must")),
|
|
72
|
+
Gate("G7", "Tests are protected (no silent weakening)", True, _lv("should", "must", "must")),
|
|
73
|
+
Gate("G8", "Trustworthy evals (AI/model components)", False, _lv("should", "must", "must")),
|
|
74
|
+
Gate("G9", "AI-use provenance recorded", False, _lv("should", "must", "must")),
|
|
75
|
+
Gate("G10", "Evidence package assembled", False, _lv("should", "must", "must")),
|
|
76
|
+
Gate("G11", "Waivers explicit and time-boxed", False, _lv("should", "should", "must")),
|
|
77
|
+
)
|
|
78
|
+
|
|
79
|
+
GATE_BY_ID = {g.id: g for g in GATES}
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def gates_for_tier(tier: int) -> list[Gate]:
|
|
83
|
+
"""The gates that apply at a given tier (1-4): those whose level for the tier's profile != 'na'."""
|
|
84
|
+
if tier not in PROFILE_KEY_BY_TIER:
|
|
85
|
+
raise ValueError(f"tier must be 1-4, got {tier}")
|
|
86
|
+
profile = PROFILE_KEY_BY_TIER[tier]
|
|
87
|
+
return [g for g in GATES if g.levels[profile] != "na"]
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def render_checklist(tier: int) -> str:
|
|
91
|
+
"""A Markdown evidence-package skeleton for the tier: what applies, and how each is satisfied."""
|
|
92
|
+
applicable = gates_for_tier(tier) # validates the tier first (clean ValueError on bad input)
|
|
93
|
+
profile = PROFILE_BY_TIER[tier]
|
|
94
|
+
auto = [g for g in applicable if g.automatable]
|
|
95
|
+
human = [g for g in applicable if not g.automatable]
|
|
96
|
+
|
|
97
|
+
lines = [
|
|
98
|
+
f"# Oracle Gate — evidence package (Tier {tier} · {profile} profile)",
|
|
99
|
+
"",
|
|
100
|
+
f"{len(applicable)} gates apply at this tier: {len(auto)} machine-checkable, "
|
|
101
|
+
f"{len(human)} require human evidence (never auto-passed).",
|
|
102
|
+
"",
|
|
103
|
+
"## Machine-checkable gates",
|
|
104
|
+
]
|
|
105
|
+
for g in auto:
|
|
106
|
+
lines.append(f"- [ ] **{g.id}** — {g.title} _(tool can run; attach the report)_")
|
|
107
|
+
lines += ["", "## Human-evidence gates (sign-off required — a tool cannot verify these)"]
|
|
108
|
+
for g in human:
|
|
109
|
+
lines.append(f"- [ ] **{g.id}** — {g.title} _(attach the artifact + approver)_")
|
|
110
|
+
lines += [
|
|
111
|
+
"",
|
|
112
|
+
"## Conformance statement",
|
|
113
|
+
f"> Meets The Oracle Gate, **{profile}** profile, when every box above is checked with "
|
|
114
|
+
"attached evidence. Anything short ships under an explicit, time-boxed waiver (G11).",
|
|
115
|
+
]
|
|
116
|
+
return "\n".join(lines)
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def schema() -> dict:
|
|
120
|
+
"""The machine-readable gate/profile schema — so other tools can build against it."""
|
|
121
|
+
return {
|
|
122
|
+
"framework": "The Oracle Gate",
|
|
123
|
+
"profiles": {k: PROFILE_NAME[k] for k in PROFILE_ORDER},
|
|
124
|
+
"tier_to_profile": {str(t): PROFILE_KEY_BY_TIER[t] for t in PROFILE_KEY_BY_TIER},
|
|
125
|
+
"levels": ["must", "should", "na"],
|
|
126
|
+
"gates": [
|
|
127
|
+
{"id": g.id, "title": g.title, "automatable": g.automatable, "levels": g.levels,
|
|
128
|
+
"metric": g.metric}
|
|
129
|
+
for g in GATES
|
|
130
|
+
],
|
|
131
|
+
}
|