envcert 0.1.0__tar.gz → 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {envcert-0.1.0 → envcert-0.2.0}/PKG-INFO +18 -9
- {envcert-0.1.0 → envcert-0.2.0}/README.md +17 -8
- {envcert-0.1.0 → envcert-0.2.0}/envcert/__init__.py +1 -1
- {envcert-0.1.0 → envcert-0.2.0}/envcert/adapters/base.py +3 -0
- {envcert-0.1.0 → envcert-0.2.0}/envcert/adapters/harbor.py +11 -0
- envcert-0.2.0/envcert/adapters/inspect_adapter.py +69 -0
- envcert-0.2.0/envcert/adapters/nemo_gym.py +77 -0
- {envcert-0.1.0 → envcert-0.2.0}/envcert/attestations/intoto.py +59 -0
- {envcert-0.1.0 → envcert-0.2.0}/envcert/checks/base.py +5 -2
- {envcert-0.1.0 → envcert-0.2.0}/envcert/checks/dynamic_checks.py +85 -8
- envcert-0.2.0/envcert/checks/external_checks.py +135 -0
- {envcert-0.1.0 → envcert-0.2.0}/envcert/checks/mutation.py +73 -8
- {envcert-0.1.0 → envcert-0.2.0}/envcert/checks/static_checks.py +103 -9
- {envcert-0.1.0 → envcert-0.2.0}/envcert/cli/main.py +68 -3
- {envcert-0.1.0 → envcert-0.2.0}/envcert/core/config.py +18 -2
- {envcert-0.1.0 → envcert-0.2.0}/envcert/core/engine.py +55 -2
- {envcert-0.1.0 → envcert-0.2.0}/envcert/runners/base.py +1 -0
- {envcert-0.1.0 → envcert-0.2.0}/envcert/runners/docker_runner.py +19 -4
- envcert-0.2.0/envcert/runners/external_runner.py +75 -0
- {envcert-0.1.0 → envcert-0.2.0}/envcert/schemas/report.schema.json +1 -1
- {envcert-0.1.0 → envcert-0.2.0}/pyproject.toml +1 -1
- {envcert-0.1.0 → envcert-0.2.0}/.gitignore +0 -0
- {envcert-0.1.0 → envcert-0.2.0}/LICENSE +0 -0
- {envcert-0.1.0 → envcert-0.2.0}/envcert/__main__.py +0 -0
- {envcert-0.1.0 → envcert-0.2.0}/envcert/adapters/__init__.py +0 -0
- {envcert-0.1.0 → envcert-0.2.0}/envcert/adapters/generic.py +0 -0
- {envcert-0.1.0 → envcert-0.2.0}/envcert/attestations/__init__.py +0 -0
- {envcert-0.1.0 → envcert-0.2.0}/envcert/checks/__init__.py +0 -0
- {envcert-0.1.0 → envcert-0.2.0}/envcert/cli/__init__.py +0 -0
- {envcert-0.1.0 → envcert-0.2.0}/envcert/core/__init__.py +0 -0
- {envcert-0.1.0 → envcert-0.2.0}/envcert/core/digests.py +0 -0
- {envcert-0.1.0 → envcert-0.2.0}/envcert/core/errors.py +0 -0
- {envcert-0.1.0 → envcert-0.2.0}/envcert/core/models.py +0 -0
- {envcert-0.1.0 → envcert-0.2.0}/envcert/reports/__init__.py +0 -0
- {envcert-0.1.0 → envcert-0.2.0}/envcert/reports/sarif.py +0 -0
- {envcert-0.1.0 → envcert-0.2.0}/envcert/reports/text.py +0 -0
- {envcert-0.1.0 → envcert-0.2.0}/envcert/runners/__init__.py +0 -0
- {envcert-0.1.0 → envcert-0.2.0}/envcert/runners/local_runner.py +0 -0
- {envcert-0.1.0 → envcert-0.2.0}/envcert/schemas/__init__.py +0 -0
- {envcert-0.1.0 → envcert-0.2.0}/envcert/utils/__init__.py +0 -0
- {envcert-0.1.0 → envcert-0.2.0}/envcert/utils/dockerfile.py +0 -0
- {envcert-0.1.0 → envcert-0.2.0}/envcert/utils/files.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: envcert
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.2.0
|
|
4
4
|
Summary: Independent, extensible integrity verification and evidence generation for AI training and evaluation environments
|
|
5
5
|
Project-URL: Homepage, https://github.com/EricMaxwellnetizen/envcert
|
|
6
6
|
Project-URL: Repository, https://github.com/EricMaxwellnetizen/envcert
|
|
@@ -62,10 +62,10 @@ else can reproduce that*.
|
|
|
62
62
|
Requires Python 3.11+.
|
|
63
63
|
|
|
64
64
|
```bash
|
|
65
|
-
pip install envcert #
|
|
66
|
-
|
|
65
|
+
pip install envcert # from PyPI
|
|
66
|
+
pipx install envcert # or isolated
|
|
67
|
+
# From a checkout of this repository:
|
|
67
68
|
pip install .
|
|
68
|
-
pipx install . # or isolated
|
|
69
69
|
```
|
|
70
70
|
|
|
71
71
|
The core has **zero runtime dependencies**: static analysis works offline,
|
|
@@ -111,7 +111,7 @@ The intentionally vulnerable example produces (among others):
|
|
|
111
111
|
|
|
112
112
|
## What it checks
|
|
113
113
|
|
|
114
|
-
|
|
114
|
+
33 checks with stable public IDs (full catalog:
|
|
115
115
|
`docs/architecture/CHECK_CATALOG.md`, or `envcert list-checks -v`):
|
|
116
116
|
|
|
117
117
|
| Area | Checks | Examples |
|
|
@@ -122,8 +122,10 @@ The intentionally vulnerable example produces (among others):
|
|
|
122
122
|
| Docker | EC030–EC033 | unpinned base images, root user, remote ADD / curl\|sh, undeclared network policy |
|
|
123
123
|
| Dependencies | EC040–EC041 | unpinned pip deps, unverified downloads |
|
|
124
124
|
| Verifier quality | EC050–EC054 | missing/empty verifier, no reward production, unconditional success, assertion-free tests, self-reported rewards |
|
|
125
|
-
|
|
|
126
|
-
|
|
|
125
|
+
| Multi-step | EC004 | per-step instruction/verifier completeness (plus per-step EC002) |
|
|
126
|
+
| Dynamic (opt-in) | EC060–EC062 | gold run, no-op negative control (docker/local/external), repeatability |
|
|
127
|
+
| Mutation (opt-in) | EC063–EC064 | AST mutants of a Python solution; literal mutants of shell solutions under Docker |
|
|
128
|
+
| External companions (opt-in) | EC070–EC072 | Hadolint, Trivy, Semgrep — run only when installed and `--external-tools` is passed |
|
|
127
129
|
|
|
128
130
|
Every finding carries evidence, a severity, a confidence, a remediation and
|
|
129
131
|
stated limitations. Heuristics report WARN — EnvCert does not present a
|
|
@@ -174,9 +176,16 @@ input/config · `3` internal/execution error.
|
|
|
174
176
|
|
|
175
177
|
| Format | Status |
|
|
176
178
|
|---|---|
|
|
177
|
-
| Harbor task directories (task.toml schema 1.x) | Implemented and fixture-tested |
|
|
179
|
+
| Harbor task directories (task.toml schema 1.x, incl. multi-step `steps/`) | Implemented and fixture-tested |
|
|
178
180
|
| Generic Docker task bundles (Dockerfile + tests/solution) | Implemented, reduced-confidence heuristics |
|
|
179
|
-
| NeMo Gym
|
|
181
|
+
| NeMo Gym datasets (dataset.toml / verifier.py / JSONL rows) | Implemented (static; Resources-Server HTTP probing is roadmap) — `examples/nemo-gym-dataset` |
|
|
182
|
+
| Inspect AI evaluations (@task Python files) | Implemented (static) — `examples/inspect-eval` |
|
|
183
|
+
|
|
184
|
+
Dynamic execution modes: `docker` (restricted), `local-python` (trusted
|
|
185
|
+
pytest tasks, no sandbox), `external` (your disposable-VM/remote runner via
|
|
186
|
+
a documented JSON protocol). OCI image digests can be bound with
|
|
187
|
+
`--image`/`--image-digest`; Cosign OCI attestation is available via
|
|
188
|
+
`envcert attest --image name@sha256:...` and `envcert verify-attestation`.
|
|
180
189
|
|
|
181
190
|
## Configuration
|
|
182
191
|
|
|
@@ -30,10 +30,10 @@ else can reproduce that*.
|
|
|
30
30
|
Requires Python 3.11+.
|
|
31
31
|
|
|
32
32
|
```bash
|
|
33
|
-
pip install envcert #
|
|
34
|
-
|
|
33
|
+
pip install envcert # from PyPI
|
|
34
|
+
pipx install envcert # or isolated
|
|
35
|
+
# From a checkout of this repository:
|
|
35
36
|
pip install .
|
|
36
|
-
pipx install . # or isolated
|
|
37
37
|
```
|
|
38
38
|
|
|
39
39
|
The core has **zero runtime dependencies**: static analysis works offline,
|
|
@@ -79,7 +79,7 @@ The intentionally vulnerable example produces (among others):
|
|
|
79
79
|
|
|
80
80
|
## What it checks
|
|
81
81
|
|
|
82
|
-
|
|
82
|
+
33 checks with stable public IDs (full catalog:
|
|
83
83
|
`docs/architecture/CHECK_CATALOG.md`, or `envcert list-checks -v`):
|
|
84
84
|
|
|
85
85
|
| Area | Checks | Examples |
|
|
@@ -90,8 +90,10 @@ The intentionally vulnerable example produces (among others):
|
|
|
90
90
|
| Docker | EC030–EC033 | unpinned base images, root user, remote ADD / curl\|sh, undeclared network policy |
|
|
91
91
|
| Dependencies | EC040–EC041 | unpinned pip deps, unverified downloads |
|
|
92
92
|
| Verifier quality | EC050–EC054 | missing/empty verifier, no reward production, unconditional success, assertion-free tests, self-reported rewards |
|
|
93
|
-
|
|
|
94
|
-
|
|
|
93
|
+
| Multi-step | EC004 | per-step instruction/verifier completeness (plus per-step EC002) |
|
|
94
|
+
| Dynamic (opt-in) | EC060–EC062 | gold run, no-op negative control (docker/local/external), repeatability |
|
|
95
|
+
| Mutation (opt-in) | EC063–EC064 | AST mutants of a Python solution; literal mutants of shell solutions under Docker |
|
|
96
|
+
| External companions (opt-in) | EC070–EC072 | Hadolint, Trivy, Semgrep — run only when installed and `--external-tools` is passed |
|
|
95
97
|
|
|
96
98
|
Every finding carries evidence, a severity, a confidence, a remediation and
|
|
97
99
|
stated limitations. Heuristics report WARN — EnvCert does not present a
|
|
@@ -142,9 +144,16 @@ input/config · `3` internal/execution error.
|
|
|
142
144
|
|
|
143
145
|
| Format | Status |
|
|
144
146
|
|---|---|
|
|
145
|
-
| Harbor task directories (task.toml schema 1.x) | Implemented and fixture-tested |
|
|
147
|
+
| Harbor task directories (task.toml schema 1.x, incl. multi-step `steps/`) | Implemented and fixture-tested |
|
|
146
148
|
| Generic Docker task bundles (Dockerfile + tests/solution) | Implemented, reduced-confidence heuristics |
|
|
147
|
-
| NeMo Gym
|
|
149
|
+
| NeMo Gym datasets (dataset.toml / verifier.py / JSONL rows) | Implemented (static; Resources-Server HTTP probing is roadmap) — `examples/nemo-gym-dataset` |
|
|
150
|
+
| Inspect AI evaluations (@task Python files) | Implemented (static) — `examples/inspect-eval` |
|
|
151
|
+
|
|
152
|
+
Dynamic execution modes: `docker` (restricted), `local-python` (trusted
|
|
153
|
+
pytest tasks, no sandbox), `external` (your disposable-VM/remote runner via
|
|
154
|
+
a documented JSON protocol). OCI image digests can be bound with
|
|
155
|
+
`--image`/`--image-digest`; Cosign OCI attestation is available via
|
|
156
|
+
`envcert attest --image name@sha256:...` and `envcert verify-attestation`.
|
|
148
157
|
|
|
149
158
|
## Configuration
|
|
150
159
|
|
|
@@ -26,6 +26,9 @@ class TaskContext:
|
|
|
26
26
|
solution_dir: Path | None = None
|
|
27
27
|
environment_dir: Path | None = None
|
|
28
28
|
warnings: list[str] = field(default_factory=list)
|
|
29
|
+
verifier_paths: list[str] = field(default_factory=list) # framework-specific verifier/scorer files (non-Harbor)
|
|
30
|
+
steps: list[dict] = field(default_factory=list) # multi-step: [{name, instruction, tests, solution}]
|
|
31
|
+
dataset_files: list[str] = field(default_factory=list)
|
|
29
32
|
_text_cache: dict[str, str | None] = field(default_factory=dict, repr=False)
|
|
30
33
|
|
|
31
34
|
def text(self, rel: str) -> str | None:
|
|
@@ -67,6 +67,17 @@ def load(root: Path) -> TaskContext:
|
|
|
67
67
|
ctx.tests_dir = root / "tests"
|
|
68
68
|
if (root / "solution").is_dir():
|
|
69
69
|
ctx.solution_dir = root / "solution"
|
|
70
|
+
steps_dir = root / "steps"
|
|
71
|
+
if steps_dir.is_dir():
|
|
72
|
+
for step in sorted(p for p in steps_dir.iterdir() if p.is_dir()):
|
|
73
|
+
rel = f"steps/{step.name}"
|
|
74
|
+
ctx.steps.append({
|
|
75
|
+
"name": step.name,
|
|
76
|
+
"instruction": f"{rel}/instruction.md" if (step / "instruction.md").is_file() else None,
|
|
77
|
+
"tests": f"{rel}/tests" if (step / "tests").is_dir() else None,
|
|
78
|
+
"test_entrypoints": [f"{rel}/tests/{n}" for n in ("test.sh", "test.bat", "verifier.py") if (step / "tests" / n).is_file()],
|
|
79
|
+
"solution": f"{rel}/solution" if (step / "solution").is_dir() else None,
|
|
80
|
+
})
|
|
70
81
|
env_dir = root / "environment"
|
|
71
82
|
if env_dir.is_dir():
|
|
72
83
|
ctx.environment_dir = env_dir
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
"""Inspect AI adapter.
|
|
2
|
+
|
|
3
|
+
Verified against inspect.aisi.org.uk (Tasks / Datasets / Scorers, 2026-11):
|
|
4
|
+
an Inspect evaluation is a Python file defining ``@task`` functions that
|
|
5
|
+
return ``Task(dataset=..., solver=..., scorer=...)``. Samples carry
|
|
6
|
+
``input`` (shown to the model) and ``target`` (grading ground truth, not
|
|
7
|
+
shown as input). Scorers include exact/includes/match/pattern and
|
|
8
|
+
model-graded scorers — the latter are non-deterministic and model-dependent,
|
|
9
|
+
which EnvCert flags as a reproducibility limitation rather than a defect.
|
|
10
|
+
|
|
11
|
+
This adapter statically locates ``@task`` files, records scorer usage, and
|
|
12
|
+
maps the task file as the verifier path so the generic verifier-structure
|
|
13
|
+
checks apply. It never imports or executes the eval file.
|
|
14
|
+
"""
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
import re
|
|
18
|
+
from pathlib import Path
|
|
19
|
+
|
|
20
|
+
from envcert.adapters.base import TaskContext, scan_files
|
|
21
|
+
from envcert.core.errors import InvalidInputError
|
|
22
|
+
|
|
23
|
+
TASK_RE = re.compile(r"from\s+inspect_ai\s+import|import\s+inspect_ai|@task\b")
|
|
24
|
+
SCORER_RE = re.compile(r"scorer\s*=\s*([A-Za-z_][\w.]*)")
|
|
25
|
+
MODEL_GRADED = ("model_graded_qa", "model_graded_fact", "model_graded")
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def find_task_files(root: Path, files: list[str]) -> list[str]:
|
|
29
|
+
out = []
|
|
30
|
+
for rel in files:
|
|
31
|
+
if not rel.endswith(".py"):
|
|
32
|
+
continue
|
|
33
|
+
try:
|
|
34
|
+
text = (root / rel).read_text(encoding="utf-8", errors="replace")[:20000]
|
|
35
|
+
except OSError:
|
|
36
|
+
continue
|
|
37
|
+
if "inspect_ai" in text and ("@task" in text or "Task(" in text):
|
|
38
|
+
out.append(rel)
|
|
39
|
+
return out
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def is_inspect(root: Path) -> bool:
|
|
43
|
+
if not root.is_dir():
|
|
44
|
+
return False
|
|
45
|
+
files = [p.relative_to(root).as_posix() for p in root.rglob("*.py") if ".venv" not in p.parts]
|
|
46
|
+
return bool(find_task_files(root, files[:200]))
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def load(root: Path) -> TaskContext:
|
|
50
|
+
root = root.resolve()
|
|
51
|
+
if not root.is_dir():
|
|
52
|
+
raise InvalidInputError(f"task path is not a directory: {root}")
|
|
53
|
+
ctx = TaskContext(root=root, adapter="inspect", task_name=root.name)
|
|
54
|
+
scan_files(ctx)
|
|
55
|
+
task_files = find_task_files(root, ctx.files)
|
|
56
|
+
ctx.verifier_paths = task_files
|
|
57
|
+
ctx.dataset_files = [f for f in ctx.files if f.endswith((".jsonl", ".json", ".csv"))]
|
|
58
|
+
if task_files:
|
|
59
|
+
text = ctx.text(task_files[0]) or ""
|
|
60
|
+
scorers = SCORER_RE.findall(text)
|
|
61
|
+
ctx.config = {"scorers": scorers, "task_files": task_files}
|
|
62
|
+
if any(any(s.startswith(m) for m in MODEL_GRADED) for s in scorers):
|
|
63
|
+
ctx.warnings.append("model-graded scorer detected: grading is model-dependent and non-deterministic; results need scorer/model version pinning for reproducibility")
|
|
64
|
+
if not scorers:
|
|
65
|
+
ctx.warnings.append("no scorer= argument detected in the @task file; the Task may rely on a default solver/scorer or compose them elsewhere")
|
|
66
|
+
else:
|
|
67
|
+
ctx.warnings.append("no @task file with inspect_ai imports found; Inspect detection was heuristic")
|
|
68
|
+
ctx.warnings.append("Inspect adapter is static-only: samples' targets are framework-hidden at runtime, so target-leakage verdicts are heuristic")
|
|
69
|
+
return ctx
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
"""NVIDIA NeMo Gym adapter.
|
|
2
|
+
|
|
3
|
+
Verified against docs.nvidia.com/nemo/gym (2026-11): Gym reads tasks in two
|
|
4
|
+
forms — Harbor task folders (handled by the Harbor adapter) and *JSONL rows
|
|
5
|
+
plus a verifier*: a dataset directory / resources layout where a shared
|
|
6
|
+
Python ``verify()`` scores rows, plus ``dataset.toml`` with a ``[gym]`` table
|
|
7
|
+
at a dataset root whose root-level ``verifier.py`` grades every single-step
|
|
8
|
+
task without its own ``tests/``. This adapter covers the non-Harbor forms:
|
|
9
|
+
|
|
10
|
+
- a dataset root with ``dataset.toml`` and/or ``verifier.py`` and task
|
|
11
|
+
subfolders and/or ``*.jsonl`` row files;
|
|
12
|
+
- a JSONL dataset file directory whose rows carry
|
|
13
|
+
``responses_create_params`` (the prompt the agent sees) plus verifier
|
|
14
|
+
fields such as ``expected_answer``.
|
|
15
|
+
|
|
16
|
+
Row files are data: ``expected_answer``-style fields are verifier-owned
|
|
17
|
+
ground truth. EnvCert checks that they are not *also* embedded in the
|
|
18
|
+
agent-visible prompt (see EC014's framework note) and that a ``verify``
|
|
19
|
+
implementation exists and is substantive. Resources Servers that run as
|
|
20
|
+
HTTP services are only partially visible statically — the adapter records
|
|
21
|
+
that warning.
|
|
22
|
+
"""
|
|
23
|
+
from __future__ import annotations
|
|
24
|
+
|
|
25
|
+
import json
|
|
26
|
+
from pathlib import Path
|
|
27
|
+
|
|
28
|
+
from envcert.adapters.base import TaskContext, scan_files
|
|
29
|
+
from envcert.adapters.harbor import parse_toml
|
|
30
|
+
from envcert.core.errors import InvalidInputError
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def is_nemo_gym(root: Path) -> bool:
|
|
34
|
+
if (root / "dataset.toml").is_file() or (root / "verifier.py").is_file():
|
|
35
|
+
return True
|
|
36
|
+
return any(p.suffix == ".jsonl" for p in root.glob("*.jsonl"))
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def load(root: Path) -> TaskContext:
|
|
40
|
+
root = root.resolve()
|
|
41
|
+
if not root.is_dir():
|
|
42
|
+
raise InvalidInputError(f"task path is not a directory: {root}")
|
|
43
|
+
ctx = TaskContext(root=root, adapter="nemo-gym", task_name=root.name)
|
|
44
|
+
scan_files(ctx)
|
|
45
|
+
if (root / "dataset.toml").is_file():
|
|
46
|
+
ctx.config_path = root / "dataset.toml"
|
|
47
|
+
ctx.config = parse_toml(root / "dataset.toml")
|
|
48
|
+
dataset = ctx.config.get("dataset") or {}
|
|
49
|
+
if isinstance(dataset, dict) and dataset.get("name"):
|
|
50
|
+
ctx.task_name = str(dataset["name"])
|
|
51
|
+
if (root / "verifier.py").is_file():
|
|
52
|
+
ctx.verifier_paths = ["verifier.py"]
|
|
53
|
+
ctx.tests_dir = root # verifier lives at dataset root in this layout
|
|
54
|
+
ctx.dataset_files = [f for f in ctx.files if f.endswith(".jsonl")]
|
|
55
|
+
# task subfolders graded by the shared verifier
|
|
56
|
+
sub_tasks = [f for f in ctx.files if f.endswith("task.toml")]
|
|
57
|
+
if sub_tasks:
|
|
58
|
+
ctx.warnings.append(f"dataset contains {len(sub_tasks)} Harbor task subfolder(s); per-folder auditing is available by running envcert on each subfolder (dataset-level checks ran here)")
|
|
59
|
+
if not ctx.verifier_paths:
|
|
60
|
+
ctx.warnings.append("no root verifier.py found; verification may live in a Resources Server (HTTP service), which static analysis can only partially inspect")
|
|
61
|
+
# validate JSONL rows cheaply and record row counts as evidence
|
|
62
|
+
for rel in ctx.dataset_files:
|
|
63
|
+
path = root / rel
|
|
64
|
+
rows, bad = 0, 0
|
|
65
|
+
try:
|
|
66
|
+
for line in path.read_text(encoding="utf-8", errors="replace").splitlines():
|
|
67
|
+
if not line.strip():
|
|
68
|
+
continue
|
|
69
|
+
rows += 1
|
|
70
|
+
try:
|
|
71
|
+
json.loads(line)
|
|
72
|
+
except ValueError:
|
|
73
|
+
bad += 1
|
|
74
|
+
except OSError:
|
|
75
|
+
bad += 1
|
|
76
|
+
ctx.warnings.append(f"dataset file {rel}: {rows} rows, {bad} invalid JSON lines" if bad else f"dataset file {rel}: {rows} rows")
|
|
77
|
+
return ctx
|
|
@@ -36,6 +36,9 @@ def build_statement(report: dict[str, Any], report_bytes: bytes) -> dict[str, An
|
|
|
36
36
|
subjects.append({"name": "verifier", "digest": {"sha256": env["verifier_digest_sha256"]}})
|
|
37
37
|
if env.get("solution_digest_sha256"):
|
|
38
38
|
subjects.append({"name": "reference-solution", "digest": {"sha256": env["solution_digest_sha256"]}})
|
|
39
|
+
if env.get("image_digest"):
|
|
40
|
+
digest = str(env["image_digest"]).removeprefix("sha256:")
|
|
41
|
+
subjects.append({"name": env.get("image_reference") or "oci-image", "digest": {"sha256": digest}})
|
|
39
42
|
failing = [f for f in report.get("findings", []) if f.get("status") in ("FAIL", "WARN")]
|
|
40
43
|
predicate: dict[str, Any] = {
|
|
41
44
|
"result": "PASSED" if policy.get("passed") else "FAILED",
|
|
@@ -74,6 +77,62 @@ def cosign_available() -> str | None:
|
|
|
74
77
|
return shutil.which("cosign")
|
|
75
78
|
|
|
76
79
|
|
|
80
|
+
def cosign_attest_image(predicate_path: Path, image: str, key_path: str | None = None) -> tuple[bool, str]:
|
|
81
|
+
"""Attach the statement as an OCI attestation on `image` via cosign attest.
|
|
82
|
+
|
|
83
|
+
The predicate file must contain the in-toto *predicate* (this module's
|
|
84
|
+
caller extracts it). `image` should be referenced by digest
|
|
85
|
+
(name@sha256:...) so the attestation binds to immutable content.
|
|
86
|
+
"""
|
|
87
|
+
if "@sha256:" not in image:
|
|
88
|
+
return False, f"refusing to attest a mutable image reference {image!r}: use name@sha256:<digest> so the attestation binds to immutable content"
|
|
89
|
+
exe = cosign_available()
|
|
90
|
+
if not exe:
|
|
91
|
+
return False, "cosign not found on PATH; OCI attestation not performed (statement file was still written)"
|
|
92
|
+
cmd = [exe, "attest", "--yes", "--predicate", str(predicate_path),
|
|
93
|
+
"--type", TEST_RESULT_PREDICATE, image]
|
|
94
|
+
if key_path:
|
|
95
|
+
cmd[2:2] = ["--key", key_path]
|
|
96
|
+
proc = subprocess.run(cmd, capture_output=True, text=True, timeout=300)
|
|
97
|
+
if proc.returncode != 0:
|
|
98
|
+
return False, f"cosign attest failed: {(proc.stderr or proc.stdout)[:500]}"
|
|
99
|
+
return True, f"cosign attestation attached to {image} (predicate type {TEST_RESULT_PREDICATE})"
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def cosign_verify_attestation(image: str, key_path: str | None = None, policy: str | None = None) -> tuple[bool, str]:
|
|
103
|
+
exe = cosign_available()
|
|
104
|
+
if not exe:
|
|
105
|
+
return False, "cosign not found on PATH; cryptographic verification not performed"
|
|
106
|
+
cmd = [exe, "verify-attestation", "--type", TEST_RESULT_PREDICATE]
|
|
107
|
+
if key_path:
|
|
108
|
+
cmd += ["--key", key_path]
|
|
109
|
+
else:
|
|
110
|
+
cmd += ["--certificate-identity-regexp", ".*", "--certificate-oidc-issuer-regexp", ".*"]
|
|
111
|
+
if policy:
|
|
112
|
+
cmd += ["--policy", policy]
|
|
113
|
+
cmd.append(image)
|
|
114
|
+
proc = subprocess.run(cmd, capture_output=True, text=True, timeout=300)
|
|
115
|
+
if proc.returncode != 0:
|
|
116
|
+
return False, f"cosign verify-attestation failed: {(proc.stderr or proc.stdout)[:500]}"
|
|
117
|
+
return True, f"cosign verify-attestation passed for {image}"
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def verify_statement_local(statement: dict[str, Any]) -> list[str]:
|
|
121
|
+
"""Structural verification only (no cryptography): type, subjects, result."""
|
|
122
|
+
problems = []
|
|
123
|
+
if statement.get("_type") != STATEMENT_TYPE:
|
|
124
|
+
problems.append(f"unexpected _type: {statement.get('_type')!r}")
|
|
125
|
+
if statement.get("predicateType") != TEST_RESULT_PREDICATE:
|
|
126
|
+
problems.append(f"unexpected predicateType: {statement.get('predicateType')!r}")
|
|
127
|
+
subjects = statement.get("subject") or []
|
|
128
|
+
if not subjects or not all((s.get("digest") or {}).get("sha256") for s in subjects):
|
|
129
|
+
problems.append("statement has no sha256-bound subjects")
|
|
130
|
+
predicate = statement.get("predicate") or {}
|
|
131
|
+
if predicate.get("result") not in ("PASSED", "FAILED"):
|
|
132
|
+
problems.append("predicate.result is not PASSED/FAILED")
|
|
133
|
+
return problems
|
|
134
|
+
|
|
135
|
+
|
|
77
136
|
def sign_with_cosign(envelope_path: Path, key_path: str | None = None) -> tuple[bool, str]:
|
|
78
137
|
exe = cosign_available()
|
|
79
138
|
if not exe:
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
3
|
from collections.abc import Callable
|
|
4
|
-
from dataclasses import dataclass
|
|
4
|
+
from dataclasses import dataclass, field
|
|
5
5
|
|
|
6
6
|
from envcert.adapters.base import TaskContext
|
|
7
7
|
from envcert.core.models import Confidence, Finding, Severity, Status
|
|
@@ -22,10 +22,13 @@ class CheckInfo:
|
|
|
22
22
|
@dataclass
|
|
23
23
|
class CheckContext:
|
|
24
24
|
task: TaskContext
|
|
25
|
-
execution: str = "none" # none | docker | local-python
|
|
25
|
+
execution: str = "none" # none | docker | local-python | external
|
|
26
26
|
repetitions: int = 3
|
|
27
27
|
enable_mutation: bool = False
|
|
28
28
|
max_mutants: int = 10
|
|
29
|
+
external_tools: bool = False
|
|
30
|
+
runner_command: str = ""
|
|
31
|
+
shared: dict = field(default_factory=dict) # checks stash run artefacts (e.g. image digest) for the report
|
|
29
32
|
|
|
30
33
|
|
|
31
34
|
CheckFn = Callable[[CheckContext], list[Finding]]
|
|
@@ -24,6 +24,10 @@ EC062 = CheckInfo("EC062", "Verifier outcome is repeatable across repetitions",
|
|
|
24
24
|
"Re-running gold verification should give a consistent reward/exit outcome.",
|
|
25
25
|
"Remove nondeterminism (time, randomness, network, ordering) from the task or verifier.",
|
|
26
26
|
"Compares reward/exit outcome, not bitwise-identical stdout.", dynamic=True)
|
|
27
|
+
EC064 = CheckInfo("EC064", "Shell-solution mutation: verifier rejects mutated reference solutions (docker)", "mutation", Severity.MEDIUM,
|
|
28
|
+
"Mutating literal outputs in solution/solve.sh should make the verifier reject the run; surviving mutants signal a weak verifier.",
|
|
29
|
+
"Strengthen verifier assertions on the mutated outputs.",
|
|
30
|
+
"Docker execution only; only literal-bearing shell solutions can be mutated. Mutant validity is heuristic.", dynamic=True)
|
|
27
31
|
EC063 = CheckInfo("EC063", "Mutation testing: verifier rejects mutated (incorrect) solutions", "mutation", Severity.MEDIUM,
|
|
28
32
|
"Mutated variants of a correct Python solution should be rejected; the rejection rate is a verifier-strength signal.",
|
|
29
33
|
"Add assertions covering the mutated behaviour; do not treat the rate as proof of correctness.",
|
|
@@ -39,9 +43,17 @@ def _is_python_task(ctx) -> bool:
|
|
|
39
43
|
def _run_variant(c: CheckContext, variant: str):
|
|
40
44
|
ctx = c.task
|
|
41
45
|
if c.execution == "docker":
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
46
|
+
result = DockerRunner().run_harbor_task(ctx.root, variant=variant)
|
|
47
|
+
if result.image_digest:
|
|
48
|
+
c.shared["image_digest"] = result.image_digest
|
|
49
|
+
return result, None
|
|
50
|
+
if c.execution == "external":
|
|
51
|
+
from envcert.runners.external_runner import ExternalRunner
|
|
52
|
+
|
|
53
|
+
result = ExternalRunner(c.runner_command).run_harbor_task(ctx.root, variant=variant)
|
|
54
|
+
if result.image_digest:
|
|
55
|
+
c.shared["image_digest"] = result.image_digest
|
|
56
|
+
return result, None
|
|
45
57
|
if c.execution == "local-python":
|
|
46
58
|
if not _is_python_task(ctx):
|
|
47
59
|
return None, "local-python execution requires solution/*.py plus tests/test_*.py (pytest-style task)"
|
|
@@ -75,16 +87,17 @@ def check_gold(c: CheckContext) -> list:
|
|
|
75
87
|
def check_negative_control(c: CheckContext) -> list:
|
|
76
88
|
if c.execution == "none":
|
|
77
89
|
return [not_run(EC061, "Dynamic checks are opt-in: re-run with --execution docker or --execution local-python.")]
|
|
78
|
-
if c.execution == "docker":
|
|
79
|
-
return [not_applicable(EC061, "Docker negative controls are not automated in this version: a no-op's validity is task-specific and Harbor artifact transfer differs per task. Run the gold check and test a no-op manually, or use local-python for pytest tasks.")]
|
|
80
90
|
result, why = _run_variant(c, "noop")
|
|
81
91
|
if result is None:
|
|
82
92
|
return [not_applicable(EC061, f"Negative control not supported here: {why}.")]
|
|
83
93
|
if result.error and result.reward is None:
|
|
84
94
|
return [finding(EC061, Status.ERROR, f"Negative-control execution failed to run: {result.error}", severity=Severity.INFO, evidence={"error": result.error})]
|
|
85
|
-
|
|
95
|
+
# Reward, when present, is authoritative (a docker harness exits 0 even
|
|
96
|
+
# when the reward is 0); only fall back to exit status without a reward.
|
|
97
|
+
noop_passed = (result.reward >= 1.0) if result.reward is not None else result.ok
|
|
98
|
+
if noop_passed:
|
|
86
99
|
return [finding(EC061, Status.WARN,
|
|
87
|
-
"No-op (emptied) solution PASSED the verifier. If a no-op should not solve this task, the verifier is too weak; if no action is a legitimate solution, record that and ignore this warning.",
|
|
100
|
+
"No-op (skipped/emptied) solution PASSED the verifier. If a no-op should not solve this task, the verifier is too weak; if no action is a legitimate solution, record that and ignore this warning.",
|
|
88
101
|
severity=Severity.HIGH, evidence={"reward": result.reward, "exit_code": result.exit_code, "variant": "noop"})]
|
|
89
102
|
return [passed(EC061, "No-op negative control was rejected by the verifier (non-passing outcome).",
|
|
90
103
|
evidence={"reward": result.reward, "exit_code": result.exit_code, "variant": "noop"})]
|
|
@@ -94,8 +107,19 @@ def check_negative_control(c: CheckContext) -> list:
|
|
|
94
107
|
def check_repeatability(c: CheckContext) -> list:
|
|
95
108
|
if c.execution == "none":
|
|
96
109
|
return [not_run(EC062, "Dynamic checks are opt-in: re-run with --execution docker or --execution local-python.")]
|
|
110
|
+
if c.execution in ("docker", "external"):
|
|
111
|
+
outcomes = []
|
|
112
|
+
for _ in range(max(2, min(c.repetitions, 3))):
|
|
113
|
+
r, _why = _run_variant(c, "gold")
|
|
114
|
+
if r is None:
|
|
115
|
+
return [not_applicable(EC062, f"Repeatability not supported here: {_why}.")]
|
|
116
|
+
outcomes.append({"reward": r.reward, "exit_code": r.exit_code, "error": r.error})
|
|
117
|
+
rewards = {o["reward"] for o in outcomes}
|
|
118
|
+
if len(rewards) == 1 and None not in rewards:
|
|
119
|
+
return [passed(EC062, f"Gold reward was consistent across {len(outcomes)} {c.execution} runs (reward={outcomes[0]['reward']}).", evidence={"runs": outcomes})]
|
|
120
|
+
return [finding(EC062, Status.WARN, f"Gold outcome varied across {len(outcomes)} {c.execution} runs: {outcomes}. Investigate nondeterminism.", evidence={"runs": outcomes})]
|
|
97
121
|
if c.execution != "local-python":
|
|
98
|
-
return [not_applicable(EC062, "Repeatability
|
|
122
|
+
return [not_applicable(EC062, "Repeatability requires an execution mode.")]
|
|
99
123
|
if not _is_python_task(c.task):
|
|
100
124
|
return [not_applicable(EC062, "Repeatability requires a pytest-style Python task under local-python execution.")]
|
|
101
125
|
runner = LocalPythonRunner()
|
|
@@ -130,3 +154,56 @@ def check_mutation(c: CheckContext) -> list:
|
|
|
130
154
|
evidence=ev)]
|
|
131
155
|
return [passed(EC063, f"Mutation rejection rate {rate:.0%} ({report['rejected']}/{report['eligible']} incorrect variants rejected). This is a strength signal, not proof of verifier correctness.",
|
|
132
156
|
evidence=ev)]
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
@check(EC064)
|
|
160
|
+
def check_shell_mutation(c: CheckContext) -> list:
|
|
161
|
+
if not c.enable_mutation:
|
|
162
|
+
return [not_run(EC064, "Shell-solution mutation is opt-in: re-run with --mutation and --execution docker.")]
|
|
163
|
+
if c.execution != "docker":
|
|
164
|
+
return [not_applicable(EC064, "Shell-solution mutation in this version runs under --execution docker only.")]
|
|
165
|
+
if c.task.adapter != "harbor":
|
|
166
|
+
return [not_applicable(EC064, "Shell-solution mutation supports Harbor tasks with solution/solve.sh.")]
|
|
167
|
+
import re
|
|
168
|
+
import shutil
|
|
169
|
+
import tempfile
|
|
170
|
+
from pathlib import Path
|
|
171
|
+
|
|
172
|
+
solve_rel = "solution/solve.sh"
|
|
173
|
+
if not c.task.has(solve_rel):
|
|
174
|
+
return [not_applicable(EC064, "No solution/solve.sh to mutate.")]
|
|
175
|
+
original = c.task.text(solve_rel) or ""
|
|
176
|
+
mutants: list[tuple[str, str]] = []
|
|
177
|
+
for m in re.finditer(r"'([^']{3,})'|\"([^\"]{3,})\"", original):
|
|
178
|
+
val = m.group(1) or m.group(2)
|
|
179
|
+
mutants.append((f"string literal {val[:30]!r} mutated", original.replace(val, val + "-MUTATED", 1)))
|
|
180
|
+
if len(mutants) >= 3:
|
|
181
|
+
break
|
|
182
|
+
if not mutants:
|
|
183
|
+
for m in re.finditer(r"\b(\d+)\b", original):
|
|
184
|
+
mutants.append((f"number {m.group(1)} incremented", original[:m.start(1)] + str(int(m.group(1)) + 1) + original[m.end(1):]))
|
|
185
|
+
if len(mutants) >= 2:
|
|
186
|
+
break
|
|
187
|
+
if not mutants:
|
|
188
|
+
return [not_applicable(EC064, "solution/solve.sh contains no string/number literals to mutate.")]
|
|
189
|
+
runner = DockerRunner()
|
|
190
|
+
if not runner.info().available:
|
|
191
|
+
return [finding(EC064, Status.ERROR, f"Docker unavailable for shell mutation: {runner.info().reason}", severity=Severity.INFO, evidence={"runner": "docker"})]
|
|
192
|
+
gold = runner.run_harbor_task(c.task.root, variant="gold")
|
|
193
|
+
if gold.reward is not None and gold.reward < 1.0:
|
|
194
|
+
return [not_applicable(EC064, f"Gold run does not pass (reward={gold.reward}); mutation would be meaningless.")]
|
|
195
|
+
results = []
|
|
196
|
+
rejected = 0
|
|
197
|
+
for desc, mutated in mutants[: c.max_mutants]:
|
|
198
|
+
with tempfile.TemporaryDirectory(prefix="envcert-shellmut-") as td:
|
|
199
|
+
dest = Path(td) / "task"
|
|
200
|
+
shutil.copytree(c.task.root, dest, ignore=shutil.ignore_patterns(".git"))
|
|
201
|
+
(dest / solve_rel).write_text(mutated, encoding="utf-8")
|
|
202
|
+
r = runner.run_harbor_task(dest, variant="gold")
|
|
203
|
+
survived = r.reward is not None and r.reward >= 1.0
|
|
204
|
+
results.append({"change": desc, "reward": r.reward, "outcome": "survived" if survived else "rejected"})
|
|
205
|
+
rejected += 0 if survived else 1
|
|
206
|
+
rate = rejected / len(results) if results else None
|
|
207
|
+
if rate is not None and rate < 0.5:
|
|
208
|
+
return [finding(EC064, Status.WARN, f"Shell-solution mutation rejection rate is {rate:.0%} ({rejected}/{len(results)}); mutated (wrong) solutions still pass.", evidence={"mutants": results, "rejection_rate": rate})]
|
|
209
|
+
return [passed(EC064, f"Shell-solution mutation rejection rate {rate:.0%} ({rejected}/{len(results)}). A strength signal, not proof of verifier correctness.", evidence={"mutants": results, "rejection_rate": rate})]
|