envcert 0.1.0__tar.gz → 0.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. {envcert-0.1.0 → envcert-0.2.0}/PKG-INFO +18 -9
  2. {envcert-0.1.0 → envcert-0.2.0}/README.md +17 -8
  3. {envcert-0.1.0 → envcert-0.2.0}/envcert/__init__.py +1 -1
  4. {envcert-0.1.0 → envcert-0.2.0}/envcert/adapters/base.py +3 -0
  5. {envcert-0.1.0 → envcert-0.2.0}/envcert/adapters/harbor.py +11 -0
  6. envcert-0.2.0/envcert/adapters/inspect_adapter.py +69 -0
  7. envcert-0.2.0/envcert/adapters/nemo_gym.py +77 -0
  8. {envcert-0.1.0 → envcert-0.2.0}/envcert/attestations/intoto.py +59 -0
  9. {envcert-0.1.0 → envcert-0.2.0}/envcert/checks/base.py +5 -2
  10. {envcert-0.1.0 → envcert-0.2.0}/envcert/checks/dynamic_checks.py +85 -8
  11. envcert-0.2.0/envcert/checks/external_checks.py +135 -0
  12. {envcert-0.1.0 → envcert-0.2.0}/envcert/checks/mutation.py +73 -8
  13. {envcert-0.1.0 → envcert-0.2.0}/envcert/checks/static_checks.py +103 -9
  14. {envcert-0.1.0 → envcert-0.2.0}/envcert/cli/main.py +68 -3
  15. {envcert-0.1.0 → envcert-0.2.0}/envcert/core/config.py +18 -2
  16. {envcert-0.1.0 → envcert-0.2.0}/envcert/core/engine.py +55 -2
  17. {envcert-0.1.0 → envcert-0.2.0}/envcert/runners/base.py +1 -0
  18. {envcert-0.1.0 → envcert-0.2.0}/envcert/runners/docker_runner.py +19 -4
  19. envcert-0.2.0/envcert/runners/external_runner.py +75 -0
  20. {envcert-0.1.0 → envcert-0.2.0}/envcert/schemas/report.schema.json +1 -1
  21. {envcert-0.1.0 → envcert-0.2.0}/pyproject.toml +1 -1
  22. {envcert-0.1.0 → envcert-0.2.0}/.gitignore +0 -0
  23. {envcert-0.1.0 → envcert-0.2.0}/LICENSE +0 -0
  24. {envcert-0.1.0 → envcert-0.2.0}/envcert/__main__.py +0 -0
  25. {envcert-0.1.0 → envcert-0.2.0}/envcert/adapters/__init__.py +0 -0
  26. {envcert-0.1.0 → envcert-0.2.0}/envcert/adapters/generic.py +0 -0
  27. {envcert-0.1.0 → envcert-0.2.0}/envcert/attestations/__init__.py +0 -0
  28. {envcert-0.1.0 → envcert-0.2.0}/envcert/checks/__init__.py +0 -0
  29. {envcert-0.1.0 → envcert-0.2.0}/envcert/cli/__init__.py +0 -0
  30. {envcert-0.1.0 → envcert-0.2.0}/envcert/core/__init__.py +0 -0
  31. {envcert-0.1.0 → envcert-0.2.0}/envcert/core/digests.py +0 -0
  32. {envcert-0.1.0 → envcert-0.2.0}/envcert/core/errors.py +0 -0
  33. {envcert-0.1.0 → envcert-0.2.0}/envcert/core/models.py +0 -0
  34. {envcert-0.1.0 → envcert-0.2.0}/envcert/reports/__init__.py +0 -0
  35. {envcert-0.1.0 → envcert-0.2.0}/envcert/reports/sarif.py +0 -0
  36. {envcert-0.1.0 → envcert-0.2.0}/envcert/reports/text.py +0 -0
  37. {envcert-0.1.0 → envcert-0.2.0}/envcert/runners/__init__.py +0 -0
  38. {envcert-0.1.0 → envcert-0.2.0}/envcert/runners/local_runner.py +0 -0
  39. {envcert-0.1.0 → envcert-0.2.0}/envcert/schemas/__init__.py +0 -0
  40. {envcert-0.1.0 → envcert-0.2.0}/envcert/utils/__init__.py +0 -0
  41. {envcert-0.1.0 → envcert-0.2.0}/envcert/utils/dockerfile.py +0 -0
  42. {envcert-0.1.0 → envcert-0.2.0}/envcert/utils/files.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: envcert
3
- Version: 0.1.0
3
+ Version: 0.2.0
4
4
  Summary: Independent, extensible integrity verification and evidence generation for AI training and evaluation environments
5
5
  Project-URL: Homepage, https://github.com/EricMaxwellnetizen/envcert
6
6
  Project-URL: Repository, https://github.com/EricMaxwellnetizen/envcert
@@ -62,10 +62,10 @@ else can reproduce that*.
62
62
  Requires Python 3.11+.
63
63
 
64
64
  ```bash
65
- pip install envcert # once published to PyPI
66
- # From this repository (current):
65
+ pip install envcert # from PyPI
66
+ pipx install envcert # or isolated
67
+ # From a checkout of this repository:
67
68
  pip install .
68
- pipx install . # or isolated
69
69
  ```
70
70
 
71
71
  The core has **zero runtime dependencies**: static analysis works offline,
@@ -111,7 +111,7 @@ The intentionally vulnerable example produces (among others):
111
111
 
112
112
  ## What it checks
113
113
 
114
- 28 checks with stable public IDs (full catalog:
114
+ 33 checks with stable public IDs (full catalog:
115
115
  `docs/architecture/CHECK_CATALOG.md`, or `envcert list-checks -v`):
116
116
 
117
117
  | Area | Checks | Examples |
@@ -122,8 +122,10 @@ The intentionally vulnerable example produces (among others):
122
122
  | Docker | EC030–EC033 | unpinned base images, root user, remote ADD / curl\|sh, undeclared network policy |
123
123
  | Dependencies | EC040–EC041 | unpinned pip deps, unverified downloads |
124
124
  | Verifier quality | EC050–EC054 | missing/empty verifier, no reward production, unconditional success, assertion-free tests, self-reported rewards |
125
- | Dynamic (opt-in) | EC060–EC062 | gold run, no-op negative control, repeatability |
126
- | Mutation (opt-in) | EC063 | AST mutants of a Python solution; rejection rate as a strength signal |
125
+ | Multi-step | EC004 | per-step instruction/verifier completeness (plus per-step EC002) |
126
+ | Dynamic (opt-in) | EC060–EC062 | gold run, no-op negative control (docker/local/external), repeatability |
127
+ | Mutation (opt-in) | EC063–EC064 | AST mutants of a Python solution; literal mutants of shell solutions under Docker |
128
+ | External companions (opt-in) | EC070–EC072 | Hadolint, Trivy, Semgrep — run only when installed and `--external-tools` is passed |
127
129
 
128
130
  Every finding carries evidence, a severity, a confidence, a remediation and
129
131
  stated limitations. Heuristics report WARN — EnvCert does not present a
@@ -174,9 +176,16 @@ input/config · `3` internal/execution error.
174
176
 
175
177
  | Format | Status |
176
178
  |---|---|
177
- | Harbor task directories (task.toml schema 1.x) | Implemented and fixture-tested |
179
+ | Harbor task directories (task.toml schema 1.x, incl. multi-step `steps/`) | Implemented and fixture-tested |
178
180
  | Generic Docker task bundles (Dockerfile + tests/solution) | Implemented, reduced-confidence heuristics |
179
- | NeMo Gym JSONL/resources layouts, Inspect AI tasks | Not yet — roadmap, not promised (`docs/architecture/ROADMAP.md`) |
181
+ | NeMo Gym datasets (dataset.toml / verifier.py / JSONL rows) | Implemented (static; Resources-Server HTTP probing is roadmap) — `examples/nemo-gym-dataset` |
182
+ | Inspect AI evaluations (@task Python files) | Implemented (static) — `examples/inspect-eval` |
183
+
184
+ Dynamic execution modes: `docker` (restricted), `local-python` (trusted
185
+ pytest tasks, no sandbox), `external` (your disposable-VM/remote runner via
186
+ a documented JSON protocol). OCI image digests can be bound with
187
+ `--image`/`--image-digest`; Cosign OCI attestation is available via
188
+ `envcert attest --image name@sha256:...` and `envcert verify-attestation`.
180
189
 
181
190
  ## Configuration
182
191
 
@@ -30,10 +30,10 @@ else can reproduce that*.
30
30
  Requires Python 3.11+.
31
31
 
32
32
  ```bash
33
- pip install envcert # once published to PyPI
34
- # From this repository (current):
33
+ pip install envcert # from PyPI
34
+ pipx install envcert # or isolated
35
+ # From a checkout of this repository:
35
36
  pip install .
36
- pipx install . # or isolated
37
37
  ```
38
38
 
39
39
  The core has **zero runtime dependencies**: static analysis works offline,
@@ -79,7 +79,7 @@ The intentionally vulnerable example produces (among others):
79
79
 
80
80
  ## What it checks
81
81
 
82
- 28 checks with stable public IDs (full catalog:
82
+ 33 checks with stable public IDs (full catalog:
83
83
  `docs/architecture/CHECK_CATALOG.md`, or `envcert list-checks -v`):
84
84
 
85
85
  | Area | Checks | Examples |
@@ -90,8 +90,10 @@ The intentionally vulnerable example produces (among others):
90
90
  | Docker | EC030–EC033 | unpinned base images, root user, remote ADD / curl\|sh, undeclared network policy |
91
91
  | Dependencies | EC040–EC041 | unpinned pip deps, unverified downloads |
92
92
  | Verifier quality | EC050–EC054 | missing/empty verifier, no reward production, unconditional success, assertion-free tests, self-reported rewards |
93
- | Dynamic (opt-in) | EC060–EC062 | gold run, no-op negative control, repeatability |
94
- | Mutation (opt-in) | EC063 | AST mutants of a Python solution; rejection rate as a strength signal |
93
+ | Multi-step | EC004 | per-step instruction/verifier completeness (plus per-step EC002) |
94
+ | Dynamic (opt-in) | EC060–EC062 | gold run, no-op negative control (docker/local/external), repeatability |
95
+ | Mutation (opt-in) | EC063–EC064 | AST mutants of a Python solution; literal mutants of shell solutions under Docker |
96
+ | External companions (opt-in) | EC070–EC072 | Hadolint, Trivy, Semgrep — run only when installed and `--external-tools` is passed |
95
97
 
96
98
  Every finding carries evidence, a severity, a confidence, a remediation and
97
99
  stated limitations. Heuristics report WARN — EnvCert does not present a
@@ -142,9 +144,16 @@ input/config · `3` internal/execution error.
142
144
 
143
145
  | Format | Status |
144
146
  |---|---|
145
- | Harbor task directories (task.toml schema 1.x) | Implemented and fixture-tested |
147
+ | Harbor task directories (task.toml schema 1.x, incl. multi-step `steps/`) | Implemented and fixture-tested |
146
148
  | Generic Docker task bundles (Dockerfile + tests/solution) | Implemented, reduced-confidence heuristics |
147
- | NeMo Gym JSONL/resources layouts, Inspect AI tasks | Not yet — roadmap, not promised (`docs/architecture/ROADMAP.md`) |
149
+ | NeMo Gym datasets (dataset.toml / verifier.py / JSONL rows) | Implemented (static; Resources-Server HTTP probing is roadmap) — `examples/nemo-gym-dataset` |
150
+ | Inspect AI evaluations (@task Python files) | Implemented (static) — `examples/inspect-eval` |
151
+
152
+ Dynamic execution modes: `docker` (restricted), `local-python` (trusted
153
+ pytest tasks, no sandbox), `external` (your disposable-VM/remote runner via
154
+ a documented JSON protocol). OCI image digests can be bound with
155
+ `--image`/`--image-digest`; Cosign OCI attestation is available via
156
+ `envcert attest --image name@sha256:...` and `envcert verify-attestation`.
148
157
 
149
158
  ## Configuration
150
159
 
@@ -1,5 +1,5 @@
1
1
  """EnvCert: integrity verification and evidence for AI evaluation environments."""
2
2
 
3
- __version__ = "0.1.0"
3
+ __version__ = "0.2.0"
4
4
  REPORT_SCHEMA_VERSION = "1.0.0"
5
5
  __all__ = ["__version__", "REPORT_SCHEMA_VERSION"]
@@ -26,6 +26,9 @@ class TaskContext:
26
26
  solution_dir: Path | None = None
27
27
  environment_dir: Path | None = None
28
28
  warnings: list[str] = field(default_factory=list)
29
+ verifier_paths: list[str] = field(default_factory=list) # framework-specific verifier/scorer files (non-Harbor)
30
+ steps: list[dict] = field(default_factory=list) # multi-step: [{name, instruction, tests, solution}]
31
+ dataset_files: list[str] = field(default_factory=list)
29
32
  _text_cache: dict[str, str | None] = field(default_factory=dict, repr=False)
30
33
 
31
34
  def text(self, rel: str) -> str | None:
@@ -67,6 +67,17 @@ def load(root: Path) -> TaskContext:
67
67
  ctx.tests_dir = root / "tests"
68
68
  if (root / "solution").is_dir():
69
69
  ctx.solution_dir = root / "solution"
70
+ steps_dir = root / "steps"
71
+ if steps_dir.is_dir():
72
+ for step in sorted(p for p in steps_dir.iterdir() if p.is_dir()):
73
+ rel = f"steps/{step.name}"
74
+ ctx.steps.append({
75
+ "name": step.name,
76
+ "instruction": f"{rel}/instruction.md" if (step / "instruction.md").is_file() else None,
77
+ "tests": f"{rel}/tests" if (step / "tests").is_dir() else None,
78
+ "test_entrypoints": [f"{rel}/tests/{n}" for n in ("test.sh", "test.bat", "verifier.py") if (step / "tests" / n).is_file()],
79
+ "solution": f"{rel}/solution" if (step / "solution").is_dir() else None,
80
+ })
70
81
  env_dir = root / "environment"
71
82
  if env_dir.is_dir():
72
83
  ctx.environment_dir = env_dir
@@ -0,0 +1,69 @@
1
+ """Inspect AI adapter.
2
+
3
+ Verified against inspect.aisi.org.uk (Tasks / Datasets / Scorers, 2026-11):
4
+ an Inspect evaluation is a Python file defining ``@task`` functions that
5
+ return ``Task(dataset=..., solver=..., scorer=...)``. Samples carry
6
+ ``input`` (shown to the model) and ``target`` (grading ground truth, not
7
+ shown as input). Scorers include exact/includes/match/pattern and
8
+ model-graded scorers — the latter are non-deterministic and model-dependent,
9
+ which EnvCert flags as a reproducibility limitation rather than a defect.
10
+
11
+ This adapter statically locates ``@task`` files, records scorer usage, and
12
+ maps the task file as the verifier path so the generic verifier-structure
13
+ checks apply. It never imports or executes the eval file.
14
+ """
15
+ from __future__ import annotations
16
+
17
+ import re
18
+ from pathlib import Path
19
+
20
+ from envcert.adapters.base import TaskContext, scan_files
21
+ from envcert.core.errors import InvalidInputError
22
+
23
+ TASK_RE = re.compile(r"from\s+inspect_ai\s+import|import\s+inspect_ai|@task\b")
24
+ SCORER_RE = re.compile(r"scorer\s*=\s*([A-Za-z_][\w.]*)")
25
+ MODEL_GRADED = ("model_graded_qa", "model_graded_fact", "model_graded")
26
+
27
+
28
+ def find_task_files(root: Path, files: list[str]) -> list[str]:
29
+ out = []
30
+ for rel in files:
31
+ if not rel.endswith(".py"):
32
+ continue
33
+ try:
34
+ text = (root / rel).read_text(encoding="utf-8", errors="replace")[:20000]
35
+ except OSError:
36
+ continue
37
+ if "inspect_ai" in text and ("@task" in text or "Task(" in text):
38
+ out.append(rel)
39
+ return out
40
+
41
+
42
+ def is_inspect(root: Path) -> bool:
43
+ if not root.is_dir():
44
+ return False
45
+ files = [p.relative_to(root).as_posix() for p in root.rglob("*.py") if ".venv" not in p.parts]
46
+ return bool(find_task_files(root, files[:200]))
47
+
48
+
49
+ def load(root: Path) -> TaskContext:
50
+ root = root.resolve()
51
+ if not root.is_dir():
52
+ raise InvalidInputError(f"task path is not a directory: {root}")
53
+ ctx = TaskContext(root=root, adapter="inspect", task_name=root.name)
54
+ scan_files(ctx)
55
+ task_files = find_task_files(root, ctx.files)
56
+ ctx.verifier_paths = task_files
57
+ ctx.dataset_files = [f for f in ctx.files if f.endswith((".jsonl", ".json", ".csv"))]
58
+ if task_files:
59
+ text = ctx.text(task_files[0]) or ""
60
+ scorers = SCORER_RE.findall(text)
61
+ ctx.config = {"scorers": scorers, "task_files": task_files}
62
+ if any(any(s.startswith(m) for m in MODEL_GRADED) for s in scorers):
63
+ ctx.warnings.append("model-graded scorer detected: grading is model-dependent and non-deterministic; results need scorer/model version pinning for reproducibility")
64
+ if not scorers:
65
+ ctx.warnings.append("no scorer= argument detected in the @task file; the Task may rely on a default solver/scorer or compose them elsewhere")
66
+ else:
67
+ ctx.warnings.append("no @task file with inspect_ai imports found; Inspect detection was heuristic")
68
+ ctx.warnings.append("Inspect adapter is static-only: samples' targets are framework-hidden at runtime, so target-leakage verdicts are heuristic")
69
+ return ctx
@@ -0,0 +1,77 @@
1
+ """NVIDIA NeMo Gym adapter.
2
+
3
+ Verified against docs.nvidia.com/nemo/gym (2026-11): Gym reads tasks in two
4
+ forms — Harbor task folders (handled by the Harbor adapter) and *JSONL rows
5
+ plus a verifier*: a dataset directory / resources layout where a shared
6
+ Python ``verify()`` scores rows, plus ``dataset.toml`` with a ``[gym]`` table
7
+ at a dataset root whose root-level ``verifier.py`` grades every single-step
8
+ task without its own ``tests/``. This adapter covers the non-Harbor forms:
9
+
10
+ - a dataset root with ``dataset.toml`` and/or ``verifier.py`` and task
11
+ subfolders and/or ``*.jsonl`` row files;
12
+ - a JSONL dataset file directory whose rows carry
13
+ ``responses_create_params`` (the prompt the agent sees) plus verifier
14
+ fields such as ``expected_answer``.
15
+
16
+ Row files are data: ``expected_answer``-style fields are verifier-owned
17
+ ground truth. EnvCert checks that they are not *also* embedded in the
18
+ agent-visible prompt (see EC014's framework note) and that a ``verify``
19
+ implementation exists and is substantive. Resources Servers that run as
20
+ HTTP services are only partially visible statically — the adapter records
21
+ that warning.
22
+ """
23
+ from __future__ import annotations
24
+
25
+ import json
26
+ from pathlib import Path
27
+
28
+ from envcert.adapters.base import TaskContext, scan_files
29
+ from envcert.adapters.harbor import parse_toml
30
+ from envcert.core.errors import InvalidInputError
31
+
32
+
33
+ def is_nemo_gym(root: Path) -> bool:
34
+ if (root / "dataset.toml").is_file() or (root / "verifier.py").is_file():
35
+ return True
36
+ return any(p.suffix == ".jsonl" for p in root.glob("*.jsonl"))
37
+
38
+
39
+ def load(root: Path) -> TaskContext:
40
+ root = root.resolve()
41
+ if not root.is_dir():
42
+ raise InvalidInputError(f"task path is not a directory: {root}")
43
+ ctx = TaskContext(root=root, adapter="nemo-gym", task_name=root.name)
44
+ scan_files(ctx)
45
+ if (root / "dataset.toml").is_file():
46
+ ctx.config_path = root / "dataset.toml"
47
+ ctx.config = parse_toml(root / "dataset.toml")
48
+ dataset = ctx.config.get("dataset") or {}
49
+ if isinstance(dataset, dict) and dataset.get("name"):
50
+ ctx.task_name = str(dataset["name"])
51
+ if (root / "verifier.py").is_file():
52
+ ctx.verifier_paths = ["verifier.py"]
53
+ ctx.tests_dir = root # verifier lives at dataset root in this layout
54
+ ctx.dataset_files = [f for f in ctx.files if f.endswith(".jsonl")]
55
+ # task subfolders graded by the shared verifier
56
+ sub_tasks = [f for f in ctx.files if f.endswith("task.toml")]
57
+ if sub_tasks:
58
+ ctx.warnings.append(f"dataset contains {len(sub_tasks)} Harbor task subfolder(s); per-folder auditing is available by running envcert on each subfolder (dataset-level checks ran here)")
59
+ if not ctx.verifier_paths:
60
+ ctx.warnings.append("no root verifier.py found; verification may live in a Resources Server (HTTP service), which static analysis can only partially inspect")
61
+ # validate JSONL rows cheaply and record row counts as evidence
62
+ for rel in ctx.dataset_files:
63
+ path = root / rel
64
+ rows, bad = 0, 0
65
+ try:
66
+ for line in path.read_text(encoding="utf-8", errors="replace").splitlines():
67
+ if not line.strip():
68
+ continue
69
+ rows += 1
70
+ try:
71
+ json.loads(line)
72
+ except ValueError:
73
+ bad += 1
74
+ except OSError:
75
+ bad += 1
76
+ ctx.warnings.append(f"dataset file {rel}: {rows} rows, {bad} invalid JSON lines" if bad else f"dataset file {rel}: {rows} rows")
77
+ return ctx
@@ -36,6 +36,9 @@ def build_statement(report: dict[str, Any], report_bytes: bytes) -> dict[str, An
36
36
  subjects.append({"name": "verifier", "digest": {"sha256": env["verifier_digest_sha256"]}})
37
37
  if env.get("solution_digest_sha256"):
38
38
  subjects.append({"name": "reference-solution", "digest": {"sha256": env["solution_digest_sha256"]}})
39
+ if env.get("image_digest"):
40
+ digest = str(env["image_digest"]).removeprefix("sha256:")
41
+ subjects.append({"name": env.get("image_reference") or "oci-image", "digest": {"sha256": digest}})
39
42
  failing = [f for f in report.get("findings", []) if f.get("status") in ("FAIL", "WARN")]
40
43
  predicate: dict[str, Any] = {
41
44
  "result": "PASSED" if policy.get("passed") else "FAILED",
@@ -74,6 +77,62 @@ def cosign_available() -> str | None:
74
77
  return shutil.which("cosign")
75
78
 
76
79
 
80
+ def cosign_attest_image(predicate_path: Path, image: str, key_path: str | None = None) -> tuple[bool, str]:
81
+ """Attach the statement as an OCI attestation on `image` via cosign attest.
82
+
83
+ The predicate file must contain the in-toto *predicate* (this module's
84
+ caller extracts it). `image` should be referenced by digest
85
+ (name@sha256:...) so the attestation binds to immutable content.
86
+ """
87
+ if "@sha256:" not in image:
88
+ return False, f"refusing to attest a mutable image reference {image!r}: use name@sha256:<digest> so the attestation binds to immutable content"
89
+ exe = cosign_available()
90
+ if not exe:
91
+ return False, "cosign not found on PATH; OCI attestation not performed (statement file was still written)"
92
+ cmd = [exe, "attest", "--yes", "--predicate", str(predicate_path),
93
+ "--type", TEST_RESULT_PREDICATE, image]
94
+ if key_path:
95
+ cmd[2:2] = ["--key", key_path]
96
+ proc = subprocess.run(cmd, capture_output=True, text=True, timeout=300)
97
+ if proc.returncode != 0:
98
+ return False, f"cosign attest failed: {(proc.stderr or proc.stdout)[:500]}"
99
+ return True, f"cosign attestation attached to {image} (predicate type {TEST_RESULT_PREDICATE})"
100
+
101
+
102
+ def cosign_verify_attestation(image: str, key_path: str | None = None, policy: str | None = None) -> tuple[bool, str]:
103
+ exe = cosign_available()
104
+ if not exe:
105
+ return False, "cosign not found on PATH; cryptographic verification not performed"
106
+ cmd = [exe, "verify-attestation", "--type", TEST_RESULT_PREDICATE]
107
+ if key_path:
108
+ cmd += ["--key", key_path]
109
+ else:
110
+ cmd += ["--certificate-identity-regexp", ".*", "--certificate-oidc-issuer-regexp", ".*"]
111
+ if policy:
112
+ cmd += ["--policy", policy]
113
+ cmd.append(image)
114
+ proc = subprocess.run(cmd, capture_output=True, text=True, timeout=300)
115
+ if proc.returncode != 0:
116
+ return False, f"cosign verify-attestation failed: {(proc.stderr or proc.stdout)[:500]}"
117
+ return True, f"cosign verify-attestation passed for {image}"
118
+
119
+
120
+ def verify_statement_local(statement: dict[str, Any]) -> list[str]:
121
+ """Structural verification only (no cryptography): type, subjects, result."""
122
+ problems = []
123
+ if statement.get("_type") != STATEMENT_TYPE:
124
+ problems.append(f"unexpected _type: {statement.get('_type')!r}")
125
+ if statement.get("predicateType") != TEST_RESULT_PREDICATE:
126
+ problems.append(f"unexpected predicateType: {statement.get('predicateType')!r}")
127
+ subjects = statement.get("subject") or []
128
+ if not subjects or not all((s.get("digest") or {}).get("sha256") for s in subjects):
129
+ problems.append("statement has no sha256-bound subjects")
130
+ predicate = statement.get("predicate") or {}
131
+ if predicate.get("result") not in ("PASSED", "FAILED"):
132
+ problems.append("predicate.result is not PASSED/FAILED")
133
+ return problems
134
+
135
+
77
136
  def sign_with_cosign(envelope_path: Path, key_path: str | None = None) -> tuple[bool, str]:
78
137
  exe = cosign_available()
79
138
  if not exe:
@@ -1,7 +1,7 @@
1
1
  from __future__ import annotations
2
2
 
3
3
  from collections.abc import Callable
4
- from dataclasses import dataclass
4
+ from dataclasses import dataclass, field
5
5
 
6
6
  from envcert.adapters.base import TaskContext
7
7
  from envcert.core.models import Confidence, Finding, Severity, Status
@@ -22,10 +22,13 @@ class CheckInfo:
22
22
  @dataclass
23
23
  class CheckContext:
24
24
  task: TaskContext
25
- execution: str = "none" # none | docker | local-python
25
+ execution: str = "none" # none | docker | local-python | external
26
26
  repetitions: int = 3
27
27
  enable_mutation: bool = False
28
28
  max_mutants: int = 10
29
+ external_tools: bool = False
30
+ runner_command: str = ""
31
+ shared: dict = field(default_factory=dict) # checks stash run artefacts (e.g. image digest) for the report
29
32
 
30
33
 
31
34
  CheckFn = Callable[[CheckContext], list[Finding]]
@@ -24,6 +24,10 @@ EC062 = CheckInfo("EC062", "Verifier outcome is repeatable across repetitions",
24
24
  "Re-running gold verification should give a consistent reward/exit outcome.",
25
25
  "Remove nondeterminism (time, randomness, network, ordering) from the task or verifier.",
26
26
  "Compares reward/exit outcome, not bitwise-identical stdout.", dynamic=True)
27
+ EC064 = CheckInfo("EC064", "Shell-solution mutation: verifier rejects mutated reference solutions (docker)", "mutation", Severity.MEDIUM,
28
+ "Mutating literal outputs in solution/solve.sh should make the verifier reject the run; surviving mutants signal a weak verifier.",
29
+ "Strengthen verifier assertions on the mutated outputs.",
30
+ "Docker execution only; only literal-bearing shell solutions can be mutated. Mutant validity is heuristic.", dynamic=True)
27
31
  EC063 = CheckInfo("EC063", "Mutation testing: verifier rejects mutated (incorrect) solutions", "mutation", Severity.MEDIUM,
28
32
  "Mutated variants of a correct Python solution should be rejected; the rejection rate is a verifier-strength signal.",
29
33
  "Add assertions covering the mutated behaviour; do not treat the rate as proof of correctness.",
@@ -39,9 +43,17 @@ def _is_python_task(ctx) -> bool:
39
43
  def _run_variant(c: CheckContext, variant: str):
40
44
  ctx = c.task
41
45
  if c.execution == "docker":
42
- if variant != "gold":
43
- return None, "docker runner currently supports the gold run only; negative controls need per-task artifact manipulation"
44
- return DockerRunner().run_harbor_task(ctx.root), None
46
+ result = DockerRunner().run_harbor_task(ctx.root, variant=variant)
47
+ if result.image_digest:
48
+ c.shared["image_digest"] = result.image_digest
49
+ return result, None
50
+ if c.execution == "external":
51
+ from envcert.runners.external_runner import ExternalRunner
52
+
53
+ result = ExternalRunner(c.runner_command).run_harbor_task(ctx.root, variant=variant)
54
+ if result.image_digest:
55
+ c.shared["image_digest"] = result.image_digest
56
+ return result, None
45
57
  if c.execution == "local-python":
46
58
  if not _is_python_task(ctx):
47
59
  return None, "local-python execution requires solution/*.py plus tests/test_*.py (pytest-style task)"
@@ -75,16 +87,17 @@ def check_gold(c: CheckContext) -> list:
75
87
  def check_negative_control(c: CheckContext) -> list:
76
88
  if c.execution == "none":
77
89
  return [not_run(EC061, "Dynamic checks are opt-in: re-run with --execution docker or --execution local-python.")]
78
- if c.execution == "docker":
79
- return [not_applicable(EC061, "Docker negative controls are not automated in this version: a no-op's validity is task-specific and Harbor artifact transfer differs per task. Run the gold check and test a no-op manually, or use local-python for pytest tasks.")]
80
90
  result, why = _run_variant(c, "noop")
81
91
  if result is None:
82
92
  return [not_applicable(EC061, f"Negative control not supported here: {why}.")]
83
93
  if result.error and result.reward is None:
84
94
  return [finding(EC061, Status.ERROR, f"Negative-control execution failed to run: {result.error}", severity=Severity.INFO, evidence={"error": result.error})]
85
- if result.ok or (result.reward is not None and result.reward >= 1.0):
95
+ # Reward, when present, is authoritative (a docker harness exits 0 even
96
+ # when the reward is 0); only fall back to exit status without a reward.
97
+ noop_passed = (result.reward >= 1.0) if result.reward is not None else result.ok
98
+ if noop_passed:
86
99
  return [finding(EC061, Status.WARN,
87
- "No-op (emptied) solution PASSED the verifier. If a no-op should not solve this task, the verifier is too weak; if no action is a legitimate solution, record that and ignore this warning.",
100
+ "No-op (skipped/emptied) solution PASSED the verifier. If a no-op should not solve this task, the verifier is too weak; if no action is a legitimate solution, record that and ignore this warning.",
88
101
  severity=Severity.HIGH, evidence={"reward": result.reward, "exit_code": result.exit_code, "variant": "noop"})]
89
102
  return [passed(EC061, "No-op negative control was rejected by the verifier (non-passing outcome).",
90
103
  evidence={"reward": result.reward, "exit_code": result.exit_code, "variant": "noop"})]
@@ -94,8 +107,19 @@ def check_negative_control(c: CheckContext) -> list:
94
107
  def check_repeatability(c: CheckContext) -> list:
95
108
  if c.execution == "none":
96
109
  return [not_run(EC062, "Dynamic checks are opt-in: re-run with --execution docker or --execution local-python.")]
110
+ if c.execution in ("docker", "external"):
111
+ outcomes = []
112
+ for _ in range(max(2, min(c.repetitions, 3))):
113
+ r, _why = _run_variant(c, "gold")
114
+ if r is None:
115
+ return [not_applicable(EC062, f"Repeatability not supported here: {_why}.")]
116
+ outcomes.append({"reward": r.reward, "exit_code": r.exit_code, "error": r.error})
117
+ rewards = {o["reward"] for o in outcomes}
118
+ if len(rewards) == 1 and None not in rewards:
119
+ return [passed(EC062, f"Gold reward was consistent across {len(outcomes)} {c.execution} runs (reward={outcomes[0]['reward']}).", evidence={"runs": outcomes})]
120
+ return [finding(EC062, Status.WARN, f"Gold outcome varied across {len(outcomes)} {c.execution} runs: {outcomes}. Investigate nondeterminism.", evidence={"runs": outcomes})]
97
121
  if c.execution != "local-python":
98
- return [not_applicable(EC062, "Repeatability repetitions are currently automated for local-python tasks only; for docker, re-run EC060 manually with different seeds/times.")]
122
+ return [not_applicable(EC062, "Repeatability requires an execution mode.")]
99
123
  if not _is_python_task(c.task):
100
124
  return [not_applicable(EC062, "Repeatability requires a pytest-style Python task under local-python execution.")]
101
125
  runner = LocalPythonRunner()
@@ -130,3 +154,56 @@ def check_mutation(c: CheckContext) -> list:
130
154
  evidence=ev)]
131
155
  return [passed(EC063, f"Mutation rejection rate {rate:.0%} ({report['rejected']}/{report['eligible']} incorrect variants rejected). This is a strength signal, not proof of verifier correctness.",
132
156
  evidence=ev)]
157
+
158
+
159
+ @check(EC064)
160
+ def check_shell_mutation(c: CheckContext) -> list:
161
+ if not c.enable_mutation:
162
+ return [not_run(EC064, "Shell-solution mutation is opt-in: re-run with --mutation and --execution docker.")]
163
+ if c.execution != "docker":
164
+ return [not_applicable(EC064, "Shell-solution mutation in this version runs under --execution docker only.")]
165
+ if c.task.adapter != "harbor":
166
+ return [not_applicable(EC064, "Shell-solution mutation supports Harbor tasks with solution/solve.sh.")]
167
+ import re
168
+ import shutil
169
+ import tempfile
170
+ from pathlib import Path
171
+
172
+ solve_rel = "solution/solve.sh"
173
+ if not c.task.has(solve_rel):
174
+ return [not_applicable(EC064, "No solution/solve.sh to mutate.")]
175
+ original = c.task.text(solve_rel) or ""
176
+ mutants: list[tuple[str, str]] = []
177
+ for m in re.finditer(r"'([^']{3,})'|\"([^\"]{3,})\"", original):
178
+ val = m.group(1) or m.group(2)
179
+ mutants.append((f"string literal {val[:30]!r} mutated", original.replace(val, val + "-MUTATED", 1)))
180
+ if len(mutants) >= 3:
181
+ break
182
+ if not mutants:
183
+ for m in re.finditer(r"\b(\d+)\b", original):
184
+ mutants.append((f"number {m.group(1)} incremented", original[:m.start(1)] + str(int(m.group(1)) + 1) + original[m.end(1):]))
185
+ if len(mutants) >= 2:
186
+ break
187
+ if not mutants:
188
+ return [not_applicable(EC064, "solution/solve.sh contains no string/number literals to mutate.")]
189
+ runner = DockerRunner()
190
+ if not runner.info().available:
191
+ return [finding(EC064, Status.ERROR, f"Docker unavailable for shell mutation: {runner.info().reason}", severity=Severity.INFO, evidence={"runner": "docker"})]
192
+ gold = runner.run_harbor_task(c.task.root, variant="gold")
193
+ if gold.reward is not None and gold.reward < 1.0:
194
+ return [not_applicable(EC064, f"Gold run does not pass (reward={gold.reward}); mutation would be meaningless.")]
195
+ results = []
196
+ rejected = 0
197
+ for desc, mutated in mutants[: c.max_mutants]:
198
+ with tempfile.TemporaryDirectory(prefix="envcert-shellmut-") as td:
199
+ dest = Path(td) / "task"
200
+ shutil.copytree(c.task.root, dest, ignore=shutil.ignore_patterns(".git"))
201
+ (dest / solve_rel).write_text(mutated, encoding="utf-8")
202
+ r = runner.run_harbor_task(dest, variant="gold")
203
+ survived = r.reward is not None and r.reward >= 1.0
204
+ results.append({"change": desc, "reward": r.reward, "outcome": "survived" if survived else "rejected"})
205
+ rejected += 0 if survived else 1
206
+ rate = rejected / len(results) if results else None
207
+ if rate is not None and rate < 0.5:
208
+ return [finding(EC064, Status.WARN, f"Shell-solution mutation rejection rate is {rate:.0%} ({rejected}/{len(results)}); mutated (wrong) solutions still pass.", evidence={"mutants": results, "rejection_rate": rate})]
209
+ return [passed(EC064, f"Shell-solution mutation rejection rate {rate:.0%} ({rejected}/{len(results)}). A strength signal, not proof of verifier correctness.", evidence={"mutants": results, "rejection_rate": rate})]