envcert 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
envcert/__init__.py ADDED
@@ -0,0 +1,5 @@
1
+ """EnvCert: integrity verification and evidence for AI evaluation environments."""
2
+
3
+ __version__ = "0.1.0"
4
+ REPORT_SCHEMA_VERSION = "1.0.0"
5
+ __all__ = ["__version__", "REPORT_SCHEMA_VERSION"]
envcert/__main__.py ADDED
@@ -0,0 +1,6 @@
1
+ import sys
2
+
3
+ from envcert.cli.main import main
4
+
5
+ if __name__ == "__main__":
6
+ sys.exit(main())
File without changes
@@ -0,0 +1,67 @@
1
+ from __future__ import annotations
2
+
3
+ from dataclasses import dataclass, field
4
+ from pathlib import Path
5
+ from typing import Any
6
+
7
+ from envcert.utils.dockerfile import Dockerfile, parse_dockerfile
8
+ from envcert.utils.files import iter_task_files, read_text_safe
9
+
10
+
11
+ @dataclass
12
+ class TaskContext:
13
+ root: Path
14
+ adapter: str
15
+ task_name: str = ""
16
+ config: dict[str, Any] = field(default_factory=dict)
17
+ config_path: Path | None = None
18
+ files: list[str] = field(default_factory=list)
19
+ symlinks: list[str] = field(default_factory=list)
20
+ instruction_path: Path | None = None
21
+ dockerfile_path: Path | None = None
22
+ dockerfile: Dockerfile | None = None
23
+ dockerignore_patterns: list[str] = field(default_factory=list)
24
+ build_context: Path | None = None
25
+ tests_dir: Path | None = None
26
+ solution_dir: Path | None = None
27
+ environment_dir: Path | None = None
28
+ warnings: list[str] = field(default_factory=list)
29
+ _text_cache: dict[str, str | None] = field(default_factory=dict, repr=False)
30
+
31
+ def text(self, rel: str) -> str | None:
32
+ if rel not in self._text_cache:
33
+ self._text_cache[rel] = read_text_safe(self.root / rel)
34
+ return self._text_cache[rel]
35
+
36
+ def abs(self, rel: str) -> Path:
37
+ return self.root / rel
38
+
39
+ def files_under(self, prefix: str) -> list[str]:
40
+ prefix = prefix.strip("/") + "/"
41
+ return [f for f in self.files if f == prefix[:-1] or f.startswith(prefix)]
42
+
43
+ def has(self, rel: str) -> bool:
44
+ return rel in self.files
45
+
46
+
47
+ def load_dockerfile(ctx: TaskContext, dockerfile_rel: str, context_rel: str) -> None:
48
+ ctx.dockerfile_path = ctx.root / dockerfile_rel
49
+ text = ctx.text(dockerfile_rel)
50
+ if text is not None:
51
+ ctx.dockerfile = parse_dockerfile(text)
52
+ ctx.build_context = ctx.root / context_rel if context_rel else ctx.root
53
+ ignore = ctx.build_context / ".dockerignore"
54
+ if ignore.is_file():
55
+ content = read_text_safe(ignore) or ""
56
+ ctx.dockerignore_patterns = [ln.strip() for ln in content.splitlines() if ln.strip() and not ln.strip().startswith("#")]
57
+ env_ignore = ctx.root / ".dockerignore"
58
+ if not ctx.dockerignore_patterns and env_ignore.is_file():
59
+ content = read_text_safe(env_ignore) or ""
60
+ ctx.dockerignore_patterns = [ln.strip() for ln in content.splitlines() if ln.strip() and not ln.strip().startswith("#")]
61
+
62
+
63
+ def scan_files(ctx: TaskContext) -> None:
64
+ for rel, _path, is_link in iter_task_files(ctx.root):
65
+ ctx.files.append(rel)
66
+ if is_link:
67
+ ctx.symlinks.append(rel)
@@ -0,0 +1,40 @@
1
+ """Generic adapter for Docker-based task bundles that are not Harbor tasks.
2
+
3
+ It recognises a root Dockerfile plus tests/ and/or solution/ directories. It
4
+ deliberately reports reduced confidence: framework-specific semantics (where
5
+ the verifier runs, what the agent can see) are unknown.
6
+ """
7
+ from __future__ import annotations
8
+
9
+ from pathlib import Path
10
+
11
+ from envcert.adapters.base import TaskContext, load_dockerfile, scan_files
12
+ from envcert.core.errors import InvalidInputError
13
+
14
+
15
+ def load(root: Path) -> TaskContext:
16
+ root = root.resolve()
17
+ if not root.is_dir():
18
+ raise InvalidInputError(f"task path is not a directory: {root}")
19
+ ctx = TaskContext(root=root, adapter="generic", task_name=root.name)
20
+ scan_files(ctx)
21
+ if (root / "instruction.md").is_file():
22
+ ctx.instruction_path = root / "instruction.md"
23
+ if (root / "tests").is_dir():
24
+ ctx.tests_dir = root / "tests"
25
+ if (root / "solution").is_dir():
26
+ ctx.solution_dir = root / "solution"
27
+ if (root / "environment").is_dir():
28
+ ctx.environment_dir = root / "environment"
29
+ if (root / "Dockerfile").is_file():
30
+ load_dockerfile(ctx, "Dockerfile", ".")
31
+ elif (root / "environment" / "Dockerfile").is_file():
32
+ load_dockerfile(ctx, "environment/Dockerfile", "environment")
33
+ ctx.warnings.append("generic adapter: framework semantics unknown; boundary findings are heuristic")
34
+ return ctx
35
+
36
+
37
+ def detect(root: Path) -> str:
38
+ if (root / "task.toml").is_file():
39
+ return "harbor"
40
+ return "generic"
@@ -0,0 +1,96 @@
1
+ """Harbor adapter.
2
+
3
+ Verified against Harbor's current docs (docs.harborframework.com, schema 1.4):
4
+ a task is a directory with task.toml + instruction.md, environment/ (Dockerfile
5
+ or docker-compose.yaml, or [environment].docker_image instead), tests/test.sh
6
+ (or test.bat / verifier.py) writing /logs/verifier/reward.txt|json, and an
7
+ optional solution/solve.sh. Multi-step tasks use steps/<name>/ with their own
8
+ instruction/tests/solution. Variations handled: missing environment/ when a
9
+ prebuilt docker_image is declared, verifier.py, test.bat, steps/.
10
+ """
11
+ from __future__ import annotations
12
+
13
+ from pathlib import Path
14
+ from typing import Any
15
+
16
+ from envcert.adapters.base import TaskContext, load_dockerfile, scan_files
17
+ from envcert.core.errors import InvalidInputError
18
+
19
+ try: # Python 3.11+
20
+ import tomllib
21
+ except ModuleNotFoundError: # pragma: no cover
22
+ import tomli as tomllib # type: ignore[no-redef]
23
+
24
+
25
+ def is_harbor_task(root: Path) -> bool:
26
+ return (root / "task.toml").is_file()
27
+
28
+
29
+ def parse_toml(path: Path) -> dict[str, Any]:
30
+ try:
31
+ with path.open("rb") as fh:
32
+ return tomllib.load(fh)
33
+ except Exception as exc:
34
+ raise InvalidInputError(f"invalid TOML in {path}: {exc}") from exc
35
+
36
+
37
+ def load(root: Path) -> TaskContext:
38
+ root = root.resolve()
39
+ if not root.is_dir():
40
+ raise InvalidInputError(f"task path is not a directory: {root}")
41
+ ctx = TaskContext(root=root, adapter="harbor")
42
+ scan_files(ctx)
43
+ cfg_path = root / "task.toml"
44
+ if cfg_path.is_file():
45
+ ctx.config_path = cfg_path
46
+ raw = cfg_path.read_text(encoding="utf-8", errors="replace")
47
+ import re as _re
48
+
49
+ if "{{" in raw or "{%" in raw or _re.search(r"\{[A-Za-z_][A-Za-z0-9_]*\}", raw):
50
+ # Adapter task-templates (e.g. in the Harbor repo) are unrendered
51
+ # Jinja templates, not runnable tasks. Degrade honestly instead
52
+ # of failing TOML parsing.
53
+ ctx.config = {}
54
+ ctx.warnings.append("task.toml appears to be an unrendered template (Jinja placeholders); config-dependent checks are heuristic until the template is rendered")
55
+ else:
56
+ ctx.config = parse_toml(cfg_path)
57
+ task_table = ctx.config.get("task") or {}
58
+ if isinstance(task_table, dict):
59
+ ctx.task_name = str(task_table.get("name") or root.name)
60
+ else:
61
+ ctx.task_name = root.name
62
+ if (root / "instruction.md").is_file():
63
+ ctx.instruction_path = root / "instruction.md"
64
+ elif (root / "steps").is_dir():
65
+ ctx.warnings.append("multi-step task: top-level instruction.md absent; per-step instructions are not individually audited in this version")
66
+ if (root / "tests").is_dir():
67
+ ctx.tests_dir = root / "tests"
68
+ if (root / "solution").is_dir():
69
+ ctx.solution_dir = root / "solution"
70
+ env_dir = root / "environment"
71
+ if env_dir.is_dir():
72
+ ctx.environment_dir = env_dir
73
+ if (env_dir / "Dockerfile").is_file():
74
+ load_dockerfile(ctx, "environment/Dockerfile", "environment")
75
+ elif (env_dir / "docker-compose.yaml").is_file() or (env_dir / "docker-compose.yml").is_file():
76
+ ctx.warnings.append("environment uses docker-compose; Dockerfile-specific checks are limited")
77
+ return ctx
78
+
79
+
80
+ def verifier_entrypoints(ctx: TaskContext) -> list[str]:
81
+ names = ("tests/test.sh", "tests/test.bat", "tests/test.ps1", "tests/verifier.py")
82
+ found = [n for n in names if ctx.has(n)]
83
+ # per-step verifiers
84
+ for f in ctx.files:
85
+ if f.startswith("steps/") and f.rsplit("/", 1)[-1] in ("test.sh", "test.bat", "verifier.py"):
86
+ found.append(f)
87
+ return found
88
+
89
+
90
+ def solution_entrypoints(ctx: TaskContext) -> list[str]:
91
+ names = ("solution/solve.sh", "solution/solve.bat", "solution/solve.ps1", "solution/solve.py", "solution/answer.md")
92
+ found = [n for n in names if ctx.has(n)]
93
+ for f in ctx.files:
94
+ if f.startswith("steps/") and "/solution/" in f:
95
+ found.append(f)
96
+ return found
File without changes
@@ -0,0 +1,87 @@
1
+ """in-toto attestation generation (unsigned statement + optional cosign).
2
+
3
+ Uses the established in-toto Statement v1 and the vetted Test Result
4
+ predicate (https://in-toto.io/attestation/test-result/v0.1) instead of
5
+ inventing a new predicate. EnvCert-specific detail (per-check results,
6
+ bundle/verifier digests, profile, limitations) is carried in the Test Result
7
+ `configurationSource.digest` + an `externalParameters`-style extension key is
8
+ avoided; instead the full report digest binds the statement to the exact
9
+ report file, and consumers fetch the report by digest. Signing is optional
10
+ via Cosign sign-blob when the binary is present; an unsigned statement is
11
+ explicitly labelled unsigned and is never described as verified.
12
+ """
13
+ from __future__ import annotations
14
+
15
+ import base64
16
+ import hashlib
17
+ import json
18
+ import shutil
19
+ import subprocess
20
+ from pathlib import Path
21
+ from typing import Any
22
+
23
+ STATEMENT_TYPE = "https://in-toto.io/Statement/v1"
24
+ TEST_RESULT_PREDICATE = "https://in-toto.io/attestation/test-result/v0.1"
25
+
26
+
27
+ def build_statement(report: dict[str, Any], report_bytes: bytes) -> dict[str, Any]:
28
+ env = report.get("environment") or {}
29
+ policy = report.get("policy") or {}
30
+ report_digest = hashlib.sha256(report_bytes).hexdigest()
31
+ subjects = [{
32
+ "name": env.get("task_name") or "envcert-task",
33
+ "digest": {"sha256": env.get("bundle_digest_sha256") or report_digest},
34
+ }]
35
+ if env.get("verifier_digest_sha256"):
36
+ subjects.append({"name": "verifier", "digest": {"sha256": env["verifier_digest_sha256"]}})
37
+ if env.get("solution_digest_sha256"):
38
+ subjects.append({"name": "reference-solution", "digest": {"sha256": env["solution_digest_sha256"]}})
39
+ failing = [f for f in report.get("findings", []) if f.get("status") in ("FAIL", "WARN")]
40
+ predicate: dict[str, Any] = {
41
+ "result": "PASSED" if policy.get("passed") else "FAILED",
42
+ "configurationSource": {
43
+ "uri": f"envcert://profile/{(report.get('execution') or {}).get('profile', 'harbor')}",
44
+ "digest": {"sha256": report_digest},
45
+ },
46
+ "metadata": {
47
+ "buildStartedOn": report.get("generated_at"),
48
+ "completeness": {"parameters": True, "environment": False, "materials": False},
49
+ },
50
+ # Extension documented in docs/architecture/REPORT_SCHEMA.md: the Test
51
+ # Result predicate allows additional fields for tool-specific detail.
52
+ "x-envcert": {
53
+ "envcert_version": report.get("envcert_version"),
54
+ "report_schema_version": report.get("schema_version"),
55
+ "report_digest_sha256": report_digest,
56
+ "adapter": env.get("adapter"),
57
+ "assurance_context": report.get("assurance_context"),
58
+ "checks_executed": report.get("checks_executed"),
59
+ "failing_check_ids": sorted({f["check_id"] for f in failing}),
60
+ "limitations": report.get("limitations"),
61
+ "signed": False,
62
+ },
63
+ }
64
+ return {"_type": STATEMENT_TYPE, "subject": subjects, "predicateType": TEST_RESULT_PREDICATE, "predicate": predicate}
65
+
66
+
67
+ def dsse_envelope_unsigned(statement: dict[str, Any]) -> dict[str, Any]:
68
+ payload = base64.b64encode(json.dumps(statement, sort_keys=True).encode()).decode()
69
+ return {"payloadType": "application/vnd.in-toto+json", "payload": payload, "signatures": [],
70
+ "_envcert_note": "UNSIGNED envelope: signatures is empty. Sign with cosign sign-blob or a DSSE signer before treating this as cryptographic evidence."}
71
+
72
+
73
+ def cosign_available() -> str | None:
74
+ return shutil.which("cosign")
75
+
76
+
77
+ def sign_with_cosign(envelope_path: Path, key_path: str | None = None) -> tuple[bool, str]:
78
+ exe = cosign_available()
79
+ if not exe:
80
+ return False, "cosign not found on PATH; attestation left unsigned (this is not an error — install cosign to sign)"
81
+ cmd = [exe, "sign-blob", "--output-signature", str(envelope_path) + ".sig", str(envelope_path)]
82
+ if key_path:
83
+ cmd[2:2] = ["--key", key_path]
84
+ proc = subprocess.run(cmd, capture_output=True, text=True, timeout=120)
85
+ if proc.returncode != 0:
86
+ return False, f"cosign sign-blob failed: {(proc.stderr or proc.stdout)[:400]}"
87
+ return True, f"signed with cosign; signature at {envelope_path}.sig"
File without changes
envcert/checks/base.py ADDED
@@ -0,0 +1,68 @@
1
+ from __future__ import annotations
2
+
3
+ from collections.abc import Callable
4
+ from dataclasses import dataclass
5
+
6
+ from envcert.adapters.base import TaskContext
7
+ from envcert.core.models import Confidence, Finding, Severity, Status
8
+
9
+
10
+ @dataclass(frozen=True)
11
+ class CheckInfo:
12
+ check_id: str
13
+ title: str
14
+ category: str
15
+ severity: Severity # severity assigned when the check fails
16
+ description: str
17
+ remediation: str
18
+ limitations: str = ""
19
+ dynamic: bool = False
20
+
21
+
22
+ @dataclass
23
+ class CheckContext:
24
+ task: TaskContext
25
+ execution: str = "none" # none | docker | local-python
26
+ repetitions: int = 3
27
+ enable_mutation: bool = False
28
+ max_mutants: int = 10
29
+
30
+
31
+ CheckFn = Callable[[CheckContext], list[Finding]]
32
+
33
+ REGISTRY: dict[str, tuple[CheckInfo, CheckFn]] = {}
34
+
35
+
36
+ def check(info: CheckInfo):
37
+ def deco(fn: CheckFn) -> CheckFn:
38
+ if info.check_id in REGISTRY:
39
+ raise ValueError(f"duplicate check id {info.check_id}")
40
+ REGISTRY[info.check_id] = (info, fn)
41
+ return fn
42
+
43
+ return deco
44
+
45
+
46
+ def finding(info: CheckInfo, status: Status, description: str = "", *, severity: Severity | None = None,
47
+ artifact: str = "", path: str | None = None, line: int | None = None,
48
+ evidence: dict | None = None, confidence: Confidence = Confidence.MEDIUM,
49
+ remediation: str = "", limitations: str = "") -> Finding:
50
+ return Finding(
51
+ check_id=info.check_id, title=info.title, description=description or info.description,
52
+ category=info.category, severity=severity or (info.severity if status in (Status.FAIL, Status.WARN) else Severity.INFO),
53
+ status=status, artifact=artifact, path=path, line=line, evidence=evidence or {},
54
+ remediation=remediation or info.remediation, confidence=confidence,
55
+ limitations=limitations or info.limitations,
56
+ )
57
+
58
+
59
+ def passed(info: CheckInfo, description: str = "", **kw) -> Finding:
60
+ return finding(info, Status.PASS, description, severity=Severity.INFO, **kw)
61
+
62
+
63
+ def not_run(info: CheckInfo, reason: str) -> Finding:
64
+ return finding(info, Status.NOT_RUN, reason, severity=Severity.INFO, confidence=Confidence.HIGH)
65
+
66
+
67
+ def not_applicable(info: CheckInfo, reason: str) -> Finding:
68
+ return finding(info, Status.NOT_APPLICABLE, reason, severity=Severity.INFO, confidence=Confidence.HIGH)
@@ -0,0 +1,132 @@
1
+ """Dynamic checks. Explicitly opt-in via execution mode.
2
+
3
+ - docker: build + run a Harbor task in the restricted DockerRunner.
4
+ - local-python: run pytest-style tests/ against solution/*.py in a temp copy
5
+ (no sandbox — trusted tasks only, recorded in the report).
6
+ - none (default): every dynamic check returns NOT_RUN, never PASS.
7
+ """
8
+ from __future__ import annotations
9
+
10
+ from envcert.checks.base import CheckContext, CheckInfo, check, finding, not_applicable, not_run, passed
11
+ from envcert.core.models import Severity, Status
12
+ from envcert.runners.docker_runner import DockerRunner
13
+ from envcert.runners.local_runner import LocalPythonRunner
14
+
15
+ EC060 = CheckInfo("EC060", "Reference (gold) solution passes the verifier", "dynamic", Severity.HIGH,
16
+ "Executing the reference solution should produce the expected (passing) reward.",
17
+ "Fix the reference solution or the verifier so a correct solution passes.",
18
+ "Only supported for docker (Harbor shell tasks) and local-python (pytest tasks) execution.", dynamic=True)
19
+ EC061 = CheckInfo("EC061", "Negative control (no-op/empty) is rejected by the verifier", "dynamic", Severity.HIGH,
20
+ "A no-op or empty submission should not pass, unless the task semantics say otherwise.",
21
+ "Strengthen the verifier so trivial submissions fail.",
22
+ "Some tasks legitimately pass with no action; then this returns NOT_APPLICABLE/WARN for human review, not a hard failure.", dynamic=True)
23
+ EC062 = CheckInfo("EC062", "Verifier outcome is repeatable across repetitions", "dynamic", Severity.MEDIUM,
24
+ "Re-running gold verification should give a consistent reward/exit outcome.",
25
+ "Remove nondeterminism (time, randomness, network, ordering) from the task or verifier.",
26
+ "Compares reward/exit outcome, not bitwise-identical stdout.", dynamic=True)
27
+ EC063 = CheckInfo("EC063", "Mutation testing: verifier rejects mutated (incorrect) solutions", "mutation", Severity.MEDIUM,
28
+ "Mutated variants of a correct Python solution should be rejected; the rejection rate is a verifier-strength signal.",
29
+ "Add assertions covering the mutated behaviour; do not treat the rate as proof of correctness.",
30
+ "Equivalent/invalid mutants and the small mutant set limit this signal. Python/pytest tasks only.", dynamic=True)
31
+
32
+
33
+ def _is_python_task(ctx) -> bool:
34
+ has_py_solution = any(f.startswith("solution/") and f.endswith(".py") for f in ctx.files)
35
+ has_py_tests = any(f.startswith("tests/") and f.endswith(".py") and "test" in f.rsplit("/", 1)[-1] for f in ctx.files)
36
+ return has_py_solution and has_py_tests
37
+
38
+
39
+ def _run_variant(c: CheckContext, variant: str):
40
+ ctx = c.task
41
+ if c.execution == "docker":
42
+ if variant != "gold":
43
+ return None, "docker runner currently supports the gold run only; negative controls need per-task artifact manipulation"
44
+ return DockerRunner().run_harbor_task(ctx.root), None
45
+ if c.execution == "local-python":
46
+ if not _is_python_task(ctx):
47
+ return None, "local-python execution requires solution/*.py plus tests/test_*.py (pytest-style task)"
48
+ return LocalPythonRunner().run_task(ctx.root, solution_variant=variant), None
49
+ return None, f"execution mode is '{c.execution}'"
50
+
51
+
52
+ @check(EC060)
53
+ def check_gold(c: CheckContext) -> list:
54
+ if c.execution == "none":
55
+ return [not_run(EC060, "Dynamic checks are opt-in: re-run with --execution docker or --execution local-python.")]
56
+ result, why = _run_variant(c, "gold")
57
+ if result is None:
58
+ return [not_applicable(EC060, f"Gold run not supported here: {why}.")]
59
+ if result.error and result.reward is None:
60
+ return [finding(EC060, Status.ERROR, f"Gold execution failed to run: {result.error}",
61
+ severity=Severity.INFO, evidence={"error": result.error, "runner": c.execution})]
62
+ if result.reward is not None:
63
+ if result.reward >= 1.0 or (c.execution == "local-python" and result.ok):
64
+ return [passed(EC060, f"Reference solution passed verification (reward={result.reward}).",
65
+ evidence={"reward": result.reward, "exit_code": result.exit_code, "runner": c.execution, "duration_sec": result.duration_sec})]
66
+ return [finding(EC060, Status.FAIL, f"Reference solution did NOT pass verification (reward={result.reward}, exit={result.exit_code}). Either the solution or the verifier is broken.",
67
+ evidence={"reward": result.reward, "exit_code": result.exit_code, "stderr_tail": result.stderr[-1000:]})]
68
+ if result.ok:
69
+ return [passed(EC060, "Reference solution run exited successfully.", evidence={"exit_code": result.exit_code, "runner": c.execution})]
70
+ return [finding(EC060, Status.FAIL, f"Reference solution run failed (exit={result.exit_code}): {result.error or result.stderr[-500:]}",
71
+ evidence={"exit_code": result.exit_code, "stderr_tail": result.stderr[-1000:]})]
72
+
73
+
74
+ @check(EC061)
75
+ def check_negative_control(c: CheckContext) -> list:
76
+ if c.execution == "none":
77
+ return [not_run(EC061, "Dynamic checks are opt-in: re-run with --execution docker or --execution local-python.")]
78
+ if c.execution == "docker":
79
+ return [not_applicable(EC061, "Docker negative controls are not automated in this version: a no-op's validity is task-specific and Harbor artifact transfer differs per task. Run the gold check and test a no-op manually, or use local-python for pytest tasks.")]
80
+ result, why = _run_variant(c, "noop")
81
+ if result is None:
82
+ return [not_applicable(EC061, f"Negative control not supported here: {why}.")]
83
+ if result.error and result.reward is None:
84
+ return [finding(EC061, Status.ERROR, f"Negative-control execution failed to run: {result.error}", severity=Severity.INFO, evidence={"error": result.error})]
85
+ if result.ok or (result.reward is not None and result.reward >= 1.0):
86
+ return [finding(EC061, Status.WARN,
87
+ "No-op (emptied) solution PASSED the verifier. If a no-op should not solve this task, the verifier is too weak; if no action is a legitimate solution, record that and ignore this warning.",
88
+ severity=Severity.HIGH, evidence={"reward": result.reward, "exit_code": result.exit_code, "variant": "noop"})]
89
+ return [passed(EC061, "No-op negative control was rejected by the verifier (non-passing outcome).",
90
+ evidence={"reward": result.reward, "exit_code": result.exit_code, "variant": "noop"})]
91
+
92
+
93
+ @check(EC062)
94
+ def check_repeatability(c: CheckContext) -> list:
95
+ if c.execution == "none":
96
+ return [not_run(EC062, "Dynamic checks are opt-in: re-run with --execution docker or --execution local-python.")]
97
+ if c.execution != "local-python":
98
+ return [not_applicable(EC062, "Repeatability repetitions are currently automated for local-python tasks only; for docker, re-run EC060 manually with different seeds/times.")]
99
+ if not _is_python_task(c.task):
100
+ return [not_applicable(EC062, "Repeatability requires a pytest-style Python task under local-python execution.")]
101
+ runner = LocalPythonRunner()
102
+ outcomes = []
103
+ for _ in range(max(2, c.repetitions)):
104
+ r = runner.run_task(c.task.root, solution_variant="gold")
105
+ outcomes.append({"reward": r.reward, "exit_code": r.exit_code, "error": r.error})
106
+ rewards = {o["reward"] for o in outcomes}
107
+ if len(rewards) == 1 and None not in rewards:
108
+ return [passed(EC062, f"Gold outcome was consistent across {len(outcomes)} runs (reward={outcomes[0]['reward']}).",
109
+ evidence={"runs": outcomes})]
110
+ return [finding(EC062, Status.WARN, f"Gold outcome varied across {len(outcomes)} runs: {outcomes}. Investigate nondeterminism.",
111
+ evidence={"runs": outcomes})]
112
+
113
+
114
+ @check(EC063)
115
+ def check_mutation(c: CheckContext) -> list:
116
+ if not c.enable_mutation:
117
+ return [not_run(EC063, "Mutation testing is opt-in: re-run with --mutation (and --execution local-python).")]
118
+ if c.execution != "local-python" or not _is_python_task(c.task):
119
+ return [not_applicable(EC063, "Mutation testing in this version supports pytest-style Python tasks under --execution local-python only.")]
120
+ from envcert.checks.mutation import run_mutation
121
+
122
+ report = run_mutation(c.task.root, max_mutants=c.max_mutants)
123
+ if report["eligible"] == 0:
124
+ return [not_applicable(EC063, f"No eligible mutants could be generated/applied: {report['notes']}")]
125
+ rate = report["rejection_rate"]
126
+ ev = {"tested": report["eligible"], "rejected": report["rejected"], "survived": report["survived"],
127
+ "invalid": report["invalid"], "rejection_rate": rate, "mutants": report["mutants"][:20]}
128
+ if rate is not None and rate < 0.5:
129
+ return [finding(EC063, Status.WARN, f"Mutation rejection rate is {rate:.0%} ({report['rejected']}/{report['eligible']}); most incorrect variants survive, so the verifier is weak for the mutated behaviour.",
130
+ evidence=ev)]
131
+ return [passed(EC063, f"Mutation rejection rate {rate:.0%} ({report['rejected']}/{report['eligible']} incorrect variants rejected). This is a strength signal, not proof of verifier correctness.",
132
+ evidence=ev)]
@@ -0,0 +1,142 @@
1
+ """Small AST mutation framework for Python solutions (pytest tasks).
2
+
3
+ Operators: arithmetic swap (+<->-, *<->/), comparison flip (==/!=, </>=),
4
+ numeric constant +1, boolean flip, return-None. Each mutant is applied to a
5
+ temp copy and the verifier (pytest) is run; a mutant is *rejected* when the
6
+ verifier fails, *survived* when it still passes, *invalid* when it does not
7
+ parse/apply. Equivalent mutants cannot be detected automatically — the
8
+ report says so and the rate must not be read as proof of correctness.
9
+ """
10
+ from __future__ import annotations
11
+
12
+ import ast
13
+ import copy
14
+ import shutil
15
+ import tempfile
16
+ from pathlib import Path
17
+
18
+ from envcert.runners.local_runner import LocalPythonRunner
19
+
20
+ ARITH = {ast.Add: ast.Sub, ast.Sub: ast.Add, ast.Mult: ast.Div, ast.Div: ast.Mult}
21
+ CMP = {ast.Eq: ast.NotEq, ast.NotEq: ast.Eq, ast.Lt: ast.GtE, ast.Gt: ast.LtE, ast.LtE: ast.Gt, ast.GtE: ast.Lt}
22
+
23
+
24
+ class _Mutator(ast.NodeTransformer):
25
+ def __init__(self, index: int):
26
+ self.index = index
27
+ self.count = -1
28
+ self.applied = ""
29
+
30
+ def _hit(self) -> bool:
31
+ self.count += 1
32
+ return self.count == self.index
33
+
34
+ def visit_BinOp(self, node):
35
+ self.generic_visit(node)
36
+ if type(node.op) in ARITH and self._hit():
37
+ self.applied = f"{type(node.op).__name__}->{ARITH[type(node.op)].__name__} (line {node.lineno})"
38
+ node.op = ARITH[type(node.op)]()
39
+ return node
40
+
41
+ def visit_Compare(self, node):
42
+ self.generic_visit(node)
43
+ if node.ops and type(node.ops[0]) in CMP and self._hit():
44
+ self.applied = f"compare {type(node.ops[0]).__name__}->{CMP[type(node.ops[0])].__name__} (line {node.lineno})"
45
+ node.ops[0] = CMP[type(node.ops[0])]()
46
+ return node
47
+
48
+ def visit_Constant(self, node):
49
+ if isinstance(node.value, bool):
50
+ if self._hit():
51
+ self.applied = f"bool {node.value}->{not node.value} (line {node.lineno})"
52
+ node.value = not node.value
53
+ elif isinstance(node.value, (int, float)) and node.value != 0:
54
+ if self._hit():
55
+ self.applied = f"constant {node.value}->{node.value + 1} (line {node.lineno})"
56
+ node.value = node.value + 1
57
+ return node
58
+
59
+ def visit_Return(self, node):
60
+ self.generic_visit(node)
61
+ if node.value is not None and self._hit():
62
+ self.applied = f"return -> None (line {node.lineno})"
63
+ node.value = ast.Constant(value=None)
64
+ return node
65
+
66
+
67
+ def _count_sites(tree: ast.AST) -> int:
68
+ n = 0
69
+ for node in ast.walk(tree):
70
+ if isinstance(node, ast.BinOp) and type(node.op) in ARITH:
71
+ n += 1
72
+ elif isinstance(node, ast.Compare) and node.ops and type(node.ops[0]) in CMP:
73
+ n += 1
74
+ elif isinstance(node, ast.Constant) and (isinstance(node.value, bool) or (isinstance(node.value, (int, float)) and node.value != 0)):
75
+ n += 1
76
+ elif isinstance(node, ast.Return) and node.value is not None:
77
+ n += 1
78
+ return n
79
+
80
+
81
+ def run_mutation(task_root: Path, max_mutants: int = 10, timeout: int = 60) -> dict:
82
+ task_root = Path(task_root)
83
+ runner = LocalPythonRunner()
84
+ mutants: list[dict] = []
85
+ eligible = rejected = survived = invalid = 0
86
+ notes = ""
87
+ sol_files = sorted((task_root / "solution").rglob("*.py")) if (task_root / "solution").is_dir() else []
88
+ if not sol_files:
89
+ return {"eligible": 0, "rejected": 0, "survived": 0, "invalid": 0, "rejection_rate": None,
90
+ "mutants": [], "notes": "no solution/*.py files"}
91
+ # gold must pass first, otherwise mutants are meaningless
92
+ gold = runner.run_task(task_root, "gold", timeout=timeout)
93
+ if not gold.ok:
94
+ return {"eligible": 0, "rejected": 0, "survived": 0, "invalid": 0, "rejection_rate": None,
95
+ "mutants": [], "notes": "gold solution does not pass; mutation skipped"}
96
+ budget = max_mutants
97
+ for sol in sol_files:
98
+ if budget <= 0:
99
+ break
100
+ source = sol.read_text(encoding="utf-8", errors="replace")
101
+ try:
102
+ tree = ast.parse(source)
103
+ except SyntaxError:
104
+ invalid += 1
105
+ continue
106
+ sites = _count_sites(tree)
107
+ for idx in range(sites):
108
+ if budget <= 0:
109
+ break
110
+ mut = _Mutator(idx)
111
+ new_tree = mut.visit(copy.deepcopy(tree))
112
+ if not mut.applied:
113
+ continue
114
+ budget -= 1
115
+ try:
116
+ mutated = ast.unparse(ast.fix_missing_locations(new_tree))
117
+ ast.parse(mutated)
118
+ except Exception: # noqa: BLE001
119
+ invalid += 1
120
+ mutants.append({"file": sol.name, "change": mut.applied, "outcome": "invalid"})
121
+ continue
122
+ eligible += 1
123
+ with tempfile.TemporaryDirectory(prefix="envcert-mut-") as td:
124
+ ws = Path(td) / "ws"
125
+ runner.prepare_workspace(task_root, ws, "gold")
126
+ target = ws / sol.name
127
+ # gold workspace copies solution files flat; overwrite the mutated one
128
+ if not target.exists(): # nested layout fallback
129
+ shutil.copytree(task_root / "solution", ws / "solution", dirs_exist_ok=True)
130
+ target = ws / "solution" / sol.name
131
+ target.write_text(mutated, encoding="utf-8")
132
+ result = runner.run_python_verifier(ws, timeout=timeout)
133
+ outcome = "survived" if result.ok else "rejected"
134
+ if outcome == "rejected":
135
+ rejected += 1
136
+ else:
137
+ survived += 1
138
+ mutants.append({"file": sol.name, "change": mut.applied, "outcome": outcome})
139
+ rate = (rejected / eligible) if eligible else None
140
+ return {"eligible": eligible, "rejected": rejected, "survived": survived, "invalid": invalid,
141
+ "rejection_rate": rate, "mutants": mutants,
142
+ "notes": notes or "equivalent mutants may inflate the survived count; interpret as a signal only"}