envcert 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- envcert/__init__.py +5 -0
- envcert/__main__.py +6 -0
- envcert/adapters/__init__.py +0 -0
- envcert/adapters/base.py +67 -0
- envcert/adapters/generic.py +40 -0
- envcert/adapters/harbor.py +96 -0
- envcert/attestations/__init__.py +0 -0
- envcert/attestations/intoto.py +87 -0
- envcert/checks/__init__.py +0 -0
- envcert/checks/base.py +68 -0
- envcert/checks/dynamic_checks.py +132 -0
- envcert/checks/mutation.py +142 -0
- envcert/checks/static_checks.py +796 -0
- envcert/cli/__init__.py +0 -0
- envcert/cli/main.py +269 -0
- envcert/core/__init__.py +0 -0
- envcert/core/config.py +139 -0
- envcert/core/digests.py +45 -0
- envcert/core/engine.py +141 -0
- envcert/core/errors.py +10 -0
- envcert/core/models.py +72 -0
- envcert/reports/__init__.py +0 -0
- envcert/reports/sarif.py +51 -0
- envcert/reports/text.py +35 -0
- envcert/runners/__init__.py +0 -0
- envcert/runners/base.py +32 -0
- envcert/runners/docker_runner.py +95 -0
- envcert/runners/local_runner.py +79 -0
- envcert/schemas/__init__.py +0 -0
- envcert/schemas/report.schema.json +65 -0
- envcert/utils/__init__.py +0 -0
- envcert/utils/dockerfile.py +105 -0
- envcert/utils/files.py +86 -0
- envcert-0.1.0.dist-info/METADATA +204 -0
- envcert-0.1.0.dist-info/RECORD +38 -0
- envcert-0.1.0.dist-info/WHEEL +4 -0
- envcert-0.1.0.dist-info/entry_points.txt +2 -0
- envcert-0.1.0.dist-info/licenses/LICENSE +187 -0
envcert/__init__.py
ADDED
envcert/__main__.py
ADDED
|
File without changes
|
envcert/adapters/base.py
ADDED
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from dataclasses import dataclass, field
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
from typing import Any
|
|
6
|
+
|
|
7
|
+
from envcert.utils.dockerfile import Dockerfile, parse_dockerfile
|
|
8
|
+
from envcert.utils.files import iter_task_files, read_text_safe
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
@dataclass
|
|
12
|
+
class TaskContext:
|
|
13
|
+
root: Path
|
|
14
|
+
adapter: str
|
|
15
|
+
task_name: str = ""
|
|
16
|
+
config: dict[str, Any] = field(default_factory=dict)
|
|
17
|
+
config_path: Path | None = None
|
|
18
|
+
files: list[str] = field(default_factory=list)
|
|
19
|
+
symlinks: list[str] = field(default_factory=list)
|
|
20
|
+
instruction_path: Path | None = None
|
|
21
|
+
dockerfile_path: Path | None = None
|
|
22
|
+
dockerfile: Dockerfile | None = None
|
|
23
|
+
dockerignore_patterns: list[str] = field(default_factory=list)
|
|
24
|
+
build_context: Path | None = None
|
|
25
|
+
tests_dir: Path | None = None
|
|
26
|
+
solution_dir: Path | None = None
|
|
27
|
+
environment_dir: Path | None = None
|
|
28
|
+
warnings: list[str] = field(default_factory=list)
|
|
29
|
+
_text_cache: dict[str, str | None] = field(default_factory=dict, repr=False)
|
|
30
|
+
|
|
31
|
+
def text(self, rel: str) -> str | None:
|
|
32
|
+
if rel not in self._text_cache:
|
|
33
|
+
self._text_cache[rel] = read_text_safe(self.root / rel)
|
|
34
|
+
return self._text_cache[rel]
|
|
35
|
+
|
|
36
|
+
def abs(self, rel: str) -> Path:
|
|
37
|
+
return self.root / rel
|
|
38
|
+
|
|
39
|
+
def files_under(self, prefix: str) -> list[str]:
|
|
40
|
+
prefix = prefix.strip("/") + "/"
|
|
41
|
+
return [f for f in self.files if f == prefix[:-1] or f.startswith(prefix)]
|
|
42
|
+
|
|
43
|
+
def has(self, rel: str) -> bool:
|
|
44
|
+
return rel in self.files
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def load_dockerfile(ctx: TaskContext, dockerfile_rel: str, context_rel: str) -> None:
|
|
48
|
+
ctx.dockerfile_path = ctx.root / dockerfile_rel
|
|
49
|
+
text = ctx.text(dockerfile_rel)
|
|
50
|
+
if text is not None:
|
|
51
|
+
ctx.dockerfile = parse_dockerfile(text)
|
|
52
|
+
ctx.build_context = ctx.root / context_rel if context_rel else ctx.root
|
|
53
|
+
ignore = ctx.build_context / ".dockerignore"
|
|
54
|
+
if ignore.is_file():
|
|
55
|
+
content = read_text_safe(ignore) or ""
|
|
56
|
+
ctx.dockerignore_patterns = [ln.strip() for ln in content.splitlines() if ln.strip() and not ln.strip().startswith("#")]
|
|
57
|
+
env_ignore = ctx.root / ".dockerignore"
|
|
58
|
+
if not ctx.dockerignore_patterns and env_ignore.is_file():
|
|
59
|
+
content = read_text_safe(env_ignore) or ""
|
|
60
|
+
ctx.dockerignore_patterns = [ln.strip() for ln in content.splitlines() if ln.strip() and not ln.strip().startswith("#")]
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def scan_files(ctx: TaskContext) -> None:
|
|
64
|
+
for rel, _path, is_link in iter_task_files(ctx.root):
|
|
65
|
+
ctx.files.append(rel)
|
|
66
|
+
if is_link:
|
|
67
|
+
ctx.symlinks.append(rel)
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
"""Generic adapter for Docker-based task bundles that are not Harbor tasks.
|
|
2
|
+
|
|
3
|
+
It recognises a root Dockerfile plus tests/ and/or solution/ directories. It
|
|
4
|
+
deliberately reports reduced confidence: framework-specific semantics (where
|
|
5
|
+
the verifier runs, what the agent can see) are unknown.
|
|
6
|
+
"""
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
|
|
11
|
+
from envcert.adapters.base import TaskContext, load_dockerfile, scan_files
|
|
12
|
+
from envcert.core.errors import InvalidInputError
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def load(root: Path) -> TaskContext:
|
|
16
|
+
root = root.resolve()
|
|
17
|
+
if not root.is_dir():
|
|
18
|
+
raise InvalidInputError(f"task path is not a directory: {root}")
|
|
19
|
+
ctx = TaskContext(root=root, adapter="generic", task_name=root.name)
|
|
20
|
+
scan_files(ctx)
|
|
21
|
+
if (root / "instruction.md").is_file():
|
|
22
|
+
ctx.instruction_path = root / "instruction.md"
|
|
23
|
+
if (root / "tests").is_dir():
|
|
24
|
+
ctx.tests_dir = root / "tests"
|
|
25
|
+
if (root / "solution").is_dir():
|
|
26
|
+
ctx.solution_dir = root / "solution"
|
|
27
|
+
if (root / "environment").is_dir():
|
|
28
|
+
ctx.environment_dir = root / "environment"
|
|
29
|
+
if (root / "Dockerfile").is_file():
|
|
30
|
+
load_dockerfile(ctx, "Dockerfile", ".")
|
|
31
|
+
elif (root / "environment" / "Dockerfile").is_file():
|
|
32
|
+
load_dockerfile(ctx, "environment/Dockerfile", "environment")
|
|
33
|
+
ctx.warnings.append("generic adapter: framework semantics unknown; boundary findings are heuristic")
|
|
34
|
+
return ctx
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def detect(root: Path) -> str:
|
|
38
|
+
if (root / "task.toml").is_file():
|
|
39
|
+
return "harbor"
|
|
40
|
+
return "generic"
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
"""Harbor adapter.
|
|
2
|
+
|
|
3
|
+
Verified against Harbor's current docs (docs.harborframework.com, schema 1.4):
|
|
4
|
+
a task is a directory with task.toml + instruction.md, environment/ (Dockerfile
|
|
5
|
+
or docker-compose.yaml, or [environment].docker_image instead), tests/test.sh
|
|
6
|
+
(or test.bat / verifier.py) writing /logs/verifier/reward.txt|json, and an
|
|
7
|
+
optional solution/solve.sh. Multi-step tasks use steps/<name>/ with their own
|
|
8
|
+
instruction/tests/solution. Variations handled: missing environment/ when a
|
|
9
|
+
prebuilt docker_image is declared, verifier.py, test.bat, steps/.
|
|
10
|
+
"""
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
from typing import Any
|
|
15
|
+
|
|
16
|
+
from envcert.adapters.base import TaskContext, load_dockerfile, scan_files
|
|
17
|
+
from envcert.core.errors import InvalidInputError
|
|
18
|
+
|
|
19
|
+
try: # Python 3.11+
|
|
20
|
+
import tomllib
|
|
21
|
+
except ModuleNotFoundError: # pragma: no cover
|
|
22
|
+
import tomli as tomllib # type: ignore[no-redef]
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def is_harbor_task(root: Path) -> bool:
|
|
26
|
+
return (root / "task.toml").is_file()
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def parse_toml(path: Path) -> dict[str, Any]:
|
|
30
|
+
try:
|
|
31
|
+
with path.open("rb") as fh:
|
|
32
|
+
return tomllib.load(fh)
|
|
33
|
+
except Exception as exc:
|
|
34
|
+
raise InvalidInputError(f"invalid TOML in {path}: {exc}") from exc
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def load(root: Path) -> TaskContext:
|
|
38
|
+
root = root.resolve()
|
|
39
|
+
if not root.is_dir():
|
|
40
|
+
raise InvalidInputError(f"task path is not a directory: {root}")
|
|
41
|
+
ctx = TaskContext(root=root, adapter="harbor")
|
|
42
|
+
scan_files(ctx)
|
|
43
|
+
cfg_path = root / "task.toml"
|
|
44
|
+
if cfg_path.is_file():
|
|
45
|
+
ctx.config_path = cfg_path
|
|
46
|
+
raw = cfg_path.read_text(encoding="utf-8", errors="replace")
|
|
47
|
+
import re as _re
|
|
48
|
+
|
|
49
|
+
if "{{" in raw or "{%" in raw or _re.search(r"\{[A-Za-z_][A-Za-z0-9_]*\}", raw):
|
|
50
|
+
# Adapter task-templates (e.g. in the Harbor repo) are unrendered
|
|
51
|
+
# Jinja templates, not runnable tasks. Degrade honestly instead
|
|
52
|
+
# of failing TOML parsing.
|
|
53
|
+
ctx.config = {}
|
|
54
|
+
ctx.warnings.append("task.toml appears to be an unrendered template (Jinja placeholders); config-dependent checks are heuristic until the template is rendered")
|
|
55
|
+
else:
|
|
56
|
+
ctx.config = parse_toml(cfg_path)
|
|
57
|
+
task_table = ctx.config.get("task") or {}
|
|
58
|
+
if isinstance(task_table, dict):
|
|
59
|
+
ctx.task_name = str(task_table.get("name") or root.name)
|
|
60
|
+
else:
|
|
61
|
+
ctx.task_name = root.name
|
|
62
|
+
if (root / "instruction.md").is_file():
|
|
63
|
+
ctx.instruction_path = root / "instruction.md"
|
|
64
|
+
elif (root / "steps").is_dir():
|
|
65
|
+
ctx.warnings.append("multi-step task: top-level instruction.md absent; per-step instructions are not individually audited in this version")
|
|
66
|
+
if (root / "tests").is_dir():
|
|
67
|
+
ctx.tests_dir = root / "tests"
|
|
68
|
+
if (root / "solution").is_dir():
|
|
69
|
+
ctx.solution_dir = root / "solution"
|
|
70
|
+
env_dir = root / "environment"
|
|
71
|
+
if env_dir.is_dir():
|
|
72
|
+
ctx.environment_dir = env_dir
|
|
73
|
+
if (env_dir / "Dockerfile").is_file():
|
|
74
|
+
load_dockerfile(ctx, "environment/Dockerfile", "environment")
|
|
75
|
+
elif (env_dir / "docker-compose.yaml").is_file() or (env_dir / "docker-compose.yml").is_file():
|
|
76
|
+
ctx.warnings.append("environment uses docker-compose; Dockerfile-specific checks are limited")
|
|
77
|
+
return ctx
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def verifier_entrypoints(ctx: TaskContext) -> list[str]:
|
|
81
|
+
names = ("tests/test.sh", "tests/test.bat", "tests/test.ps1", "tests/verifier.py")
|
|
82
|
+
found = [n for n in names if ctx.has(n)]
|
|
83
|
+
# per-step verifiers
|
|
84
|
+
for f in ctx.files:
|
|
85
|
+
if f.startswith("steps/") and f.rsplit("/", 1)[-1] in ("test.sh", "test.bat", "verifier.py"):
|
|
86
|
+
found.append(f)
|
|
87
|
+
return found
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def solution_entrypoints(ctx: TaskContext) -> list[str]:
|
|
91
|
+
names = ("solution/solve.sh", "solution/solve.bat", "solution/solve.ps1", "solution/solve.py", "solution/answer.md")
|
|
92
|
+
found = [n for n in names if ctx.has(n)]
|
|
93
|
+
for f in ctx.files:
|
|
94
|
+
if f.startswith("steps/") and "/solution/" in f:
|
|
95
|
+
found.append(f)
|
|
96
|
+
return found
|
|
File without changes
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
"""in-toto attestation generation (unsigned statement + optional cosign).
|
|
2
|
+
|
|
3
|
+
Uses the established in-toto Statement v1 and the vetted Test Result
|
|
4
|
+
predicate (https://in-toto.io/attestation/test-result/v0.1) instead of
|
|
5
|
+
inventing a new predicate. EnvCert-specific detail (per-check results,
|
|
6
|
+
bundle/verifier digests, profile, limitations) is carried in the Test Result
|
|
7
|
+
`configurationSource.digest` + an `externalParameters`-style extension key is
|
|
8
|
+
avoided; instead the full report digest binds the statement to the exact
|
|
9
|
+
report file, and consumers fetch the report by digest. Signing is optional
|
|
10
|
+
via Cosign sign-blob when the binary is present; an unsigned statement is
|
|
11
|
+
explicitly labelled unsigned and is never described as verified.
|
|
12
|
+
"""
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import base64
|
|
16
|
+
import hashlib
|
|
17
|
+
import json
|
|
18
|
+
import shutil
|
|
19
|
+
import subprocess
|
|
20
|
+
from pathlib import Path
|
|
21
|
+
from typing import Any
|
|
22
|
+
|
|
23
|
+
STATEMENT_TYPE = "https://in-toto.io/Statement/v1"
|
|
24
|
+
TEST_RESULT_PREDICATE = "https://in-toto.io/attestation/test-result/v0.1"
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def build_statement(report: dict[str, Any], report_bytes: bytes) -> dict[str, Any]:
|
|
28
|
+
env = report.get("environment") or {}
|
|
29
|
+
policy = report.get("policy") or {}
|
|
30
|
+
report_digest = hashlib.sha256(report_bytes).hexdigest()
|
|
31
|
+
subjects = [{
|
|
32
|
+
"name": env.get("task_name") or "envcert-task",
|
|
33
|
+
"digest": {"sha256": env.get("bundle_digest_sha256") or report_digest},
|
|
34
|
+
}]
|
|
35
|
+
if env.get("verifier_digest_sha256"):
|
|
36
|
+
subjects.append({"name": "verifier", "digest": {"sha256": env["verifier_digest_sha256"]}})
|
|
37
|
+
if env.get("solution_digest_sha256"):
|
|
38
|
+
subjects.append({"name": "reference-solution", "digest": {"sha256": env["solution_digest_sha256"]}})
|
|
39
|
+
failing = [f for f in report.get("findings", []) if f.get("status") in ("FAIL", "WARN")]
|
|
40
|
+
predicate: dict[str, Any] = {
|
|
41
|
+
"result": "PASSED" if policy.get("passed") else "FAILED",
|
|
42
|
+
"configurationSource": {
|
|
43
|
+
"uri": f"envcert://profile/{(report.get('execution') or {}).get('profile', 'harbor')}",
|
|
44
|
+
"digest": {"sha256": report_digest},
|
|
45
|
+
},
|
|
46
|
+
"metadata": {
|
|
47
|
+
"buildStartedOn": report.get("generated_at"),
|
|
48
|
+
"completeness": {"parameters": True, "environment": False, "materials": False},
|
|
49
|
+
},
|
|
50
|
+
# Extension documented in docs/architecture/REPORT_SCHEMA.md: the Test
|
|
51
|
+
# Result predicate allows additional fields for tool-specific detail.
|
|
52
|
+
"x-envcert": {
|
|
53
|
+
"envcert_version": report.get("envcert_version"),
|
|
54
|
+
"report_schema_version": report.get("schema_version"),
|
|
55
|
+
"report_digest_sha256": report_digest,
|
|
56
|
+
"adapter": env.get("adapter"),
|
|
57
|
+
"assurance_context": report.get("assurance_context"),
|
|
58
|
+
"checks_executed": report.get("checks_executed"),
|
|
59
|
+
"failing_check_ids": sorted({f["check_id"] for f in failing}),
|
|
60
|
+
"limitations": report.get("limitations"),
|
|
61
|
+
"signed": False,
|
|
62
|
+
},
|
|
63
|
+
}
|
|
64
|
+
return {"_type": STATEMENT_TYPE, "subject": subjects, "predicateType": TEST_RESULT_PREDICATE, "predicate": predicate}
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def dsse_envelope_unsigned(statement: dict[str, Any]) -> dict[str, Any]:
|
|
68
|
+
payload = base64.b64encode(json.dumps(statement, sort_keys=True).encode()).decode()
|
|
69
|
+
return {"payloadType": "application/vnd.in-toto+json", "payload": payload, "signatures": [],
|
|
70
|
+
"_envcert_note": "UNSIGNED envelope: signatures is empty. Sign with cosign sign-blob or a DSSE signer before treating this as cryptographic evidence."}
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def cosign_available() -> str | None:
|
|
74
|
+
return shutil.which("cosign")
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def sign_with_cosign(envelope_path: Path, key_path: str | None = None) -> tuple[bool, str]:
|
|
78
|
+
exe = cosign_available()
|
|
79
|
+
if not exe:
|
|
80
|
+
return False, "cosign not found on PATH; attestation left unsigned (this is not an error — install cosign to sign)"
|
|
81
|
+
cmd = [exe, "sign-blob", "--output-signature", str(envelope_path) + ".sig", str(envelope_path)]
|
|
82
|
+
if key_path:
|
|
83
|
+
cmd[2:2] = ["--key", key_path]
|
|
84
|
+
proc = subprocess.run(cmd, capture_output=True, text=True, timeout=120)
|
|
85
|
+
if proc.returncode != 0:
|
|
86
|
+
return False, f"cosign sign-blob failed: {(proc.stderr or proc.stdout)[:400]}"
|
|
87
|
+
return True, f"signed with cosign; signature at {envelope_path}.sig"
|
|
File without changes
|
envcert/checks/base.py
ADDED
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from collections.abc import Callable
|
|
4
|
+
from dataclasses import dataclass
|
|
5
|
+
|
|
6
|
+
from envcert.adapters.base import TaskContext
|
|
7
|
+
from envcert.core.models import Confidence, Finding, Severity, Status
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
@dataclass(frozen=True)
|
|
11
|
+
class CheckInfo:
|
|
12
|
+
check_id: str
|
|
13
|
+
title: str
|
|
14
|
+
category: str
|
|
15
|
+
severity: Severity # severity assigned when the check fails
|
|
16
|
+
description: str
|
|
17
|
+
remediation: str
|
|
18
|
+
limitations: str = ""
|
|
19
|
+
dynamic: bool = False
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
@dataclass
|
|
23
|
+
class CheckContext:
|
|
24
|
+
task: TaskContext
|
|
25
|
+
execution: str = "none" # none | docker | local-python
|
|
26
|
+
repetitions: int = 3
|
|
27
|
+
enable_mutation: bool = False
|
|
28
|
+
max_mutants: int = 10
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
CheckFn = Callable[[CheckContext], list[Finding]]
|
|
32
|
+
|
|
33
|
+
REGISTRY: dict[str, tuple[CheckInfo, CheckFn]] = {}
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def check(info: CheckInfo):
|
|
37
|
+
def deco(fn: CheckFn) -> CheckFn:
|
|
38
|
+
if info.check_id in REGISTRY:
|
|
39
|
+
raise ValueError(f"duplicate check id {info.check_id}")
|
|
40
|
+
REGISTRY[info.check_id] = (info, fn)
|
|
41
|
+
return fn
|
|
42
|
+
|
|
43
|
+
return deco
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def finding(info: CheckInfo, status: Status, description: str = "", *, severity: Severity | None = None,
|
|
47
|
+
artifact: str = "", path: str | None = None, line: int | None = None,
|
|
48
|
+
evidence: dict | None = None, confidence: Confidence = Confidence.MEDIUM,
|
|
49
|
+
remediation: str = "", limitations: str = "") -> Finding:
|
|
50
|
+
return Finding(
|
|
51
|
+
check_id=info.check_id, title=info.title, description=description or info.description,
|
|
52
|
+
category=info.category, severity=severity or (info.severity if status in (Status.FAIL, Status.WARN) else Severity.INFO),
|
|
53
|
+
status=status, artifact=artifact, path=path, line=line, evidence=evidence or {},
|
|
54
|
+
remediation=remediation or info.remediation, confidence=confidence,
|
|
55
|
+
limitations=limitations or info.limitations,
|
|
56
|
+
)
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def passed(info: CheckInfo, description: str = "", **kw) -> Finding:
|
|
60
|
+
return finding(info, Status.PASS, description, severity=Severity.INFO, **kw)
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def not_run(info: CheckInfo, reason: str) -> Finding:
|
|
64
|
+
return finding(info, Status.NOT_RUN, reason, severity=Severity.INFO, confidence=Confidence.HIGH)
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def not_applicable(info: CheckInfo, reason: str) -> Finding:
|
|
68
|
+
return finding(info, Status.NOT_APPLICABLE, reason, severity=Severity.INFO, confidence=Confidence.HIGH)
|
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
"""Dynamic checks. Explicitly opt-in via execution mode.
|
|
2
|
+
|
|
3
|
+
- docker: build + run a Harbor task in the restricted DockerRunner.
|
|
4
|
+
- local-python: run pytest-style tests/ against solution/*.py in a temp copy
|
|
5
|
+
(no sandbox — trusted tasks only, recorded in the report).
|
|
6
|
+
- none (default): every dynamic check returns NOT_RUN, never PASS.
|
|
7
|
+
"""
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from envcert.checks.base import CheckContext, CheckInfo, check, finding, not_applicable, not_run, passed
|
|
11
|
+
from envcert.core.models import Severity, Status
|
|
12
|
+
from envcert.runners.docker_runner import DockerRunner
|
|
13
|
+
from envcert.runners.local_runner import LocalPythonRunner
|
|
14
|
+
|
|
15
|
+
EC060 = CheckInfo("EC060", "Reference (gold) solution passes the verifier", "dynamic", Severity.HIGH,
|
|
16
|
+
"Executing the reference solution should produce the expected (passing) reward.",
|
|
17
|
+
"Fix the reference solution or the verifier so a correct solution passes.",
|
|
18
|
+
"Only supported for docker (Harbor shell tasks) and local-python (pytest tasks) execution.", dynamic=True)
|
|
19
|
+
EC061 = CheckInfo("EC061", "Negative control (no-op/empty) is rejected by the verifier", "dynamic", Severity.HIGH,
|
|
20
|
+
"A no-op or empty submission should not pass, unless the task semantics say otherwise.",
|
|
21
|
+
"Strengthen the verifier so trivial submissions fail.",
|
|
22
|
+
"Some tasks legitimately pass with no action; then this returns NOT_APPLICABLE/WARN for human review, not a hard failure.", dynamic=True)
|
|
23
|
+
EC062 = CheckInfo("EC062", "Verifier outcome is repeatable across repetitions", "dynamic", Severity.MEDIUM,
|
|
24
|
+
"Re-running gold verification should give a consistent reward/exit outcome.",
|
|
25
|
+
"Remove nondeterminism (time, randomness, network, ordering) from the task or verifier.",
|
|
26
|
+
"Compares reward/exit outcome, not bitwise-identical stdout.", dynamic=True)
|
|
27
|
+
EC063 = CheckInfo("EC063", "Mutation testing: verifier rejects mutated (incorrect) solutions", "mutation", Severity.MEDIUM,
|
|
28
|
+
"Mutated variants of a correct Python solution should be rejected; the rejection rate is a verifier-strength signal.",
|
|
29
|
+
"Add assertions covering the mutated behaviour; do not treat the rate as proof of correctness.",
|
|
30
|
+
"Equivalent/invalid mutants and the small mutant set limit this signal. Python/pytest tasks only.", dynamic=True)
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _is_python_task(ctx) -> bool:
|
|
34
|
+
has_py_solution = any(f.startswith("solution/") and f.endswith(".py") for f in ctx.files)
|
|
35
|
+
has_py_tests = any(f.startswith("tests/") and f.endswith(".py") and "test" in f.rsplit("/", 1)[-1] for f in ctx.files)
|
|
36
|
+
return has_py_solution and has_py_tests
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _run_variant(c: CheckContext, variant: str):
|
|
40
|
+
ctx = c.task
|
|
41
|
+
if c.execution == "docker":
|
|
42
|
+
if variant != "gold":
|
|
43
|
+
return None, "docker runner currently supports the gold run only; negative controls need per-task artifact manipulation"
|
|
44
|
+
return DockerRunner().run_harbor_task(ctx.root), None
|
|
45
|
+
if c.execution == "local-python":
|
|
46
|
+
if not _is_python_task(ctx):
|
|
47
|
+
return None, "local-python execution requires solution/*.py plus tests/test_*.py (pytest-style task)"
|
|
48
|
+
return LocalPythonRunner().run_task(ctx.root, solution_variant=variant), None
|
|
49
|
+
return None, f"execution mode is '{c.execution}'"
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
@check(EC060)
|
|
53
|
+
def check_gold(c: CheckContext) -> list:
|
|
54
|
+
if c.execution == "none":
|
|
55
|
+
return [not_run(EC060, "Dynamic checks are opt-in: re-run with --execution docker or --execution local-python.")]
|
|
56
|
+
result, why = _run_variant(c, "gold")
|
|
57
|
+
if result is None:
|
|
58
|
+
return [not_applicable(EC060, f"Gold run not supported here: {why}.")]
|
|
59
|
+
if result.error and result.reward is None:
|
|
60
|
+
return [finding(EC060, Status.ERROR, f"Gold execution failed to run: {result.error}",
|
|
61
|
+
severity=Severity.INFO, evidence={"error": result.error, "runner": c.execution})]
|
|
62
|
+
if result.reward is not None:
|
|
63
|
+
if result.reward >= 1.0 or (c.execution == "local-python" and result.ok):
|
|
64
|
+
return [passed(EC060, f"Reference solution passed verification (reward={result.reward}).",
|
|
65
|
+
evidence={"reward": result.reward, "exit_code": result.exit_code, "runner": c.execution, "duration_sec": result.duration_sec})]
|
|
66
|
+
return [finding(EC060, Status.FAIL, f"Reference solution did NOT pass verification (reward={result.reward}, exit={result.exit_code}). Either the solution or the verifier is broken.",
|
|
67
|
+
evidence={"reward": result.reward, "exit_code": result.exit_code, "stderr_tail": result.stderr[-1000:]})]
|
|
68
|
+
if result.ok:
|
|
69
|
+
return [passed(EC060, "Reference solution run exited successfully.", evidence={"exit_code": result.exit_code, "runner": c.execution})]
|
|
70
|
+
return [finding(EC060, Status.FAIL, f"Reference solution run failed (exit={result.exit_code}): {result.error or result.stderr[-500:]}",
|
|
71
|
+
evidence={"exit_code": result.exit_code, "stderr_tail": result.stderr[-1000:]})]
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
@check(EC061)
|
|
75
|
+
def check_negative_control(c: CheckContext) -> list:
|
|
76
|
+
if c.execution == "none":
|
|
77
|
+
return [not_run(EC061, "Dynamic checks are opt-in: re-run with --execution docker or --execution local-python.")]
|
|
78
|
+
if c.execution == "docker":
|
|
79
|
+
return [not_applicable(EC061, "Docker negative controls are not automated in this version: a no-op's validity is task-specific and Harbor artifact transfer differs per task. Run the gold check and test a no-op manually, or use local-python for pytest tasks.")]
|
|
80
|
+
result, why = _run_variant(c, "noop")
|
|
81
|
+
if result is None:
|
|
82
|
+
return [not_applicable(EC061, f"Negative control not supported here: {why}.")]
|
|
83
|
+
if result.error and result.reward is None:
|
|
84
|
+
return [finding(EC061, Status.ERROR, f"Negative-control execution failed to run: {result.error}", severity=Severity.INFO, evidence={"error": result.error})]
|
|
85
|
+
if result.ok or (result.reward is not None and result.reward >= 1.0):
|
|
86
|
+
return [finding(EC061, Status.WARN,
|
|
87
|
+
"No-op (emptied) solution PASSED the verifier. If a no-op should not solve this task, the verifier is too weak; if no action is a legitimate solution, record that and ignore this warning.",
|
|
88
|
+
severity=Severity.HIGH, evidence={"reward": result.reward, "exit_code": result.exit_code, "variant": "noop"})]
|
|
89
|
+
return [passed(EC061, "No-op negative control was rejected by the verifier (non-passing outcome).",
|
|
90
|
+
evidence={"reward": result.reward, "exit_code": result.exit_code, "variant": "noop"})]
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
@check(EC062)
|
|
94
|
+
def check_repeatability(c: CheckContext) -> list:
|
|
95
|
+
if c.execution == "none":
|
|
96
|
+
return [not_run(EC062, "Dynamic checks are opt-in: re-run with --execution docker or --execution local-python.")]
|
|
97
|
+
if c.execution != "local-python":
|
|
98
|
+
return [not_applicable(EC062, "Repeatability repetitions are currently automated for local-python tasks only; for docker, re-run EC060 manually with different seeds/times.")]
|
|
99
|
+
if not _is_python_task(c.task):
|
|
100
|
+
return [not_applicable(EC062, "Repeatability requires a pytest-style Python task under local-python execution.")]
|
|
101
|
+
runner = LocalPythonRunner()
|
|
102
|
+
outcomes = []
|
|
103
|
+
for _ in range(max(2, c.repetitions)):
|
|
104
|
+
r = runner.run_task(c.task.root, solution_variant="gold")
|
|
105
|
+
outcomes.append({"reward": r.reward, "exit_code": r.exit_code, "error": r.error})
|
|
106
|
+
rewards = {o["reward"] for o in outcomes}
|
|
107
|
+
if len(rewards) == 1 and None not in rewards:
|
|
108
|
+
return [passed(EC062, f"Gold outcome was consistent across {len(outcomes)} runs (reward={outcomes[0]['reward']}).",
|
|
109
|
+
evidence={"runs": outcomes})]
|
|
110
|
+
return [finding(EC062, Status.WARN, f"Gold outcome varied across {len(outcomes)} runs: {outcomes}. Investigate nondeterminism.",
|
|
111
|
+
evidence={"runs": outcomes})]
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
@check(EC063)
|
|
115
|
+
def check_mutation(c: CheckContext) -> list:
|
|
116
|
+
if not c.enable_mutation:
|
|
117
|
+
return [not_run(EC063, "Mutation testing is opt-in: re-run with --mutation (and --execution local-python).")]
|
|
118
|
+
if c.execution != "local-python" or not _is_python_task(c.task):
|
|
119
|
+
return [not_applicable(EC063, "Mutation testing in this version supports pytest-style Python tasks under --execution local-python only.")]
|
|
120
|
+
from envcert.checks.mutation import run_mutation
|
|
121
|
+
|
|
122
|
+
report = run_mutation(c.task.root, max_mutants=c.max_mutants)
|
|
123
|
+
if report["eligible"] == 0:
|
|
124
|
+
return [not_applicable(EC063, f"No eligible mutants could be generated/applied: {report['notes']}")]
|
|
125
|
+
rate = report["rejection_rate"]
|
|
126
|
+
ev = {"tested": report["eligible"], "rejected": report["rejected"], "survived": report["survived"],
|
|
127
|
+
"invalid": report["invalid"], "rejection_rate": rate, "mutants": report["mutants"][:20]}
|
|
128
|
+
if rate is not None and rate < 0.5:
|
|
129
|
+
return [finding(EC063, Status.WARN, f"Mutation rejection rate is {rate:.0%} ({report['rejected']}/{report['eligible']}); most incorrect variants survive, so the verifier is weak for the mutated behaviour.",
|
|
130
|
+
evidence=ev)]
|
|
131
|
+
return [passed(EC063, f"Mutation rejection rate {rate:.0%} ({report['rejected']}/{report['eligible']} incorrect variants rejected). This is a strength signal, not proof of verifier correctness.",
|
|
132
|
+
evidence=ev)]
|
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
"""Small AST mutation framework for Python solutions (pytest tasks).
|
|
2
|
+
|
|
3
|
+
Operators: arithmetic swap (+<->-, *<->/), comparison flip (==/!=, </>=),
|
|
4
|
+
numeric constant +1, boolean flip, return-None. Each mutant is applied to a
|
|
5
|
+
temp copy and the verifier (pytest) is run; a mutant is *rejected* when the
|
|
6
|
+
verifier fails, *survived* when it still passes, *invalid* when it does not
|
|
7
|
+
parse/apply. Equivalent mutants cannot be detected automatically — the
|
|
8
|
+
report says so and the rate must not be read as proof of correctness.
|
|
9
|
+
"""
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import ast
|
|
13
|
+
import copy
|
|
14
|
+
import shutil
|
|
15
|
+
import tempfile
|
|
16
|
+
from pathlib import Path
|
|
17
|
+
|
|
18
|
+
from envcert.runners.local_runner import LocalPythonRunner
|
|
19
|
+
|
|
20
|
+
ARITH = {ast.Add: ast.Sub, ast.Sub: ast.Add, ast.Mult: ast.Div, ast.Div: ast.Mult}
|
|
21
|
+
CMP = {ast.Eq: ast.NotEq, ast.NotEq: ast.Eq, ast.Lt: ast.GtE, ast.Gt: ast.LtE, ast.LtE: ast.Gt, ast.GtE: ast.Lt}
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class _Mutator(ast.NodeTransformer):
|
|
25
|
+
def __init__(self, index: int):
|
|
26
|
+
self.index = index
|
|
27
|
+
self.count = -1
|
|
28
|
+
self.applied = ""
|
|
29
|
+
|
|
30
|
+
def _hit(self) -> bool:
|
|
31
|
+
self.count += 1
|
|
32
|
+
return self.count == self.index
|
|
33
|
+
|
|
34
|
+
def visit_BinOp(self, node):
|
|
35
|
+
self.generic_visit(node)
|
|
36
|
+
if type(node.op) in ARITH and self._hit():
|
|
37
|
+
self.applied = f"{type(node.op).__name__}->{ARITH[type(node.op)].__name__} (line {node.lineno})"
|
|
38
|
+
node.op = ARITH[type(node.op)]()
|
|
39
|
+
return node
|
|
40
|
+
|
|
41
|
+
def visit_Compare(self, node):
|
|
42
|
+
self.generic_visit(node)
|
|
43
|
+
if node.ops and type(node.ops[0]) in CMP and self._hit():
|
|
44
|
+
self.applied = f"compare {type(node.ops[0]).__name__}->{CMP[type(node.ops[0])].__name__} (line {node.lineno})"
|
|
45
|
+
node.ops[0] = CMP[type(node.ops[0])]()
|
|
46
|
+
return node
|
|
47
|
+
|
|
48
|
+
def visit_Constant(self, node):
|
|
49
|
+
if isinstance(node.value, bool):
|
|
50
|
+
if self._hit():
|
|
51
|
+
self.applied = f"bool {node.value}->{not node.value} (line {node.lineno})"
|
|
52
|
+
node.value = not node.value
|
|
53
|
+
elif isinstance(node.value, (int, float)) and node.value != 0:
|
|
54
|
+
if self._hit():
|
|
55
|
+
self.applied = f"constant {node.value}->{node.value + 1} (line {node.lineno})"
|
|
56
|
+
node.value = node.value + 1
|
|
57
|
+
return node
|
|
58
|
+
|
|
59
|
+
def visit_Return(self, node):
|
|
60
|
+
self.generic_visit(node)
|
|
61
|
+
if node.value is not None and self._hit():
|
|
62
|
+
self.applied = f"return -> None (line {node.lineno})"
|
|
63
|
+
node.value = ast.Constant(value=None)
|
|
64
|
+
return node
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def _count_sites(tree: ast.AST) -> int:
|
|
68
|
+
n = 0
|
|
69
|
+
for node in ast.walk(tree):
|
|
70
|
+
if isinstance(node, ast.BinOp) and type(node.op) in ARITH:
|
|
71
|
+
n += 1
|
|
72
|
+
elif isinstance(node, ast.Compare) and node.ops and type(node.ops[0]) in CMP:
|
|
73
|
+
n += 1
|
|
74
|
+
elif isinstance(node, ast.Constant) and (isinstance(node.value, bool) or (isinstance(node.value, (int, float)) and node.value != 0)):
|
|
75
|
+
n += 1
|
|
76
|
+
elif isinstance(node, ast.Return) and node.value is not None:
|
|
77
|
+
n += 1
|
|
78
|
+
return n
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def run_mutation(task_root: Path, max_mutants: int = 10, timeout: int = 60) -> dict:
|
|
82
|
+
task_root = Path(task_root)
|
|
83
|
+
runner = LocalPythonRunner()
|
|
84
|
+
mutants: list[dict] = []
|
|
85
|
+
eligible = rejected = survived = invalid = 0
|
|
86
|
+
notes = ""
|
|
87
|
+
sol_files = sorted((task_root / "solution").rglob("*.py")) if (task_root / "solution").is_dir() else []
|
|
88
|
+
if not sol_files:
|
|
89
|
+
return {"eligible": 0, "rejected": 0, "survived": 0, "invalid": 0, "rejection_rate": None,
|
|
90
|
+
"mutants": [], "notes": "no solution/*.py files"}
|
|
91
|
+
# gold must pass first, otherwise mutants are meaningless
|
|
92
|
+
gold = runner.run_task(task_root, "gold", timeout=timeout)
|
|
93
|
+
if not gold.ok:
|
|
94
|
+
return {"eligible": 0, "rejected": 0, "survived": 0, "invalid": 0, "rejection_rate": None,
|
|
95
|
+
"mutants": [], "notes": "gold solution does not pass; mutation skipped"}
|
|
96
|
+
budget = max_mutants
|
|
97
|
+
for sol in sol_files:
|
|
98
|
+
if budget <= 0:
|
|
99
|
+
break
|
|
100
|
+
source = sol.read_text(encoding="utf-8", errors="replace")
|
|
101
|
+
try:
|
|
102
|
+
tree = ast.parse(source)
|
|
103
|
+
except SyntaxError:
|
|
104
|
+
invalid += 1
|
|
105
|
+
continue
|
|
106
|
+
sites = _count_sites(tree)
|
|
107
|
+
for idx in range(sites):
|
|
108
|
+
if budget <= 0:
|
|
109
|
+
break
|
|
110
|
+
mut = _Mutator(idx)
|
|
111
|
+
new_tree = mut.visit(copy.deepcopy(tree))
|
|
112
|
+
if not mut.applied:
|
|
113
|
+
continue
|
|
114
|
+
budget -= 1
|
|
115
|
+
try:
|
|
116
|
+
mutated = ast.unparse(ast.fix_missing_locations(new_tree))
|
|
117
|
+
ast.parse(mutated)
|
|
118
|
+
except Exception: # noqa: BLE001
|
|
119
|
+
invalid += 1
|
|
120
|
+
mutants.append({"file": sol.name, "change": mut.applied, "outcome": "invalid"})
|
|
121
|
+
continue
|
|
122
|
+
eligible += 1
|
|
123
|
+
with tempfile.TemporaryDirectory(prefix="envcert-mut-") as td:
|
|
124
|
+
ws = Path(td) / "ws"
|
|
125
|
+
runner.prepare_workspace(task_root, ws, "gold")
|
|
126
|
+
target = ws / sol.name
|
|
127
|
+
# gold workspace copies solution files flat; overwrite the mutated one
|
|
128
|
+
if not target.exists(): # nested layout fallback
|
|
129
|
+
shutil.copytree(task_root / "solution", ws / "solution", dirs_exist_ok=True)
|
|
130
|
+
target = ws / "solution" / sol.name
|
|
131
|
+
target.write_text(mutated, encoding="utf-8")
|
|
132
|
+
result = runner.run_python_verifier(ws, timeout=timeout)
|
|
133
|
+
outcome = "survived" if result.ok else "rejected"
|
|
134
|
+
if outcome == "rejected":
|
|
135
|
+
rejected += 1
|
|
136
|
+
else:
|
|
137
|
+
survived += 1
|
|
138
|
+
mutants.append({"file": sol.name, "change": mut.applied, "outcome": outcome})
|
|
139
|
+
rate = (rejected / eligible) if eligible else None
|
|
140
|
+
return {"eligible": eligible, "rejected": rejected, "survived": survived, "invalid": invalid,
|
|
141
|
+
"rejection_rate": rate, "mutants": mutants,
|
|
142
|
+
"notes": notes or "equivalent mutants may inflate the survived count; interpret as a signal only"}
|