envcert 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- envcert/__init__.py +5 -0
- envcert/__main__.py +6 -0
- envcert/adapters/__init__.py +0 -0
- envcert/adapters/base.py +67 -0
- envcert/adapters/generic.py +40 -0
- envcert/adapters/harbor.py +96 -0
- envcert/attestations/__init__.py +0 -0
- envcert/attestations/intoto.py +87 -0
- envcert/checks/__init__.py +0 -0
- envcert/checks/base.py +68 -0
- envcert/checks/dynamic_checks.py +132 -0
- envcert/checks/mutation.py +142 -0
- envcert/checks/static_checks.py +796 -0
- envcert/cli/__init__.py +0 -0
- envcert/cli/main.py +269 -0
- envcert/core/__init__.py +0 -0
- envcert/core/config.py +139 -0
- envcert/core/digests.py +45 -0
- envcert/core/engine.py +141 -0
- envcert/core/errors.py +10 -0
- envcert/core/models.py +72 -0
- envcert/reports/__init__.py +0 -0
- envcert/reports/sarif.py +51 -0
- envcert/reports/text.py +35 -0
- envcert/runners/__init__.py +0 -0
- envcert/runners/base.py +32 -0
- envcert/runners/docker_runner.py +95 -0
- envcert/runners/local_runner.py +79 -0
- envcert/schemas/__init__.py +0 -0
- envcert/schemas/report.schema.json +65 -0
- envcert/utils/__init__.py +0 -0
- envcert/utils/dockerfile.py +105 -0
- envcert/utils/files.py +86 -0
- envcert-0.1.0.dist-info/METADATA +204 -0
- envcert-0.1.0.dist-info/RECORD +38 -0
- envcert-0.1.0.dist-info/WHEEL +4 -0
- envcert-0.1.0.dist-info/entry_points.txt +2 -0
- envcert-0.1.0.dist-info/licenses/LICENSE +187 -0
|
@@ -0,0 +1,796 @@
|
|
|
1
|
+
"""Static checks. Every check returns at least one Finding; PASS is explicit.
|
|
2
|
+
|
|
3
|
+
Design rules enforced here (from the product spec):
|
|
4
|
+
- Do not flag a solution merely because it exists in the source tree; decide
|
|
5
|
+
whether it is plausibly inside the Docker build context / agent-visible.
|
|
6
|
+
- Do not flag conftest.py / sitecustomize.py merely for existing; decide
|
|
7
|
+
whether it sits on an import/execution path the verifier plausibly uses.
|
|
8
|
+
- Where runtime visibility cannot be proven statically, say so (WARN,
|
|
9
|
+
medium/low confidence) instead of claiming a confirmed leak.
|
|
10
|
+
"""
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import ast
|
|
14
|
+
import re
|
|
15
|
+
|
|
16
|
+
from envcert.adapters import harbor as harbor_adapter
|
|
17
|
+
from envcert.checks.base import CheckContext, CheckInfo, check, finding, not_applicable, passed
|
|
18
|
+
from envcert.core.models import Confidence, Severity, Status
|
|
19
|
+
from envcert.utils.dockerfile import copy_sources, image_ref_pinned
|
|
20
|
+
from envcert.utils.files import dockerignore_match
|
|
21
|
+
|
|
22
|
+
# ---------------------------------------------------------------- structure
|
|
23
|
+
|
|
24
|
+
EC001 = CheckInfo("EC001", "Task structure is recognisable and complete", "structure", Severity.HIGH,
|
|
25
|
+
"Task has the files its framework requires (config, instruction, verifier, environment).",
|
|
26
|
+
"Add the missing task.toml / instruction.md / tests / environment files for your framework.",
|
|
27
|
+
"Only Harbor and generic Docker layouts are recognised in this version.")
|
|
28
|
+
EC002 = CheckInfo("EC002", "Instruction is present and non-trivial", "structure", Severity.MEDIUM,
|
|
29
|
+
"instruction.md exists, is non-empty and contains no unresolved placeholders.",
|
|
30
|
+
"Write a complete instruction.md without TODO/placeholder text.")
|
|
31
|
+
EC003 = CheckInfo("EC003", "Reference solution is present (or its absence is explicit)", "structure", Severity.LOW,
|
|
32
|
+
"A reference/oracle solution exists so the verifier can be sanity-checked.",
|
|
33
|
+
"Add solution/solve.sh (Harbor) or document why no reference solution exists.",
|
|
34
|
+
"Some frameworks source solutions externally; then this is informational.")
|
|
35
|
+
|
|
36
|
+
# ---------------------------------------------------------------- leakage
|
|
37
|
+
|
|
38
|
+
EC010 = CheckInfo("EC010", "No exposed version-control metadata in agent-visible paths", "leakage", Severity.HIGH,
|
|
39
|
+
".git data inside the build context or environment can leak hidden reference material and history.",
|
|
40
|
+
"Exclude .git via .dockerignore, do not COPY it into the image, and strip VCS data from published images.",
|
|
41
|
+
"Static analysis cannot prove the agent can read the path at runtime; exposure is assessed from the build context and COPY instructions.")
|
|
42
|
+
EC011 = CheckInfo("EC011", "Reference solution is not copied into the agent environment", "leakage", Severity.CRITICAL,
|
|
43
|
+
"Solution/answer material must not be included in the Docker build context or copied into agent-visible paths.",
|
|
44
|
+
"Keep solution/ outside the build context, exclude it in .dockerignore, and never COPY solution material into the image.",
|
|
45
|
+
"Runtime mounts configured outside this task directory cannot be seen statically.")
|
|
46
|
+
EC012 = CheckInfo("EC012", "No known answer / gold files in agent-visible paths", "leakage", Severity.HIGH,
|
|
47
|
+
"Files with answer-like names (gold patch, expected output, answers) inside the environment can be read by the agent.",
|
|
48
|
+
"Move expected outputs and gold patches into tests/ (uploaded only at verification time) or remove them.",
|
|
49
|
+
"Name-based heuristic; a file with an answer-like name may be an innocent fixture.")
|
|
50
|
+
EC013 = CheckInfo("EC013", "No hard-coded secrets or credentials in task material", "leakage", Severity.HIGH,
|
|
51
|
+
"Hard-coded API keys/tokens/passwords in task files can leak and indicate exposed evaluation data.",
|
|
52
|
+
"Use ${VAR} host-environment references (Harbor) or CI secrets; rotate any committed credential.",
|
|
53
|
+
"Pattern-based; example/dummy values are flagged for human review, not asserted to be live credentials.")
|
|
54
|
+
EC014 = CheckInfo("EC014", "Instruction does not disclose solution or hidden-test content", "leakage", Severity.MEDIUM,
|
|
55
|
+
"The instruction given to the agent must not contain the reference solution or hidden expected outputs.",
|
|
56
|
+
"Remove solution code / expected outputs from instruction.md; describe the task, not the answer.",
|
|
57
|
+
"Heuristic overlap detection; short common tokens are ignored.")
|
|
58
|
+
|
|
59
|
+
# ---------------------------------------------------------------- boundary
|
|
60
|
+
|
|
61
|
+
EC020 = CheckInfo("EC020", "Verifier runs separated from the agent where the framework supports it", "boundary", Severity.MEDIUM,
|
|
62
|
+
"Shared agent/verifier execution lets agent state influence verification. Harbor supports verifier.environment_mode='separate'.",
|
|
63
|
+
"Set [verifier] environment_mode = \"separate\" (with declared artifacts), or document why shared verification is required.",
|
|
64
|
+
"Some tasks legitimately need shared state (live services); this is a warning, not proof of exploitability.")
|
|
65
|
+
EC021 = CheckInfo("EC021", "No agent-controlled Python import shadowing on verifier paths", "boundary", Severity.HIGH,
|
|
66
|
+
"sitecustomize/usercustomize/conftest or shadow modules in agent-writable, verifier-imported paths can alter verifier behaviour.",
|
|
67
|
+
"Keep import-shadowing files out of the agent environment; run the verifier with a clean PYTHONPATH and isolated rootdir.",
|
|
68
|
+
"Whether pytest/Python actually imports the file depends on runtime cwd/PYTHONPATH, which is assessed heuristically.")
|
|
69
|
+
EC022 = CheckInfo("EC022", "No executable shadowing of verifier tooling", "boundary", Severity.HIGH,
|
|
70
|
+
"Agent-placed executables named like verifier tools (pytest, python, test.sh) earlier on PATH can replace the real tool.",
|
|
71
|
+
"Do not ship tool-named executables in the agent workdir; invoke verifiers by absolute path.",
|
|
72
|
+
"PATH order at verification time is inferred from the Dockerfile/task config.")
|
|
73
|
+
EC023 = CheckInfo("EC023", "Verifier/test directories are not made world-writable in the image", "boundary", Severity.HIGH,
|
|
74
|
+
"chmod 777 / a+w on verifier, test or system paths lets the agent tamper with verification.",
|
|
75
|
+
"Use least-privilege permissions; make /tests read-only where the framework allows.",
|
|
76
|
+
"Runtime framework uploads may reset permissions; this checks what the task itself declares.")
|
|
77
|
+
EC024 = CheckInfo("EC024", "No shared writable mounts or Docker socket exposure", "boundary", Severity.CRITICAL,
|
|
78
|
+
"Host mounts, docker.sock or privileged settings in compose/task config collapse the agent/host boundary.",
|
|
79
|
+
"Remove docker.sock and host mounts; never run task containers privileged.",
|
|
80
|
+
"Only mounts declared inside this task directory are visible.")
|
|
81
|
+
|
|
82
|
+
# ---------------------------------------------------------------- docker
|
|
83
|
+
|
|
84
|
+
EC030 = CheckInfo("EC030", "Base images are pinned to a digest or specific tag", "docker", Severity.MEDIUM,
|
|
85
|
+
"Floating tags (latest / untagged) make builds non-reproducible and silently change the environment.",
|
|
86
|
+
"Pin base images as name:tag@sha256:<digest>.",
|
|
87
|
+
"Tag pinning without a digest is reported as a warning-level reproducibility issue, not a vulnerability.")
|
|
88
|
+
EC031 = CheckInfo("EC031", "Container does not default to root without justification", "docker", Severity.LOW,
|
|
89
|
+
"Running as root widens the impact of agent code; a non-root USER is defence in depth.",
|
|
90
|
+
"Add a non-root USER for the agent phase where the task allows it.",
|
|
91
|
+
"Many benchmark tasks legitimately need root; classified as hardening, not benchmark-integrity failure.")
|
|
92
|
+
EC032 = CheckInfo("EC032", "No dangerous Dockerfile patterns (remote ADD, curl|sh, privileged hints)", "docker", Severity.HIGH,
|
|
93
|
+
"Remote ADD and pipe-to-shell installs execute unreviewed remote code at build time.",
|
|
94
|
+
"Download with a pinned URL + checksum verification, or vendor the artifact.",
|
|
95
|
+
"Build-time network content cannot be inspected statically.")
|
|
96
|
+
EC033 = CheckInfo("EC033", "Network policy is declared and not silently unrestricted", "docker", Severity.MEDIUM,
|
|
97
|
+
"Harbor defaults to public network when network_mode is omitted; unrestricted egress enables answer retrieval.",
|
|
98
|
+
"Set [environment] network_mode = \"no-network\" or \"allowlist\" with allowed_hosts, unless the task truly needs public access.",
|
|
99
|
+
"A task may legitimately need network access; the finding reports the declared policy and its risk.")
|
|
100
|
+
|
|
101
|
+
# ---------------------------------------------------------------- dependencies
|
|
102
|
+
|
|
103
|
+
EC040 = CheckInfo("EC040", "Python dependencies are pinned / locked", "dependencies", Severity.MEDIUM,
|
|
104
|
+
"Unpinned pip installs make the environment non-reproducible and vulnerable to silent upstream changes.",
|
|
105
|
+
"Pin versions with == (or hashes) and commit a lockfile (uv.lock / poetry.lock / Pipfile.lock).",
|
|
106
|
+
"A missing lockfile is a reproducibility signal, not a confirmed vulnerability.")
|
|
107
|
+
EC041 = CheckInfo("EC041", "External downloads at build/run time are pinned or checksummed", "dependencies", Severity.MEDIUM,
|
|
108
|
+
"Unversioned downloads (git clone without commit, unversioned URLs) can change the environment silently.",
|
|
109
|
+
"Pin commits/versions and verify checksums for downloaded artifacts.")
|
|
110
|
+
|
|
111
|
+
# ---------------------------------------------------------------- verifier
|
|
112
|
+
|
|
113
|
+
EC050 = CheckInfo("EC050", "Verifier exists and is non-empty", "verifier", Severity.CRITICAL,
|
|
114
|
+
"A missing or empty verifier cannot distinguish correct from incorrect work.",
|
|
115
|
+
"Provide tests/test.sh (Harbor) that runs real assertions and writes a reward.")
|
|
116
|
+
EC051 = CheckInfo("EC051", "Verifier produces a reward in the framework's expected location", "verifier", Severity.HIGH,
|
|
117
|
+
"Harbor verifiers must write /logs/verifier/reward.txt or reward.json; otherwise every run scores 0 / is masked.",
|
|
118
|
+
"Write the numeric reward to /logs/verifier/reward.txt (or reward.json) on every code path.",
|
|
119
|
+
"Dynamic reward production is only proven by execution; this checks the code mentions the contract.")
|
|
120
|
+
EC052 = CheckInfo("EC052", "No unconditional-success path in the verifier", "verifier", Severity.CRITICAL,
|
|
121
|
+
"A verifier that always reports success (echo 1 without conditions, assert True only, sys.exit(0)) accepts any submission.",
|
|
122
|
+
"Gate the success reward on real assertions; ensure failures write reward 0.",
|
|
123
|
+
"AST/text heuristics cannot prove semantic correctness; flagged patterns need human confirmation.")
|
|
124
|
+
EC053 = CheckInfo("EC053", "Verifier contains substantive assertions", "verifier", Severity.HIGH,
|
|
125
|
+
"Verifiers that only check file existence or trivial substrings accept incorrect solutions (see BenchJack V-class flaws).",
|
|
126
|
+
"Assert on computed outputs/behaviour, include negative cases, and compare against expected values.",
|
|
127
|
+
"Assertion counting is a strength signal, not proof the assertions are semantically correct.")
|
|
128
|
+
EC054 = CheckInfo("EC054", "Verifier does not trust agent-controlled result files without validation", "verifier", Severity.HIGH,
|
|
129
|
+
"Reading a score/reward the agent itself wrote, or parsing agent stdout loosely, lets the agent report its own grade.",
|
|
130
|
+
"Compute the reward inside the verifier from raw artifacts; validate any agent-produced file strictly.",
|
|
131
|
+
"Heuristic; some tasks legitimately read agent output files — they must validate content, which needs human review.")
|
|
132
|
+
|
|
133
|
+
ALL_STATIC = [EC001, EC002, EC003, EC010, EC011, EC012, EC013, EC014, EC020, EC021, EC022, EC023, EC024,
|
|
134
|
+
EC030, EC031, EC032, EC033, EC040, EC041, EC050, EC051, EC052, EC053, EC054]
|
|
135
|
+
|
|
136
|
+
SECRET_PATTERNS = [
|
|
137
|
+
(re.compile(r"(?i)(api[_-]?key|secret|token|password|passwd)\s*[:=]\s*['\"][^'\"${}\s]{8,}['\"]"), "credential assignment"),
|
|
138
|
+
(re.compile(r"sk-[A-Za-z0-9]{20,}"), "OpenAI-style key"),
|
|
139
|
+
(re.compile(r"ghp_[A-Za-z0-9]{30,}"), "GitHub token"),
|
|
140
|
+
(re.compile(r"AKIA[0-9A-Z]{16}"), "AWS access key id"),
|
|
141
|
+
(re.compile(r"-----BEGIN (RSA |EC |OPENSSH )?PRIVATE KEY-----"), "private key"),
|
|
142
|
+
]
|
|
143
|
+
ANSWER_NAMES = re.compile(r"(?i)(gold|answer|answers|expected[_-]?output|solution|oracle|ground[_-]?truth|reference[_-]?patch)")
|
|
144
|
+
SHADOW_MODULES = {"sitecustomize.py", "usercustomize.py"}
|
|
145
|
+
TOOL_NAMES = {"pytest", "pytest.exe", "python", "python3", "python.exe", "pip", "pip3", "test.sh", "bash", "sh"}
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def _verifier_files(ctx) -> list[str]:
|
|
149
|
+
out: list = []
|
|
150
|
+
for f in ctx.files:
|
|
151
|
+
if f.startswith("tests/") or "/tests/" in f:
|
|
152
|
+
out.append(f)
|
|
153
|
+
return out
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def _verifier_texts(ctx):
|
|
157
|
+
for f in _verifier_files(ctx):
|
|
158
|
+
t = ctx.text(f)
|
|
159
|
+
if t is not None:
|
|
160
|
+
yield f, t
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
# ---------------------------------------------------------------- checks
|
|
164
|
+
|
|
165
|
+
@check(EC001)
|
|
166
|
+
def check_structure(c: CheckContext) -> list:
|
|
167
|
+
ctx = c.task
|
|
168
|
+
missing = []
|
|
169
|
+
if ctx.adapter == "harbor":
|
|
170
|
+
if not ctx.config_path:
|
|
171
|
+
missing.append("task.toml")
|
|
172
|
+
if not ctx.instruction_path and not ctx.has("steps"):
|
|
173
|
+
missing.append("instruction.md")
|
|
174
|
+
entries = harbor_adapter.verifier_entrypoints(ctx)
|
|
175
|
+
if not entries:
|
|
176
|
+
missing.append("tests/test.sh (or test.bat/verifier.py)")
|
|
177
|
+
env_cfg = (ctx.config.get("environment") or {}) if isinstance(ctx.config.get("environment"), dict) else {}
|
|
178
|
+
if not ctx.environment_dir and not env_cfg.get("docker_image"):
|
|
179
|
+
missing.append("environment/ (or [environment].docker_image)")
|
|
180
|
+
else:
|
|
181
|
+
if not ctx.tests_dir and not ctx.solution_dir:
|
|
182
|
+
missing.append("tests/ or solution/")
|
|
183
|
+
if not ctx.dockerfile_path and not ctx.environment_dir:
|
|
184
|
+
missing.append("Dockerfile / environment/")
|
|
185
|
+
if missing:
|
|
186
|
+
return [finding(EC001, Status.FAIL, f"Missing required task components: {', '.join(missing)}",
|
|
187
|
+
evidence={"missing": missing, "adapter": ctx.adapter})]
|
|
188
|
+
return [passed(EC001, f"Recognised {ctx.adapter} task '{ctx.task_name}' with required components present.",
|
|
189
|
+
evidence={"adapter": ctx.adapter, "files": len(ctx.files)})]
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
@check(EC002)
|
|
193
|
+
def check_instruction(c: CheckContext) -> list:
|
|
194
|
+
ctx = c.task
|
|
195
|
+
if not ctx.instruction_path:
|
|
196
|
+
return [not_applicable(EC002, "No top-level instruction.md (multi-step or generic layout); per-step instructions not audited.")]
|
|
197
|
+
text = ctx.text("instruction.md") or ""
|
|
198
|
+
issues = []
|
|
199
|
+
if len(text.strip()) < 40:
|
|
200
|
+
issues.append("instruction is shorter than 40 characters")
|
|
201
|
+
if re.search(r"(?i)\b(TODO|TBD|Lorem ipsum|placeholder)\b", text):
|
|
202
|
+
issues.append("instruction contains placeholder text")
|
|
203
|
+
if issues:
|
|
204
|
+
return [finding(EC002, Status.WARN, "; ".join(issues), path="instruction.md", evidence={"issues": issues})]
|
|
205
|
+
return [passed(EC002, f"instruction.md present ({len(text)} chars, no placeholders).", path="instruction.md")]
|
|
206
|
+
|
|
207
|
+
|
|
208
|
+
@check(EC003)
|
|
209
|
+
def check_solution_present(c: CheckContext) -> list:
|
|
210
|
+
ctx = c.task
|
|
211
|
+
entries = harbor_adapter.solution_entrypoints(ctx) if ctx.adapter == "harbor" else (
|
|
212
|
+
[f for f in ctx.files if f.startswith("solution/")])
|
|
213
|
+
if entries:
|
|
214
|
+
return [passed(EC003, f"Reference solution present: {', '.join(entries[:5])}", evidence={"entrypoints": entries})]
|
|
215
|
+
return [finding(EC003, Status.WARN, "No reference solution found; verifier cannot be sanity-checked with a gold run.",
|
|
216
|
+
severity=Severity.LOW, evidence={"searched": "solution/"})]
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
@check(EC010)
|
|
220
|
+
def check_git_exposed(c: CheckContext) -> list:
|
|
221
|
+
ctx = c.task
|
|
222
|
+
# Files under .git are not enumerated by our walker if huge; also check dirs via known markers
|
|
223
|
+
markers = [f for f in ctx.files if ".git/" in f or "/.git" in f or f.startswith(".git") or "/.git" in ("/" + f)]
|
|
224
|
+
if ctx.build_context is not None:
|
|
225
|
+
ctx_rel = ctx.build_context.relative_to(ctx.root).as_posix() if ctx.build_context != ctx.root else ""
|
|
226
|
+
in_context = [f for f in markers if (ctx_rel == "" or f.startswith(ctx_rel + "/") or f == ctx_rel)]
|
|
227
|
+
else:
|
|
228
|
+
in_context = markers
|
|
229
|
+
ctx_prefix = ""
|
|
230
|
+
if ctx.build_context is not None and ctx.build_context != ctx.root:
|
|
231
|
+
ctx_prefix = ctx.build_context.relative_to(ctx.root).as_posix() + "/"
|
|
232
|
+
ignored = [f for f in in_context if dockerignore_match(f[len(ctx_prefix):] if ctx_prefix and f.startswith(ctx_prefix) else f, ctx.dockerignore_patterns)]
|
|
233
|
+
exposed = [f for f in in_context if f not in ignored]
|
|
234
|
+
if exposed:
|
|
235
|
+
return [finding(EC010, Status.FAIL,
|
|
236
|
+
f".git metadata is inside the Docker build context and not excluded by .dockerignore ({len(exposed)} files, e.g. {exposed[0]}). Git history may contain reference material.",
|
|
237
|
+
path=exposed[0], evidence={"examples": exposed[:10], "dockerignore_checked": bool(ctx.dockerignore_patterns)},
|
|
238
|
+
confidence=Confidence.MEDIUM)]
|
|
239
|
+
if markers and ignored:
|
|
240
|
+
return [passed(EC010, ".git exists in the source tree but is excluded from the build context by .dockerignore.",
|
|
241
|
+
evidence={"excluded_examples": ignored[:5]})]
|
|
242
|
+
if (ctx.root / ".git").exists():
|
|
243
|
+
return [passed(EC010, "A .git directory exists at the task root but no .git files are inside the build context; agent exposure not established statically.",
|
|
244
|
+
evidence={"note": "root .git outside build context"})]
|
|
245
|
+
return [passed(EC010, "No .git metadata found in the task bundle.")]
|
|
246
|
+
|
|
247
|
+
|
|
248
|
+
def _copied_into_image(ctx) -> list[dict]:
|
|
249
|
+
"""Analyse COPY/ADD: return list of {source, dest, line, instr}."""
|
|
250
|
+
out: list = []
|
|
251
|
+
if ctx.dockerfile is None:
|
|
252
|
+
return out
|
|
253
|
+
for instr in ctx.dockerfile.find("COPY", "ADD"):
|
|
254
|
+
sources, dest = copy_sources(instr)
|
|
255
|
+
for s in sources:
|
|
256
|
+
out.append({"source": s, "dest": dest or "", "line": instr.line, "instr": instr.name})
|
|
257
|
+
return out
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
def _context_rel(ctx, path_in_context: str) -> str:
|
|
261
|
+
ctx_prefix = ""
|
|
262
|
+
if ctx.build_context is not None and ctx.build_context != ctx.root:
|
|
263
|
+
ctx_prefix = ctx.build_context.relative_to(ctx.root).as_posix() + "/"
|
|
264
|
+
return ctx_prefix + path_in_context.strip("./")
|
|
265
|
+
|
|
266
|
+
|
|
267
|
+
@check(EC011)
|
|
268
|
+
def check_solution_in_image(c: CheckContext) -> list:
|
|
269
|
+
ctx = c.task
|
|
270
|
+
findings = []
|
|
271
|
+
copies = _copied_into_image(ctx)
|
|
272
|
+
ctx_prefix = ""
|
|
273
|
+
if ctx.build_context is not None and ctx.build_context != ctx.root:
|
|
274
|
+
ctx_prefix = ctx.build_context.relative_to(ctx.root).as_posix() + "/"
|
|
275
|
+
# 1. solution-like files physically inside the build context
|
|
276
|
+
in_context_solutions = []
|
|
277
|
+
for f in ctx.files:
|
|
278
|
+
if ctx_prefix and not f.startswith(ctx_prefix):
|
|
279
|
+
continue
|
|
280
|
+
rel_in_ctx = f[len(ctx_prefix):] if ctx_prefix else f
|
|
281
|
+
base = f.rsplit("/", 1)[-1]
|
|
282
|
+
if ("solution" in f.split("/") or ANSWER_NAMES.search(base)) and not f.startswith("tests/"):
|
|
283
|
+
if not dockerignore_match(rel_in_ctx, ctx.dockerignore_patterns):
|
|
284
|
+
in_context_solutions.append(f)
|
|
285
|
+
broad_copies = [cp for cp in copies if cp["source"] in (".", "./", "*", "./*", "")]
|
|
286
|
+
if in_context_solutions and (broad_copies or copies):
|
|
287
|
+
cp = (broad_copies or copies)[0]
|
|
288
|
+
findings.append(finding(EC011, Status.FAIL,
|
|
289
|
+
f"Reference/answer material inside the build context ({in_context_solutions[0]}) is plausibly copied into the image by '{cp['instr']} {cp['source']}' (Dockerfile line {cp['line']}).",
|
|
290
|
+
path="environment/Dockerfile" if ctx.dockerfile_path else None, line=cp["line"],
|
|
291
|
+
evidence={"files": in_context_solutions[:10], "copy": cp}, confidence=Confidence.MEDIUM))
|
|
292
|
+
elif in_context_solutions:
|
|
293
|
+
findings.append(finding(EC011, Status.WARN,
|
|
294
|
+
f"Answer-like files exist inside the build context ({in_context_solutions[0]}) but no COPY/ADD clearly includes them; verify they are not baked into the image.",
|
|
295
|
+
evidence={"files": in_context_solutions[:10]}, confidence=Confidence.LOW))
|
|
296
|
+
# 2. explicit COPY of solution paths
|
|
297
|
+
for cp in copies:
|
|
298
|
+
src = cp["source"].lstrip("./")
|
|
299
|
+
if "solution" in src.split("/") or ANSWER_NAMES.search(src):
|
|
300
|
+
findings.append(finding(EC011, Status.FAIL,
|
|
301
|
+
f"Dockerfile explicitly copies solution/answer material into the image: '{cp['instr']} {cp['source']} {cp['dest']}' (line {cp['line']}).",
|
|
302
|
+
line=cp["line"], evidence={"copy": cp}, confidence=Confidence.HIGH))
|
|
303
|
+
if src.startswith("../"):
|
|
304
|
+
findings.append(finding(EC011, Status.WARN,
|
|
305
|
+
f"COPY source '{cp['source']}' escapes the build context (line {cp['line']}); Docker rejects this, but it signals an attempt to include external (possibly solution) material.",
|
|
306
|
+
line=cp["line"], evidence={"copy": cp}, confidence=Confidence.MEDIUM))
|
|
307
|
+
if findings:
|
|
308
|
+
return findings
|
|
309
|
+
sol = harbor_adapter.solution_entrypoints(ctx) if ctx.adapter == "harbor" else [f for f in ctx.files if f.startswith("solution/")]
|
|
310
|
+
return [passed(EC011, "Reference solution exists outside the build context and no COPY/ADD includes solution material in the image."
|
|
311
|
+
if sol else "No solution material found in the build context or COPY/ADD instructions.",
|
|
312
|
+
evidence={"solution_entrypoints": sol, "copies_analysed": len(copies)})]
|
|
313
|
+
|
|
314
|
+
|
|
315
|
+
@check(EC012)
|
|
316
|
+
def check_answer_files_visible(c: CheckContext) -> list:
|
|
317
|
+
ctx = c.task
|
|
318
|
+
hits: list = []
|
|
319
|
+
agent_areas = ("environment/",) if ctx.environment_dir else ()
|
|
320
|
+
for f in ctx.files:
|
|
321
|
+
if f.startswith("tests/") or "/tests/" in f or f.startswith("solution/"):
|
|
322
|
+
continue
|
|
323
|
+
if agent_areas and not any(f.startswith(a) for a in agent_areas) and ctx.adapter == "harbor":
|
|
324
|
+
continue
|
|
325
|
+
base = f.rsplit("/", 1)[-1]
|
|
326
|
+
if ANSWER_NAMES.search(base) and base not in ("instruction.md",):
|
|
327
|
+
hits.append(f)
|
|
328
|
+
if hits:
|
|
329
|
+
return [finding(EC012, Status.WARN,
|
|
330
|
+
f"Answer-like filenames in agent-visible areas: {', '.join(hits[:5])}. If these contain expected outputs/gold material the agent can read them.",
|
|
331
|
+
path=hits[0], evidence={"files": hits[:20]}, confidence=Confidence.LOW)]
|
|
332
|
+
return [passed(EC012, "No answer/gold/expected-output filenames in agent-visible areas (outside tests/ and solution/).")]
|
|
333
|
+
|
|
334
|
+
|
|
335
|
+
@check(EC013)
|
|
336
|
+
def check_secrets(c: CheckContext) -> list:
|
|
337
|
+
ctx = c.task
|
|
338
|
+
hits: list = []
|
|
339
|
+
skip_ext = (".png", ".jpg", ".gif", ".pdf", ".zip", ".lock")
|
|
340
|
+
for f in ctx.files:
|
|
341
|
+
if f.endswith(skip_ext):
|
|
342
|
+
continue
|
|
343
|
+
text = ctx.text(f)
|
|
344
|
+
if not text:
|
|
345
|
+
continue
|
|
346
|
+
for i, line in enumerate(text.splitlines(), 1):
|
|
347
|
+
for pat, label in SECRET_PATTERNS:
|
|
348
|
+
if pat.search(line):
|
|
349
|
+
if "${" in line: # host-env reference, not a literal secret
|
|
350
|
+
continue
|
|
351
|
+
hits.append({"path": f, "line": i, "kind": label})
|
|
352
|
+
break
|
|
353
|
+
if hits:
|
|
354
|
+
h = hits[0]
|
|
355
|
+
sev = Severity.HIGH if h["kind"] != "credential assignment" else Severity.MEDIUM
|
|
356
|
+
return [finding(EC013, Status.WARN, f"Possible hard-coded credential ({h['kind']}) at {h['path']}:{h['line']} plus {len(hits)-1} more; review and rotate if real.",
|
|
357
|
+
severity=sev, path=str(h["path"]), line=int(h["line"]), evidence={"hits": hits[:20]}, confidence=Confidence.LOW)]
|
|
358
|
+
return [passed(EC013, "No hard-coded credential patterns found in text files.")]
|
|
359
|
+
|
|
360
|
+
|
|
361
|
+
@check(EC014)
|
|
362
|
+
def check_instruction_leak(c: CheckContext) -> list:
|
|
363
|
+
ctx = c.task
|
|
364
|
+
if not ctx.instruction_path:
|
|
365
|
+
return [not_applicable(EC014, "No instruction.md to compare against the solution.")]
|
|
366
|
+
instr = ctx.text("instruction.md") or ""
|
|
367
|
+
sol_files = [f for f in ctx.files if f.startswith("solution/") or "/solution/" in f]
|
|
368
|
+
leaked = []
|
|
369
|
+
for f in sol_files:
|
|
370
|
+
text = ctx.text(f)
|
|
371
|
+
if not text:
|
|
372
|
+
continue
|
|
373
|
+
for raw in text.splitlines():
|
|
374
|
+
line = raw.strip()
|
|
375
|
+
if len(line) >= 40 and line in instr and not line.startswith("#"):
|
|
376
|
+
leaked.append({"solution_file": f, "line": line[:120]})
|
|
377
|
+
break
|
|
378
|
+
if leaked:
|
|
379
|
+
return [finding(EC014, Status.WARN, "Instruction contains verbatim long lines from the reference solution; the answer may be disclosed to the agent.",
|
|
380
|
+
path="instruction.md", evidence={"matches": leaked[:5]}, confidence=Confidence.MEDIUM)]
|
|
381
|
+
return [passed(EC014, "No verbatim solution lines (>=40 chars) found in instruction.md.")]
|
|
382
|
+
|
|
383
|
+
|
|
384
|
+
@check(EC020)
|
|
385
|
+
def check_verifier_separation(c: CheckContext) -> list:
|
|
386
|
+
ctx = c.task
|
|
387
|
+
if ctx.adapter != "harbor":
|
|
388
|
+
return [not_applicable(EC020, "Verifier separation mode is a Harbor task.toml concept; generic adapter cannot assess it.")]
|
|
389
|
+
verifier = ctx.config.get("verifier") or {}
|
|
390
|
+
mode = verifier.get("environment_mode") if isinstance(verifier, dict) else None
|
|
391
|
+
has_env = isinstance(verifier, dict) and "environment" in verifier
|
|
392
|
+
if mode == "separate" or has_env:
|
|
393
|
+
return [passed(EC020, "Verifier is configured for a separate environment (verifier.environment_mode='separate'), so agent filesystem state does not carry into verification except via declared artifacts.",
|
|
394
|
+
evidence={"environment_mode": mode or "implied-separate"})]
|
|
395
|
+
return [finding(EC020, Status.WARN,
|
|
396
|
+
"Verifier runs in the shared agent environment (Harbor default). Agent processes/filesystem state can influence verification; prefer environment_mode='separate' unless live state must be inspected.",
|
|
397
|
+
severity=Severity.MEDIUM, evidence={"environment_mode": mode or "shared (default)"}, confidence=Confidence.MEDIUM)]
|
|
398
|
+
|
|
399
|
+
|
|
400
|
+
@check(EC021)
|
|
401
|
+
def check_import_shadowing(c: CheckContext) -> list:
|
|
402
|
+
ctx = c.task
|
|
403
|
+
risky: list = []
|
|
404
|
+
for f in ctx.files:
|
|
405
|
+
base = f.rsplit("/", 1)[-1]
|
|
406
|
+
in_tests = f.startswith("tests/") or "/tests/" in f
|
|
407
|
+
if in_tests:
|
|
408
|
+
continue
|
|
409
|
+
if base in SHADOW_MODULES:
|
|
410
|
+
risky.append((f, "auto-imported by Python at startup"))
|
|
411
|
+
elif base == "conftest.py" and (f.startswith("environment/") or ctx.adapter == "generic"):
|
|
412
|
+
risky.append((f, "pytest auto-loads conftest.py from rootdir/parents of collected tests"))
|
|
413
|
+
elif base in ("pytest.ini", "pyproject.toml", "setup.cfg", "tox.ini") and f.startswith("environment/"):
|
|
414
|
+
text = ctx.text(f) or ""
|
|
415
|
+
if "pytest" in text or base == "pytest.ini":
|
|
416
|
+
risky.append((f, "pytest configuration in agent area can alter collection/plugins"))
|
|
417
|
+
if risky:
|
|
418
|
+
f, why = risky[0]
|
|
419
|
+
return [finding(EC021, Status.WARN,
|
|
420
|
+
f"'{f}' is in an agent-visible area and {why}. If the verifier imports/collects from that path, agent code can alter verification.",
|
|
421
|
+
path=f, evidence={"files": [r[0] for r in risky], "reason": why}, confidence=Confidence.LOW)]
|
|
422
|
+
return [passed(EC021, "No sitecustomize/usercustomize/conftest/pytest-config shadowing files in agent-visible areas (such files under tests/ are verifier-owned and expected).")]
|
|
423
|
+
|
|
424
|
+
|
|
425
|
+
@check(EC022)
|
|
426
|
+
def check_exe_shadowing(c: CheckContext) -> list:
|
|
427
|
+
ctx = c.task
|
|
428
|
+
hits: list = []
|
|
429
|
+
for f in ctx.files:
|
|
430
|
+
if f.startswith("tests/") or "/tests/" in f:
|
|
431
|
+
continue
|
|
432
|
+
base = f.rsplit("/", 1)[-1]
|
|
433
|
+
if base in TOOL_NAMES and (f.startswith("environment/") or ctx.adapter == "generic"):
|
|
434
|
+
hits.append(f)
|
|
435
|
+
if ctx.dockerfile is not None:
|
|
436
|
+
for instr in ctx.dockerfile.find("COPY", "ADD"):
|
|
437
|
+
sources, dest = copy_sources(instr)
|
|
438
|
+
for s in sources:
|
|
439
|
+
if s.rsplit("/", 1)[-1] in TOOL_NAMES and dest and ("/bin/" in dest or dest.endswith("/bin")):
|
|
440
|
+
hits.append(f"Dockerfile:{instr.line} COPY {s} {dest}")
|
|
441
|
+
if hits:
|
|
442
|
+
return [finding(EC022, Status.WARN,
|
|
443
|
+
f"Tool-named executables in agent-visible locations could shadow verifier tooling on PATH: {', '.join(hits[:5])}.",
|
|
444
|
+
evidence={"files": hits[:10]}, confidence=Confidence.LOW)]
|
|
445
|
+
return [passed(EC022, "No tool-named executables (pytest/python/pip/test.sh) in agent-visible areas or copied into bin directories.")]
|
|
446
|
+
|
|
447
|
+
|
|
448
|
+
@check(EC023)
|
|
449
|
+
def check_writable_verifier(c: CheckContext) -> list:
|
|
450
|
+
ctx = c.task
|
|
451
|
+
hits: list = []
|
|
452
|
+
texts = []
|
|
453
|
+
if ctx.dockerfile_path:
|
|
454
|
+
texts.append(("environment/Dockerfile" if ctx.environment_dir else "Dockerfile", ctx.text(ctx.dockerfile_path.relative_to(ctx.root).as_posix()) or ""))
|
|
455
|
+
for f in ctx.files:
|
|
456
|
+
if f.endswith((".sh", ".toml", ".yaml", ".yml")) and not f.startswith("tests/"):
|
|
457
|
+
t = ctx.text(f)
|
|
458
|
+
if t:
|
|
459
|
+
texts.append((f, t))
|
|
460
|
+
for path, text in texts:
|
|
461
|
+
for i, line in enumerate(text.splitlines(), 1):
|
|
462
|
+
if re.search(r"chmod\s+(-[a-zA-Z]+\s+)*(777|a\+w|o\+w|ugo\+w)", line) or "privileged = true" in line:
|
|
463
|
+
hits.append({"path": path, "line": i, "text": line.strip()[:160]})
|
|
464
|
+
if hits:
|
|
465
|
+
h = hits[0]
|
|
466
|
+
return [finding(EC023, Status.WARN, f"World-writable permissions declared at {h['path']}:{h['line']} ('{h['text']}'); agent-writable verifier/system paths enable tampering.",
|
|
467
|
+
path=str(h["path"]), line=int(h["line"]), evidence={"hits": hits[:10]}, confidence=Confidence.MEDIUM)]
|
|
468
|
+
return [passed(EC023, "No chmod 777 / a+w style world-writable declarations outside tests/.")]
|
|
469
|
+
|
|
470
|
+
|
|
471
|
+
@check(EC024)
|
|
472
|
+
def check_mounts_privileged(c: CheckContext) -> list:
|
|
473
|
+
ctx = c.task
|
|
474
|
+
hits: list = []
|
|
475
|
+
for f in ctx.files:
|
|
476
|
+
if not f.endswith((".toml", ".yaml", ".yml", ".json")) and "Dockerfile" not in f:
|
|
477
|
+
continue
|
|
478
|
+
text = ctx.text(f)
|
|
479
|
+
if not text:
|
|
480
|
+
continue
|
|
481
|
+
for i, line in enumerate(text.splitlines(), 1):
|
|
482
|
+
low = line.lower()
|
|
483
|
+
if "docker.sock" in low or "/var/run/docker.sock" in low:
|
|
484
|
+
hits.append((f, i, "Docker socket mount"))
|
|
485
|
+
elif re.search(r"privileged\s*[:=]\s*true", low):
|
|
486
|
+
hits.append((f, i, "privileged container"))
|
|
487
|
+
elif re.search(r"cap_add\s*:", low) and "SYS_ADMIN" in line:
|
|
488
|
+
hits.append((f, i, "SYS_ADMIN capability"))
|
|
489
|
+
if hits:
|
|
490
|
+
f, i, why = hits[0]
|
|
491
|
+
return [finding(EC024, Status.FAIL, f"{why} declared at {f}:{i}; this collapses container isolation.",
|
|
492
|
+
severity=Severity.CRITICAL, path=f, line=i, evidence={"hits": [{"path": a, "line": b, "kind": c} for a, b, c in hits]})]
|
|
493
|
+
return [passed(EC024, "No docker.sock mounts, privileged mode or SYS_ADMIN in task configuration files.")]
|
|
494
|
+
|
|
495
|
+
|
|
496
|
+
@check(EC030)
|
|
497
|
+
def check_image_pinning(c: CheckContext) -> list:
|
|
498
|
+
ctx = c.task
|
|
499
|
+
if ctx.dockerfile is None:
|
|
500
|
+
env_cfg = ctx.config.get("environment") or {}
|
|
501
|
+
img = env_cfg.get("docker_image") if isinstance(env_cfg, dict) else None
|
|
502
|
+
if img:
|
|
503
|
+
has_digest, specific = image_ref_pinned(str(img))
|
|
504
|
+
if has_digest:
|
|
505
|
+
return [passed(EC030, f"Prebuilt image '{img}' is digest-pinned.")]
|
|
506
|
+
return [finding(EC030, Status.WARN, f"Prebuilt image '{img}' is not digest-pinned; the image can change silently.",
|
|
507
|
+
evidence={"image": img})]
|
|
508
|
+
return [not_applicable(EC030, "No Dockerfile or prebuilt image reference to assess.")]
|
|
509
|
+
if not ctx.dockerfile.froms:
|
|
510
|
+
return [finding(EC030, Status.FAIL, "Dockerfile has no FROM instruction.", severity=Severity.MEDIUM)]
|
|
511
|
+
bad: list[dict] = []
|
|
512
|
+
for instr in ctx.dockerfile.froms:
|
|
513
|
+
toks = [t for t in instr.tokens() if not t.startswith("--")]
|
|
514
|
+
if not toks:
|
|
515
|
+
continue
|
|
516
|
+
ref = toks[0]
|
|
517
|
+
if ref in ("scratch",) or "$" in ref:
|
|
518
|
+
continue
|
|
519
|
+
has_digest, specific = image_ref_pinned(ref)
|
|
520
|
+
if not has_digest:
|
|
521
|
+
bad.append({"image": ref, "line": instr.line, "specific_tag": specific})
|
|
522
|
+
if bad:
|
|
523
|
+
b = bad[0]
|
|
524
|
+
sev = Severity.MEDIUM if b["specific_tag"] else Severity.HIGH
|
|
525
|
+
return [finding(EC030, Status.WARN, f"Base image '{b['image']}' (line {b['line']}) is not pinned to a sha256 digest"
|
|
526
|
+
+ ("" if b["specific_tag"] else " and uses a floating/absent tag") + "; builds are not fully reproducible.",
|
|
527
|
+
severity=sev, path=(ctx.dockerfile_path.relative_to(ctx.root).as_posix() if ctx.dockerfile_path else None), line=int(b["line"]), evidence={"unpinned": bad})]
|
|
528
|
+
return [passed(EC030, "All base images are pinned to sha256 digests.")]
|
|
529
|
+
|
|
530
|
+
|
|
531
|
+
@check(EC031)
|
|
532
|
+
def check_root_user(c: CheckContext) -> list:
|
|
533
|
+
ctx = c.task
|
|
534
|
+
if ctx.dockerfile is None:
|
|
535
|
+
return [not_applicable(EC031, "No Dockerfile to assess for USER.")]
|
|
536
|
+
users = ctx.dockerfile.find("USER")
|
|
537
|
+
if not users:
|
|
538
|
+
return [finding(EC031, Status.WARN, "Dockerfile sets no USER; the container defaults to root. Consider a non-root agent user.",
|
|
539
|
+
severity=Severity.LOW, evidence={"user_instructions": 0})]
|
|
540
|
+
last = users[-1]
|
|
541
|
+
val = last.args.strip().split(":")[0]
|
|
542
|
+
if val in ("root", "0"):
|
|
543
|
+
return [finding(EC031, Status.WARN, f"Final USER is root (line {last.line}); hardening opportunity, not by itself a benchmark-integrity failure.",
|
|
544
|
+
severity=Severity.LOW, path=(ctx.dockerfile_path.relative_to(ctx.root).as_posix() if ctx.dockerfile_path else None), line=last.line, evidence={"user": val})]
|
|
545
|
+
return [passed(EC031, f"Final USER is non-root ('{val}').", evidence={"user": val})]
|
|
546
|
+
|
|
547
|
+
|
|
548
|
+
@check(EC032)
|
|
549
|
+
def check_dangerous_dockerfile(c: CheckContext) -> list:
|
|
550
|
+
ctx = c.task
|
|
551
|
+
if ctx.dockerfile is None:
|
|
552
|
+
return [not_applicable(EC032, "No Dockerfile to assess.")]
|
|
553
|
+
hits: list = []
|
|
554
|
+
for instr in ctx.dockerfile.instructions:
|
|
555
|
+
if instr.name == "ADD" and re.search(r"https?://", instr.args):
|
|
556
|
+
hits.append((instr.line, "remote ADD " + instr.args[:80]))
|
|
557
|
+
if instr.name == "RUN" and re.search(r"(curl|wget)\b[^|]*\|\s*(sudo\s+)?(ba)?sh\b", instr.args):
|
|
558
|
+
hits.append((instr.line, "pipe-to-shell install: " + instr.args[:100]))
|
|
559
|
+
if instr.name == "RUN" and re.search(r"\bapt-get\b", instr.args) and "--allow-unauthenticated" in instr.args:
|
|
560
|
+
hits.append((instr.line, "apt-get --allow-unauthenticated"))
|
|
561
|
+
if hits:
|
|
562
|
+
line, what = hits[0]
|
|
563
|
+
return [finding(EC032, Status.WARN, f"Dangerous build pattern at Dockerfile line {line}: {what}. Remote code executed at build time is unreviewed and unpinned.",
|
|
564
|
+
severity=Severity.HIGH, path=(ctx.dockerfile_path.relative_to(ctx.root).as_posix() if ctx.dockerfile_path else None), line=line, evidence={"hits": [{"line": a, "pattern": b} for a, b in hits]})]
|
|
565
|
+
return [passed(EC032, "No remote ADD, pipe-to-shell or unauthenticated package installs in the Dockerfile.")]
|
|
566
|
+
|
|
567
|
+
|
|
568
|
+
@check(EC033)
|
|
569
|
+
def check_network(c: CheckContext) -> list:
|
|
570
|
+
ctx = c.task
|
|
571
|
+
if ctx.adapter != "harbor":
|
|
572
|
+
return [not_applicable(EC033, "Network policy declaration is framework-specific; generic adapter cannot assess it.")]
|
|
573
|
+
env_cfg = ctx.config.get("environment") or {}
|
|
574
|
+
if not isinstance(env_cfg, dict):
|
|
575
|
+
env_cfg = {}
|
|
576
|
+
mode = env_cfg.get("network_mode")
|
|
577
|
+
allowed = env_cfg.get("allowed_hosts")
|
|
578
|
+
if mode == "no-network":
|
|
579
|
+
return [passed(EC033, "Network mode is 'no-network' for the agent environment.", evidence={"network_mode": mode})]
|
|
580
|
+
if mode == "allowlist":
|
|
581
|
+
return [passed(EC033, f"Network mode is 'allowlist' (hosts: {allowed}).", evidence={"network_mode": mode, "allowed_hosts": allowed})]
|
|
582
|
+
if mode == "public":
|
|
583
|
+
return [finding(EC033, Status.WARN, "Network mode is explicitly 'public': the agent has unrestricted egress and could retrieve answers or external help.",
|
|
584
|
+
evidence={"network_mode": mode}, confidence=Confidence.HIGH)]
|
|
585
|
+
return [finding(EC033, Status.WARN, "No [environment] network_mode declared; Harbor defaults to public network. Declare no-network/allowlist unless public access is required.",
|
|
586
|
+
severity=Severity.LOW, evidence={"network_mode": None, "default": "public"}, confidence=Confidence.HIGH)]
|
|
587
|
+
|
|
588
|
+
|
|
589
|
+
def _dependency_texts(ctx):
|
|
590
|
+
for f in ctx.files:
|
|
591
|
+
base = f.rsplit("/", 1)[-1]
|
|
592
|
+
if base in ("requirements.txt", "requirements-dev.txt", "Pipfile", "pyproject.toml", "setup.py", "setup.cfg") or f.endswith("Dockerfile"):
|
|
593
|
+
t = ctx.text(f)
|
|
594
|
+
if t is not None:
|
|
595
|
+
yield f, t
|
|
596
|
+
|
|
597
|
+
|
|
598
|
+
@check(EC040)
|
|
599
|
+
def check_dependencies(c: CheckContext) -> list:
|
|
600
|
+
ctx = c.task
|
|
601
|
+
unpinned: list[dict] = []
|
|
602
|
+
has_lock = any(f.rsplit("/", 1)[-1] in ("uv.lock", "poetry.lock", "Pipfile.lock", "requirements.lock") or f.endswith(".lock")
|
|
603
|
+
for f in ctx.files)
|
|
604
|
+
for f, text in _dependency_texts(ctx):
|
|
605
|
+
for i, line in enumerate(text.splitlines(), 1):
|
|
606
|
+
s = line.strip()
|
|
607
|
+
if f.endswith("Dockerfile"):
|
|
608
|
+
m = re.search(r"pip3?\s+install\s+(.+)", s)
|
|
609
|
+
if m:
|
|
610
|
+
for tok in m.group(1).split():
|
|
611
|
+
if tok.startswith("-") or tok in ("pip", "\\"):
|
|
612
|
+
continue
|
|
613
|
+
if "==" not in tok and ">=" not in tok and tok not in ("--upgrade",):
|
|
614
|
+
unpinned.append({"path": f, "line": i, "package": tok.strip("'\"")})
|
|
615
|
+
elif f.endswith("requirements.txt") or f.endswith("requirements-dev.txt"):
|
|
616
|
+
if s and not s.startswith(("#", "-")) and "==" not in s and "@" not in s:
|
|
617
|
+
unpinned.append({"path": f, "line": i, "package": s.split(";")[0][:60]})
|
|
618
|
+
if unpinned:
|
|
619
|
+
return [finding(EC040, Status.WARN,
|
|
620
|
+
f"{len(unpinned)} unpinned Python dependencies (e.g. '{unpinned[0]['package']}' at {unpinned[0]['path']}:{unpinned[0]['line']}); the environment can drift silently."
|
|
621
|
+
+ ("" if has_lock else " No lockfile found."),
|
|
622
|
+
evidence={"unpinned": unpinned[:20], "lockfile_present": has_lock}, confidence=Confidence.MEDIUM)]
|
|
623
|
+
if has_lock:
|
|
624
|
+
return [passed(EC040, "Dependencies are pinned and a lockfile is present.", evidence={"lockfile_present": True})]
|
|
625
|
+
return [passed(EC040, "No unpinned pip/requirements entries detected.", evidence={"lockfile_present": has_lock})]
|
|
626
|
+
|
|
627
|
+
|
|
628
|
+
@check(EC041)
|
|
629
|
+
def check_downloads(c: CheckContext) -> list:
|
|
630
|
+
ctx = c.task
|
|
631
|
+
hits: list = []
|
|
632
|
+
for f in ctx.files:
|
|
633
|
+
if not (f.endswith((".sh", "Dockerfile", ".py", ".toml")) or "Dockerfile" in f):
|
|
634
|
+
continue
|
|
635
|
+
text = ctx.text(f)
|
|
636
|
+
if not text:
|
|
637
|
+
continue
|
|
638
|
+
for i, line in enumerate(text.splitlines(), 1):
|
|
639
|
+
if re.search(r"git\s+clone\s", line) and not re.search(r"(--branch|--depth|checkout\s+[0-9a-f]{40})", line):
|
|
640
|
+
# clone followed by checkout in same file counts as pinned; checked coarsely below
|
|
641
|
+
if "checkout" not in text:
|
|
642
|
+
hits.append({"path": f, "line": i, "kind": "git clone without pinned commit/branch", "text": line.strip()[:120]})
|
|
643
|
+
if re.search(r"(curl|wget)\s+.*https?://", line) and "sha256" not in text and "checksum" not in text.lower():
|
|
644
|
+
if "|" not in line: # pipe-to-shell handled by EC032
|
|
645
|
+
hits.append({"path": f, "line": i, "kind": "download without checksum verification in file", "text": line.strip()[:120]})
|
|
646
|
+
if hits:
|
|
647
|
+
h = hits[0]
|
|
648
|
+
return [finding(EC041, Status.WARN, f"Unpinned/unverified external download at {h['path']}:{h['line']} ({h['kind']}).",
|
|
649
|
+
path=h["path"], line=h["line"], evidence={"hits": hits[:10]}, confidence=Confidence.LOW)]
|
|
650
|
+
return [passed(EC041, "No unpinned git clones or unverified downloads detected.")]
|
|
651
|
+
|
|
652
|
+
|
|
653
|
+
@check(EC050)
|
|
654
|
+
def check_verifier_exists(c: CheckContext) -> list:
|
|
655
|
+
ctx = c.task
|
|
656
|
+
entries = harbor_adapter.verifier_entrypoints(ctx) if ctx.adapter == "harbor" else _verifier_files(ctx)[:5]
|
|
657
|
+
if not entries:
|
|
658
|
+
return [finding(EC050, Status.FAIL, "No verifier entrypoint found (expected tests/test.sh, test.bat or verifier.py).",
|
|
659
|
+
severity=Severity.CRITICAL, evidence={"searched": "tests/"})]
|
|
660
|
+
empty = [e for e in entries if not (ctx.text(e) or "").strip()]
|
|
661
|
+
substantive = [e for e in entries if e not in empty]
|
|
662
|
+
if empty and not substantive:
|
|
663
|
+
return [finding(EC050, Status.FAIL, f"Verifier entrypoint(s) are empty: {', '.join(empty)}.", severity=Severity.CRITICAL, evidence={"empty": empty})]
|
|
664
|
+
extra = f" (also empty: {', '.join(empty)})" if empty else ""
|
|
665
|
+
return [passed(EC050, f"Verifier present and non-empty: {', '.join(substantive[:3])}{extra}", evidence={"entrypoints": entries})]
|
|
666
|
+
|
|
667
|
+
|
|
668
|
+
@check(EC051)
|
|
669
|
+
def check_reward_production(c: CheckContext) -> list:
|
|
670
|
+
ctx = c.task
|
|
671
|
+
if ctx.adapter != "harbor":
|
|
672
|
+
return [not_applicable(EC051, "Reward-file contract is Harbor-specific (/logs/verifier/reward.txt|json).")]
|
|
673
|
+
entries = harbor_adapter.verifier_entrypoints(ctx)
|
|
674
|
+
if not entries:
|
|
675
|
+
return [finding(EC051, Status.FAIL, "No verifier exists, so no reward can be produced.", severity=Severity.CRITICAL)]
|
|
676
|
+
mentions = []
|
|
677
|
+
for e in entries:
|
|
678
|
+
t = ctx.text(e) or ""
|
|
679
|
+
if "reward.txt" in t or "reward.json" in t or "/logs/verifier" in t:
|
|
680
|
+
mentions.append(e)
|
|
681
|
+
# helpers in tests/ may write the reward on behalf of test.sh
|
|
682
|
+
helper_mentions = [f for f, t in _verifier_texts(ctx) if "reward.txt" in t or "reward.json" in t]
|
|
683
|
+
if mentions or helper_mentions:
|
|
684
|
+
return [passed(EC051, f"Verifier code writes the Harbor reward file (found in {', '.join(list(dict.fromkeys(mentions + helper_mentions))[:3])}).",
|
|
685
|
+
evidence={"reward_writers": list(dict.fromkeys(mentions + helper_mentions))[:5]})]
|
|
686
|
+
py_entries = [e for e in entries if e.endswith("verifier.py")]
|
|
687
|
+
if py_entries:
|
|
688
|
+
return [passed(EC051, "Python verifier (verifier.py) returns the reward directly; no reward file required under NeMo Gym/Harbor python-verifier conventions.",
|
|
689
|
+
evidence={"entrypoints": py_entries})]
|
|
690
|
+
return [finding(EC051, Status.FAIL, "Verifier never mentions /logs/verifier/reward.txt or reward.json; Harbor will score every run 0 or mask it.",
|
|
691
|
+
severity=Severity.HIGH, evidence={"entrypoints": entries})]
|
|
692
|
+
|
|
693
|
+
|
|
694
|
+
def _python_verifier_analysis(ctx) -> dict:
|
|
695
|
+
info: dict = {"asserts": 0, "test_funcs": 0, "empty_test_funcs": [], "assert_true_only": False, "files": []}
|
|
696
|
+
for f, text in _verifier_texts(ctx):
|
|
697
|
+
if not f.endswith(".py"):
|
|
698
|
+
continue
|
|
699
|
+
info["files"].append(f)
|
|
700
|
+
try:
|
|
701
|
+
tree = ast.parse(text)
|
|
702
|
+
except SyntaxError:
|
|
703
|
+
continue
|
|
704
|
+
for node in ast.walk(tree):
|
|
705
|
+
if isinstance(node, ast.Assert):
|
|
706
|
+
info["asserts"] += 1
|
|
707
|
+
if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) and node.name.startswith("test"):
|
|
708
|
+
info["test_funcs"] += 1
|
|
709
|
+
asserts_in = [n for n in ast.walk(node) if isinstance(n, ast.Assert)]
|
|
710
|
+
# pytest style also uses plain calls that raise; count assert + raising helpers coarsely
|
|
711
|
+
if not asserts_in:
|
|
712
|
+
info["empty_test_funcs"].append(f"{f}:{node.name}")
|
|
713
|
+
elif all(isinstance(a.test, ast.Constant) and a.test.value is True for a in asserts_in):
|
|
714
|
+
info["assert_true_only"] = True
|
|
715
|
+
return info
|
|
716
|
+
|
|
717
|
+
|
|
718
|
+
@check(EC052)
|
|
719
|
+
def check_unconditional_success(c: CheckContext) -> list:
|
|
720
|
+
ctx = c.task
|
|
721
|
+
hits: list = []
|
|
722
|
+
for f, text in _verifier_texts(ctx):
|
|
723
|
+
lines = text.splitlines()
|
|
724
|
+
has_condition = bool(re.search(r"(?m)^\s*(if|elif|else|case|pytest|python3?\s|bash\s|\[|test\s|! )", text))
|
|
725
|
+
for i, line in enumerate(lines, 1):
|
|
726
|
+
s = line.strip()
|
|
727
|
+
if re.search(r"echo\s+\"?1(\.0)?\"?\s*>\s*\S*reward\.txt", s) and not has_condition:
|
|
728
|
+
hits.append({"path": f, "line": i, "pattern": "unconditional 'echo 1 > reward.txt'"})
|
|
729
|
+
if re.search(r"sys\.exit\(0\)|exit\(0\)|return\s+1(\.0)?\s*$", s) and f.endswith("verifier.py"):
|
|
730
|
+
# only suspicious if no assert in file
|
|
731
|
+
if "assert" not in text:
|
|
732
|
+
hits.append({"path": f, "line": i, "pattern": "verifier returns success without assertions"})
|
|
733
|
+
if re.match(r"assert\s+True\b", s):
|
|
734
|
+
hits.append({"path": f, "line": i, "pattern": "assert True"})
|
|
735
|
+
analysis = _python_verifier_analysis(ctx)
|
|
736
|
+
if analysis["assert_true_only"] and analysis["asserts"] <= 1:
|
|
737
|
+
hits.append({"path": analysis["files"][0] if analysis["files"] else "tests/", "line": None, "pattern": "only assertion is 'assert True'"})
|
|
738
|
+
if hits:
|
|
739
|
+
h = hits[0]
|
|
740
|
+
return [finding(EC052, Status.FAIL, f"Possible unconditional-success path in verifier: {h['pattern']} at {h['path']}"
|
|
741
|
+
+ (f":{h['line']}" if h["line"] else "") + ". Confirm manually; if real, any submission passes.",
|
|
742
|
+
severity=Severity.CRITICAL, path=str(h["path"]), line=int(h["line"]), evidence={"hits": hits[:10]}, confidence=Confidence.MEDIUM)]
|
|
743
|
+
return [passed(EC052, "No unconditional-success patterns (bare echo 1, assert True only, success-without-assert) detected in the verifier.")] if _verifier_files(ctx) else [not_applicable(EC052, "No verifier files to analyse.")]
|
|
744
|
+
|
|
745
|
+
|
|
746
|
+
@check(EC053)
|
|
747
|
+
def check_assertion_strength(c: CheckContext) -> list:
|
|
748
|
+
ctx = c.task
|
|
749
|
+
if not _verifier_files(ctx):
|
|
750
|
+
return [not_applicable(EC053, "No verifier files to analyse.")]
|
|
751
|
+
analysis = _python_verifier_analysis(ctx)
|
|
752
|
+
shell_text = "\n".join(t for _f, t in _verifier_texts(ctx))
|
|
753
|
+
shell_checks = len(re.findall(r"(?m)(diff\s|cmp\s|\[\s|test\s|grep\s+-|==|pytest)", shell_text))
|
|
754
|
+
total_signals = analysis["asserts"] + shell_checks
|
|
755
|
+
if analysis["assert_true_only"]:
|
|
756
|
+
return [finding(EC053, Status.WARN, "The only Python assertion(s) are 'assert True'; the verifier cannot reject incorrect solutions.", severity=Severity.HIGH, evidence={"python": analysis}, confidence=Confidence.MEDIUM)]
|
|
757
|
+
weak_only = bool(re.search(r"os\.path\.exists|Path\(.*\)\.exists\(\)|\[\s+-f\s", shell_text)) and analysis["asserts"] <= 1 and shell_checks <= 1
|
|
758
|
+
if analysis["empty_test_funcs"] and analysis["asserts"] == 0:
|
|
759
|
+
return [finding(EC053, Status.FAIL, f"Python test functions contain no assertions (e.g. {analysis['empty_test_funcs'][0]}); they cannot reject incorrect solutions.",
|
|
760
|
+
severity=Severity.HIGH, evidence=analysis, confidence=Confidence.MEDIUM)]
|
|
761
|
+
if weak_only:
|
|
762
|
+
return [finding(EC053, Status.WARN, "Verifier appears to check mainly file existence; existence checks accept incorrect content (a known benchmark flaw pattern).",
|
|
763
|
+
severity=Severity.HIGH, evidence={"python": analysis, "shell_signals": shell_checks}, confidence=Confidence.LOW)]
|
|
764
|
+
if total_signals == 0:
|
|
765
|
+
return [finding(EC053, Status.WARN, "No assertions or comparison operations detected in the verifier; its ability to reject incorrect solutions is doubtful.",
|
|
766
|
+
severity=Severity.HIGH, evidence={"python": analysis, "shell_signals": shell_checks}, confidence=Confidence.LOW)]
|
|
767
|
+
return [passed(EC053, f"Verifier contains substantive check signals (python asserts: {analysis['asserts']}, shell comparisons: {shell_checks}, test functions: {analysis['test_funcs']}).",
|
|
768
|
+
evidence={"python_asserts": analysis["asserts"], "shell_signals": shell_checks, "test_funcs": analysis["test_funcs"]})]
|
|
769
|
+
|
|
770
|
+
|
|
771
|
+
@check(EC054)
|
|
772
|
+
def check_self_reported_reward(c: CheckContext) -> list:
|
|
773
|
+
ctx = c.task
|
|
774
|
+
hits: list = []
|
|
775
|
+
for f, text in _verifier_texts(ctx):
|
|
776
|
+
for i, line in enumerate(text.splitlines(), 1):
|
|
777
|
+
if re.search(r"cat\s+\S*(reward|score|result)\.(txt|json)", line) and "logs/verifier" not in line:
|
|
778
|
+
hits.append({"path": f, "line": i, "text": line.strip()[:140]})
|
|
779
|
+
if re.search(r"(reward|score)\s*=\s*(float|int)\(\s*(open|Path)\(", line):
|
|
780
|
+
hits.append({"path": f, "line": i, "text": line.strip()[:140]})
|
|
781
|
+
if hits:
|
|
782
|
+
h = hits[0]
|
|
783
|
+
return [finding(EC054, Status.WARN, f"Verifier appears to read a reward/score value from a file at {h['path']}:{h['line']} ('{h['text']}'); if that file is agent-writable, the agent can report its own grade. Verify the source is verifier-controlled and validated.",
|
|
784
|
+
severity=Severity.HIGH, path=str(h["path"]), line=int(h["line"]), evidence={"hits": hits[:10]}, confidence=Confidence.LOW)]
|
|
785
|
+
return [passed(EC054, "Verifier does not obviously read its reward from an agent-produced file.")]
|
|
786
|
+
|
|
787
|
+
|
|
788
|
+
def register_all() -> list[CheckInfo]:
|
|
789
|
+
return [REGISTRY[i][0] for i in
|
|
790
|
+
["EC001", "EC002", "EC003", "EC010", "EC011", "EC012", "EC013", "EC014", "EC020", "EC021", "EC022",
|
|
791
|
+
"EC023", "EC024", "EC030", "EC031", "EC032", "EC033", "EC040", "EC041", "EC050", "EC051", "EC052",
|
|
792
|
+
"EC053", "EC054"]]
|
|
793
|
+
|
|
794
|
+
|
|
795
|
+
# importing this module registers checks; ensure registry import side-effect is explicit
|
|
796
|
+
from envcert.checks.base import REGISTRY # noqa: E402
|