bitexoft-verify 0.2.0__tar.gz → 0.2.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {bitexoft_verify-0.2.0 → bitexoft_verify-0.2.1}/.gitignore +1 -0
- {bitexoft_verify-0.2.0 → bitexoft_verify-0.2.1}/PKG-INFO +1 -1
- {bitexoft_verify-0.2.0 → bitexoft_verify-0.2.1}/src/bitexoft_verify/__init__.py +1 -1
- bitexoft_verify-0.2.1/src/bitexoft_verify/checks/assertions.py +129 -0
- {bitexoft_verify-0.2.0 → bitexoft_verify-0.2.1}/src/bitexoft_verify/checks/common.py +64 -17
- bitexoft_verify-0.2.1/src/bitexoft_verify/checks/execution.py +199 -0
- bitexoft_verify-0.2.1/src/bitexoft_verify/checks/isolated.py +138 -0
- {bitexoft_verify-0.2.0 → bitexoft_verify-0.2.1}/src/bitexoft_verify/cli.py +22 -4
- {bitexoft_verify-0.2.0 → bitexoft_verify-0.2.1}/src/bitexoft_verify/codes.py +6 -0
- {bitexoft_verify-0.2.0 → bitexoft_verify-0.2.1}/src/bitexoft_verify/data/issue-codes.json +19 -0
- {bitexoft_verify-0.2.0 → bitexoft_verify-0.2.1}/src/bitexoft_verify/report.py +57 -7
- bitexoft_verify-0.2.1/src/bitexoft_verify/runner.py +137 -0
- bitexoft_verify-0.2.1/src/bitexoft_verify/seeds.py +43 -0
- {bitexoft_verify-0.2.0 → bitexoft_verify-0.2.1}/tests/test_a1.py +90 -1
- bitexoft_verify-0.2.1/tests/test_a1_benign.py +309 -0
- bitexoft_verify-0.2.1/tests/test_assertions.py +96 -0
- bitexoft_verify-0.2.1/tests/test_execution.py +86 -0
- bitexoft_verify-0.2.1/tests/test_isolated.py +145 -0
- {bitexoft_verify-0.2.0 → bitexoft_verify-0.2.1}/tests/test_issues.py +11 -2
- bitexoft_verify-0.2.1/tests/test_parity.py +100 -0
- bitexoft_verify-0.2.1/tests/test_projects.py +119 -0
- bitexoft_verify-0.2.1/tests/test_report_v4.py +98 -0
- bitexoft_verify-0.2.1/tests/test_seeds.py +107 -0
- bitexoft_verify-0.2.0/src/bitexoft_verify/runner.py +0 -101
- {bitexoft_verify-0.2.0 → bitexoft_verify-0.2.1}/README.md +0 -0
- {bitexoft_verify-0.2.0 → bitexoft_verify-0.2.1}/pyproject.toml +0 -0
- {bitexoft_verify-0.2.0 → bitexoft_verify-0.2.1}/src/bitexoft_verify/__main__.py +0 -0
- {bitexoft_verify-0.2.0 → bitexoft_verify-0.2.1}/src/bitexoft_verify/checks/__init__.py +0 -0
- {bitexoft_verify-0.2.0 → bitexoft_verify-0.2.1}/src/bitexoft_verify/checks/a1.py +0 -0
- {bitexoft_verify-0.2.0 → bitexoft_verify-0.2.1}/src/bitexoft_verify/data/README.md +0 -0
- {bitexoft_verify-0.2.0 → bitexoft_verify-0.2.1}/src/bitexoft_verify/data/a1_impossible_sample.csv +0 -0
- {bitexoft_verify-0.2.0 → bitexoft_verify-0.2.1}/src/bitexoft_verify/data/a1_match_sample.csv +0 -0
- {bitexoft_verify-0.2.0 → bitexoft_verify-0.2.1}/src/bitexoft_verify/data/a2_isolated_sample.csv +0 -0
- {bitexoft_verify-0.2.0 → bitexoft_verify-0.2.1}/src/bitexoft_verify/manifest.py +0 -0
- {bitexoft_verify-0.2.0 → bitexoft_verify-0.2.1}/tests/conftest.py +0 -0
- {bitexoft_verify-0.2.0 → bitexoft_verify-0.2.1}/tests/golden/a1_checks.json +0 -0
- {bitexoft_verify-0.2.0 → bitexoft_verify-0.2.1}/tests/test_equivalence.py +0 -0
- {bitexoft_verify-0.2.0 → bitexoft_verify-0.2.1}/tests/test_version.py +0 -0
- {bitexoft_verify-0.2.0 → bitexoft_verify-0.2.1}/tools/make_fixtures.py +0 -0
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
"""One vocabulary for "this value is not what it should be" (ADR 0049; foundation 09, D5, F12).
|
|
2
|
+
|
|
3
|
+
Every check that compares a value used to word its own failure, and no two worded it alike. An assertion here
|
|
4
|
+
is a small function from a value to the list of things wrong with it; a check runs one against a value and the
|
|
5
|
+
failures land in its `Problems`, each with a stable issue code and the values behind it (`expected`, `found`), so
|
|
6
|
+
the platform writes the sentence in the learner's language and the terminal reads the same in English.
|
|
7
|
+
|
|
8
|
+
equal(expected) the value is exactly this
|
|
9
|
+
contains(needle) the text includes this
|
|
10
|
+
not_contains(needle) the text does not include this
|
|
11
|
+
matches(pattern) the text matches this regular expression (searched, not anchored)
|
|
12
|
+
all_of(a, b, ...) every one holds, and each that does not is reported: not just the first
|
|
13
|
+
annotated(label, a) says what was being looked at, in front of whatever `a` finds wrong
|
|
14
|
+
|
|
15
|
+
problems = Problems()
|
|
16
|
+
check_value(problems, annotated("verdict on the stranded frame", equal("BROKEN")), verdict)
|
|
17
|
+
if problems:
|
|
18
|
+
return problems.failing("furthest_policy", "The second policy notices a stranded player")
|
|
19
|
+
|
|
20
|
+
An assertion never raises for a value it did not expect: a value of the wrong type simply does not satisfy it, and
|
|
21
|
+
what the learner's code returned is described, not trusted.
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
from __future__ import annotations
|
|
25
|
+
|
|
26
|
+
import re
|
|
27
|
+
from collections.abc import Callable
|
|
28
|
+
from dataclasses import dataclass
|
|
29
|
+
from typing import Any
|
|
30
|
+
|
|
31
|
+
from ..report import Issue, Problems, issue
|
|
32
|
+
|
|
33
|
+
SHOWN = 80 # characters of a found value in a sentence; a long one is cut, with a mark
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _shown(value: Any) -> str:
|
|
37
|
+
text = repr(value) if not isinstance(value, str) else value
|
|
38
|
+
return text if len(text) <= SHOWN else text[: SHOWN - 1] + "…"
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
@dataclass(frozen=True, slots=True)
|
|
42
|
+
class Failure:
|
|
43
|
+
"""One thing wrong: the English sentence for the terminal and the issue for the platform."""
|
|
44
|
+
|
|
45
|
+
text: str
|
|
46
|
+
issue: Issue
|
|
47
|
+
|
|
48
|
+
def with_context(self, context: str) -> Failure:
|
|
49
|
+
return Failure(f"{context}: {self.text}", Issue(self.issue.code, {**self.issue.params, "context": context}))
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
Assertion = Callable[[Any], list[Failure]]
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def equal(expected: Any) -> Assertion:
|
|
56
|
+
"""Exact, type included: `3` is not `3.0` and `1` is not `True`."""
|
|
57
|
+
|
|
58
|
+
def run(value: Any) -> list[Failure]:
|
|
59
|
+
if value == expected and type(value) is type(expected):
|
|
60
|
+
return []
|
|
61
|
+
return [
|
|
62
|
+
Failure(
|
|
63
|
+
f"expected {_shown(expected)} but found {_shown(value)}",
|
|
64
|
+
issue("assert.not_equal", expected=_shown(expected), found=_shown(value)),
|
|
65
|
+
)
|
|
66
|
+
]
|
|
67
|
+
|
|
68
|
+
return run
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def contains(needle: str) -> Assertion:
|
|
72
|
+
def run(value: Any) -> list[Failure]:
|
|
73
|
+
if isinstance(value, str) and needle in value:
|
|
74
|
+
return []
|
|
75
|
+
return [
|
|
76
|
+
Failure(
|
|
77
|
+
f"expected to find {_shown(needle)} in {_shown(value)}",
|
|
78
|
+
issue("assert.missing", expected=_shown(needle), found=_shown(value)),
|
|
79
|
+
)
|
|
80
|
+
]
|
|
81
|
+
|
|
82
|
+
return run
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def not_contains(needle: str) -> Assertion:
|
|
86
|
+
def run(value: Any) -> list[Failure]:
|
|
87
|
+
if not isinstance(value, str) or needle not in value:
|
|
88
|
+
return []
|
|
89
|
+
return [Failure(f"did not expect to find {_shown(needle)}", issue("assert.unexpected", found=_shown(needle)))]
|
|
90
|
+
|
|
91
|
+
return run
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def matches(pattern: str) -> Assertion:
|
|
95
|
+
compiled = re.compile(pattern)
|
|
96
|
+
|
|
97
|
+
def run(value: Any) -> list[Failure]:
|
|
98
|
+
if isinstance(value, str) and compiled.search(value):
|
|
99
|
+
return []
|
|
100
|
+
return [
|
|
101
|
+
Failure(
|
|
102
|
+
f"{_shown(value)} does not match {pattern}",
|
|
103
|
+
issue("assert.no_match", expected=pattern, found=_shown(value)),
|
|
104
|
+
)
|
|
105
|
+
]
|
|
106
|
+
|
|
107
|
+
return run
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def all_of(*assertions: Assertion) -> Assertion:
|
|
111
|
+
def run(value: Any) -> list[Failure]:
|
|
112
|
+
return [failure for assertion in assertions for failure in assertion(value)]
|
|
113
|
+
|
|
114
|
+
return run
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def annotated(context: str, assertion: Assertion) -> Assertion:
|
|
118
|
+
def run(value: Any) -> list[Failure]:
|
|
119
|
+
return [failure.with_context(context) for failure in assertion(value)]
|
|
120
|
+
|
|
121
|
+
return run
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def check_value(problems: Problems, assertion: Assertion, value: Any) -> bool:
|
|
125
|
+
"""Runs `assertion` on `value`, adds what is wrong to `problems`, and says whether the value held."""
|
|
126
|
+
failures = assertion(value)
|
|
127
|
+
for failure in failures:
|
|
128
|
+
problems.add_group(failure.text, [failure.issue])
|
|
129
|
+
return not failures
|
|
@@ -9,6 +9,7 @@ from __future__ import annotations
|
|
|
9
9
|
|
|
10
10
|
import ast
|
|
11
11
|
import importlib
|
|
12
|
+
import random
|
|
12
13
|
import subprocess
|
|
13
14
|
import sys
|
|
14
15
|
from dataclasses import dataclass
|
|
@@ -17,7 +18,8 @@ from pathlib import Path
|
|
|
17
18
|
from typing import Any
|
|
18
19
|
|
|
19
20
|
from ..manifest import Manifest
|
|
20
|
-
from ..report import Check, Problems, issue
|
|
21
|
+
from ..report import Check, Problems, Trace, issue
|
|
22
|
+
from .execution import run_bounded
|
|
21
23
|
|
|
22
24
|
FRAMEWORKS = (
|
|
23
25
|
"fastapi",
|
|
@@ -47,17 +49,19 @@ class Context:
|
|
|
47
49
|
root: Path
|
|
48
50
|
manifest: Manifest
|
|
49
51
|
timeout: int = 180
|
|
52
|
+
# What a check that builds its own inputs draws them from (seeds.py). `seed_used` is how the report knows
|
|
53
|
+
# whether to say so: a run whose checks never asked for randomness carries no seed, as A1's never do.
|
|
54
|
+
seed: int = 0
|
|
55
|
+
seed_used: bool = False
|
|
56
|
+
|
|
57
|
+
def random(self, label: str) -> random.Random:
|
|
58
|
+
"""A generator for `label`: the same seed and label always give the same sequence."""
|
|
59
|
+
self.seed_used = True
|
|
60
|
+
return random.Random(f"{self.seed}:{label}")
|
|
50
61
|
|
|
51
62
|
def run(self, args: list[str], timeout: int | None = None) -> subprocess.CompletedProcess[str]:
|
|
52
|
-
"""Runs a command at the root of the learner's repository."""
|
|
53
|
-
return
|
|
54
|
-
args,
|
|
55
|
-
cwd=self.root,
|
|
56
|
-
capture_output=True,
|
|
57
|
-
text=True,
|
|
58
|
-
timeout=timeout or self.timeout,
|
|
59
|
-
check=False,
|
|
60
|
-
)
|
|
63
|
+
"""Runs a command at the root of the learner's repository, within the budget of `execution.py`."""
|
|
64
|
+
return run_bounded(args, cwd=self.root, timeout=timeout or self.timeout)
|
|
61
65
|
|
|
62
66
|
def python(self, args: list[str], timeout: int | None = None) -> subprocess.CompletedProcess[str]:
|
|
63
67
|
return self.run([sys.executable, *args], timeout=timeout)
|
|
@@ -212,20 +216,58 @@ def check_domain_has_no_io(ctx: Context, check_id: str = "boundary") -> Check:
|
|
|
212
216
|
return Check(check_id, title, "pass")
|
|
213
217
|
|
|
214
218
|
|
|
219
|
+
def trace_of(
|
|
220
|
+
args: list[str],
|
|
221
|
+
result: subprocess.CompletedProcess | subprocess.TimeoutExpired, # type: ignore[type-arg]
|
|
222
|
+
seconds: float | None = None,
|
|
223
|
+
) -> Trace:
|
|
224
|
+
"""What was run and what came back, for a check that opts in to showing it.
|
|
225
|
+
|
|
226
|
+
Takes any `CompletedProcess`: `Executed` carries the seconds and whether the output was cut, and a plain one
|
|
227
|
+
(a stand-in in a test, or a command run another way) is shown without them."""
|
|
228
|
+
if isinstance(result, subprocess.TimeoutExpired):
|
|
229
|
+
return Trace(
|
|
230
|
+
command=" ".join(args),
|
|
231
|
+
exit_code=None,
|
|
232
|
+
stdout=str(result.output or ""),
|
|
233
|
+
stderr=str(result.stderr or ""),
|
|
234
|
+
seconds=float(seconds or result.timeout),
|
|
235
|
+
truncated=False,
|
|
236
|
+
)
|
|
237
|
+
return Trace(
|
|
238
|
+
command=" ".join(args),
|
|
239
|
+
exit_code=result.returncode,
|
|
240
|
+
stdout=result.stdout or "",
|
|
241
|
+
stderr=result.stderr or "",
|
|
242
|
+
seconds=float(getattr(result, "seconds", 0.0)),
|
|
243
|
+
truncated=bool(getattr(result, "truncated", False)),
|
|
244
|
+
)
|
|
245
|
+
|
|
246
|
+
|
|
215
247
|
def check_tests_pass(ctx: Context, check_id: str = "tests") -> Check:
|
|
216
248
|
title = "Your test suite passes"
|
|
249
|
+
command = [sys.executable, "-m", "pytest", "-q", "--no-header", "-x"]
|
|
217
250
|
try:
|
|
218
|
-
result = ctx.
|
|
219
|
-
except subprocess.TimeoutExpired:
|
|
251
|
+
result = ctx.run(command, timeout=ctx.timeout)
|
|
252
|
+
except subprocess.TimeoutExpired as expired:
|
|
253
|
+
# "timeout", not "fail": the runner turns it back into "fail" for a frozen delivery (ADR 0047).
|
|
220
254
|
return Check(
|
|
221
255
|
check_id,
|
|
222
256
|
title,
|
|
223
|
-
"
|
|
257
|
+
"timeout",
|
|
224
258
|
f"the tests did not finish within {ctx.timeout}s",
|
|
225
259
|
(issue("tests.timeout", seconds=ctx.timeout),),
|
|
260
|
+
trace_of(command, expired, ctx.timeout),
|
|
226
261
|
)
|
|
227
262
|
if result.returncode == 5:
|
|
228
|
-
return Check(
|
|
263
|
+
return Check(
|
|
264
|
+
check_id,
|
|
265
|
+
title,
|
|
266
|
+
"fail",
|
|
267
|
+
"no tests were collected",
|
|
268
|
+
(issue("tests.none_collected"),),
|
|
269
|
+
trace_of(command, result),
|
|
270
|
+
)
|
|
229
271
|
if result.returncode != 0:
|
|
230
272
|
tail = (result.stdout or result.stderr).strip().splitlines()[-3:]
|
|
231
273
|
return Check(
|
|
@@ -234,6 +276,7 @@ def check_tests_pass(ctx: Context, check_id: str = "tests") -> Check:
|
|
|
234
276
|
"fail",
|
|
235
277
|
" / ".join(tail) or "pytest returned a non-zero exit code",
|
|
236
278
|
(issue("tests.failed", detail=" / ".join(tail)),),
|
|
279
|
+
trace_of(command, result),
|
|
237
280
|
)
|
|
238
281
|
return Check(check_id, title, "pass")
|
|
239
282
|
|
|
@@ -257,7 +300,10 @@ def check_tests_cover_invariants(ctx: Context, error_names: list[str], check_id:
|
|
|
257
300
|
return Check(check_id, title, "fail", "I cannot find any test files", (issue("tests.no_files"),))
|
|
258
301
|
for path in files:
|
|
259
302
|
text = path.read_text(encoding="utf-8", errors="ignore")
|
|
260
|
-
|
|
303
|
+
# `pytest.raises` and `unittest`'s `assertRaises` (with its Regex and callable forms) say a refusal is
|
|
304
|
+
# expected (ADR 0051). Case matters on purpose: a docstring's "Raises:" is not a test. A refusal caught
|
|
305
|
+
# with try/except is still not seen.
|
|
306
|
+
if ("raises" in text or "assertRaises" in text) and any(name in text for name in error_names):
|
|
261
307
|
return Check(check_id, title, "pass")
|
|
262
308
|
return Check(
|
|
263
309
|
check_id,
|
|
@@ -273,13 +319,14 @@ def check_determinism(ctx: Context, args: list[str], check_id: str = "determinis
|
|
|
273
319
|
try:
|
|
274
320
|
first = ctx.cli(args, timeout=120)
|
|
275
321
|
second = ctx.cli(args, timeout=120)
|
|
276
|
-
except subprocess.TimeoutExpired:
|
|
322
|
+
except subprocess.TimeoutExpired as expired:
|
|
277
323
|
return Check(
|
|
278
324
|
check_id,
|
|
279
325
|
title,
|
|
280
|
-
"
|
|
326
|
+
"timeout",
|
|
281
327
|
"one of the two runs did not finish within 120s",
|
|
282
328
|
(issue("determinism.timeout", seconds=120),),
|
|
329
|
+
trace_of(list(args), expired, 120),
|
|
283
330
|
)
|
|
284
331
|
if first.returncode != 0 or second.returncode != 0:
|
|
285
332
|
return Check(
|
|
@@ -0,0 +1,199 @@
|
|
|
1
|
+
"""Runs the learner's code with a budget (foundation 09, D4 and F4; ADR 0046).
|
|
2
|
+
|
|
3
|
+
Every command the verifier runs for a check goes through here, so there is one place that says what learner
|
|
4
|
+
code may do while it is being verified:
|
|
5
|
+
|
|
6
|
+
time a limit; at the limit the WHOLE process tree is killed, not only the process we started
|
|
7
|
+
output bounded in memory; a runaway print cannot exhaust it. The head and the tail are kept,
|
|
8
|
+
because the last lines (a test summary, an exit message) are what a failure message reads
|
|
9
|
+
secrets variables that look like credentials are not handed down. In CI the learner's job
|
|
10
|
+
holds a token, and the learner's code is the one thing here that is not ours
|
|
11
|
+
input closed, so a program that waits for a keyboard ends instead of hanging until the limit
|
|
12
|
+
|
|
13
|
+
It is portable: the learner works on Windows, where the group kill is `taskkill /T`, and on Linux, where it
|
|
14
|
+
is a process group. There is no memory limit: the OS gives no portable way to set one without breaking Python,
|
|
15
|
+
and a limit that is only enforced on one platform would be a promise nobody could rely on.
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
import contextlib
|
|
21
|
+
import locale
|
|
22
|
+
import os
|
|
23
|
+
import re
|
|
24
|
+
import signal
|
|
25
|
+
import subprocess
|
|
26
|
+
import sys
|
|
27
|
+
import threading
|
|
28
|
+
import time
|
|
29
|
+
from collections.abc import Mapping, Sequence
|
|
30
|
+
from pathlib import Path
|
|
31
|
+
|
|
32
|
+
# Per stream. Large enough for any real pytest report, small enough that a loop of `print` costs nothing.
|
|
33
|
+
MAX_OUTPUT_BYTES = 30_000
|
|
34
|
+
_HEAD = MAX_OUTPUT_BYTES // 2
|
|
35
|
+
_TAIL = MAX_OUTPUT_BYTES - _HEAD
|
|
36
|
+
|
|
37
|
+
# Names that hold credentials in practice. Matching on the name is crude and deliberately wide.
|
|
38
|
+
_SECRET_NAME = re.compile(r"(TOKEN|SECRET|PASSWORD|PASSWD|API_?KEY|PRIVATE_?KEY|CREDENTIAL)", re.IGNORECASE)
|
|
39
|
+
|
|
40
|
+
_WINDOWS = sys.platform == "win32"
|
|
41
|
+
# After SIGTERM, how long the group gets before SIGKILL.
|
|
42
|
+
_GRACE_SECONDS = 2.0
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def scrubbed_environment(env: Mapping[str, str] | None = None) -> dict[str, str]:
|
|
46
|
+
"""The environment for learner code: the current one minus anything that looks like a credential."""
|
|
47
|
+
source = os.environ if env is None else env
|
|
48
|
+
return {name: value for name, value in source.items() if not _SECRET_NAME.search(name)}
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
class Executed(subprocess.CompletedProcess): # type: ignore[type-arg]
|
|
52
|
+
"""What `subprocess.run` returned, plus what the budget did to it."""
|
|
53
|
+
|
|
54
|
+
def __init__(
|
|
55
|
+
self,
|
|
56
|
+
args: Sequence[str],
|
|
57
|
+
returncode: int,
|
|
58
|
+
stdout: str,
|
|
59
|
+
stderr: str,
|
|
60
|
+
*,
|
|
61
|
+
truncated: bool,
|
|
62
|
+
seconds: float,
|
|
63
|
+
) -> None:
|
|
64
|
+
super().__init__(args, returncode, stdout, stderr)
|
|
65
|
+
self.truncated = truncated
|
|
66
|
+
self.seconds = seconds
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
class _Bounded:
|
|
70
|
+
"""Reads a stream to its end, keeping the head and the tail of what is too long."""
|
|
71
|
+
|
|
72
|
+
def __init__(self) -> None:
|
|
73
|
+
self.head = bytearray()
|
|
74
|
+
self.tail = bytearray()
|
|
75
|
+
self.total = 0
|
|
76
|
+
|
|
77
|
+
def feed(self, chunk: bytes) -> None:
|
|
78
|
+
self.total += len(chunk)
|
|
79
|
+
room = _HEAD - len(self.head)
|
|
80
|
+
if room > 0:
|
|
81
|
+
self.head += chunk[:room]
|
|
82
|
+
chunk = chunk[room:]
|
|
83
|
+
if chunk:
|
|
84
|
+
self.tail += chunk
|
|
85
|
+
if len(self.tail) > _TAIL:
|
|
86
|
+
del self.tail[: len(self.tail) - _TAIL]
|
|
87
|
+
|
|
88
|
+
@property
|
|
89
|
+
def truncated(self) -> bool:
|
|
90
|
+
return self.total > len(self.head) + len(self.tail)
|
|
91
|
+
|
|
92
|
+
def text(self) -> str:
|
|
93
|
+
encoding = locale.getpreferredencoding(False) or "utf-8"
|
|
94
|
+
if not self.truncated:
|
|
95
|
+
return bytes(self.head + self.tail).decode(encoding, errors="replace")
|
|
96
|
+
omitted = self.total - len(self.head) - len(self.tail)
|
|
97
|
+
marker = f"\n[... {omitted} bytes of output omitted ...]\n"
|
|
98
|
+
head = bytes(self.head).decode(encoding, errors="replace")
|
|
99
|
+
tail = bytes(self.tail).decode(encoding, errors="replace")
|
|
100
|
+
return head + marker + tail
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def _pump(stream, sink: _Bounded) -> None: # noqa: ANN001 - a binary pipe
|
|
104
|
+
try:
|
|
105
|
+
while True:
|
|
106
|
+
chunk = stream.read1(65536) if hasattr(stream, "read1") else stream.read(65536)
|
|
107
|
+
if not chunk:
|
|
108
|
+
return
|
|
109
|
+
sink.feed(chunk)
|
|
110
|
+
except (OSError, ValueError):
|
|
111
|
+
return # the pipe was closed under us when the tree was killed
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def kill_tree(process: subprocess.Popen) -> None: # type: ignore[type-arg]
|
|
115
|
+
"""Ends the process and everything it started. Gives a group a moment on POSIX before it forces."""
|
|
116
|
+
if _WINDOWS:
|
|
117
|
+
subprocess.run(
|
|
118
|
+
["taskkill", "/F", "/T", "/PID", str(process.pid)],
|
|
119
|
+
capture_output=True,
|
|
120
|
+
check=False,
|
|
121
|
+
)
|
|
122
|
+
with contextlib.suppress(OSError):
|
|
123
|
+
process.kill()
|
|
124
|
+
return
|
|
125
|
+
try:
|
|
126
|
+
os.killpg(process.pid, signal.SIGTERM)
|
|
127
|
+
except (ProcessLookupError, PermissionError):
|
|
128
|
+
return
|
|
129
|
+
deadline = time.monotonic() + _GRACE_SECONDS
|
|
130
|
+
while time.monotonic() < deadline:
|
|
131
|
+
try:
|
|
132
|
+
os.killpg(process.pid, 0)
|
|
133
|
+
except ProcessLookupError:
|
|
134
|
+
return
|
|
135
|
+
time.sleep(0.05)
|
|
136
|
+
with contextlib.suppress(ProcessLookupError):
|
|
137
|
+
os.killpg(process.pid, signal.SIGKILL)
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def run_bounded(
|
|
141
|
+
args: Sequence[str],
|
|
142
|
+
*,
|
|
143
|
+
cwd: Path,
|
|
144
|
+
timeout: float,
|
|
145
|
+
env: Mapping[str, str] | None = None,
|
|
146
|
+
) -> Executed:
|
|
147
|
+
"""Runs `args` in `cwd` within the budget. Raises `subprocess.TimeoutExpired` at the limit, after the
|
|
148
|
+
whole tree has been killed, with whatever it printed so far."""
|
|
149
|
+
options: dict = {}
|
|
150
|
+
if _WINDOWS:
|
|
151
|
+
options["creationflags"] = subprocess.CREATE_NEW_PROCESS_GROUP
|
|
152
|
+
else:
|
|
153
|
+
options["start_new_session"] = True
|
|
154
|
+
|
|
155
|
+
started = time.monotonic()
|
|
156
|
+
process = subprocess.Popen(
|
|
157
|
+
list(args),
|
|
158
|
+
cwd=cwd,
|
|
159
|
+
stdin=subprocess.DEVNULL,
|
|
160
|
+
stdout=subprocess.PIPE,
|
|
161
|
+
stderr=subprocess.PIPE,
|
|
162
|
+
env=scrubbed_environment(env),
|
|
163
|
+
**options,
|
|
164
|
+
)
|
|
165
|
+
out, err = _Bounded(), _Bounded()
|
|
166
|
+
readers = [
|
|
167
|
+
threading.Thread(target=_pump, args=(process.stdout, out), daemon=True),
|
|
168
|
+
threading.Thread(target=_pump, args=(process.stderr, err), daemon=True),
|
|
169
|
+
]
|
|
170
|
+
for reader in readers:
|
|
171
|
+
reader.start()
|
|
172
|
+
|
|
173
|
+
try:
|
|
174
|
+
process.wait(timeout=timeout)
|
|
175
|
+
except subprocess.TimeoutExpired:
|
|
176
|
+
kill_tree(process)
|
|
177
|
+
process.wait()
|
|
178
|
+
for reader in readers:
|
|
179
|
+
reader.join(timeout=_GRACE_SECONDS)
|
|
180
|
+
raise subprocess.TimeoutExpired(list(args), timeout, output=out.text(), stderr=err.text()) from None
|
|
181
|
+
|
|
182
|
+
# The leader exited. A child it left running can keep the pipes open, so do not wait for them for ever;
|
|
183
|
+
# if they are still open after the grace period, something of the learner's is still running: end it.
|
|
184
|
+
# Only then: on Windows a process id that has exited can already belong to someone else.
|
|
185
|
+
for reader in readers:
|
|
186
|
+
reader.join(timeout=_GRACE_SECONDS)
|
|
187
|
+
if any(reader.is_alive() for reader in readers):
|
|
188
|
+
kill_tree(process)
|
|
189
|
+
for reader in readers:
|
|
190
|
+
reader.join(timeout=_GRACE_SECONDS)
|
|
191
|
+
|
|
192
|
+
return Executed(
|
|
193
|
+
list(args),
|
|
194
|
+
process.returncode,
|
|
195
|
+
out.text(),
|
|
196
|
+
err.text(),
|
|
197
|
+
truncated=out.truncated or err.truncated,
|
|
198
|
+
seconds=round(time.monotonic() - started, 3),
|
|
199
|
+
)
|
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
"""Calls the learner's code in a child process, under the budget of `execution.py` (ADR 0048).
|
|
2
|
+
|
|
3
|
+
A1's checks import the learner's symbols into the verifier's own process (`common.import_symbol`) and call them
|
|
4
|
+
there. That is fine while the verifier only reads what a learner built: a hang there is a bug of the learner's
|
|
5
|
+
that the whole run waits on, and A1 is frozen (ADR 0009), so it stays as it is.
|
|
6
|
+
|
|
7
|
+
From A2 on, a check hands the learner's code something of its own (a stub policy the learner has never seen) and
|
|
8
|
+
calls it. If that code loops, raises `SystemExit` or kills the interpreter, it must take the child with it and
|
|
9
|
+
nothing else. So a check that calls learner code runs its snippet here:
|
|
10
|
+
|
|
11
|
+
- the snippet is the verifier's, written as a string and run in a fresh interpreter whose `sys.path` starts
|
|
12
|
+
with the learner's root; it can `load("package.module:Symbol")` and sets `result` to a JSON value
|
|
13
|
+
- what the learner prints cannot be mistaken for the answer: the answer is the last thing on stdout, between
|
|
14
|
+
two markers, and the budget keeps the tail of a long output (`execution.py`)
|
|
15
|
+
- four outcomes, which a check turns into its own words: `ok`, `raised` (the learner's code raised),
|
|
16
|
+
`timeout` and `crashed` (the interpreter ended without answering)
|
|
17
|
+
|
|
18
|
+
It runs whatever interpreter the verifier runs under, as `Context.python` does, so the learner's dependencies are
|
|
19
|
+
the ones installed in that environment.
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
from __future__ import annotations
|
|
23
|
+
|
|
24
|
+
import base64
|
|
25
|
+
import json
|
|
26
|
+
import subprocess
|
|
27
|
+
import sys
|
|
28
|
+
import tempfile
|
|
29
|
+
from dataclasses import dataclass
|
|
30
|
+
from pathlib import Path
|
|
31
|
+
from typing import Any, Literal
|
|
32
|
+
|
|
33
|
+
from ..report import Trace
|
|
34
|
+
from .common import Context, trace_of
|
|
35
|
+
from .execution import Executed, run_bounded
|
|
36
|
+
|
|
37
|
+
START = "<<<bitexoft-result>>>"
|
|
38
|
+
END = "<<<bitexoft-end>>>"
|
|
39
|
+
|
|
40
|
+
# Runs in the child. Reads the check's snippet and payload from files (stdin is closed by the budget, and a
|
|
41
|
+
# command line has a length limit), imports from the learner's root first, and answers between markers.
|
|
42
|
+
#
|
|
43
|
+
# The answer is base64, so the markers cannot appear inside it. They used to: from Python 3.13 a traceback of code run
|
|
44
|
+
# with `-c` shows its source lines, and the line that writes the markers went into the answer's own `traceback` text, so
|
|
45
|
+
# the parent read a marker from the middle of the answer. An exception message or a learner's `print` can hold them too.
|
|
46
|
+
_BOOT = f"""
|
|
47
|
+
import base64, json, sys, traceback
|
|
48
|
+
root, payload_path, script_path = sys.argv[1:4]
|
|
49
|
+
sys.path.insert(0, root)
|
|
50
|
+
from bitexoft_verify.checks.common import import_symbol
|
|
51
|
+
def emit(answer):
|
|
52
|
+
sys.stdout.write("\\n{START}" + base64.b64encode(json.dumps(answer).encode("utf-8")).decode("ascii") + "{END}\\n")
|
|
53
|
+
sys.stdout.flush()
|
|
54
|
+
try:
|
|
55
|
+
with open(payload_path, encoding="utf-8") as handle:
|
|
56
|
+
payload = json.load(handle)
|
|
57
|
+
with open(script_path, encoding="utf-8") as handle:
|
|
58
|
+
source = handle.read()
|
|
59
|
+
scope = {{"payload": payload, "load": import_symbol, "result": None}}
|
|
60
|
+
exec(compile(source, "<check>", "exec"), scope)
|
|
61
|
+
emit({{"ok": scope["result"]}})
|
|
62
|
+
except BaseException as error:
|
|
63
|
+
emit({{
|
|
64
|
+
"raised": type(error).__name__,
|
|
65
|
+
"message": str(error)[:500],
|
|
66
|
+
"traceback": "".join(traceback.format_exception(error))[-1500:],
|
|
67
|
+
}})
|
|
68
|
+
"""
|
|
69
|
+
|
|
70
|
+
Kind = Literal["ok", "raised", "timeout", "crashed"]
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
@dataclass(frozen=True, slots=True)
|
|
74
|
+
class ChildOutcome:
|
|
75
|
+
kind: Kind
|
|
76
|
+
value: Any = None # `ok`: what the snippet set `result` to
|
|
77
|
+
error: str | None = None # `raised`: the exception's type name
|
|
78
|
+
message: str = "" # `raised`: its message
|
|
79
|
+
detail: str = "" # `raised`: the end of the traceback; `crashed`: the end of stderr
|
|
80
|
+
exit_code: int | None = None
|
|
81
|
+
command: tuple[str, ...] = ()
|
|
82
|
+
executed: Executed | subprocess.TimeoutExpired | None = None
|
|
83
|
+
|
|
84
|
+
def trace(self) -> Trace | None:
|
|
85
|
+
"""What was run and what came back, for a check that opts in to showing it."""
|
|
86
|
+
if self.executed is None:
|
|
87
|
+
return None
|
|
88
|
+
return trace_of(list(self.command), self.executed)
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def _answer(stdout: str) -> dict[str, Any] | None:
|
|
92
|
+
"""The last well-formed answer between the markers, or None if the child never gave one."""
|
|
93
|
+
end = stdout.rfind(END)
|
|
94
|
+
start = stdout.rfind(START, 0, end) if end != -1 else -1
|
|
95
|
+
if start == -1:
|
|
96
|
+
return None
|
|
97
|
+
try:
|
|
98
|
+
answer = json.loads(base64.b64decode(stdout[start + len(START) : end], validate=True))
|
|
99
|
+
except ValueError: # not base64 (binascii.Error is one), not UTF-8 or not JSON
|
|
100
|
+
return None
|
|
101
|
+
return answer if isinstance(answer, dict) else None
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def run_in_child(root: Path, script: str, payload: Any = None, timeout: float = 30) -> ChildOutcome:
|
|
105
|
+
"""Runs `script` in a fresh interpreter at `root`, with `payload` visible to it, and says what happened."""
|
|
106
|
+
with tempfile.TemporaryDirectory(prefix="bitexoft-child-") as folder:
|
|
107
|
+
payload_path = Path(folder) / "payload.json"
|
|
108
|
+
script_path = Path(folder) / "script.py"
|
|
109
|
+
payload_path.write_text(json.dumps(payload), encoding="utf-8")
|
|
110
|
+
script_path.write_text(script, encoding="utf-8")
|
|
111
|
+
command = (sys.executable, "-c", _BOOT, str(root.resolve()), str(payload_path), str(script_path))
|
|
112
|
+
# What a person reads in a trace: not the bootstrap, not temporary paths.
|
|
113
|
+
shown = (sys.executable, "-c", "<bitexoft: call the code under test>")
|
|
114
|
+
try:
|
|
115
|
+
executed = run_bounded(list(command), cwd=root, timeout=timeout)
|
|
116
|
+
except subprocess.TimeoutExpired as expired:
|
|
117
|
+
return ChildOutcome("timeout", command=shown, executed=expired)
|
|
118
|
+
|
|
119
|
+
answer = _answer(executed.stdout)
|
|
120
|
+
if answer is None:
|
|
121
|
+
tail = "\n".join((executed.stderr or "").strip().splitlines()[-6:])
|
|
122
|
+
return ChildOutcome("crashed", detail=tail, exit_code=executed.returncode, command=shown, executed=executed)
|
|
123
|
+
if "ok" in answer:
|
|
124
|
+
return ChildOutcome("ok", value=answer["ok"], exit_code=executed.returncode, command=shown, executed=executed)
|
|
125
|
+
return ChildOutcome(
|
|
126
|
+
"raised",
|
|
127
|
+
error=str(answer.get("raised", "")),
|
|
128
|
+
message=str(answer.get("message", "")),
|
|
129
|
+
detail=str(answer.get("traceback", "")),
|
|
130
|
+
exit_code=executed.returncode,
|
|
131
|
+
command=shown,
|
|
132
|
+
executed=executed,
|
|
133
|
+
)
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def call_in_child(ctx: Context, script: str, payload: Any = None, timeout: float | None = None) -> ChildOutcome:
|
|
137
|
+
"""`run_in_child` at the learner's repository, within the check's own budget unless one is given."""
|
|
138
|
+
return run_in_child(ctx.root, script, payload, timeout if timeout is not None else ctx.timeout)
|