bitexoft-verify 0.2.0__tar.gz → 0.2.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. {bitexoft_verify-0.2.0 → bitexoft_verify-0.2.1}/.gitignore +1 -0
  2. {bitexoft_verify-0.2.0 → bitexoft_verify-0.2.1}/PKG-INFO +1 -1
  3. {bitexoft_verify-0.2.0 → bitexoft_verify-0.2.1}/src/bitexoft_verify/__init__.py +1 -1
  4. bitexoft_verify-0.2.1/src/bitexoft_verify/checks/assertions.py +129 -0
  5. {bitexoft_verify-0.2.0 → bitexoft_verify-0.2.1}/src/bitexoft_verify/checks/common.py +64 -17
  6. bitexoft_verify-0.2.1/src/bitexoft_verify/checks/execution.py +199 -0
  7. bitexoft_verify-0.2.1/src/bitexoft_verify/checks/isolated.py +138 -0
  8. {bitexoft_verify-0.2.0 → bitexoft_verify-0.2.1}/src/bitexoft_verify/cli.py +22 -4
  9. {bitexoft_verify-0.2.0 → bitexoft_verify-0.2.1}/src/bitexoft_verify/codes.py +6 -0
  10. {bitexoft_verify-0.2.0 → bitexoft_verify-0.2.1}/src/bitexoft_verify/data/issue-codes.json +19 -0
  11. {bitexoft_verify-0.2.0 → bitexoft_verify-0.2.1}/src/bitexoft_verify/report.py +57 -7
  12. bitexoft_verify-0.2.1/src/bitexoft_verify/runner.py +137 -0
  13. bitexoft_verify-0.2.1/src/bitexoft_verify/seeds.py +43 -0
  14. {bitexoft_verify-0.2.0 → bitexoft_verify-0.2.1}/tests/test_a1.py +90 -1
  15. bitexoft_verify-0.2.1/tests/test_a1_benign.py +309 -0
  16. bitexoft_verify-0.2.1/tests/test_assertions.py +96 -0
  17. bitexoft_verify-0.2.1/tests/test_execution.py +86 -0
  18. bitexoft_verify-0.2.1/tests/test_isolated.py +145 -0
  19. {bitexoft_verify-0.2.0 → bitexoft_verify-0.2.1}/tests/test_issues.py +11 -2
  20. bitexoft_verify-0.2.1/tests/test_parity.py +100 -0
  21. bitexoft_verify-0.2.1/tests/test_projects.py +119 -0
  22. bitexoft_verify-0.2.1/tests/test_report_v4.py +98 -0
  23. bitexoft_verify-0.2.1/tests/test_seeds.py +107 -0
  24. bitexoft_verify-0.2.0/src/bitexoft_verify/runner.py +0 -101
  25. {bitexoft_verify-0.2.0 → bitexoft_verify-0.2.1}/README.md +0 -0
  26. {bitexoft_verify-0.2.0 → bitexoft_verify-0.2.1}/pyproject.toml +0 -0
  27. {bitexoft_verify-0.2.0 → bitexoft_verify-0.2.1}/src/bitexoft_verify/__main__.py +0 -0
  28. {bitexoft_verify-0.2.0 → bitexoft_verify-0.2.1}/src/bitexoft_verify/checks/__init__.py +0 -0
  29. {bitexoft_verify-0.2.0 → bitexoft_verify-0.2.1}/src/bitexoft_verify/checks/a1.py +0 -0
  30. {bitexoft_verify-0.2.0 → bitexoft_verify-0.2.1}/src/bitexoft_verify/data/README.md +0 -0
  31. {bitexoft_verify-0.2.0 → bitexoft_verify-0.2.1}/src/bitexoft_verify/data/a1_impossible_sample.csv +0 -0
  32. {bitexoft_verify-0.2.0 → bitexoft_verify-0.2.1}/src/bitexoft_verify/data/a1_match_sample.csv +0 -0
  33. {bitexoft_verify-0.2.0 → bitexoft_verify-0.2.1}/src/bitexoft_verify/data/a2_isolated_sample.csv +0 -0
  34. {bitexoft_verify-0.2.0 → bitexoft_verify-0.2.1}/src/bitexoft_verify/manifest.py +0 -0
  35. {bitexoft_verify-0.2.0 → bitexoft_verify-0.2.1}/tests/conftest.py +0 -0
  36. {bitexoft_verify-0.2.0 → bitexoft_verify-0.2.1}/tests/golden/a1_checks.json +0 -0
  37. {bitexoft_verify-0.2.0 → bitexoft_verify-0.2.1}/tests/test_equivalence.py +0 -0
  38. {bitexoft_verify-0.2.0 → bitexoft_verify-0.2.1}/tests/test_version.py +0 -0
  39. {bitexoft_verify-0.2.0 → bitexoft_verify-0.2.1}/tools/make_fixtures.py +0 -0
@@ -11,6 +11,7 @@ __pycache__/
11
11
 
12
12
  # generated
13
13
  app/dist/
14
+ app/shots/
14
15
  dist/
15
16
  build/
16
17
  app/src/content/catalog.json
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: bitexoft-verify
3
- Version: 0.2.0
3
+ Version: 0.2.1
4
4
  Summary: Verifica las entregas de los proyectos de Bitexoft: la misma herramienta en tu máquina y en CI.
5
5
  Author: Bitexoft
6
6
  License: Proprietary
@@ -1,3 +1,3 @@
1
1
  """bitexoft-verify: the platform's only definition of "correct"."""
2
2
 
3
- __version__ = "0.2.0"
3
+ __version__ = "0.2.1"
@@ -0,0 +1,129 @@
1
+ """One vocabulary for "this value is not what it should be" (ADR 0049; foundation 09, D5, F12).
2
+
3
+ Every check that compares a value used to word its own failure, and no two worded it alike. An assertion here
4
+ is a small function from a value to the list of things wrong with it; a check runs one against a value and the
5
+ failures land in its `Problems`, each with a stable issue code and the values behind it (`expected`, `found`), so
6
+ the platform writes the sentence in the learner's language and the terminal reads the same in English.
7
+
8
+ equal(expected) the value is exactly this
9
+ contains(needle) the text includes this
10
+ not_contains(needle) the text does not include this
11
+ matches(pattern) the text matches this regular expression (searched, not anchored)
12
+ all_of(a, b, ...) every one holds, and each that does not is reported: not just the first
13
+ annotated(label, a) says what was being looked at, in front of whatever `a` finds wrong
14
+
15
+ problems = Problems()
16
+ check_value(problems, annotated("verdict on the stranded frame", equal("BROKEN")), verdict)
17
+ if problems:
18
+ return problems.failing("furthest_policy", "The second policy notices a stranded player")
19
+
20
+ An assertion never raises for a value it did not expect: a value of the wrong type simply does not satisfy it, and
21
+ what the learner's code returned is described, not trusted.
22
+ """
23
+
24
+ from __future__ import annotations
25
+
26
+ import re
27
+ from collections.abc import Callable
28
+ from dataclasses import dataclass
29
+ from typing import Any
30
+
31
+ from ..report import Issue, Problems, issue
32
+
33
+ SHOWN = 80 # characters of a found value in a sentence; a long one is cut, with a mark
34
+
35
+
36
+ def _shown(value: Any) -> str:
37
+ text = repr(value) if not isinstance(value, str) else value
38
+ return text if len(text) <= SHOWN else text[: SHOWN - 1] + "…"
39
+
40
+
41
+ @dataclass(frozen=True, slots=True)
42
+ class Failure:
43
+ """One thing wrong: the English sentence for the terminal and the issue for the platform."""
44
+
45
+ text: str
46
+ issue: Issue
47
+
48
+ def with_context(self, context: str) -> Failure:
49
+ return Failure(f"{context}: {self.text}", Issue(self.issue.code, {**self.issue.params, "context": context}))
50
+
51
+
52
+ Assertion = Callable[[Any], list[Failure]]
53
+
54
+
55
+ def equal(expected: Any) -> Assertion:
56
+ """Exact, type included: `3` is not `3.0` and `1` is not `True`."""
57
+
58
+ def run(value: Any) -> list[Failure]:
59
+ if value == expected and type(value) is type(expected):
60
+ return []
61
+ return [
62
+ Failure(
63
+ f"expected {_shown(expected)} but found {_shown(value)}",
64
+ issue("assert.not_equal", expected=_shown(expected), found=_shown(value)),
65
+ )
66
+ ]
67
+
68
+ return run
69
+
70
+
71
+ def contains(needle: str) -> Assertion:
72
+ def run(value: Any) -> list[Failure]:
73
+ if isinstance(value, str) and needle in value:
74
+ return []
75
+ return [
76
+ Failure(
77
+ f"expected to find {_shown(needle)} in {_shown(value)}",
78
+ issue("assert.missing", expected=_shown(needle), found=_shown(value)),
79
+ )
80
+ ]
81
+
82
+ return run
83
+
84
+
85
+ def not_contains(needle: str) -> Assertion:
86
+ def run(value: Any) -> list[Failure]:
87
+ if not isinstance(value, str) or needle not in value:
88
+ return []
89
+ return [Failure(f"did not expect to find {_shown(needle)}", issue("assert.unexpected", found=_shown(needle)))]
90
+
91
+ return run
92
+
93
+
94
+ def matches(pattern: str) -> Assertion:
95
+ compiled = re.compile(pattern)
96
+
97
+ def run(value: Any) -> list[Failure]:
98
+ if isinstance(value, str) and compiled.search(value):
99
+ return []
100
+ return [
101
+ Failure(
102
+ f"{_shown(value)} does not match {pattern}",
103
+ issue("assert.no_match", expected=pattern, found=_shown(value)),
104
+ )
105
+ ]
106
+
107
+ return run
108
+
109
+
110
+ def all_of(*assertions: Assertion) -> Assertion:
111
+ def run(value: Any) -> list[Failure]:
112
+ return [failure for assertion in assertions for failure in assertion(value)]
113
+
114
+ return run
115
+
116
+
117
+ def annotated(context: str, assertion: Assertion) -> Assertion:
118
+ def run(value: Any) -> list[Failure]:
119
+ return [failure.with_context(context) for failure in assertion(value)]
120
+
121
+ return run
122
+
123
+
124
+ def check_value(problems: Problems, assertion: Assertion, value: Any) -> bool:
125
+ """Runs `assertion` on `value`, adds what is wrong to `problems`, and says whether the value held."""
126
+ failures = assertion(value)
127
+ for failure in failures:
128
+ problems.add_group(failure.text, [failure.issue])
129
+ return not failures
@@ -9,6 +9,7 @@ from __future__ import annotations
9
9
 
10
10
  import ast
11
11
  import importlib
12
+ import random
12
13
  import subprocess
13
14
  import sys
14
15
  from dataclasses import dataclass
@@ -17,7 +18,8 @@ from pathlib import Path
17
18
  from typing import Any
18
19
 
19
20
  from ..manifest import Manifest
20
- from ..report import Check, Problems, issue
21
+ from ..report import Check, Problems, Trace, issue
22
+ from .execution import run_bounded
21
23
 
22
24
  FRAMEWORKS = (
23
25
  "fastapi",
@@ -47,17 +49,19 @@ class Context:
47
49
  root: Path
48
50
  manifest: Manifest
49
51
  timeout: int = 180
52
+ # What a check that builds its own inputs draws them from (seeds.py). `seed_used` is how the report knows
53
+ # whether to say so: a run whose checks never asked for randomness carries no seed, as A1's never do.
54
+ seed: int = 0
55
+ seed_used: bool = False
56
+
57
+ def random(self, label: str) -> random.Random:
58
+ """A generator for `label`: the same seed and label always give the same sequence."""
59
+ self.seed_used = True
60
+ return random.Random(f"{self.seed}:{label}")
50
61
 
51
62
  def run(self, args: list[str], timeout: int | None = None) -> subprocess.CompletedProcess[str]:
52
- """Runs a command at the root of the learner's repository."""
53
- return subprocess.run(
54
- args,
55
- cwd=self.root,
56
- capture_output=True,
57
- text=True,
58
- timeout=timeout or self.timeout,
59
- check=False,
60
- )
63
+ """Runs a command at the root of the learner's repository, within the budget of `execution.py`."""
64
+ return run_bounded(args, cwd=self.root, timeout=timeout or self.timeout)
61
65
 
62
66
  def python(self, args: list[str], timeout: int | None = None) -> subprocess.CompletedProcess[str]:
63
67
  return self.run([sys.executable, *args], timeout=timeout)
@@ -212,20 +216,58 @@ def check_domain_has_no_io(ctx: Context, check_id: str = "boundary") -> Check:
212
216
  return Check(check_id, title, "pass")
213
217
 
214
218
 
219
+ def trace_of(
220
+ args: list[str],
221
+ result: subprocess.CompletedProcess | subprocess.TimeoutExpired, # type: ignore[type-arg]
222
+ seconds: float | None = None,
223
+ ) -> Trace:
224
+ """What was run and what came back, for a check that opts in to showing it.
225
+
226
+ Takes any `CompletedProcess`: `Executed` carries the seconds and whether the output was cut, and a plain one
227
+ (a stand-in in a test, or a command run another way) is shown without them."""
228
+ if isinstance(result, subprocess.TimeoutExpired):
229
+ return Trace(
230
+ command=" ".join(args),
231
+ exit_code=None,
232
+ stdout=str(result.output or ""),
233
+ stderr=str(result.stderr or ""),
234
+ seconds=float(seconds or result.timeout),
235
+ truncated=False,
236
+ )
237
+ return Trace(
238
+ command=" ".join(args),
239
+ exit_code=result.returncode,
240
+ stdout=result.stdout or "",
241
+ stderr=result.stderr or "",
242
+ seconds=float(getattr(result, "seconds", 0.0)),
243
+ truncated=bool(getattr(result, "truncated", False)),
244
+ )
245
+
246
+
215
247
  def check_tests_pass(ctx: Context, check_id: str = "tests") -> Check:
216
248
  title = "Your test suite passes"
249
+ command = [sys.executable, "-m", "pytest", "-q", "--no-header", "-x"]
217
250
  try:
218
- result = ctx.python(["-m", "pytest", "-q", "--no-header", "-x"], timeout=ctx.timeout)
219
- except subprocess.TimeoutExpired:
251
+ result = ctx.run(command, timeout=ctx.timeout)
252
+ except subprocess.TimeoutExpired as expired:
253
+ # "timeout", not "fail": the runner turns it back into "fail" for a frozen delivery (ADR 0047).
220
254
  return Check(
221
255
  check_id,
222
256
  title,
223
- "fail",
257
+ "timeout",
224
258
  f"the tests did not finish within {ctx.timeout}s",
225
259
  (issue("tests.timeout", seconds=ctx.timeout),),
260
+ trace_of(command, expired, ctx.timeout),
226
261
  )
227
262
  if result.returncode == 5:
228
- return Check(check_id, title, "fail", "no tests were collected", (issue("tests.none_collected"),))
263
+ return Check(
264
+ check_id,
265
+ title,
266
+ "fail",
267
+ "no tests were collected",
268
+ (issue("tests.none_collected"),),
269
+ trace_of(command, result),
270
+ )
229
271
  if result.returncode != 0:
230
272
  tail = (result.stdout or result.stderr).strip().splitlines()[-3:]
231
273
  return Check(
@@ -234,6 +276,7 @@ def check_tests_pass(ctx: Context, check_id: str = "tests") -> Check:
234
276
  "fail",
235
277
  " / ".join(tail) or "pytest returned a non-zero exit code",
236
278
  (issue("tests.failed", detail=" / ".join(tail)),),
279
+ trace_of(command, result),
237
280
  )
238
281
  return Check(check_id, title, "pass")
239
282
 
@@ -257,7 +300,10 @@ def check_tests_cover_invariants(ctx: Context, error_names: list[str], check_id:
257
300
  return Check(check_id, title, "fail", "I cannot find any test files", (issue("tests.no_files"),))
258
301
  for path in files:
259
302
  text = path.read_text(encoding="utf-8", errors="ignore")
260
- if "raises" in text and any(name in text for name in error_names):
303
+ # `pytest.raises` and `unittest`'s `assertRaises` (with its Regex and callable forms) say a refusal is
304
+ # expected (ADR 0051). Case matters on purpose: a docstring's "Raises:" is not a test. A refusal caught
305
+ # with try/except is still not seen.
306
+ if ("raises" in text or "assertRaises" in text) and any(name in text for name in error_names):
261
307
  return Check(check_id, title, "pass")
262
308
  return Check(
263
309
  check_id,
@@ -273,13 +319,14 @@ def check_determinism(ctx: Context, args: list[str], check_id: str = "determinis
273
319
  try:
274
320
  first = ctx.cli(args, timeout=120)
275
321
  second = ctx.cli(args, timeout=120)
276
- except subprocess.TimeoutExpired:
322
+ except subprocess.TimeoutExpired as expired:
277
323
  return Check(
278
324
  check_id,
279
325
  title,
280
- "fail",
326
+ "timeout",
281
327
  "one of the two runs did not finish within 120s",
282
328
  (issue("determinism.timeout", seconds=120),),
329
+ trace_of(list(args), expired, 120),
283
330
  )
284
331
  if first.returncode != 0 or second.returncode != 0:
285
332
  return Check(
@@ -0,0 +1,199 @@
1
+ """Runs the learner's code with a budget (foundation 09, D4 and F4; ADR 0046).
2
+
3
+ Every command the verifier runs for a check goes through here, so there is one place that says what learner
4
+ code may do while it is being verified:
5
+
6
+ time a limit; at the limit the WHOLE process tree is killed, not only the process we started
7
+ output bounded in memory; a runaway print cannot exhaust it. The head and the tail are kept,
8
+ because the last lines (a test summary, an exit message) are what a failure message reads
9
+ secrets variables that look like credentials are not handed down. In CI the learner's job
10
+ holds a token, and the learner's code is the one thing here that is not ours
11
+ input closed, so a program that waits for a keyboard ends instead of hanging until the limit
12
+
13
+ It is portable: the learner works on Windows, where the group kill is `taskkill /T`, and on Linux, where it
14
+ is a process group. There is no memory limit: the OS gives no portable way to set one without breaking Python,
15
+ and a limit that is only enforced on one platform would be a promise nobody could rely on.
16
+ """
17
+
18
+ from __future__ import annotations
19
+
20
+ import contextlib
21
+ import locale
22
+ import os
23
+ import re
24
+ import signal
25
+ import subprocess
26
+ import sys
27
+ import threading
28
+ import time
29
+ from collections.abc import Mapping, Sequence
30
+ from pathlib import Path
31
+
32
+ # Per stream. Large enough for any real pytest report, small enough that a loop of `print` costs nothing.
33
+ MAX_OUTPUT_BYTES = 30_000
34
+ _HEAD = MAX_OUTPUT_BYTES // 2
35
+ _TAIL = MAX_OUTPUT_BYTES - _HEAD
36
+
37
+ # Names that hold credentials in practice. Matching on the name is crude and deliberately wide.
38
+ _SECRET_NAME = re.compile(r"(TOKEN|SECRET|PASSWORD|PASSWD|API_?KEY|PRIVATE_?KEY|CREDENTIAL)", re.IGNORECASE)
39
+
40
+ _WINDOWS = sys.platform == "win32"
41
+ # After SIGTERM, how long the group gets before SIGKILL.
42
+ _GRACE_SECONDS = 2.0
43
+
44
+
45
+ def scrubbed_environment(env: Mapping[str, str] | None = None) -> dict[str, str]:
46
+ """The environment for learner code: the current one minus anything that looks like a credential."""
47
+ source = os.environ if env is None else env
48
+ return {name: value for name, value in source.items() if not _SECRET_NAME.search(name)}
49
+
50
+
51
+ class Executed(subprocess.CompletedProcess): # type: ignore[type-arg]
52
+ """What `subprocess.run` returned, plus what the budget did to it."""
53
+
54
+ def __init__(
55
+ self,
56
+ args: Sequence[str],
57
+ returncode: int,
58
+ stdout: str,
59
+ stderr: str,
60
+ *,
61
+ truncated: bool,
62
+ seconds: float,
63
+ ) -> None:
64
+ super().__init__(args, returncode, stdout, stderr)
65
+ self.truncated = truncated
66
+ self.seconds = seconds
67
+
68
+
69
+ class _Bounded:
70
+ """Reads a stream to its end, keeping the head and the tail of what is too long."""
71
+
72
+ def __init__(self) -> None:
73
+ self.head = bytearray()
74
+ self.tail = bytearray()
75
+ self.total = 0
76
+
77
+ def feed(self, chunk: bytes) -> None:
78
+ self.total += len(chunk)
79
+ room = _HEAD - len(self.head)
80
+ if room > 0:
81
+ self.head += chunk[:room]
82
+ chunk = chunk[room:]
83
+ if chunk:
84
+ self.tail += chunk
85
+ if len(self.tail) > _TAIL:
86
+ del self.tail[: len(self.tail) - _TAIL]
87
+
88
+ @property
89
+ def truncated(self) -> bool:
90
+ return self.total > len(self.head) + len(self.tail)
91
+
92
+ def text(self) -> str:
93
+ encoding = locale.getpreferredencoding(False) or "utf-8"
94
+ if not self.truncated:
95
+ return bytes(self.head + self.tail).decode(encoding, errors="replace")
96
+ omitted = self.total - len(self.head) - len(self.tail)
97
+ marker = f"\n[... {omitted} bytes of output omitted ...]\n"
98
+ head = bytes(self.head).decode(encoding, errors="replace")
99
+ tail = bytes(self.tail).decode(encoding, errors="replace")
100
+ return head + marker + tail
101
+
102
+
103
+ def _pump(stream, sink: _Bounded) -> None: # noqa: ANN001 - a binary pipe
104
+ try:
105
+ while True:
106
+ chunk = stream.read1(65536) if hasattr(stream, "read1") else stream.read(65536)
107
+ if not chunk:
108
+ return
109
+ sink.feed(chunk)
110
+ except (OSError, ValueError):
111
+ return # the pipe was closed under us when the tree was killed
112
+
113
+
114
+ def kill_tree(process: subprocess.Popen) -> None: # type: ignore[type-arg]
115
+ """Ends the process and everything it started. Gives a group a moment on POSIX before it forces."""
116
+ if _WINDOWS:
117
+ subprocess.run(
118
+ ["taskkill", "/F", "/T", "/PID", str(process.pid)],
119
+ capture_output=True,
120
+ check=False,
121
+ )
122
+ with contextlib.suppress(OSError):
123
+ process.kill()
124
+ return
125
+ try:
126
+ os.killpg(process.pid, signal.SIGTERM)
127
+ except (ProcessLookupError, PermissionError):
128
+ return
129
+ deadline = time.monotonic() + _GRACE_SECONDS
130
+ while time.monotonic() < deadline:
131
+ try:
132
+ os.killpg(process.pid, 0)
133
+ except ProcessLookupError:
134
+ return
135
+ time.sleep(0.05)
136
+ with contextlib.suppress(ProcessLookupError):
137
+ os.killpg(process.pid, signal.SIGKILL)
138
+
139
+
140
+ def run_bounded(
141
+ args: Sequence[str],
142
+ *,
143
+ cwd: Path,
144
+ timeout: float,
145
+ env: Mapping[str, str] | None = None,
146
+ ) -> Executed:
147
+ """Runs `args` in `cwd` within the budget. Raises `subprocess.TimeoutExpired` at the limit, after the
148
+ whole tree has been killed, with whatever it printed so far."""
149
+ options: dict = {}
150
+ if _WINDOWS:
151
+ options["creationflags"] = subprocess.CREATE_NEW_PROCESS_GROUP
152
+ else:
153
+ options["start_new_session"] = True
154
+
155
+ started = time.monotonic()
156
+ process = subprocess.Popen(
157
+ list(args),
158
+ cwd=cwd,
159
+ stdin=subprocess.DEVNULL,
160
+ stdout=subprocess.PIPE,
161
+ stderr=subprocess.PIPE,
162
+ env=scrubbed_environment(env),
163
+ **options,
164
+ )
165
+ out, err = _Bounded(), _Bounded()
166
+ readers = [
167
+ threading.Thread(target=_pump, args=(process.stdout, out), daemon=True),
168
+ threading.Thread(target=_pump, args=(process.stderr, err), daemon=True),
169
+ ]
170
+ for reader in readers:
171
+ reader.start()
172
+
173
+ try:
174
+ process.wait(timeout=timeout)
175
+ except subprocess.TimeoutExpired:
176
+ kill_tree(process)
177
+ process.wait()
178
+ for reader in readers:
179
+ reader.join(timeout=_GRACE_SECONDS)
180
+ raise subprocess.TimeoutExpired(list(args), timeout, output=out.text(), stderr=err.text()) from None
181
+
182
+ # The leader exited. A child it left running can keep the pipes open, so do not wait for them for ever;
183
+ # if they are still open after the grace period, something of the learner's is still running: end it.
184
+ # Only then: on Windows a process id that has exited can already belong to someone else.
185
+ for reader in readers:
186
+ reader.join(timeout=_GRACE_SECONDS)
187
+ if any(reader.is_alive() for reader in readers):
188
+ kill_tree(process)
189
+ for reader in readers:
190
+ reader.join(timeout=_GRACE_SECONDS)
191
+
192
+ return Executed(
193
+ list(args),
194
+ process.returncode,
195
+ out.text(),
196
+ err.text(),
197
+ truncated=out.truncated or err.truncated,
198
+ seconds=round(time.monotonic() - started, 3),
199
+ )
@@ -0,0 +1,138 @@
1
+ """Calls the learner's code in a child process, under the budget of `execution.py` (ADR 0048).
2
+
3
+ A1's checks import the learner's symbols into the verifier's own process (`common.import_symbol`) and call them
4
+ there. That is fine while the verifier only reads what a learner built: a hang there is a bug of the learner's
5
+ that the whole run waits on, and A1 is frozen (ADR 0009), so it stays as it is.
6
+
7
+ From A2 on, a check hands the learner's code something of its own (a stub policy the learner has never seen) and
8
+ calls it. If that code loops, raises `SystemExit` or kills the interpreter, it must take the child with it and
9
+ nothing else. So a check that calls learner code runs its snippet here:
10
+
11
+ - the snippet is the verifier's, written as a string and run in a fresh interpreter whose `sys.path` starts
12
+ with the learner's root; it can `load("package.module:Symbol")` and sets `result` to a JSON value
13
+ - what the learner prints cannot be mistaken for the answer: the answer is the last thing on stdout, between
14
+ two markers, and the budget keeps the tail of a long output (`execution.py`)
15
+ - four outcomes, which a check turns into its own words: `ok`, `raised` (the learner's code raised),
16
+ `timeout` and `crashed` (the interpreter ended without answering)
17
+
18
+ It runs whatever interpreter the verifier runs under, as `Context.python` does, so the learner's dependencies are
19
+ the ones installed in that environment.
20
+ """
21
+
22
+ from __future__ import annotations
23
+
24
+ import base64
25
+ import json
26
+ import subprocess
27
+ import sys
28
+ import tempfile
29
+ from dataclasses import dataclass
30
+ from pathlib import Path
31
+ from typing import Any, Literal
32
+
33
+ from ..report import Trace
34
+ from .common import Context, trace_of
35
+ from .execution import Executed, run_bounded
36
+
37
+ START = "<<<bitexoft-result>>>"
38
+ END = "<<<bitexoft-end>>>"
39
+
40
+ # Runs in the child. Reads the check's snippet and payload from files (stdin is closed by the budget, and a
41
+ # command line has a length limit), imports from the learner's root first, and answers between markers.
42
+ #
43
+ # The answer is base64, so the markers cannot appear inside it. They used to: from Python 3.13 a traceback of code run
44
+ # with `-c` shows its source lines, and the line that writes the markers went into the answer's own `traceback` text, so
45
+ # the parent read a marker from the middle of the answer. An exception message or a learner's `print` can hold them too.
46
+ _BOOT = f"""
47
+ import base64, json, sys, traceback
48
+ root, payload_path, script_path = sys.argv[1:4]
49
+ sys.path.insert(0, root)
50
+ from bitexoft_verify.checks.common import import_symbol
51
+ def emit(answer):
52
+ sys.stdout.write("\\n{START}" + base64.b64encode(json.dumps(answer).encode("utf-8")).decode("ascii") + "{END}\\n")
53
+ sys.stdout.flush()
54
+ try:
55
+ with open(payload_path, encoding="utf-8") as handle:
56
+ payload = json.load(handle)
57
+ with open(script_path, encoding="utf-8") as handle:
58
+ source = handle.read()
59
+ scope = {{"payload": payload, "load": import_symbol, "result": None}}
60
+ exec(compile(source, "<check>", "exec"), scope)
61
+ emit({{"ok": scope["result"]}})
62
+ except BaseException as error:
63
+ emit({{
64
+ "raised": type(error).__name__,
65
+ "message": str(error)[:500],
66
+ "traceback": "".join(traceback.format_exception(error))[-1500:],
67
+ }})
68
+ """
69
+
70
+ Kind = Literal["ok", "raised", "timeout", "crashed"]
71
+
72
+
73
+ @dataclass(frozen=True, slots=True)
74
+ class ChildOutcome:
75
+ kind: Kind
76
+ value: Any = None # `ok`: what the snippet set `result` to
77
+ error: str | None = None # `raised`: the exception's type name
78
+ message: str = "" # `raised`: its message
79
+ detail: str = "" # `raised`: the end of the traceback; `crashed`: the end of stderr
80
+ exit_code: int | None = None
81
+ command: tuple[str, ...] = ()
82
+ executed: Executed | subprocess.TimeoutExpired | None = None
83
+
84
+ def trace(self) -> Trace | None:
85
+ """What was run and what came back, for a check that opts in to showing it."""
86
+ if self.executed is None:
87
+ return None
88
+ return trace_of(list(self.command), self.executed)
89
+
90
+
91
+ def _answer(stdout: str) -> dict[str, Any] | None:
92
+ """The last well-formed answer between the markers, or None if the child never gave one."""
93
+ end = stdout.rfind(END)
94
+ start = stdout.rfind(START, 0, end) if end != -1 else -1
95
+ if start == -1:
96
+ return None
97
+ try:
98
+ answer = json.loads(base64.b64decode(stdout[start + len(START) : end], validate=True))
99
+ except ValueError: # not base64 (binascii.Error is one), not UTF-8 or not JSON
100
+ return None
101
+ return answer if isinstance(answer, dict) else None
102
+
103
+
104
+ def run_in_child(root: Path, script: str, payload: Any = None, timeout: float = 30) -> ChildOutcome:
105
+ """Runs `script` in a fresh interpreter at `root`, with `payload` visible to it, and says what happened."""
106
+ with tempfile.TemporaryDirectory(prefix="bitexoft-child-") as folder:
107
+ payload_path = Path(folder) / "payload.json"
108
+ script_path = Path(folder) / "script.py"
109
+ payload_path.write_text(json.dumps(payload), encoding="utf-8")
110
+ script_path.write_text(script, encoding="utf-8")
111
+ command = (sys.executable, "-c", _BOOT, str(root.resolve()), str(payload_path), str(script_path))
112
+ # What a person reads in a trace: not the bootstrap, not temporary paths.
113
+ shown = (sys.executable, "-c", "<bitexoft: call the code under test>")
114
+ try:
115
+ executed = run_bounded(list(command), cwd=root, timeout=timeout)
116
+ except subprocess.TimeoutExpired as expired:
117
+ return ChildOutcome("timeout", command=shown, executed=expired)
118
+
119
+ answer = _answer(executed.stdout)
120
+ if answer is None:
121
+ tail = "\n".join((executed.stderr or "").strip().splitlines()[-6:])
122
+ return ChildOutcome("crashed", detail=tail, exit_code=executed.returncode, command=shown, executed=executed)
123
+ if "ok" in answer:
124
+ return ChildOutcome("ok", value=answer["ok"], exit_code=executed.returncode, command=shown, executed=executed)
125
+ return ChildOutcome(
126
+ "raised",
127
+ error=str(answer.get("raised", "")),
128
+ message=str(answer.get("message", "")),
129
+ detail=str(answer.get("traceback", "")),
130
+ exit_code=executed.returncode,
131
+ command=shown,
132
+ executed=executed,
133
+ )
134
+
135
+
136
+ def call_in_child(ctx: Context, script: str, payload: Any = None, timeout: float | None = None) -> ChildOutcome:
137
+ """`run_in_child` at the learner's repository, within the check's own budget unless one is given."""
138
+ return run_in_child(ctx.root, script, payload, timeout if timeout is not None else ctx.timeout)