rolloutdiff 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- rolloutdiff/__init__.py +130 -0
- rolloutdiff/_aria_replay.py +346 -0
- rolloutdiff/_benchmarks/m0-2026-11-06.json +318 -0
- rolloutdiff/_dependence.py +195 -0
- rolloutdiff/_feature_values.py +79 -0
- rolloutdiff/_replay_container.py +568 -0
- rolloutdiff/_report_validation.py +758 -0
- rolloutdiff/_retrospective_policy.py +33 -0
- rolloutdiff/_sampling.py +127 -0
- rolloutdiff/_series_validation.py +67 -0
- rolloutdiff/_watch_report.py +436 -0
- rolloutdiff/_watch_storage.py +800 -0
- rolloutdiff/_web/assets/index-D8uGEQ9w.js +12 -0
- rolloutdiff/_web/assets/index-Dvo9nx9b.css +1 -0
- rolloutdiff/_web/assets/mona-sans-OFL.txt +93 -0
- rolloutdiff/_web/assets/mona-sans-latin-ext-wght-normal-BqSk0Z3x.woff2 +0 -0
- rolloutdiff/_web/assets/mona-sans-latin-wght-normal-Pz49MTQZ.woff2 +0 -0
- rolloutdiff/_web/assets/mona-sans-vietnamese-wght-normal-DsUdksa4.woff2 +0 -0
- rolloutdiff/_web/assets/third-party-notices.txt +112 -0
- rolloutdiff/_web/index.html +14 -0
- rolloutdiff/adapters.py +264 -0
- rolloutdiff/app.py +956 -0
- rolloutdiff/app_evidence.py +678 -0
- rolloutdiff/benchmark_reference.py +458 -0
- rolloutdiff/cli.py +1292 -0
- rolloutdiff/comparison.py +572 -0
- rolloutdiff/comparison_report.py +982 -0
- rolloutdiff/corroboration.py +287 -0
- rolloutdiff/detect.py +307 -0
- rolloutdiff/events.py +173 -0
- rolloutdiff/evidence.py +285 -0
- rolloutdiff/exemplars.py +240 -0
- rolloutdiff/features.py +823 -0
- rolloutdiff/html_report.py +726 -0
- rolloutdiff/inspect_logs.py +383 -0
- rolloutdiff/inspection.py +207 -0
- rolloutdiff/integrations.py +552 -0
- rolloutdiff/investigation.py +1146 -0
- rolloutdiff/notifications.py +406 -0
- rolloutdiff/pipeline.py +809 -0
- rolloutdiff/producer.py +663 -0
- rolloutdiff/prompt_mix.py +258 -0
- rolloutdiff/ranking.py +388 -0
- rolloutdiff/replay.py +589 -0
- rolloutdiff/replication.py +572 -0
- rolloutdiff/report.py +838 -0
- rolloutdiff/retrospective.py +386 -0
- rolloutdiff/schema.py +109 -0
- rolloutdiff/sensitivity.py +202 -0
- rolloutdiff/series.py +430 -0
- rolloutdiff/tool_features.py +422 -0
- rolloutdiff/trajectories.py +236 -0
- rolloutdiff/watch-compatibility.json +13 -0
- rolloutdiff/watch.py +742 -0
- rolloutdiff-0.1.0.dist-info/METADATA +158 -0
- rolloutdiff-0.1.0.dist-info/RECORD +62 -0
- rolloutdiff-0.1.0.dist-info/WHEEL +5 -0
- rolloutdiff-0.1.0.dist-info/entry_points.txt +2 -0
- rolloutdiff-0.1.0.dist-info/licenses/LICENSE +202 -0
- rolloutdiff-0.1.0.dist-info/licenses/frontend/public/assets/mona-sans-OFL.txt +93 -0
- rolloutdiff-0.1.0.dist-info/licenses/frontend/public/assets/third-party-notices.txt +112 -0
- rolloutdiff-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,568 @@
|
|
|
1
|
+
"""Bounded local Docker transport; container output has no verdict authority."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
import os
|
|
7
|
+
import re
|
|
8
|
+
import selectors
|
|
9
|
+
import shutil
|
|
10
|
+
import signal
|
|
11
|
+
import subprocess
|
|
12
|
+
import sys
|
|
13
|
+
import tempfile
|
|
14
|
+
import time
|
|
15
|
+
import uuid
|
|
16
|
+
from contextlib import suppress
|
|
17
|
+
from dataclasses import dataclass, field
|
|
18
|
+
from pathlib import Path
|
|
19
|
+
from urllib.parse import urlsplit
|
|
20
|
+
|
|
21
|
+
DEFAULT_IMAGE = (
|
|
22
|
+
"sha256:57cd7c3a7a273101a6485ba99423ee568157882804b1124b4dd04266317710de"
|
|
23
|
+
)
|
|
24
|
+
_IMAGE_PATH = (
|
|
25
|
+
"/usr/local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
|
|
26
|
+
)
|
|
27
|
+
_OUTPUT_LIMIT = 1024 * 1024
|
|
28
|
+
_OWNER_LABEL = "org.rolloutdiff.replay-owner"
|
|
29
|
+
_ULIMITS = {"cpu": 3, "core": 0, "nofile": 64, "fsize": 1048576}
|
|
30
|
+
_HOST_CONFIG = {
|
|
31
|
+
"NetworkMode": "none",
|
|
32
|
+
"ReadonlyRootfs": True,
|
|
33
|
+
"Privileged": False,
|
|
34
|
+
"CapDrop": ["ALL"],
|
|
35
|
+
"SecurityOpt": ["no-new-privileges"],
|
|
36
|
+
"IpcMode": "none",
|
|
37
|
+
"PidMode": "",
|
|
38
|
+
"UTSMode": "",
|
|
39
|
+
"CgroupnsMode": "private",
|
|
40
|
+
"Memory": 1073741824,
|
|
41
|
+
"MemorySwap": 1073741824,
|
|
42
|
+
"NanoCpus": 1000000000,
|
|
43
|
+
"PidsLimit": 16,
|
|
44
|
+
"Tmpfs": {"/tmp": "rw,noexec,nosuid,nodev,size=16777216"},
|
|
45
|
+
"PublishAllPorts": False,
|
|
46
|
+
"AutoRemove": False,
|
|
47
|
+
"Runtime": "runc",
|
|
48
|
+
"LogConfig": {"Type": "none", "Config": {}},
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
@dataclass(frozen=True)
|
|
53
|
+
class SandboxConfig:
|
|
54
|
+
image: str = DEFAULT_IMAGE
|
|
55
|
+
docker_host: str = "unix:///var/run/docker.sock"
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
@dataclass(frozen=True)
|
|
59
|
+
class ContainerResult:
|
|
60
|
+
status: str
|
|
61
|
+
exit_code: int | None = None
|
|
62
|
+
oom_killed: bool | None = None
|
|
63
|
+
elapsed_seconds: float = 0.0
|
|
64
|
+
stdout: bytes = b""
|
|
65
|
+
stderr: bytes = b""
|
|
66
|
+
config_verified: bool = False
|
|
67
|
+
cleanup_verified: bool = False
|
|
68
|
+
error: str | None = None
|
|
69
|
+
metadata: dict = field(default_factory=dict)
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
@dataclass(frozen=True)
|
|
73
|
+
class _ProcessResult:
|
|
74
|
+
returncode: int | None
|
|
75
|
+
stdout: bytes
|
|
76
|
+
stderr: bytes
|
|
77
|
+
status: str = "completed"
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
class _InfrastructureError(Exception):
|
|
81
|
+
pass
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def _invoke(args, *, env, input_data=b"", timeout, output_limit):
|
|
85
|
+
"""Drain both pipes incrementally and stop before retaining excess output."""
|
|
86
|
+
process = subprocess.Popen(
|
|
87
|
+
args,
|
|
88
|
+
stdin=subprocess.PIPE,
|
|
89
|
+
stdout=subprocess.PIPE,
|
|
90
|
+
stderr=subprocess.PIPE,
|
|
91
|
+
env=env,
|
|
92
|
+
close_fds=True,
|
|
93
|
+
start_new_session=True,
|
|
94
|
+
)
|
|
95
|
+
try:
|
|
96
|
+
buffers = {"stdout": bytearray(), "stderr": bytearray()}
|
|
97
|
+
status, written, retained = "completed", 0, 0
|
|
98
|
+
deadline = time.monotonic() + timeout
|
|
99
|
+
with selectors.DefaultSelector() as selector:
|
|
100
|
+
for stream, event, name in (
|
|
101
|
+
(process.stdin, selectors.EVENT_WRITE, "stdin"),
|
|
102
|
+
(process.stdout, selectors.EVENT_READ, "stdout"),
|
|
103
|
+
(process.stderr, selectors.EVENT_READ, "stderr"),
|
|
104
|
+
):
|
|
105
|
+
os.set_blocking(stream.fileno(), False)
|
|
106
|
+
if name == "stdin" and not input_data:
|
|
107
|
+
stream.close()
|
|
108
|
+
else:
|
|
109
|
+
selector.register(stream, event, name)
|
|
110
|
+
while selector.get_map():
|
|
111
|
+
remaining = deadline - time.monotonic()
|
|
112
|
+
if remaining <= 0:
|
|
113
|
+
status = "timeout"
|
|
114
|
+
break
|
|
115
|
+
for key, _ in selector.select(min(remaining, 0.1)):
|
|
116
|
+
if key.data == "stdin":
|
|
117
|
+
try:
|
|
118
|
+
written += os.write(
|
|
119
|
+
key.fd,
|
|
120
|
+
memoryview(input_data)[written : written + 65536],
|
|
121
|
+
)
|
|
122
|
+
except BrokenPipeError:
|
|
123
|
+
written = len(input_data)
|
|
124
|
+
if written == len(input_data):
|
|
125
|
+
selector.unregister(key.fileobj)
|
|
126
|
+
key.fileobj.close()
|
|
127
|
+
else:
|
|
128
|
+
chunk = os.read(key.fd, 65536)
|
|
129
|
+
if not chunk:
|
|
130
|
+
selector.unregister(key.fileobj)
|
|
131
|
+
key.fileobj.close()
|
|
132
|
+
continue
|
|
133
|
+
allowed = output_limit - retained
|
|
134
|
+
buffers[key.data].extend(chunk[:allowed])
|
|
135
|
+
retained += min(allowed, len(chunk))
|
|
136
|
+
if len(chunk) > allowed:
|
|
137
|
+
status = "output_limit"
|
|
138
|
+
break
|
|
139
|
+
if status != "completed":
|
|
140
|
+
break
|
|
141
|
+
if status == "completed":
|
|
142
|
+
try:
|
|
143
|
+
process.wait(timeout=max(0.001, deadline - time.monotonic()))
|
|
144
|
+
except subprocess.TimeoutExpired:
|
|
145
|
+
status = "timeout"
|
|
146
|
+
finally:
|
|
147
|
+
original_error = sys.exception()
|
|
148
|
+
interruption = cleanup_failure = None
|
|
149
|
+
killed = False
|
|
150
|
+
try:
|
|
151
|
+
for _ in range(2):
|
|
152
|
+
try:
|
|
153
|
+
if not killed and process.poll() is None:
|
|
154
|
+
with suppress(ProcessLookupError):
|
|
155
|
+
os.killpg(process.pid, signal.SIGKILL)
|
|
156
|
+
killed = True
|
|
157
|
+
process.wait(timeout=1)
|
|
158
|
+
except (KeyboardInterrupt, SystemExit) as exc:
|
|
159
|
+
if interruption is None:
|
|
160
|
+
interruption = exc
|
|
161
|
+
cleanup_failure = exc
|
|
162
|
+
except (OSError, subprocess.SubprocessError) as exc:
|
|
163
|
+
cleanup_failure = exc
|
|
164
|
+
break
|
|
165
|
+
else:
|
|
166
|
+
cleanup_failure = None
|
|
167
|
+
break
|
|
168
|
+
finally:
|
|
169
|
+
for stream in (process.stdin, process.stdout, process.stderr):
|
|
170
|
+
stream.close()
|
|
171
|
+
if cleanup_failure is not None:
|
|
172
|
+
error = (
|
|
173
|
+
original_error
|
|
174
|
+
or interruption
|
|
175
|
+
or _InfrastructureError(
|
|
176
|
+
"owned subprocess cleanup could not be verified"
|
|
177
|
+
)
|
|
178
|
+
)
|
|
179
|
+
error.process_cleanup_failure = {"pid": process.pid, "reaped": False}
|
|
180
|
+
error.add_note(
|
|
181
|
+
f"Owned subprocess cleanup is unverified (pid {process.pid})"
|
|
182
|
+
)
|
|
183
|
+
if error is cleanup_failure:
|
|
184
|
+
raise error
|
|
185
|
+
raise error from cleanup_failure
|
|
186
|
+
if interruption is not None and original_error is None:
|
|
187
|
+
raise interruption
|
|
188
|
+
return _ProcessResult(
|
|
189
|
+
process.returncode, bytes(buffers["stdout"]), bytes(buffers["stderr"]), status
|
|
190
|
+
)
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
def _checked_json(result, operation):
|
|
194
|
+
if result.status != "completed" or result.returncode != 0:
|
|
195
|
+
raise _InfrastructureError(f"Docker {operation} failed")
|
|
196
|
+
value = json.loads(result.stdout)
|
|
197
|
+
if not isinstance(value, dict):
|
|
198
|
+
raise _InfrastructureError(f"Docker {operation} returned invalid metadata")
|
|
199
|
+
return value
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
def _validate_image(image, config):
|
|
203
|
+
settings = image["Config"]
|
|
204
|
+
values = settings.get("Env", [])
|
|
205
|
+
env = dict(item.split("=", 1) for item in values)
|
|
206
|
+
if (
|
|
207
|
+
image["Id"] != config.image
|
|
208
|
+
or image["Os"] != "linux"
|
|
209
|
+
or image["Architecture"] not in {"arm64", "amd64"}
|
|
210
|
+
or settings.get("Entrypoint")
|
|
211
|
+
or settings.get("Volumes")
|
|
212
|
+
or len(env) != len(values)
|
|
213
|
+
or set(env) != {"PATH", "LANG", "GPG_KEY", "PYTHON_VERSION", "PYTHON_SHA256"}
|
|
214
|
+
or env["PATH"] != _IMAGE_PATH
|
|
215
|
+
or env["LANG"] != "C.UTF-8"
|
|
216
|
+
or not re.fullmatch(r"3\.12\.\d+", env["PYTHON_VERSION"])
|
|
217
|
+
or not re.fullmatch(r"[A-F0-9]{40}", env["GPG_KEY"])
|
|
218
|
+
or not re.fullmatch(r"[a-f0-9]{64}", env["PYTHON_SHA256"])
|
|
219
|
+
):
|
|
220
|
+
raise _InfrastructureError("local image configuration is unsupported")
|
|
221
|
+
return env
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
def _validate_daemon(info):
|
|
225
|
+
if (
|
|
226
|
+
info.get("OSType") != "linux"
|
|
227
|
+
or info.get("CgroupVersion") != "2"
|
|
228
|
+
or "name=seccomp,profile=builtin" not in info.get("SecurityOptions", [])
|
|
229
|
+
or "runc" not in info.get("Runtimes", {})
|
|
230
|
+
or any(
|
|
231
|
+
info.get(key) is not True
|
|
232
|
+
for key in (
|
|
233
|
+
"MemoryLimit",
|
|
234
|
+
"SwapLimit",
|
|
235
|
+
"PidsLimit",
|
|
236
|
+
"CpuCfsQuota",
|
|
237
|
+
"CpuCfsPeriod",
|
|
238
|
+
)
|
|
239
|
+
)
|
|
240
|
+
):
|
|
241
|
+
raise _InfrastructureError("Docker isolation prerequisites are unavailable")
|
|
242
|
+
|
|
243
|
+
|
|
244
|
+
def _verify_container(container, image, command, owner):
|
|
245
|
+
host, settings = container["HostConfig"], container["Config"]
|
|
246
|
+
ulimits = {item["Name"]: (item["Soft"], item["Hard"]) for item in host["Ulimits"]}
|
|
247
|
+
if (
|
|
248
|
+
any(host.get(key) != value for key, value in _HOST_CONFIG.items())
|
|
249
|
+
or any(
|
|
250
|
+
host.get(key)
|
|
251
|
+
for key in (
|
|
252
|
+
"CapAdd",
|
|
253
|
+
"Binds",
|
|
254
|
+
"Mounts",
|
|
255
|
+
"Devices",
|
|
256
|
+
"DeviceRequests",
|
|
257
|
+
"VolumesFrom",
|
|
258
|
+
"Links",
|
|
259
|
+
"ExtraHosts",
|
|
260
|
+
"PortBindings",
|
|
261
|
+
)
|
|
262
|
+
)
|
|
263
|
+
or ulimits != {name: (limit, limit) for name, limit in _ULIMITS.items()}
|
|
264
|
+
or container.get("Mounts")
|
|
265
|
+
or not re.fullmatch(r"[a-f0-9]{64}", container["Id"])
|
|
266
|
+
or container["Image"] != image["Id"]
|
|
267
|
+
or settings.get("User") != "65534:65534"
|
|
268
|
+
or settings.get("WorkingDir") != "/tmp"
|
|
269
|
+
or settings.get("Env") != image["Config"]["Env"]
|
|
270
|
+
or settings.get("Entrypoint")
|
|
271
|
+
or settings.get("Volumes")
|
|
272
|
+
or settings.get("Cmd") != command
|
|
273
|
+
or settings.get("Labels", {}).get(_OWNER_LABEL) != owner
|
|
274
|
+
or settings.get("OpenStdin") is not True
|
|
275
|
+
or settings.get("Tty") is not False
|
|
276
|
+
or settings.get("Healthcheck", {}).get("Test") != ["NONE"]
|
|
277
|
+
or container["State"].get("Status") != "created"
|
|
278
|
+
):
|
|
279
|
+
raise _InfrastructureError("created container failed isolation verification")
|
|
280
|
+
|
|
281
|
+
|
|
282
|
+
def _cleanup(command, name, owner, *, creation_resolved, known_id, recovery):
|
|
283
|
+
if known_id is not None:
|
|
284
|
+
with suppress(OSError, subprocess.SubprocessError, _InfrastructureError):
|
|
285
|
+
command(["rm", "--force", "--volumes", known_id])
|
|
286
|
+
for remove in (known_id is None, False):
|
|
287
|
+
result = command(["container", "inspect", "--format", "{{json .}}", name])
|
|
288
|
+
if (
|
|
289
|
+
result.status == "completed"
|
|
290
|
+
and result.returncode == 1
|
|
291
|
+
and (
|
|
292
|
+
b"No such container" in result.stderr
|
|
293
|
+
or b"No such object" in result.stderr
|
|
294
|
+
)
|
|
295
|
+
):
|
|
296
|
+
return creation_resolved
|
|
297
|
+
try:
|
|
298
|
+
container = _checked_json(result, "cleanup inspection")
|
|
299
|
+
identity = container["Id"]
|
|
300
|
+
if container["Config"].get("Labels", {}).get(
|
|
301
|
+
_OWNER_LABEL
|
|
302
|
+
) != owner or not re.fullmatch(r"[a-f0-9]{64}", identity):
|
|
303
|
+
return False
|
|
304
|
+
recovery["container_id"] = identity
|
|
305
|
+
except (OSError, ValueError, KeyError, TypeError, _InfrastructureError):
|
|
306
|
+
return False
|
|
307
|
+
if remove:
|
|
308
|
+
creation_resolved = True
|
|
309
|
+
with suppress(OSError, subprocess.SubprocessError, _InfrastructureError):
|
|
310
|
+
command(["rm", "--force", "--volumes", identity])
|
|
311
|
+
return False
|
|
312
|
+
|
|
313
|
+
|
|
314
|
+
def run_container(
|
|
315
|
+
script: str, payload: bytes, config: SandboxConfig, *, execution_authorized: bool
|
|
316
|
+
) -> ContainerResult:
|
|
317
|
+
"""Run an authorized payload; only daemon state supplies execution status.
|
|
318
|
+
|
|
319
|
+
Cancellation propagates after bounded cleanup, with its ContainerResult
|
|
320
|
+
attached as ``replay_result`` for recovery when absence is unverified.
|
|
321
|
+
"""
|
|
322
|
+
if execution_authorized is not True:
|
|
323
|
+
raise ValueError("container execution requires explicit authorization")
|
|
324
|
+
endpoint = urlsplit(config.docker_host)
|
|
325
|
+
if (
|
|
326
|
+
endpoint.scheme != "unix"
|
|
327
|
+
or endpoint.netloc
|
|
328
|
+
or not endpoint.path.startswith("/")
|
|
329
|
+
or endpoint.query
|
|
330
|
+
or endpoint.fragment
|
|
331
|
+
or "%" in config.docker_host
|
|
332
|
+
or any(ord(char) < 33 for char in config.docker_host)
|
|
333
|
+
):
|
|
334
|
+
raise ValueError("Docker endpoint must be an absolute local unix socket")
|
|
335
|
+
if not re.fullmatch(r"sha256:[a-f0-9]{64}", config.image):
|
|
336
|
+
raise ValueError("image must be an immutable local sha256 identity")
|
|
337
|
+
if config.image != DEFAULT_IMAGE:
|
|
338
|
+
raise ValueError(
|
|
339
|
+
"unsupported image: this slice requires the reviewed Python image"
|
|
340
|
+
)
|
|
341
|
+
if not isinstance(payload, bytes) or len(payload) > 2 * 1024 * 1024:
|
|
342
|
+
raise ValueError("container input exceeds the 2 MiB limit")
|
|
343
|
+
if not isinstance(script, str) or len(script.encode()) > 65536 or "\0" in script:
|
|
344
|
+
raise ValueError("container runner must be at most 64 KiB without NUL bytes")
|
|
345
|
+
started = time.monotonic()
|
|
346
|
+
status, error = "infrastructure_error", None
|
|
347
|
+
stdout, stderr, exit_code, oom = b"", b"", None, None
|
|
348
|
+
verified, cleaned, creation_attempted = False, True, False
|
|
349
|
+
creation_resolved = False
|
|
350
|
+
container_id = None
|
|
351
|
+
interruption = None
|
|
352
|
+
metadata = {}
|
|
353
|
+
owner = uuid.uuid4().hex
|
|
354
|
+
name = "rolloutdiff-replay-" + owner
|
|
355
|
+
python_command = ["/usr/local/bin/python3", "-I", "-S", "-c", script]
|
|
356
|
+
with tempfile.TemporaryDirectory(prefix="rolloutdiff-docker-") as directory:
|
|
357
|
+
Path(directory, "config.json").write_text('{"auths": {}}')
|
|
358
|
+
docker = shutil.which("docker")
|
|
359
|
+
if docker is None:
|
|
360
|
+
return ContainerResult(
|
|
361
|
+
"infrastructure_error",
|
|
362
|
+
error="Docker CLI is unavailable",
|
|
363
|
+
cleanup_verified=True,
|
|
364
|
+
)
|
|
365
|
+
base = [docker, "--config", directory, "--host", config.docker_host]
|
|
366
|
+
env = {"HOME": directory, "PATH": os.defpath}
|
|
367
|
+
|
|
368
|
+
def command(args):
|
|
369
|
+
return _invoke(
|
|
370
|
+
base + args,
|
|
371
|
+
env=env,
|
|
372
|
+
timeout=10 if args[0] == "create" else 3,
|
|
373
|
+
output_limit=_OUTPUT_LIMIT,
|
|
374
|
+
)
|
|
375
|
+
|
|
376
|
+
def cleanup_command(args):
|
|
377
|
+
nonlocal interruption
|
|
378
|
+
# A cancelled CLI command has an unknown daemon outcome. Retry once;
|
|
379
|
+
# the final inspection still decides whether cleanup is established.
|
|
380
|
+
for _ in range(2):
|
|
381
|
+
try:
|
|
382
|
+
return command(args)
|
|
383
|
+
except (KeyboardInterrupt, SystemExit) as exc:
|
|
384
|
+
if interruption is None:
|
|
385
|
+
interruption = exc
|
|
386
|
+
if hasattr(exc, "process_cleanup_failure"):
|
|
387
|
+
metadata["process_cleanup_failure"] = (
|
|
388
|
+
exc.process_cleanup_failure
|
|
389
|
+
)
|
|
390
|
+
except (
|
|
391
|
+
OSError,
|
|
392
|
+
subprocess.SubprocessError,
|
|
393
|
+
_InfrastructureError,
|
|
394
|
+
) as exc:
|
|
395
|
+
if hasattr(exc, "process_cleanup_failure"):
|
|
396
|
+
metadata["process_cleanup_failure"] = (
|
|
397
|
+
exc.process_cleanup_failure
|
|
398
|
+
)
|
|
399
|
+
return _ProcessResult(None, b"", b"", "infrastructure_error")
|
|
400
|
+
return _ProcessResult(None, b"", b"", "interrupted")
|
|
401
|
+
|
|
402
|
+
try:
|
|
403
|
+
info = _checked_json(command(["info", "--format", "{{json .}}"]), "info")
|
|
404
|
+
_validate_daemon(info)
|
|
405
|
+
image = _checked_json(
|
|
406
|
+
command(["image", "inspect", "--format", "{{json .}}", config.image]),
|
|
407
|
+
"image inspection",
|
|
408
|
+
)
|
|
409
|
+
image_env = _validate_image(image, config)
|
|
410
|
+
metadata = {
|
|
411
|
+
"image_id": image["Id"],
|
|
412
|
+
"architecture": image["Architecture"],
|
|
413
|
+
"python_version": image_env["PYTHON_VERSION"],
|
|
414
|
+
"daemon_version": info.get("ServerVersion"),
|
|
415
|
+
"cgroup_version": info["CgroupVersion"],
|
|
416
|
+
"runtime": "runc",
|
|
417
|
+
"limits": {
|
|
418
|
+
"memory_bytes": 1073741824,
|
|
419
|
+
"cpus": 1,
|
|
420
|
+
"pids": 16,
|
|
421
|
+
"wall_seconds": 6,
|
|
422
|
+
"output_bytes": _OUTPUT_LIMIT,
|
|
423
|
+
"stdin_bytes": 2097152,
|
|
424
|
+
"tmp_bytes": 16777216,
|
|
425
|
+
},
|
|
426
|
+
}
|
|
427
|
+
create = [
|
|
428
|
+
"create",
|
|
429
|
+
"--name",
|
|
430
|
+
name,
|
|
431
|
+
"--label",
|
|
432
|
+
f"{_OWNER_LABEL}={owner}",
|
|
433
|
+
"--pull=never",
|
|
434
|
+
"--interactive",
|
|
435
|
+
"--network=none",
|
|
436
|
+
"--read-only",
|
|
437
|
+
"--tmpfs=/tmp:rw,noexec,nosuid,nodev,size=16777216",
|
|
438
|
+
"--ipc=none",
|
|
439
|
+
"--user=65534:65534",
|
|
440
|
+
"--cap-drop=ALL",
|
|
441
|
+
"--security-opt=no-new-privileges",
|
|
442
|
+
"--cgroupns=private",
|
|
443
|
+
"--memory=1073741824",
|
|
444
|
+
"--memory-swap=1073741824",
|
|
445
|
+
"--cpus=1",
|
|
446
|
+
"--pids-limit=16",
|
|
447
|
+
"--log-driver=none",
|
|
448
|
+
"--runtime=runc",
|
|
449
|
+
"--no-healthcheck",
|
|
450
|
+
"--workdir=/tmp",
|
|
451
|
+
]
|
|
452
|
+
for key, limit in _ULIMITS.items():
|
|
453
|
+
create.append(f"--ulimit={key}={limit}:{limit}")
|
|
454
|
+
metadata["container_name"] = name
|
|
455
|
+
creation_attempted = True
|
|
456
|
+
created = command([*create, config.image, *python_command])
|
|
457
|
+
creation_resolved = (
|
|
458
|
+
created.status == "completed" and created.returncode == 0
|
|
459
|
+
)
|
|
460
|
+
if created.status != "completed" or created.returncode != 0:
|
|
461
|
+
raise _InfrastructureError("Docker create failed")
|
|
462
|
+
container = _checked_json(
|
|
463
|
+
command(["container", "inspect", "--format", "{{json .}}", name]),
|
|
464
|
+
"configuration inspection",
|
|
465
|
+
)
|
|
466
|
+
_verify_container(container, image, python_command, owner)
|
|
467
|
+
verified = True
|
|
468
|
+
container_id = container["Id"]
|
|
469
|
+
metadata["container_id"] = container_id
|
|
470
|
+
result = _invoke(
|
|
471
|
+
[*base, "start", "--attach", "--interactive", name],
|
|
472
|
+
env=env,
|
|
473
|
+
input_data=payload,
|
|
474
|
+
timeout=6,
|
|
475
|
+
output_limit=_OUTPUT_LIMIT,
|
|
476
|
+
)
|
|
477
|
+
stdout, stderr = result.stdout, result.stderr
|
|
478
|
+
if result.status in {"timeout", "output_limit"}:
|
|
479
|
+
status = result.status
|
|
480
|
+
else:
|
|
481
|
+
state = _checked_json(
|
|
482
|
+
command(["container", "inspect", "--format", "{{json .}}", name]),
|
|
483
|
+
"outcome inspection",
|
|
484
|
+
)["State"]
|
|
485
|
+
if (
|
|
486
|
+
state.get("Status") != "exited"
|
|
487
|
+
or state.get("Running") is not False
|
|
488
|
+
or state.get("Error")
|
|
489
|
+
or type(state.get("ExitCode")) is not int
|
|
490
|
+
or type(state.get("OOMKilled")) is not bool
|
|
491
|
+
):
|
|
492
|
+
raise _InfrastructureError(
|
|
493
|
+
"container outcome could not be verified"
|
|
494
|
+
)
|
|
495
|
+
exit_code, oom = state["ExitCode"], state["OOMKilled"]
|
|
496
|
+
status = "completed" if exit_code == 0 else "nonzero_exit"
|
|
497
|
+
except _InfrastructureError as exc:
|
|
498
|
+
error = str(exc)
|
|
499
|
+
if hasattr(exc, "process_cleanup_failure"):
|
|
500
|
+
metadata["process_cleanup_failure"] = exc.process_cleanup_failure
|
|
501
|
+
except (
|
|
502
|
+
OSError,
|
|
503
|
+
ValueError,
|
|
504
|
+
KeyError,
|
|
505
|
+
TypeError,
|
|
506
|
+
subprocess.SubprocessError,
|
|
507
|
+
) as exc:
|
|
508
|
+
error = "Docker transport or metadata validation failed"
|
|
509
|
+
if hasattr(exc, "process_cleanup_failure"):
|
|
510
|
+
metadata["process_cleanup_failure"] = exc.process_cleanup_failure
|
|
511
|
+
except (KeyboardInterrupt, SystemExit) as exc:
|
|
512
|
+
interruption = exc
|
|
513
|
+
error = "Docker execution interrupted"
|
|
514
|
+
if hasattr(exc, "process_cleanup_failure"):
|
|
515
|
+
metadata["process_cleanup_failure"] = exc.process_cleanup_failure
|
|
516
|
+
finally:
|
|
517
|
+
if creation_attempted:
|
|
518
|
+
cleaned = False
|
|
519
|
+
for _ in range(2):
|
|
520
|
+
try:
|
|
521
|
+
cleaned = _cleanup(
|
|
522
|
+
cleanup_command,
|
|
523
|
+
name,
|
|
524
|
+
owner,
|
|
525
|
+
creation_resolved=creation_resolved,
|
|
526
|
+
known_id=container_id,
|
|
527
|
+
recovery=metadata,
|
|
528
|
+
)
|
|
529
|
+
break
|
|
530
|
+
except (KeyboardInterrupt, SystemExit) as exc:
|
|
531
|
+
if interruption is None:
|
|
532
|
+
interruption = exc
|
|
533
|
+
except (
|
|
534
|
+
OSError,
|
|
535
|
+
ValueError,
|
|
536
|
+
KeyError,
|
|
537
|
+
TypeError,
|
|
538
|
+
subprocess.SubprocessError,
|
|
539
|
+
_InfrastructureError,
|
|
540
|
+
):
|
|
541
|
+
break
|
|
542
|
+
if "process_cleanup_failure" in metadata:
|
|
543
|
+
cleaned = False
|
|
544
|
+
if not cleaned:
|
|
545
|
+
status, error = (
|
|
546
|
+
"infrastructure_error",
|
|
547
|
+
"container cleanup could not be verified",
|
|
548
|
+
)
|
|
549
|
+
result = ContainerResult(
|
|
550
|
+
status,
|
|
551
|
+
exit_code,
|
|
552
|
+
oom,
|
|
553
|
+
time.monotonic() - started,
|
|
554
|
+
stdout,
|
|
555
|
+
stderr,
|
|
556
|
+
verified,
|
|
557
|
+
cleaned,
|
|
558
|
+
error,
|
|
559
|
+
metadata,
|
|
560
|
+
)
|
|
561
|
+
if interruption is not None:
|
|
562
|
+
interruption.replay_result = result
|
|
563
|
+
if not result.cleanup_verified:
|
|
564
|
+
interruption.add_note(
|
|
565
|
+
"Container cleanup is unverified; recovery identity is in replay_result"
|
|
566
|
+
)
|
|
567
|
+
raise interruption
|
|
568
|
+
return result
|