manifold-cli 0.0.39__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- manifold_cli/__init__.py +3 -0
- manifold_cli/__main__.py +4 -0
- manifold_cli/bridge.py +70 -0
- manifold_cli/cli.py +54 -0
- manifold_cli/clients/__init__.py +0 -0
- manifold_cli/clients/auth_flow.py +169 -0
- manifold_cli/clients/credentials.py +53 -0
- manifold_cli/clients/docker_engine.py +43 -0
- manifold_cli/clients/image.py +130 -0
- manifold_cli/clients/manifold.py +1288 -0
- manifold_cli/clients/runner_home.py +165 -0
- manifold_cli/clients/workos.py +99 -0
- manifold_cli/commands/__init__.py +0 -0
- manifold_cli/commands/auth.py +101 -0
- manifold_cli/commands/benchmark.py +539 -0
- manifold_cli/commands/organization.py +154 -0
- manifold_cli/commands/policy.py +749 -0
- manifold_cli/commands/run.py +337 -0
- manifold_cli/commands/run_log.py +291 -0
- manifold_cli/commands/runner.py +852 -0
- manifold_cli/commands/table.py +101 -0
- manifold_cli/commands/token.py +144 -0
- manifold_cli/dependencies.py +116 -0
- manifold_cli/models/__init__.py +0 -0
- manifold_cli/models/auth.py +49 -0
- manifold_cli/models/benchmark.py +87 -0
- manifold_cli/models/catalog.py +189 -0
- manifold_cli/models/credentials.py +19 -0
- manifold_cli/models/errors.py +121 -0
- manifold_cli/models/log.py +114 -0
- manifold_cli/models/observability.py +121 -0
- manifold_cli/models/organization.py +19 -0
- manifold_cli/models/pagination.py +59 -0
- manifold_cli/models/policy.py +49 -0
- manifold_cli/models/run.py +381 -0
- manifold_cli/models/runner.py +186 -0
- manifold_cli/models/runner_home.py +69 -0
- manifold_cli/models/task_queue.py +40 -0
- manifold_cli/models/token.py +39 -0
- manifold_cli/replay/__init__.py +68 -0
- manifold_cli/replay/log.py +814 -0
- manifold_cli/runner/__init__.py +72 -0
- manifold_cli/runner/api_retry.py +36 -0
- manifold_cli/runner/benchmark/__init__.py +800 -0
- manifold_cli/runner/benchmark/live_view.py +248 -0
- manifold_cli/runner/benchmark/preassigned_scheduling.py +325 -0
- manifold_cli/runner/benchmark/recording.py +1208 -0
- manifold_cli/runner/benchmark/scheduling.py +256 -0
- manifold_cli/runner/benchmark/task_queue_scheduling.py +717 -0
- manifold_cli/runner/benchmark/task_server.py +123 -0
- manifold_cli/runner/benchmark/video.py +124 -0
- manifold_cli/runner/benchmark_steps.py +178 -0
- manifold_cli/runner/cancellation.py +22 -0
- manifold_cli/runner/capture.py +282 -0
- manifold_cli/runner/container.py +370 -0
- manifold_cli/runner/current_dispatch.py +25 -0
- manifold_cli/runner/daemon.py +520 -0
- manifold_cli/runner/dispatch.py +38 -0
- manifold_cli/runner/gpu.py +182 -0
- manifold_cli/runner/heartbeat.py +202 -0
- manifold_cli/runner/loopback.py +102 -0
- manifold_cli/runner/loopback_bench.py +183 -0
- manifold_cli/runner/loopback_policy.py +39 -0
- manifold_cli/runner/output.py +218 -0
- manifold_cli/runner/pairing_acceptance.py +57 -0
- manifold_cli/runner/plan.py +369 -0
- manifold_cli/runner/policy.py +350 -0
- manifold_cli/runner/policy_steps.py +164 -0
- manifold_cli/runner/preparation.py +47 -0
- manifold_cli/runner/process.py +219 -0
- manifold_cli/runner/ram.py +17 -0
- manifold_cli/runner/resource_usage.py +265 -0
- manifold_cli/runner/served.py +218 -0
- manifold_cli/runner/service.py +120 -0
- manifold_cli/runner/side_failure.py +68 -0
- manifold_cli/runner/signature.py +98 -0
- manifold_cli/runner/source.py +66 -0
- manifold_cli/runner/stages.py +241 -0
- manifold_cli/runner/step_timing.py +229 -0
- manifold_cli/runner/worker.py +534 -0
- manifold_cli/runner/workload.py +401 -0
- manifold_cli/services/__init__.py +0 -0
- manifold_cli/services/auth.py +89 -0
- manifold_cli/services/benchmark.py +357 -0
- manifold_cli/services/organization.py +22 -0
- manifold_cli/services/policy.py +234 -0
- manifold_cli/services/run.py +304 -0
- manifold_cli/services/runner.py +207 -0
- manifold_cli/services/runner_daemon.py +146 -0
- manifold_cli/services/token.py +52 -0
- manifold_cli/settings.py +30 -0
- manifold_cli/transport/__init__.py +31 -0
- manifold_cli/transport/diagnostics.py +72 -0
- manifold_cli/transport/interfaces.py +69 -0
- manifold_cli/transport/iroh.py +317 -0
- manifold_cli/transport/registry.py +34 -0
- manifold_cli/transport/tcp.py +146 -0
- manifold_cli-0.0.39.dist-info/METADATA +406 -0
- manifold_cli-0.0.39.dist-info/RECORD +102 -0
- manifold_cli-0.0.39.dist-info/WHEEL +4 -0
- manifold_cli-0.0.39.dist-info/entry_points.txt +2 -0
- manifold_cli-0.0.39.dist-info/licenses/LICENSE +373 -0
|
@@ -0,0 +1,534 @@
|
|
|
1
|
+
"""Running one worker from its plan: setup, spawn, readiness, teardown.
|
|
2
|
+
|
|
3
|
+
One worker and nothing about the run it belongs to. Claiming one side of a run
|
|
4
|
+
and shipping what its worker produces both land later.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
import asyncio
|
|
8
|
+
import contextlib
|
|
9
|
+
import logging
|
|
10
|
+
import sys
|
|
11
|
+
from collections.abc import AsyncIterator, Callable
|
|
12
|
+
from contextlib import AbstractContextManager, asynccontextmanager, nullcontext
|
|
13
|
+
from dataclasses import dataclass
|
|
14
|
+
from functools import partial
|
|
15
|
+
|
|
16
|
+
from rich.markup import escape
|
|
17
|
+
|
|
18
|
+
from manifold_cli.models.log import LogSource
|
|
19
|
+
from manifold_cli.runner.capture import ByteSink, CapturedOutput, WorkerCapture, drain
|
|
20
|
+
from manifold_cli.runner.plan import (
|
|
21
|
+
CommandProbe,
|
|
22
|
+
ProcessOnly,
|
|
23
|
+
RunnerError,
|
|
24
|
+
SetupCommand,
|
|
25
|
+
TcpProbe,
|
|
26
|
+
WireBinding,
|
|
27
|
+
WorkerPlan,
|
|
28
|
+
)
|
|
29
|
+
from manifold_cli.runner.process import (
|
|
30
|
+
TERMINATE_GRACE_SECONDS,
|
|
31
|
+
CommandResult,
|
|
32
|
+
ProcessHandle,
|
|
33
|
+
Spawner,
|
|
34
|
+
)
|
|
35
|
+
from manifold_cli.runner.stages import IndentedEcho, SideStages
|
|
36
|
+
from manifold_cli.transport.tcp import TcpConnector
|
|
37
|
+
|
|
38
|
+
logger = logging.getLogger(__name__)
|
|
39
|
+
|
|
40
|
+
PROBE_INTERVAL_SECONDS = 0.5
|
|
41
|
+
PROBE_CONNECT_TIMEOUT_SECONDS = 1.0
|
|
42
|
+
PROBE_COMMAND_TIMEOUT_SECONDS = 10.0
|
|
43
|
+
TEARDOWN_TIMEOUT_SECONDS = 30.0
|
|
44
|
+
# How long a container gets to exit on SIGTERM before docker kills it.
|
|
45
|
+
STOP_GRACE_SECONDS = 5
|
|
46
|
+
# The pause between two attempts to remove a container that docker still holds.
|
|
47
|
+
REMOVE_RETRY_INTERVAL_SECONDS = 1.0
|
|
48
|
+
|
|
49
|
+
# What a runner reports for a check it cannot start: 127 for a command it cannot
|
|
50
|
+
# find, 126 for one it cannot execute. A shell and `docker exec` both use them. A
|
|
51
|
+
# probe that exits either way is answering for the check itself rather than for
|
|
52
|
+
# the worker, so it is fatal. Docker's own failures -- no such container, a
|
|
53
|
+
# daemon it cannot reach -- exit 1, and stay part of the poll.
|
|
54
|
+
PROBE_UNRUNNABLE_EXIT_CODES = (126, 127)
|
|
55
|
+
|
|
56
|
+
# How much of a failed command's complaint the error carries.
|
|
57
|
+
STDERR_TAIL = 400
|
|
58
|
+
|
|
59
|
+
# How long docker gets to answer for a container that has already exited.
|
|
60
|
+
INSPECT_TIMEOUT_SECONDS = 10.0
|
|
61
|
+
|
|
62
|
+
# How much of a worker's last output reaches a message.
|
|
63
|
+
LOG_TAIL_CHARS = 600
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
class WorkerStartError(RunnerError):
|
|
67
|
+
"""A worker never became ready.
|
|
68
|
+
|
|
69
|
+
Carries a message naming what failed: a setup command, the spawn itself, or
|
|
70
|
+
the readiness probe.
|
|
71
|
+
"""
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
class SetupStepError(WorkerStartError):
|
|
75
|
+
"""A setup command failed, timed out or could not run."""
|
|
76
|
+
|
|
77
|
+
def __init__(self, message: str, *, phase: str) -> None:
|
|
78
|
+
super().__init__(message)
|
|
79
|
+
self.phase = phase
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
@dataclass(frozen=True)
|
|
83
|
+
class WorkerExit:
|
|
84
|
+
"""How a worker ended, and what it last said.
|
|
85
|
+
|
|
86
|
+
`log_tail` is the end of what the pumps read, for a worker of any kind.
|
|
87
|
+
`oom_killed` is docker's answer about a container and stays False for a
|
|
88
|
+
worker that is not one. The exit code does not carry the reason on its own:
|
|
89
|
+
the kernel's memory killer and a stop that outran its grace period both
|
|
90
|
+
leave 137.
|
|
91
|
+
"""
|
|
92
|
+
|
|
93
|
+
code: int
|
|
94
|
+
oom_killed: bool = False
|
|
95
|
+
log_tail: str = ""
|
|
96
|
+
|
|
97
|
+
@property
|
|
98
|
+
def cause(self) -> str:
|
|
99
|
+
"""How the worker ended, as a fragment naming its own doing."""
|
|
100
|
+
if self.oom_killed:
|
|
101
|
+
return f"ran out of memory and was killed (exit {self.code})"
|
|
102
|
+
return f"exited {self.code}"
|
|
103
|
+
|
|
104
|
+
@property
|
|
105
|
+
def detail(self) -> str:
|
|
106
|
+
"""What the side reports: the cause, and the end of the worker's output."""
|
|
107
|
+
return with_tail(f"The worker {self.cause}.", self.log_tail)
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
@asynccontextmanager
|
|
111
|
+
async def running_worker(
|
|
112
|
+
plan: WorkerPlan,
|
|
113
|
+
*,
|
|
114
|
+
spawner: Spawner,
|
|
115
|
+
captured: WorkerCapture,
|
|
116
|
+
stages: SideStages | None = None,
|
|
117
|
+
) -> AsyncIterator[ProcessHandle]:
|
|
118
|
+
"""Run `plan`'s setup, spawn its worker, and yield it once it is ready.
|
|
119
|
+
|
|
120
|
+
What the worker writes is drained into `captured` for as long as it runs, and
|
|
121
|
+
echoed on to this process's own streams as it passes.
|
|
122
|
+
|
|
123
|
+
Given `stages`, it logs each setup command that names its stage, and the
|
|
124
|
+
wait from the spawn until the worker is ready. The echo of the worker's
|
|
125
|
+
output is then indented under those stages.
|
|
126
|
+
|
|
127
|
+
The worker is terminated and `plan.teardown` runs on the way out, whatever
|
|
128
|
+
happened: a failed setup command, a probe that timed out, or the body
|
|
129
|
+
raising. A container the docker client left behind holds its GPU, so
|
|
130
|
+
teardown does not wait on the worker having started.
|
|
131
|
+
|
|
132
|
+
Raises WorkerStartError if the worker does not come up.
|
|
133
|
+
"""
|
|
134
|
+
process: ProcessHandle | None = None
|
|
135
|
+
try:
|
|
136
|
+
await run_setup(spawner, plan.setup, stages=stages)
|
|
137
|
+
with _loading(plan, stages):
|
|
138
|
+
try:
|
|
139
|
+
process = await spawner.spawn(
|
|
140
|
+
plan.argv, workdir=plan.workdir, env=plan.env
|
|
141
|
+
)
|
|
142
|
+
except OSError as exc:
|
|
143
|
+
raise WorkerStartError(f"Could not spawn the worker: {exc}") from exc
|
|
144
|
+
_start_pumps(process, captured, indented=stages is not None)
|
|
145
|
+
await _await_ready(plan, process, spawner, captured)
|
|
146
|
+
yield process
|
|
147
|
+
finally:
|
|
148
|
+
# NOTE: shielded, so a cancellation arriving while the cleanup is
|
|
149
|
+
# already awaiting does not abandon it half-done. `CancelledError` is a
|
|
150
|
+
# BaseException, so the suppression in `_run_teardown` would not catch
|
|
151
|
+
# one, and the container would outlive the runner holding its GPU.
|
|
152
|
+
await asyncio.shield(_stop_worker(spawner, plan, process))
|
|
153
|
+
# The worker is gone, so its pipes end and the pumps with them.
|
|
154
|
+
await asyncio.shield(captured.settle())
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def _loading(
|
|
158
|
+
plan: WorkerPlan, stages: SideStages | None
|
|
159
|
+
) -> AbstractContextManager[None]:
|
|
160
|
+
"""The stage from the spawn until the worker is ready.
|
|
161
|
+
|
|
162
|
+
Logged only where there is a wait: a worker without a probe is ready once
|
|
163
|
+
its process is up.
|
|
164
|
+
"""
|
|
165
|
+
if stages is None or isinstance(plan.readiness, ProcessOnly):
|
|
166
|
+
return nullcontext()
|
|
167
|
+
side = stages.side.value
|
|
168
|
+
return stages.step(f"Loading the {side}", f"{side.capitalize()} loaded")
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
def _start_pumps(
|
|
172
|
+
process: ProcessHandle, captured: WorkerCapture, *, indented: bool = False
|
|
173
|
+
) -> None:
|
|
174
|
+
"""Read each of the worker's pipes, echoing what arrives on the way past.
|
|
175
|
+
|
|
176
|
+
A pipe is absent for a handle nothing piped, which is what a test double
|
|
177
|
+
gives back, and there is nothing to read for it.
|
|
178
|
+
"""
|
|
179
|
+
for reader, stream, source in (
|
|
180
|
+
(process.stdout, sys.stdout, LogSource.STDOUT),
|
|
181
|
+
(process.stderr, sys.stderr, LogSource.STDERR),
|
|
182
|
+
):
|
|
183
|
+
if reader is not None:
|
|
184
|
+
captured.pump(reader, echo_for(stream, indented=indented), source)
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
def echo_for(stream: object, *, indented: bool = False) -> ByteSink | None:
|
|
188
|
+
"""The binary half of `stream`, where it has one.
|
|
189
|
+
|
|
190
|
+
`indented` sets each line under the stage lines, dimmed where `stream` is
|
|
191
|
+
a terminal.
|
|
192
|
+
|
|
193
|
+
NOTE: the echo writes bytes so that nothing a worker printed is read as
|
|
194
|
+
console markup. A stream standing in for the real one during a test may hold
|
|
195
|
+
no binary half at all, and an echo is not what such a test is checking.
|
|
196
|
+
"""
|
|
197
|
+
buffer = getattr(stream, "buffer", None)
|
|
198
|
+
if not isinstance(buffer, ByteSink):
|
|
199
|
+
return None
|
|
200
|
+
if not indented:
|
|
201
|
+
return buffer
|
|
202
|
+
isatty = getattr(stream, "isatty", None)
|
|
203
|
+
return IndentedEcho(buffer, dim=callable(isatty) and bool(isatty()))
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
async def classify_exit(
|
|
207
|
+
plan: WorkerPlan, code: int, *, spawner: Spawner, captured: WorkerCapture
|
|
208
|
+
) -> WorkerExit:
|
|
209
|
+
"""Explain `code` from what the worker said and what docker recorded.
|
|
210
|
+
|
|
211
|
+
The tail is what the pumps kept, so a worker the runner started directly is
|
|
212
|
+
explained as fully as a container: a loopback worker's own last words now
|
|
213
|
+
reach the message, where reading a container's log could only ever have
|
|
214
|
+
covered the workers docker started.
|
|
215
|
+
|
|
216
|
+
The pumps are settled first, because a worker that exited before anything
|
|
217
|
+
read its pipes has said its last words into one of them. Closing the streams
|
|
218
|
+
comes after that settling, so every line the worker wrote is numbered before
|
|
219
|
+
the exit code that ends them.
|
|
220
|
+
|
|
221
|
+
Only `oom_killed` is docker's to answer, and it needs the container to still
|
|
222
|
+
be there - so call this before leaving `running_worker`, whose teardown
|
|
223
|
+
removes it. A plan that started no container leaves the flag False.
|
|
224
|
+
|
|
225
|
+
Nothing here raises: this explains a failure that is already being reported,
|
|
226
|
+
and must not displace it.
|
|
227
|
+
"""
|
|
228
|
+
await captured.settle()
|
|
229
|
+
captured.close(code)
|
|
230
|
+
container_id = _container_id(plan)
|
|
231
|
+
oom_killed = container_id is not None and await _was_oom_killed(
|
|
232
|
+
spawner, container_id
|
|
233
|
+
)
|
|
234
|
+
return WorkerExit(
|
|
235
|
+
code=code,
|
|
236
|
+
oom_killed=oom_killed,
|
|
237
|
+
log_tail=captured.tail(limit=LOG_TAIL_CHARS),
|
|
238
|
+
)
|
|
239
|
+
|
|
240
|
+
|
|
241
|
+
async def _stop_worker(
|
|
242
|
+
spawner: Spawner, plan: WorkerPlan, process: ProcessHandle | None
|
|
243
|
+
) -> None:
|
|
244
|
+
"""Terminate the worker, then run teardown whether or not that worked."""
|
|
245
|
+
try:
|
|
246
|
+
if process is not None:
|
|
247
|
+
await process.terminate(grace_s=TERMINATE_GRACE_SECONDS)
|
|
248
|
+
finally:
|
|
249
|
+
await _remove_by_cid(spawner, plan)
|
|
250
|
+
await _run_teardown(spawner, plan)
|
|
251
|
+
|
|
252
|
+
|
|
253
|
+
async def _remove_by_cid(spawner: Spawner, plan: WorkerPlan) -> None:
|
|
254
|
+
"""Stop and remove what this attempt started, by the id the spawn recorded.
|
|
255
|
+
|
|
256
|
+
An absent or empty file means this attempt never started one, so there is
|
|
257
|
+
nothing to stop.
|
|
258
|
+
|
|
259
|
+
The runner removes the container rather than leaving it to `docker run
|
|
260
|
+
--rm`, which discards the exit state the moment the worker exits -- before
|
|
261
|
+
`classify_exit` can ask whether the kernel killed it for memory.
|
|
262
|
+
|
|
263
|
+
The removal repeats until docker has no container with the id, for at most
|
|
264
|
+
TEARDOWN_TIMEOUT_SECONDS of retries after the stop. A container still
|
|
265
|
+
present after that is logged as a warning.
|
|
266
|
+
"""
|
|
267
|
+
if plan.cidfile is None:
|
|
268
|
+
return
|
|
269
|
+
container_id = _container_id(plan)
|
|
270
|
+
if container_id is not None:
|
|
271
|
+
with contextlib.suppress(Exception):
|
|
272
|
+
await spawner.run(
|
|
273
|
+
("docker", "stop", f"--time={STOP_GRACE_SECONDS}", container_id),
|
|
274
|
+
timeout_s=TEARDOWN_TIMEOUT_SECONDS,
|
|
275
|
+
)
|
|
276
|
+
try:
|
|
277
|
+
async with asyncio.timeout(TEARDOWN_TIMEOUT_SECONDS):
|
|
278
|
+
while not await _remove_container(spawner, container_id):
|
|
279
|
+
await asyncio.sleep(REMOVE_RETRY_INTERVAL_SECONDS)
|
|
280
|
+
except TimeoutError:
|
|
281
|
+
logger.warning(
|
|
282
|
+
"Docker did not remove container %s within %.0f s of retries.",
|
|
283
|
+
container_id,
|
|
284
|
+
TEARDOWN_TIMEOUT_SECONDS,
|
|
285
|
+
)
|
|
286
|
+
with contextlib.suppress(OSError):
|
|
287
|
+
plan.cidfile.unlink()
|
|
288
|
+
|
|
289
|
+
|
|
290
|
+
async def _remove_container(spawner: Spawner, container_id: str) -> bool:
|
|
291
|
+
"""Run `docker rm --force` once. Return True when docker removed the container."""
|
|
292
|
+
try:
|
|
293
|
+
result = await spawner.run(
|
|
294
|
+
("docker", "rm", "--force", container_id),
|
|
295
|
+
timeout_s=TEARDOWN_TIMEOUT_SECONDS,
|
|
296
|
+
)
|
|
297
|
+
except Exception:
|
|
298
|
+
return False
|
|
299
|
+
return result.exit_code == 0
|
|
300
|
+
|
|
301
|
+
|
|
302
|
+
def _container_id(plan: WorkerPlan) -> str | None:
|
|
303
|
+
"""The container this attempt started, as docker recorded it.
|
|
304
|
+
|
|
305
|
+
None where the plan starts no container, and where the spawn never got as
|
|
306
|
+
far as writing the file.
|
|
307
|
+
"""
|
|
308
|
+
if plan.cidfile is None:
|
|
309
|
+
return None
|
|
310
|
+
try:
|
|
311
|
+
container_id = plan.cidfile.read_text().strip()
|
|
312
|
+
except OSError:
|
|
313
|
+
return None
|
|
314
|
+
if len(container_id) == 0:
|
|
315
|
+
return None
|
|
316
|
+
return container_id
|
|
317
|
+
|
|
318
|
+
|
|
319
|
+
async def _was_oom_killed(spawner: Spawner, container_id: str) -> bool:
|
|
320
|
+
"""Whether the kernel killed the container for memory, as docker saw it.
|
|
321
|
+
|
|
322
|
+
False for anything the inspect cannot answer, so a docker that has already
|
|
323
|
+
forgotten the container reports an ordinary exit rather than failing.
|
|
324
|
+
"""
|
|
325
|
+
try:
|
|
326
|
+
result = await spawner.run(
|
|
327
|
+
("docker", "inspect", "--format", "{{.State.OOMKilled}}", container_id),
|
|
328
|
+
timeout_s=INSPECT_TIMEOUT_SECONDS,
|
|
329
|
+
)
|
|
330
|
+
except (OSError, TimeoutError):
|
|
331
|
+
return False
|
|
332
|
+
return result.exit_code == 0 and result.stdout.strip() == "true"
|
|
333
|
+
|
|
334
|
+
|
|
335
|
+
async def run_setup(
|
|
336
|
+
spawner: Spawner,
|
|
337
|
+
setup: tuple[SetupCommand, ...],
|
|
338
|
+
*,
|
|
339
|
+
output: Callable[[str], None] | None = None,
|
|
340
|
+
stages: SideStages | None = None,
|
|
341
|
+
) -> None:
|
|
342
|
+
"""Run setup commands in order and optionally report each output line.
|
|
343
|
+
|
|
344
|
+
Given `stages`, each command that names its stage is logged as one.
|
|
345
|
+
|
|
346
|
+
Raises SetupStepError on failure.
|
|
347
|
+
"""
|
|
348
|
+
for step in setup:
|
|
349
|
+
with _setup_stage(step, stages):
|
|
350
|
+
await _run_setup_command(spawner, step, output=output)
|
|
351
|
+
|
|
352
|
+
|
|
353
|
+
def _setup_stage(
|
|
354
|
+
step: SetupCommand, stages: SideStages | None
|
|
355
|
+
) -> AbstractContextManager[None]:
|
|
356
|
+
if stages is None or not step.started:
|
|
357
|
+
return nullcontext()
|
|
358
|
+
return stages.step(escape(step.started), escape(step.completed))
|
|
359
|
+
|
|
360
|
+
|
|
361
|
+
async def _run_setup_command(
|
|
362
|
+
spawner: Spawner,
|
|
363
|
+
step: SetupCommand,
|
|
364
|
+
*,
|
|
365
|
+
output: Callable[[str], None] | None,
|
|
366
|
+
) -> None:
|
|
367
|
+
"""Run one setup command, raising SetupStepError on failure."""
|
|
368
|
+
try:
|
|
369
|
+
result = (
|
|
370
|
+
await spawner.run(step.argv, timeout_s=step.timeout_s)
|
|
371
|
+
if output is None
|
|
372
|
+
else await _run_streaming_setup(spawner, step, output=output)
|
|
373
|
+
)
|
|
374
|
+
except TimeoutError as exc:
|
|
375
|
+
raise SetupStepError(
|
|
376
|
+
f"The {step.phase} step did not finish within {step.timeout_s:g}s.",
|
|
377
|
+
phase=step.phase,
|
|
378
|
+
) from exc
|
|
379
|
+
except OSError as exc:
|
|
380
|
+
raise SetupStepError(
|
|
381
|
+
f"The {step.phase} step could not run: {exc}", phase=step.phase
|
|
382
|
+
) from exc
|
|
383
|
+
if result.exit_code != 0:
|
|
384
|
+
raise SetupStepError(
|
|
385
|
+
f"The {step.phase} step exited {result.exit_code}: "
|
|
386
|
+
f"{result.stderr.strip()[-STDERR_TAIL:]}",
|
|
387
|
+
phase=step.phase,
|
|
388
|
+
)
|
|
389
|
+
|
|
390
|
+
|
|
391
|
+
async def _run_streaming_setup(
|
|
392
|
+
spawner: Spawner,
|
|
393
|
+
step: SetupCommand,
|
|
394
|
+
*,
|
|
395
|
+
output: Callable[[str], None],
|
|
396
|
+
) -> CommandResult:
|
|
397
|
+
"""Run one setup command and pass each output line to `output`."""
|
|
398
|
+
process = await spawner.spawn(step.argv)
|
|
399
|
+
captured = CapturedOutput()
|
|
400
|
+
pumps = [
|
|
401
|
+
asyncio.create_task(drain(reader, captured=captured, echo=None, kept=output))
|
|
402
|
+
for reader in (process.stdout, process.stderr)
|
|
403
|
+
if reader is not None
|
|
404
|
+
]
|
|
405
|
+
waiting = asyncio.create_task(process.wait())
|
|
406
|
+
tasks = [waiting, *pumps]
|
|
407
|
+
try:
|
|
408
|
+
async with asyncio.timeout(step.timeout_s):
|
|
409
|
+
await asyncio.gather(*tasks)
|
|
410
|
+
except (TimeoutError, asyncio.CancelledError):
|
|
411
|
+
await _stop_streaming_setup(process, tasks)
|
|
412
|
+
raise
|
|
413
|
+
except Exception as exc:
|
|
414
|
+
await _stop_streaming_setup(process, tasks)
|
|
415
|
+
raise SetupStepError(
|
|
416
|
+
f"The {step.phase} step output failed: {exc or type(exc).__name__}",
|
|
417
|
+
phase=step.phase,
|
|
418
|
+
) from exc
|
|
419
|
+
return CommandResult(
|
|
420
|
+
exit_code=waiting.result(),
|
|
421
|
+
stderr=captured.tail(limit=STDERR_TAIL),
|
|
422
|
+
)
|
|
423
|
+
|
|
424
|
+
|
|
425
|
+
async def _stop_streaming_setup(
|
|
426
|
+
process: ProcessHandle, tasks: list[asyncio.Task[object]]
|
|
427
|
+
) -> None:
|
|
428
|
+
"""Terminate a setup process and stop reading its output."""
|
|
429
|
+
await process.terminate()
|
|
430
|
+
for task in tasks:
|
|
431
|
+
if not task.done():
|
|
432
|
+
task.cancel()
|
|
433
|
+
await asyncio.gather(*tasks, return_exceptions=True)
|
|
434
|
+
|
|
435
|
+
|
|
436
|
+
async def _await_ready(
|
|
437
|
+
plan: WorkerPlan,
|
|
438
|
+
process: ProcessHandle,
|
|
439
|
+
spawner: Spawner,
|
|
440
|
+
captured: WorkerCapture,
|
|
441
|
+
) -> None:
|
|
442
|
+
"""Poll the plan's probe until it answers, or the deadline passes.
|
|
443
|
+
|
|
444
|
+
Each attempt is bounded on its own, so an address that swallows the SYN or a
|
|
445
|
+
docker daemon that never answers cannot hold one attempt past the deadline.
|
|
446
|
+
|
|
447
|
+
A worker that dies while loading is the case an operator most needs a reason
|
|
448
|
+
for, so the error carries the same diagnosis a completed run reports.
|
|
449
|
+
"""
|
|
450
|
+
readiness = plan.readiness
|
|
451
|
+
if isinstance(readiness, ProcessOnly):
|
|
452
|
+
return
|
|
453
|
+
if isinstance(readiness, TcpProbe):
|
|
454
|
+
target = readiness.wire.address
|
|
455
|
+
attempt = partial(accepts_a_connection, readiness.wire)
|
|
456
|
+
else:
|
|
457
|
+
target = readiness.target
|
|
458
|
+
attempt = partial(_probe_answers, spawner, readiness)
|
|
459
|
+
loop = asyncio.get_running_loop()
|
|
460
|
+
deadline = loop.time() + readiness.timeout_s
|
|
461
|
+
while True:
|
|
462
|
+
if process.returncode is not None:
|
|
463
|
+
ended = await classify_exit(
|
|
464
|
+
plan, process.returncode, spawner=spawner, captured=captured
|
|
465
|
+
)
|
|
466
|
+
raise WorkerStartError(
|
|
467
|
+
with_tail(
|
|
468
|
+
f"The worker {ended.cause} before it was ready.", ended.log_tail
|
|
469
|
+
)
|
|
470
|
+
)
|
|
471
|
+
if await attempt():
|
|
472
|
+
return
|
|
473
|
+
remaining = deadline - loop.time()
|
|
474
|
+
if remaining <= 0:
|
|
475
|
+
raise WorkerStartError(
|
|
476
|
+
f"The worker was not ready on {target} within {readiness.timeout_s:g}s."
|
|
477
|
+
)
|
|
478
|
+
await asyncio.sleep(min(PROBE_INTERVAL_SECONDS, remaining))
|
|
479
|
+
|
|
480
|
+
|
|
481
|
+
async def accepts_a_connection(wire: WireBinding) -> bool:
|
|
482
|
+
"""Whether the worker's own address accepts a connection."""
|
|
483
|
+
connector = TcpConnector(wire.host, wire.port)
|
|
484
|
+
try:
|
|
485
|
+
stream = await asyncio.wait_for(
|
|
486
|
+
connector.connect(), timeout=PROBE_CONNECT_TIMEOUT_SECONDS
|
|
487
|
+
)
|
|
488
|
+
except OSError:
|
|
489
|
+
return False
|
|
490
|
+
await stream.close()
|
|
491
|
+
return True
|
|
492
|
+
|
|
493
|
+
|
|
494
|
+
async def _probe_answers(spawner: Spawner, probe: CommandProbe) -> bool:
|
|
495
|
+
"""Whether the probe command exited zero, raising for one that cannot run."""
|
|
496
|
+
try:
|
|
497
|
+
result = await spawner.run(probe.argv, timeout_s=PROBE_COMMAND_TIMEOUT_SECONDS)
|
|
498
|
+
except TimeoutError:
|
|
499
|
+
return False
|
|
500
|
+
except OSError as exc:
|
|
501
|
+
raise WorkerStartError(
|
|
502
|
+
f"The readiness check for {probe.target} could not run: {exc}"
|
|
503
|
+
) from exc
|
|
504
|
+
if result.exit_code in PROBE_UNRUNNABLE_EXIT_CODES:
|
|
505
|
+
# NOTE: `docker exec` writes the command it could not start to stdout,
|
|
506
|
+
# so stderr alone leaves this error with nothing to name.
|
|
507
|
+
complaint = result.stderr.strip()
|
|
508
|
+
if len(complaint) == 0:
|
|
509
|
+
complaint = result.stdout.strip()
|
|
510
|
+
raise WorkerStartError(
|
|
511
|
+
f"The readiness check for {probe.target} could not run "
|
|
512
|
+
f"(exit {result.exit_code}): {complaint[-STDERR_TAIL:]}"
|
|
513
|
+
)
|
|
514
|
+
return result.exit_code == 0
|
|
515
|
+
|
|
516
|
+
|
|
517
|
+
async def _run_teardown(spawner: Spawner, plan: WorkerPlan) -> None:
|
|
518
|
+
"""Run every teardown command, best effort.
|
|
519
|
+
|
|
520
|
+
A teardown command reports the state it found rather than a fault of its
|
|
521
|
+
own -- `docker stop` on a container already removed exits non-zero -- and it
|
|
522
|
+
must not displace whatever failure brought us here, so nothing it does is
|
|
523
|
+
checked or raised.
|
|
524
|
+
"""
|
|
525
|
+
for argv in plan.teardown:
|
|
526
|
+
with contextlib.suppress(Exception):
|
|
527
|
+
await spawner.run(argv, timeout_s=TEARDOWN_TIMEOUT_SECONDS)
|
|
528
|
+
|
|
529
|
+
|
|
530
|
+
def with_tail(message: str, log_tail: str) -> str:
|
|
531
|
+
"""`message`, and the worker's last output where there was any."""
|
|
532
|
+
if len(log_tail) == 0:
|
|
533
|
+
return message
|
|
534
|
+
return f"{message} Last output: {log_tail}"
|