manifold-cli 0.0.39__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (102) hide show
  1. manifold_cli/__init__.py +3 -0
  2. manifold_cli/__main__.py +4 -0
  3. manifold_cli/bridge.py +70 -0
  4. manifold_cli/cli.py +54 -0
  5. manifold_cli/clients/__init__.py +0 -0
  6. manifold_cli/clients/auth_flow.py +169 -0
  7. manifold_cli/clients/credentials.py +53 -0
  8. manifold_cli/clients/docker_engine.py +43 -0
  9. manifold_cli/clients/image.py +130 -0
  10. manifold_cli/clients/manifold.py +1288 -0
  11. manifold_cli/clients/runner_home.py +165 -0
  12. manifold_cli/clients/workos.py +99 -0
  13. manifold_cli/commands/__init__.py +0 -0
  14. manifold_cli/commands/auth.py +101 -0
  15. manifold_cli/commands/benchmark.py +539 -0
  16. manifold_cli/commands/organization.py +154 -0
  17. manifold_cli/commands/policy.py +749 -0
  18. manifold_cli/commands/run.py +337 -0
  19. manifold_cli/commands/run_log.py +291 -0
  20. manifold_cli/commands/runner.py +852 -0
  21. manifold_cli/commands/table.py +101 -0
  22. manifold_cli/commands/token.py +144 -0
  23. manifold_cli/dependencies.py +116 -0
  24. manifold_cli/models/__init__.py +0 -0
  25. manifold_cli/models/auth.py +49 -0
  26. manifold_cli/models/benchmark.py +87 -0
  27. manifold_cli/models/catalog.py +189 -0
  28. manifold_cli/models/credentials.py +19 -0
  29. manifold_cli/models/errors.py +121 -0
  30. manifold_cli/models/log.py +114 -0
  31. manifold_cli/models/observability.py +121 -0
  32. manifold_cli/models/organization.py +19 -0
  33. manifold_cli/models/pagination.py +59 -0
  34. manifold_cli/models/policy.py +49 -0
  35. manifold_cli/models/run.py +381 -0
  36. manifold_cli/models/runner.py +186 -0
  37. manifold_cli/models/runner_home.py +69 -0
  38. manifold_cli/models/task_queue.py +40 -0
  39. manifold_cli/models/token.py +39 -0
  40. manifold_cli/replay/__init__.py +68 -0
  41. manifold_cli/replay/log.py +814 -0
  42. manifold_cli/runner/__init__.py +72 -0
  43. manifold_cli/runner/api_retry.py +36 -0
  44. manifold_cli/runner/benchmark/__init__.py +800 -0
  45. manifold_cli/runner/benchmark/live_view.py +248 -0
  46. manifold_cli/runner/benchmark/preassigned_scheduling.py +325 -0
  47. manifold_cli/runner/benchmark/recording.py +1208 -0
  48. manifold_cli/runner/benchmark/scheduling.py +256 -0
  49. manifold_cli/runner/benchmark/task_queue_scheduling.py +717 -0
  50. manifold_cli/runner/benchmark/task_server.py +123 -0
  51. manifold_cli/runner/benchmark/video.py +124 -0
  52. manifold_cli/runner/benchmark_steps.py +178 -0
  53. manifold_cli/runner/cancellation.py +22 -0
  54. manifold_cli/runner/capture.py +282 -0
  55. manifold_cli/runner/container.py +370 -0
  56. manifold_cli/runner/current_dispatch.py +25 -0
  57. manifold_cli/runner/daemon.py +520 -0
  58. manifold_cli/runner/dispatch.py +38 -0
  59. manifold_cli/runner/gpu.py +182 -0
  60. manifold_cli/runner/heartbeat.py +202 -0
  61. manifold_cli/runner/loopback.py +102 -0
  62. manifold_cli/runner/loopback_bench.py +183 -0
  63. manifold_cli/runner/loopback_policy.py +39 -0
  64. manifold_cli/runner/output.py +218 -0
  65. manifold_cli/runner/pairing_acceptance.py +57 -0
  66. manifold_cli/runner/plan.py +369 -0
  67. manifold_cli/runner/policy.py +350 -0
  68. manifold_cli/runner/policy_steps.py +164 -0
  69. manifold_cli/runner/preparation.py +47 -0
  70. manifold_cli/runner/process.py +219 -0
  71. manifold_cli/runner/ram.py +17 -0
  72. manifold_cli/runner/resource_usage.py +265 -0
  73. manifold_cli/runner/served.py +218 -0
  74. manifold_cli/runner/service.py +120 -0
  75. manifold_cli/runner/side_failure.py +68 -0
  76. manifold_cli/runner/signature.py +98 -0
  77. manifold_cli/runner/source.py +66 -0
  78. manifold_cli/runner/stages.py +241 -0
  79. manifold_cli/runner/step_timing.py +229 -0
  80. manifold_cli/runner/worker.py +534 -0
  81. manifold_cli/runner/workload.py +401 -0
  82. manifold_cli/services/__init__.py +0 -0
  83. manifold_cli/services/auth.py +89 -0
  84. manifold_cli/services/benchmark.py +357 -0
  85. manifold_cli/services/organization.py +22 -0
  86. manifold_cli/services/policy.py +234 -0
  87. manifold_cli/services/run.py +304 -0
  88. manifold_cli/services/runner.py +207 -0
  89. manifold_cli/services/runner_daemon.py +146 -0
  90. manifold_cli/services/token.py +52 -0
  91. manifold_cli/settings.py +30 -0
  92. manifold_cli/transport/__init__.py +31 -0
  93. manifold_cli/transport/diagnostics.py +72 -0
  94. manifold_cli/transport/interfaces.py +69 -0
  95. manifold_cli/transport/iroh.py +317 -0
  96. manifold_cli/transport/registry.py +34 -0
  97. manifold_cli/transport/tcp.py +146 -0
  98. manifold_cli-0.0.39.dist-info/METADATA +406 -0
  99. manifold_cli-0.0.39.dist-info/RECORD +102 -0
  100. manifold_cli-0.0.39.dist-info/WHEEL +4 -0
  101. manifold_cli-0.0.39.dist-info/entry_points.txt +2 -0
  102. manifold_cli-0.0.39.dist-info/licenses/LICENSE +373 -0
@@ -0,0 +1,534 @@
1
+ """Running one worker from its plan: setup, spawn, readiness, teardown.
2
+
3
+ One worker and nothing about the run it belongs to. Claiming one side of a run
4
+ and shipping what its worker produces both land later.
5
+ """
6
+
7
+ import asyncio
8
+ import contextlib
9
+ import logging
10
+ import sys
11
+ from collections.abc import AsyncIterator, Callable
12
+ from contextlib import AbstractContextManager, asynccontextmanager, nullcontext
13
+ from dataclasses import dataclass
14
+ from functools import partial
15
+
16
+ from rich.markup import escape
17
+
18
+ from manifold_cli.models.log import LogSource
19
+ from manifold_cli.runner.capture import ByteSink, CapturedOutput, WorkerCapture, drain
20
+ from manifold_cli.runner.plan import (
21
+ CommandProbe,
22
+ ProcessOnly,
23
+ RunnerError,
24
+ SetupCommand,
25
+ TcpProbe,
26
+ WireBinding,
27
+ WorkerPlan,
28
+ )
29
+ from manifold_cli.runner.process import (
30
+ TERMINATE_GRACE_SECONDS,
31
+ CommandResult,
32
+ ProcessHandle,
33
+ Spawner,
34
+ )
35
+ from manifold_cli.runner.stages import IndentedEcho, SideStages
36
+ from manifold_cli.transport.tcp import TcpConnector
37
+
38
+ logger = logging.getLogger(__name__)
39
+
40
+ PROBE_INTERVAL_SECONDS = 0.5
41
+ PROBE_CONNECT_TIMEOUT_SECONDS = 1.0
42
+ PROBE_COMMAND_TIMEOUT_SECONDS = 10.0
43
+ TEARDOWN_TIMEOUT_SECONDS = 30.0
44
+ # How long a container gets to exit on SIGTERM before docker kills it.
45
+ STOP_GRACE_SECONDS = 5
46
+ # The pause between two attempts to remove a container that docker still holds.
47
+ REMOVE_RETRY_INTERVAL_SECONDS = 1.0
48
+
49
+ # What a runner reports for a check it cannot start: 127 for a command it cannot
50
+ # find, 126 for one it cannot execute. A shell and `docker exec` both use them. A
51
+ # probe that exits either way is answering for the check itself rather than for
52
+ # the worker, so it is fatal. Docker's own failures -- no such container, a
53
+ # daemon it cannot reach -- exit 1, and stay part of the poll.
54
+ PROBE_UNRUNNABLE_EXIT_CODES = (126, 127)
55
+
56
+ # How much of a failed command's complaint the error carries.
57
+ STDERR_TAIL = 400
58
+
59
+ # How long docker gets to answer for a container that has already exited.
60
+ INSPECT_TIMEOUT_SECONDS = 10.0
61
+
62
+ # How much of a worker's last output reaches a message.
63
+ LOG_TAIL_CHARS = 600
64
+
65
+
66
+ class WorkerStartError(RunnerError):
67
+ """A worker never became ready.
68
+
69
+ Carries a message naming what failed: a setup command, the spawn itself, or
70
+ the readiness probe.
71
+ """
72
+
73
+
74
+ class SetupStepError(WorkerStartError):
75
+ """A setup command failed, timed out or could not run."""
76
+
77
+ def __init__(self, message: str, *, phase: str) -> None:
78
+ super().__init__(message)
79
+ self.phase = phase
80
+
81
+
82
+ @dataclass(frozen=True)
83
+ class WorkerExit:
84
+ """How a worker ended, and what it last said.
85
+
86
+ `log_tail` is the end of what the pumps read, for a worker of any kind.
87
+ `oom_killed` is docker's answer about a container and stays False for a
88
+ worker that is not one. The exit code does not carry the reason on its own:
89
+ the kernel's memory killer and a stop that outran its grace period both
90
+ leave 137.
91
+ """
92
+
93
+ code: int
94
+ oom_killed: bool = False
95
+ log_tail: str = ""
96
+
97
+ @property
98
+ def cause(self) -> str:
99
+ """How the worker ended, as a fragment naming its own doing."""
100
+ if self.oom_killed:
101
+ return f"ran out of memory and was killed (exit {self.code})"
102
+ return f"exited {self.code}"
103
+
104
+ @property
105
+ def detail(self) -> str:
106
+ """What the side reports: the cause, and the end of the worker's output."""
107
+ return with_tail(f"The worker {self.cause}.", self.log_tail)
108
+
109
+
110
+ @asynccontextmanager
111
+ async def running_worker(
112
+ plan: WorkerPlan,
113
+ *,
114
+ spawner: Spawner,
115
+ captured: WorkerCapture,
116
+ stages: SideStages | None = None,
117
+ ) -> AsyncIterator[ProcessHandle]:
118
+ """Run `plan`'s setup, spawn its worker, and yield it once it is ready.
119
+
120
+ What the worker writes is drained into `captured` for as long as it runs, and
121
+ echoed on to this process's own streams as it passes.
122
+
123
+ Given `stages`, it logs each setup command that names its stage, and the
124
+ wait from the spawn until the worker is ready. The echo of the worker's
125
+ output is then indented under those stages.
126
+
127
+ The worker is terminated and `plan.teardown` runs on the way out, whatever
128
+ happened: a failed setup command, a probe that timed out, or the body
129
+ raising. A container the docker client left behind holds its GPU, so
130
+ teardown does not wait on the worker having started.
131
+
132
+ Raises WorkerStartError if the worker does not come up.
133
+ """
134
+ process: ProcessHandle | None = None
135
+ try:
136
+ await run_setup(spawner, plan.setup, stages=stages)
137
+ with _loading(plan, stages):
138
+ try:
139
+ process = await spawner.spawn(
140
+ plan.argv, workdir=plan.workdir, env=plan.env
141
+ )
142
+ except OSError as exc:
143
+ raise WorkerStartError(f"Could not spawn the worker: {exc}") from exc
144
+ _start_pumps(process, captured, indented=stages is not None)
145
+ await _await_ready(plan, process, spawner, captured)
146
+ yield process
147
+ finally:
148
+ # NOTE: shielded, so a cancellation arriving while the cleanup is
149
+ # already awaiting does not abandon it half-done. `CancelledError` is a
150
+ # BaseException, so the suppression in `_run_teardown` would not catch
151
+ # one, and the container would outlive the runner holding its GPU.
152
+ await asyncio.shield(_stop_worker(spawner, plan, process))
153
+ # The worker is gone, so its pipes end and the pumps with them.
154
+ await asyncio.shield(captured.settle())
155
+
156
+
157
+ def _loading(
158
+ plan: WorkerPlan, stages: SideStages | None
159
+ ) -> AbstractContextManager[None]:
160
+ """The stage from the spawn until the worker is ready.
161
+
162
+ Logged only where there is a wait: a worker without a probe is ready once
163
+ its process is up.
164
+ """
165
+ if stages is None or isinstance(plan.readiness, ProcessOnly):
166
+ return nullcontext()
167
+ side = stages.side.value
168
+ return stages.step(f"Loading the {side}", f"{side.capitalize()} loaded")
169
+
170
+
171
+ def _start_pumps(
172
+ process: ProcessHandle, captured: WorkerCapture, *, indented: bool = False
173
+ ) -> None:
174
+ """Read each of the worker's pipes, echoing what arrives on the way past.
175
+
176
+ A pipe is absent for a handle nothing piped, which is what a test double
177
+ gives back, and there is nothing to read for it.
178
+ """
179
+ for reader, stream, source in (
180
+ (process.stdout, sys.stdout, LogSource.STDOUT),
181
+ (process.stderr, sys.stderr, LogSource.STDERR),
182
+ ):
183
+ if reader is not None:
184
+ captured.pump(reader, echo_for(stream, indented=indented), source)
185
+
186
+
187
+ def echo_for(stream: object, *, indented: bool = False) -> ByteSink | None:
188
+ """The binary half of `stream`, where it has one.
189
+
190
+ `indented` sets each line under the stage lines, dimmed where `stream` is
191
+ a terminal.
192
+
193
+ NOTE: the echo writes bytes so that nothing a worker printed is read as
194
+ console markup. A stream standing in for the real one during a test may hold
195
+ no binary half at all, and an echo is not what such a test is checking.
196
+ """
197
+ buffer = getattr(stream, "buffer", None)
198
+ if not isinstance(buffer, ByteSink):
199
+ return None
200
+ if not indented:
201
+ return buffer
202
+ isatty = getattr(stream, "isatty", None)
203
+ return IndentedEcho(buffer, dim=callable(isatty) and bool(isatty()))
204
+
205
+
206
+ async def classify_exit(
207
+ plan: WorkerPlan, code: int, *, spawner: Spawner, captured: WorkerCapture
208
+ ) -> WorkerExit:
209
+ """Explain `code` from what the worker said and what docker recorded.
210
+
211
+ The tail is what the pumps kept, so a worker the runner started directly is
212
+ explained as fully as a container: a loopback worker's own last words now
213
+ reach the message, where reading a container's log could only ever have
214
+ covered the workers docker started.
215
+
216
+ The pumps are settled first, because a worker that exited before anything
217
+ read its pipes has said its last words into one of them. Closing the streams
218
+ comes after that settling, so every line the worker wrote is numbered before
219
+ the exit code that ends them.
220
+
221
+ Only `oom_killed` is docker's to answer, and it needs the container to still
222
+ be there - so call this before leaving `running_worker`, whose teardown
223
+ removes it. A plan that started no container leaves the flag False.
224
+
225
+ Nothing here raises: this explains a failure that is already being reported,
226
+ and must not displace it.
227
+ """
228
+ await captured.settle()
229
+ captured.close(code)
230
+ container_id = _container_id(plan)
231
+ oom_killed = container_id is not None and await _was_oom_killed(
232
+ spawner, container_id
233
+ )
234
+ return WorkerExit(
235
+ code=code,
236
+ oom_killed=oom_killed,
237
+ log_tail=captured.tail(limit=LOG_TAIL_CHARS),
238
+ )
239
+
240
+
241
+ async def _stop_worker(
242
+ spawner: Spawner, plan: WorkerPlan, process: ProcessHandle | None
243
+ ) -> None:
244
+ """Terminate the worker, then run teardown whether or not that worked."""
245
+ try:
246
+ if process is not None:
247
+ await process.terminate(grace_s=TERMINATE_GRACE_SECONDS)
248
+ finally:
249
+ await _remove_by_cid(spawner, plan)
250
+ await _run_teardown(spawner, plan)
251
+
252
+
253
+ async def _remove_by_cid(spawner: Spawner, plan: WorkerPlan) -> None:
254
+ """Stop and remove what this attempt started, by the id the spawn recorded.
255
+
256
+ An absent or empty file means this attempt never started one, so there is
257
+ nothing to stop.
258
+
259
+ The runner removes the container rather than leaving it to `docker run
260
+ --rm`, which discards the exit state the moment the worker exits -- before
261
+ `classify_exit` can ask whether the kernel killed it for memory.
262
+
263
+ The removal repeats until docker has no container with the id, for at most
264
+ TEARDOWN_TIMEOUT_SECONDS of retries after the stop. A container still
265
+ present after that is logged as a warning.
266
+ """
267
+ if plan.cidfile is None:
268
+ return
269
+ container_id = _container_id(plan)
270
+ if container_id is not None:
271
+ with contextlib.suppress(Exception):
272
+ await spawner.run(
273
+ ("docker", "stop", f"--time={STOP_GRACE_SECONDS}", container_id),
274
+ timeout_s=TEARDOWN_TIMEOUT_SECONDS,
275
+ )
276
+ try:
277
+ async with asyncio.timeout(TEARDOWN_TIMEOUT_SECONDS):
278
+ while not await _remove_container(spawner, container_id):
279
+ await asyncio.sleep(REMOVE_RETRY_INTERVAL_SECONDS)
280
+ except TimeoutError:
281
+ logger.warning(
282
+ "Docker did not remove container %s within %.0f s of retries.",
283
+ container_id,
284
+ TEARDOWN_TIMEOUT_SECONDS,
285
+ )
286
+ with contextlib.suppress(OSError):
287
+ plan.cidfile.unlink()
288
+
289
+
290
+ async def _remove_container(spawner: Spawner, container_id: str) -> bool:
291
+ """Run `docker rm --force` once. Return True when docker removed the container."""
292
+ try:
293
+ result = await spawner.run(
294
+ ("docker", "rm", "--force", container_id),
295
+ timeout_s=TEARDOWN_TIMEOUT_SECONDS,
296
+ )
297
+ except Exception:
298
+ return False
299
+ return result.exit_code == 0
300
+
301
+
302
+ def _container_id(plan: WorkerPlan) -> str | None:
303
+ """The container this attempt started, as docker recorded it.
304
+
305
+ None where the plan starts no container, and where the spawn never got as
306
+ far as writing the file.
307
+ """
308
+ if plan.cidfile is None:
309
+ return None
310
+ try:
311
+ container_id = plan.cidfile.read_text().strip()
312
+ except OSError:
313
+ return None
314
+ if len(container_id) == 0:
315
+ return None
316
+ return container_id
317
+
318
+
319
+ async def _was_oom_killed(spawner: Spawner, container_id: str) -> bool:
320
+ """Whether the kernel killed the container for memory, as docker saw it.
321
+
322
+ False for anything the inspect cannot answer, so a docker that has already
323
+ forgotten the container reports an ordinary exit rather than failing.
324
+ """
325
+ try:
326
+ result = await spawner.run(
327
+ ("docker", "inspect", "--format", "{{.State.OOMKilled}}", container_id),
328
+ timeout_s=INSPECT_TIMEOUT_SECONDS,
329
+ )
330
+ except (OSError, TimeoutError):
331
+ return False
332
+ return result.exit_code == 0 and result.stdout.strip() == "true"
333
+
334
+
335
+ async def run_setup(
336
+ spawner: Spawner,
337
+ setup: tuple[SetupCommand, ...],
338
+ *,
339
+ output: Callable[[str], None] | None = None,
340
+ stages: SideStages | None = None,
341
+ ) -> None:
342
+ """Run setup commands in order and optionally report each output line.
343
+
344
+ Given `stages`, each command that names its stage is logged as one.
345
+
346
+ Raises SetupStepError on failure.
347
+ """
348
+ for step in setup:
349
+ with _setup_stage(step, stages):
350
+ await _run_setup_command(spawner, step, output=output)
351
+
352
+
353
+ def _setup_stage(
354
+ step: SetupCommand, stages: SideStages | None
355
+ ) -> AbstractContextManager[None]:
356
+ if stages is None or not step.started:
357
+ return nullcontext()
358
+ return stages.step(escape(step.started), escape(step.completed))
359
+
360
+
361
+ async def _run_setup_command(
362
+ spawner: Spawner,
363
+ step: SetupCommand,
364
+ *,
365
+ output: Callable[[str], None] | None,
366
+ ) -> None:
367
+ """Run one setup command, raising SetupStepError on failure."""
368
+ try:
369
+ result = (
370
+ await spawner.run(step.argv, timeout_s=step.timeout_s)
371
+ if output is None
372
+ else await _run_streaming_setup(spawner, step, output=output)
373
+ )
374
+ except TimeoutError as exc:
375
+ raise SetupStepError(
376
+ f"The {step.phase} step did not finish within {step.timeout_s:g}s.",
377
+ phase=step.phase,
378
+ ) from exc
379
+ except OSError as exc:
380
+ raise SetupStepError(
381
+ f"The {step.phase} step could not run: {exc}", phase=step.phase
382
+ ) from exc
383
+ if result.exit_code != 0:
384
+ raise SetupStepError(
385
+ f"The {step.phase} step exited {result.exit_code}: "
386
+ f"{result.stderr.strip()[-STDERR_TAIL:]}",
387
+ phase=step.phase,
388
+ )
389
+
390
+
391
+ async def _run_streaming_setup(
392
+ spawner: Spawner,
393
+ step: SetupCommand,
394
+ *,
395
+ output: Callable[[str], None],
396
+ ) -> CommandResult:
397
+ """Run one setup command and pass each output line to `output`."""
398
+ process = await spawner.spawn(step.argv)
399
+ captured = CapturedOutput()
400
+ pumps = [
401
+ asyncio.create_task(drain(reader, captured=captured, echo=None, kept=output))
402
+ for reader in (process.stdout, process.stderr)
403
+ if reader is not None
404
+ ]
405
+ waiting = asyncio.create_task(process.wait())
406
+ tasks = [waiting, *pumps]
407
+ try:
408
+ async with asyncio.timeout(step.timeout_s):
409
+ await asyncio.gather(*tasks)
410
+ except (TimeoutError, asyncio.CancelledError):
411
+ await _stop_streaming_setup(process, tasks)
412
+ raise
413
+ except Exception as exc:
414
+ await _stop_streaming_setup(process, tasks)
415
+ raise SetupStepError(
416
+ f"The {step.phase} step output failed: {exc or type(exc).__name__}",
417
+ phase=step.phase,
418
+ ) from exc
419
+ return CommandResult(
420
+ exit_code=waiting.result(),
421
+ stderr=captured.tail(limit=STDERR_TAIL),
422
+ )
423
+
424
+
425
+ async def _stop_streaming_setup(
426
+ process: ProcessHandle, tasks: list[asyncio.Task[object]]
427
+ ) -> None:
428
+ """Terminate a setup process and stop reading its output."""
429
+ await process.terminate()
430
+ for task in tasks:
431
+ if not task.done():
432
+ task.cancel()
433
+ await asyncio.gather(*tasks, return_exceptions=True)
434
+
435
+
436
+ async def _await_ready(
437
+ plan: WorkerPlan,
438
+ process: ProcessHandle,
439
+ spawner: Spawner,
440
+ captured: WorkerCapture,
441
+ ) -> None:
442
+ """Poll the plan's probe until it answers, or the deadline passes.
443
+
444
+ Each attempt is bounded on its own, so an address that swallows the SYN or a
445
+ docker daemon that never answers cannot hold one attempt past the deadline.
446
+
447
+ A worker that dies while loading is the case an operator most needs a reason
448
+ for, so the error carries the same diagnosis a completed run reports.
449
+ """
450
+ readiness = plan.readiness
451
+ if isinstance(readiness, ProcessOnly):
452
+ return
453
+ if isinstance(readiness, TcpProbe):
454
+ target = readiness.wire.address
455
+ attempt = partial(accepts_a_connection, readiness.wire)
456
+ else:
457
+ target = readiness.target
458
+ attempt = partial(_probe_answers, spawner, readiness)
459
+ loop = asyncio.get_running_loop()
460
+ deadline = loop.time() + readiness.timeout_s
461
+ while True:
462
+ if process.returncode is not None:
463
+ ended = await classify_exit(
464
+ plan, process.returncode, spawner=spawner, captured=captured
465
+ )
466
+ raise WorkerStartError(
467
+ with_tail(
468
+ f"The worker {ended.cause} before it was ready.", ended.log_tail
469
+ )
470
+ )
471
+ if await attempt():
472
+ return
473
+ remaining = deadline - loop.time()
474
+ if remaining <= 0:
475
+ raise WorkerStartError(
476
+ f"The worker was not ready on {target} within {readiness.timeout_s:g}s."
477
+ )
478
+ await asyncio.sleep(min(PROBE_INTERVAL_SECONDS, remaining))
479
+
480
+
481
+ async def accepts_a_connection(wire: WireBinding) -> bool:
482
+ """Whether the worker's own address accepts a connection."""
483
+ connector = TcpConnector(wire.host, wire.port)
484
+ try:
485
+ stream = await asyncio.wait_for(
486
+ connector.connect(), timeout=PROBE_CONNECT_TIMEOUT_SECONDS
487
+ )
488
+ except OSError:
489
+ return False
490
+ await stream.close()
491
+ return True
492
+
493
+
494
+ async def _probe_answers(spawner: Spawner, probe: CommandProbe) -> bool:
495
+ """Whether the probe command exited zero, raising for one that cannot run."""
496
+ try:
497
+ result = await spawner.run(probe.argv, timeout_s=PROBE_COMMAND_TIMEOUT_SECONDS)
498
+ except TimeoutError:
499
+ return False
500
+ except OSError as exc:
501
+ raise WorkerStartError(
502
+ f"The readiness check for {probe.target} could not run: {exc}"
503
+ ) from exc
504
+ if result.exit_code in PROBE_UNRUNNABLE_EXIT_CODES:
505
+ # NOTE: `docker exec` writes the command it could not start to stdout,
506
+ # so stderr alone leaves this error with nothing to name.
507
+ complaint = result.stderr.strip()
508
+ if len(complaint) == 0:
509
+ complaint = result.stdout.strip()
510
+ raise WorkerStartError(
511
+ f"The readiness check for {probe.target} could not run "
512
+ f"(exit {result.exit_code}): {complaint[-STDERR_TAIL:]}"
513
+ )
514
+ return result.exit_code == 0
515
+
516
+
517
+ async def _run_teardown(spawner: Spawner, plan: WorkerPlan) -> None:
518
+ """Run every teardown command, best effort.
519
+
520
+ A teardown command reports the state it found rather than a fault of its
521
+ own -- `docker stop` on a container already removed exits non-zero -- and it
522
+ must not displace whatever failure brought us here, so nothing it does is
523
+ checked or raised.
524
+ """
525
+ for argv in plan.teardown:
526
+ with contextlib.suppress(Exception):
527
+ await spawner.run(argv, timeout_s=TEARDOWN_TIMEOUT_SECONDS)
528
+
529
+
530
+ def with_tail(message: str, log_tail: str) -> str:
531
+ """`message`, and the worker's last output where there was any."""
532
+ if len(log_tail) == 0:
533
+ return message
534
+ return f"{message} Last output: {log_tail}"