manifold-cli 0.0.39__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (102) hide show
  1. manifold_cli/__init__.py +3 -0
  2. manifold_cli/__main__.py +4 -0
  3. manifold_cli/bridge.py +70 -0
  4. manifold_cli/cli.py +54 -0
  5. manifold_cli/clients/__init__.py +0 -0
  6. manifold_cli/clients/auth_flow.py +169 -0
  7. manifold_cli/clients/credentials.py +53 -0
  8. manifold_cli/clients/docker_engine.py +43 -0
  9. manifold_cli/clients/image.py +130 -0
  10. manifold_cli/clients/manifold.py +1288 -0
  11. manifold_cli/clients/runner_home.py +165 -0
  12. manifold_cli/clients/workos.py +99 -0
  13. manifold_cli/commands/__init__.py +0 -0
  14. manifold_cli/commands/auth.py +101 -0
  15. manifold_cli/commands/benchmark.py +539 -0
  16. manifold_cli/commands/organization.py +154 -0
  17. manifold_cli/commands/policy.py +749 -0
  18. manifold_cli/commands/run.py +337 -0
  19. manifold_cli/commands/run_log.py +291 -0
  20. manifold_cli/commands/runner.py +852 -0
  21. manifold_cli/commands/table.py +101 -0
  22. manifold_cli/commands/token.py +144 -0
  23. manifold_cli/dependencies.py +116 -0
  24. manifold_cli/models/__init__.py +0 -0
  25. manifold_cli/models/auth.py +49 -0
  26. manifold_cli/models/benchmark.py +87 -0
  27. manifold_cli/models/catalog.py +189 -0
  28. manifold_cli/models/credentials.py +19 -0
  29. manifold_cli/models/errors.py +121 -0
  30. manifold_cli/models/log.py +114 -0
  31. manifold_cli/models/observability.py +121 -0
  32. manifold_cli/models/organization.py +19 -0
  33. manifold_cli/models/pagination.py +59 -0
  34. manifold_cli/models/policy.py +49 -0
  35. manifold_cli/models/run.py +381 -0
  36. manifold_cli/models/runner.py +186 -0
  37. manifold_cli/models/runner_home.py +69 -0
  38. manifold_cli/models/task_queue.py +40 -0
  39. manifold_cli/models/token.py +39 -0
  40. manifold_cli/replay/__init__.py +68 -0
  41. manifold_cli/replay/log.py +814 -0
  42. manifold_cli/runner/__init__.py +72 -0
  43. manifold_cli/runner/api_retry.py +36 -0
  44. manifold_cli/runner/benchmark/__init__.py +800 -0
  45. manifold_cli/runner/benchmark/live_view.py +248 -0
  46. manifold_cli/runner/benchmark/preassigned_scheduling.py +325 -0
  47. manifold_cli/runner/benchmark/recording.py +1208 -0
  48. manifold_cli/runner/benchmark/scheduling.py +256 -0
  49. manifold_cli/runner/benchmark/task_queue_scheduling.py +717 -0
  50. manifold_cli/runner/benchmark/task_server.py +123 -0
  51. manifold_cli/runner/benchmark/video.py +124 -0
  52. manifold_cli/runner/benchmark_steps.py +178 -0
  53. manifold_cli/runner/cancellation.py +22 -0
  54. manifold_cli/runner/capture.py +282 -0
  55. manifold_cli/runner/container.py +370 -0
  56. manifold_cli/runner/current_dispatch.py +25 -0
  57. manifold_cli/runner/daemon.py +520 -0
  58. manifold_cli/runner/dispatch.py +38 -0
  59. manifold_cli/runner/gpu.py +182 -0
  60. manifold_cli/runner/heartbeat.py +202 -0
  61. manifold_cli/runner/loopback.py +102 -0
  62. manifold_cli/runner/loopback_bench.py +183 -0
  63. manifold_cli/runner/loopback_policy.py +39 -0
  64. manifold_cli/runner/output.py +218 -0
  65. manifold_cli/runner/pairing_acceptance.py +57 -0
  66. manifold_cli/runner/plan.py +369 -0
  67. manifold_cli/runner/policy.py +350 -0
  68. manifold_cli/runner/policy_steps.py +164 -0
  69. manifold_cli/runner/preparation.py +47 -0
  70. manifold_cli/runner/process.py +219 -0
  71. manifold_cli/runner/ram.py +17 -0
  72. manifold_cli/runner/resource_usage.py +265 -0
  73. manifold_cli/runner/served.py +218 -0
  74. manifold_cli/runner/service.py +120 -0
  75. manifold_cli/runner/side_failure.py +68 -0
  76. manifold_cli/runner/signature.py +98 -0
  77. manifold_cli/runner/source.py +66 -0
  78. manifold_cli/runner/stages.py +241 -0
  79. manifold_cli/runner/step_timing.py +229 -0
  80. manifold_cli/runner/worker.py +534 -0
  81. manifold_cli/runner/workload.py +401 -0
  82. manifold_cli/services/__init__.py +0 -0
  83. manifold_cli/services/auth.py +89 -0
  84. manifold_cli/services/benchmark.py +357 -0
  85. manifold_cli/services/organization.py +22 -0
  86. manifold_cli/services/policy.py +234 -0
  87. manifold_cli/services/run.py +304 -0
  88. manifold_cli/services/runner.py +207 -0
  89. manifold_cli/services/runner_daemon.py +146 -0
  90. manifold_cli/services/token.py +52 -0
  91. manifold_cli/settings.py +30 -0
  92. manifold_cli/transport/__init__.py +31 -0
  93. manifold_cli/transport/diagnostics.py +72 -0
  94. manifold_cli/transport/interfaces.py +69 -0
  95. manifold_cli/transport/iroh.py +317 -0
  96. manifold_cli/transport/registry.py +34 -0
  97. manifold_cli/transport/tcp.py +146 -0
  98. manifold_cli-0.0.39.dist-info/METADATA +406 -0
  99. manifold_cli-0.0.39.dist-info/RECORD +102 -0
  100. manifold_cli-0.0.39.dist-info/WHEEL +4 -0
  101. manifold_cli-0.0.39.dist-info/entry_points.txt +2 -0
  102. manifold_cli-0.0.39.dist-info/licenses/LICENSE +373 -0
@@ -0,0 +1,520 @@
1
+ """The loop an installed runner runs: take the side it is handed, run it, ask again.
2
+
3
+ This is the whole of what a runner does. It sends a start request, asks whether
4
+ a side is assigned to it, drives that side to its reported outcome, and asks
5
+ again. On the stop signal it sends a stop request. Everything a side needs --
6
+ the plan, the containers, the bridge, the beating, the outcome -- belongs to
7
+ `policy.run_policy_side` and `benchmark.run_benchmark_side`; what is here is
8
+ which of the two a dispatch reaches, and what a daemon does when a poll or a
9
+ box fails it.
10
+
11
+ Nothing is chosen here. A placement pass decides which run a box serves and which
12
+ side of it, so a runner cannot prefer one run over another and cannot be refused
13
+ the side it was handed.
14
+
15
+ The loop is synchronous. The API client is, and so are the waits between polls;
16
+ only a side's workers and their bridges are not, so each side runs under an event
17
+ loop of its own that is torn down before the next poll.
18
+ """
19
+
20
+ import asyncio
21
+ import contextlib
22
+ import shutil
23
+ import signal
24
+ import time
25
+ from collections.abc import Callable, Coroutine, Iterator
26
+ from http import HTTPStatus
27
+ from pathlib import Path
28
+ from typing import Any, Protocol
29
+
30
+ from rich.markup import escape
31
+ from tenacity import RetryCallState, Retrying, retry_if_exception, wait_fixed
32
+
33
+ from manifold_cli.clients.manifold import ManifoldApiClient
34
+ from manifold_cli.models.errors import (
35
+ ManifoldApiError,
36
+ NoRunnerCredentialError,
37
+ RunnerCredentialRejectedError,
38
+ )
39
+ from manifold_cli.models.run import AssignmentDispatch, RunSide
40
+ from manifold_cli.models.runner_home import BoxFacts
41
+ from manifold_cli.runner.current_dispatch import CurrentDispatch
42
+ from manifold_cli.runner.plan import RESULTS_ROOT
43
+ from manifold_cli.runner.policy import run_policy_side
44
+ from manifold_cli.runner.process import Spawner
45
+ from manifold_cli.runner.stages import side_started
46
+ from manifold_cli.runner.workload import Log
47
+
48
+ # How long a poll may be held open. The server holds it for a window of its own
49
+ # and answers with nothing when that is up, so this has to exceed that window or
50
+ # every idle poll reads as a transport failure.
51
+ POLL_TIMEOUT_SECONDS = 90.0
52
+
53
+ # The least a daemon waits after a poll that answered with nothing. The server
54
+ # holds the request open, so this is a floor rather than an interval: it is what
55
+ # keeps a deployment that answers immediately from being asked in a tight loop.
56
+ IDLE_FLOOR_SECONDS = 1.0
57
+
58
+ # How long a daemon waits after a start request or a poll it could not make.
59
+ RETRY_SECONDS = 5.0
60
+
61
+ # How long a side gets to read the bridge its peer publishes. Shorter than
62
+ # the poll above: the side is already this box's, and a worker is waiting on
63
+ # what the read answers.
64
+ BRIDGE_POLL_SECONDS = 2.0
65
+
66
+ # Delay after the stop signal before the daemon logs that the side task is still
67
+ # running. The daemon then continues waiting while the task tears down the side
68
+ # and reports its outcome. Keep this below the service unit's TimeoutStopSec so
69
+ # the warning appears before systemd sends SIGKILL.
70
+ SHUTDOWN_TIMEOUT_SECONDS = 60.0
71
+
72
+ # How long a run's results may remain under the results root before a starting
73
+ # daemon removes them. A daemon that was killed rather than stopped did not run
74
+ # its own cleanup, and nothing else removes the results it left. Two days is
75
+ # long enough for someone to find an overnight failure and short enough that
76
+ # last week's results are not still on a tmpfs.
77
+ ORPHANED_RESULTS_MAX_AGE_SECONDS = 48 * 60 * 60
78
+
79
+
80
+ class PolicySide(Protocol):
81
+ """Runs the policy side of a run: `run_policy_side`."""
82
+
83
+ def __call__(
84
+ self,
85
+ dispatch: AssignmentDispatch,
86
+ *,
87
+ client: ManifoldApiClient,
88
+ spawner: Spawner,
89
+ box: BoxFacts,
90
+ log: Log,
91
+ ) -> Coroutine[Any, Any, None]: ...
92
+
93
+
94
+ class BenchmarkSide(Protocol):
95
+ """Runs the benchmark side of a run: `run_benchmark_side`.
96
+
97
+ Takes the bridge poll the policy side has no use for: only this side waits
98
+ on an address the other one publishes.
99
+ """
100
+
101
+ def __call__(
102
+ self,
103
+ dispatch: AssignmentDispatch,
104
+ *,
105
+ client: ManifoldApiClient,
106
+ spawner: Spawner,
107
+ box: BoxFacts,
108
+ poll_s: float,
109
+ log: Log,
110
+ ) -> Coroutine[Any, Any, None]: ...
111
+
112
+
113
+ # A callback that the daemon calls with the dispatch after the stop request and
114
+ # before it cancels the side's task.
115
+ StopSide = Callable[[AssignmentDispatch], None]
116
+
117
+ # A request about the runner itself: `ManifoldApiClient.start_runner` or
118
+ # `ManifoldApiClient.stop_runner`.
119
+ RunnerRequest = Callable[[], None]
120
+
121
+
122
+ def load_benchmark_side(
123
+ dispatch: AssignmentDispatch,
124
+ *,
125
+ client: ManifoldApiClient,
126
+ spawner: Spawner,
127
+ box: BoxFacts,
128
+ poll_s: float,
129
+ log: Log,
130
+ ) -> Coroutine[Any, Any, None]:
131
+ """Import `run_benchmark_side` when a benchmark side arrives, then run it.
132
+
133
+ The benchmark side records replays and live views, which need the
134
+ `[runner]` extra. `manifold policy serve` only ever runs a policy side,
135
+ so importing the benchmark side at the top of this module would make a
136
+ plain install fail before it serves anything.
137
+ """
138
+ from manifold_cli.runner.benchmark import run_benchmark_side
139
+
140
+ return run_benchmark_side(
141
+ dispatch, client=client, spawner=spawner, box=box, poll_s=poll_s, log=log
142
+ )
143
+
144
+
145
+ def sweep_orphaned_results(
146
+ *,
147
+ log: Log,
148
+ root: Path = RESULTS_ROOT,
149
+ max_age_s: float = ORPHANED_RESULTS_MAX_AGE_SECONDS,
150
+ now: Callable[[], float] = time.time,
151
+ ) -> None:
152
+ """Remove run directories nothing is coming back for.
153
+
154
+ Run on the way in, before this daemon polls for its first side, so every
155
+ directory under the root belongs to a run some earlier process was serving.
156
+
157
+ Select a directory by its age, not by its presence. A daemon that was
158
+ killed leaves results behind without deciding anything about them. Keep
159
+ them for a window in which someone can look, but not for a window that
160
+ lasts forever on a root that is a box's memory by default.
161
+
162
+ A directory's age is the newest thing in it, so one a leftover container is
163
+ still writing to is not swept out from under it.
164
+
165
+ Reports rather than raises. A daemon that could not tidy up still has work to
166
+ do, and a directory made by an older runner is root's rather than this
167
+ user's -- which is exactly the case an operator needs told.
168
+ """
169
+ if not root.is_dir():
170
+ return
171
+ for run_dir in sorted(root.iterdir()):
172
+ if not run_dir.is_dir():
173
+ continue
174
+ age_s = now() - _newest_mtime(run_dir)
175
+ if age_s < max_age_s:
176
+ continue
177
+ try:
178
+ shutil.rmtree(run_dir)
179
+ except OSError as exc:
180
+ log(f"Old results were left on the box: {escape(str(exc))}")
181
+ else:
182
+ log(
183
+ f"Removed the results of run [bold]{escape(run_dir.name)}[/bold], "
184
+ f"last written {age_s / 3600:.0f} hours ago."
185
+ )
186
+
187
+
188
+ def _newest_mtime(run_dir: Path) -> float:
189
+ """When anything under `run_dir` was last written, the directory included.
190
+
191
+ The directory's own mtime moves when a shard is added and not when a log
192
+ inside one is, so a walk is what answers this for a run still being written.
193
+ An unreadable entry is skipped: it still counts through its parent, which is
194
+ readable or the run directory would not have been listed.
195
+ """
196
+ newest = run_dir.stat().st_mtime
197
+ for path in run_dir.rglob("*"):
198
+ try:
199
+ newest = max(newest, path.stat().st_mtime)
200
+ except OSError:
201
+ continue
202
+ return newest
203
+
204
+
205
+ def run_daemon(
206
+ *,
207
+ client: ManifoldApiClient,
208
+ spawner: Spawner,
209
+ box: BoxFacts,
210
+ poll_timeout_s: float,
211
+ bridge_poll_s: float,
212
+ current_dispatch: CurrentDispatch,
213
+ stop_signal: signal.Signals,
214
+ start_runner: RunnerRequest,
215
+ stop_runner: RunnerRequest,
216
+ stop_side: StopSide,
217
+ log: Log,
218
+ sleep: Callable[[float], None] = time.sleep,
219
+ policy_side: PolicySide = run_policy_side,
220
+ benchmark_side: BenchmarkSide = load_benchmark_side,
221
+ ) -> None:
222
+ """Serve the sides this box is handed, one at a time, until `stop_signal`
223
+ arrives.
224
+
225
+ A box holds one policy container and one GPU, and the server places at most
226
+ one live side on a runner, so the two agree: the daemon asks again once the
227
+ side it was handed has reported.
228
+
229
+ Nothing here is fatal except a credential this box cannot use. An empty poll
230
+ is the ordinary answer for an idle runner, and a deployment that is
231
+ restarting is logged and polled again -- a daemon that exited on a 502 would
232
+ have to be restarted by hand on every box pointed at it. An unusable
233
+ credential ends the daemon loop.
234
+
235
+ The daemon sets `current_dispatch` to the dispatch while it drives the
236
+ dispatch's side, and to None otherwise.
237
+
238
+ The server places a runner only after its start request, so the daemon
239
+ sends `start_runner` before its first poll, and retries it as it retries a
240
+ poll.
241
+
242
+ When `stop_signal` arrives, the daemon sends `stop_runner` once. If a side
243
+ runs, the daemon then calls `stop_side` with the dispatch and cancels the
244
+ side's task. The daemon then returns.
245
+ """
246
+ _send_start(start_runner, log=log, sleep=sleep)
247
+ while True:
248
+ try:
249
+ with _stop_signal_raised(stop_signal):
250
+ dispatch = _next_dispatch(
251
+ client, poll_timeout_s=poll_timeout_s, log=log, sleep=sleep
252
+ )
253
+ except _StopSignalled:
254
+ log("Runner stopping.")
255
+ _send_stop(stop_runner, log=log)
256
+ return
257
+ log(side_started(dispatch.assignment.side, dispatch.run.name))
258
+ current_dispatch.set(dispatch)
259
+ try:
260
+ stopped = _drive_side(
261
+ dispatch,
262
+ _side_of(
263
+ dispatch,
264
+ client=client,
265
+ spawner=spawner,
266
+ box=box,
267
+ bridge_poll_s=bridge_poll_s,
268
+ log=log,
269
+ policy_side=policy_side,
270
+ benchmark_side=benchmark_side,
271
+ ),
272
+ stop_signal=stop_signal,
273
+ stop_runner=stop_runner,
274
+ stop_side=stop_side,
275
+ log=log,
276
+ )
277
+ finally:
278
+ current_dispatch.set(None)
279
+ if stopped:
280
+ return
281
+
282
+
283
+ def credential_unusable(exc: Exception) -> bool:
284
+ """Whether this daemon can no longer authenticate, which is terminal.
285
+
286
+ `RunnerAuth` raises its own error rather than refreshing, since a runner
287
+ token is revoked rather than rotated. A client built without it reports the
288
+ same refusal as a 401. A home removed under a running daemon raises a third
289
+ error, and is as terminal as the other two: registering again mints a new
290
+ runner and writes a home this process is not holding.
291
+
292
+ The command that starts the daemon asks the same question of what reaches
293
+ it, so the two answer it here rather than each spelling it out.
294
+ """
295
+ if isinstance(exc, RunnerCredentialRejectedError | NoRunnerCredentialError):
296
+ return True
297
+ return (
298
+ isinstance(exc, ManifoldApiError) and exc.status_code == HTTPStatus.UNAUTHORIZED
299
+ )
300
+
301
+
302
+ def _send_start(
303
+ start_runner: RunnerRequest, *, log: Log, sleep: Callable[[float], None]
304
+ ) -> None:
305
+ """Send the start request until the server accepts it.
306
+
307
+ An unusable credential is terminal, as it is for a poll. Any other failure
308
+ is logged, and the request is sent again after `RETRY_SECONDS`.
309
+ """
310
+ retrying = _create_retrying(
311
+ "send the start request to Manifold", log=log, sleep=sleep
312
+ )
313
+ retrying(start_runner)
314
+
315
+
316
+ def _next_dispatch(
317
+ client: ManifoldApiClient,
318
+ *,
319
+ poll_timeout_s: float,
320
+ log: Log,
321
+ sleep: Callable[[float], None],
322
+ ) -> AssignmentDispatch:
323
+ """Poll until a side is assigned to this runner."""
324
+ retrying = _create_retrying("poll Manifold for work", log=log, sleep=sleep)
325
+ while True:
326
+ dispatch = retrying(client.poll_for_assignment, timeout_s=poll_timeout_s)
327
+ if dispatch is not None:
328
+ return dispatch
329
+ sleep(IDLE_FLOOR_SECONDS)
330
+
331
+
332
+ def _create_retrying(
333
+ action: str, *, log: Log, sleep: Callable[[float], None]
334
+ ) -> Retrying:
335
+ """Retry a failed call every `RETRY_SECONDS`, and log each failure with
336
+ `action`.
337
+
338
+ An unusable credential is terminal, so its error is raised as it is.
339
+ """
340
+
341
+ def log_failure(retry_state: RetryCallState) -> None:
342
+ exc = retry_state.outcome.exception() if retry_state.outcome else None
343
+ log(
344
+ f"[red]Could not {action}; retrying in "
345
+ f"{RETRY_SECONDS:g}s.[/red] {escape(str(exc))}"
346
+ )
347
+
348
+ return Retrying(
349
+ sleep=sleep,
350
+ wait=wait_fixed(RETRY_SECONDS),
351
+ retry=retry_if_exception(_is_retryable_failure),
352
+ before_sleep=log_failure,
353
+ reraise=True,
354
+ )
355
+
356
+
357
+ def _is_retryable_failure(exc: BaseException) -> bool:
358
+ """Whether a failed call is retried.
359
+
360
+ Tenacity catches a BaseException around each attempt. `_StopSignalled` and
361
+ `KeyboardInterrupt` must end the wait for work, so only an `Exception` is
362
+ retried.
363
+ """
364
+ return isinstance(exc, Exception) and not credential_unusable(exc)
365
+
366
+
367
+ class _StopSignalled(BaseException):
368
+ """Raised by the handler that `_stop_signal_raised` installs.
369
+
370
+ A BaseException, so that the retry around a poll does not catch it and poll
371
+ again.
372
+ """
373
+
374
+
375
+ @contextlib.contextmanager
376
+ def _stop_signal_raised(stop_signal: signal.Signals) -> Iterator[None]:
377
+ """Raise `_StopSignalled` when `stop_signal` arrives inside the block.
378
+
379
+ The handler raises rather than sets a flag: the poll blocks for up to its
380
+ timeout, and the daemon would read a flag only after the poll returns.
381
+ Restore the previous handler on exit, so that outside the wait for work
382
+ `stop_signal` has the effect it had before.
383
+ """
384
+
385
+ def raise_stop_signalled(signum: int, frame: object) -> None:
386
+ raise _StopSignalled
387
+
388
+ previous = signal.getsignal(stop_signal)
389
+ try:
390
+ signal.signal(stop_signal, raise_stop_signalled)
391
+ yield
392
+ finally:
393
+ signal.signal(stop_signal, previous)
394
+
395
+
396
+ def _send_stop(stop_runner: RunnerRequest, *, log: Log) -> None:
397
+ """Send the stop request once.
398
+
399
+ A failure is logged rather than retried. Placement skips a runner that has
400
+ not polled within the server's liveness window, and a running side reports
401
+ its failure when the daemon cancels it.
402
+ """
403
+ try:
404
+ stop_runner()
405
+ except Exception as exc:
406
+ log(
407
+ "[red]Could not send the stop request to Manifold.[/red] "
408
+ f"{escape(str(exc) or type(exc).__name__)}"
409
+ )
410
+
411
+
412
+ def _side_of(
413
+ dispatch: AssignmentDispatch,
414
+ *,
415
+ client: ManifoldApiClient,
416
+ spawner: Spawner,
417
+ box: BoxFacts,
418
+ bridge_poll_s: float,
419
+ log: Log,
420
+ policy_side: PolicySide,
421
+ benchmark_side: BenchmarkSide,
422
+ ) -> Coroutine[Any, Any, None]:
423
+ """The half of the pairing this dispatch is for, ready to be driven."""
424
+ if dispatch.assignment.side is RunSide.POLICY:
425
+ return policy_side(dispatch, client=client, spawner=spawner, box=box, log=log)
426
+ return benchmark_side(
427
+ dispatch,
428
+ client=client,
429
+ spawner=spawner,
430
+ box=box,
431
+ poll_s=bridge_poll_s,
432
+ log=log,
433
+ )
434
+
435
+
436
+ def _drive_side(
437
+ dispatch: AssignmentDispatch,
438
+ work: Coroutine[Any, Any, None],
439
+ *,
440
+ stop_signal: signal.Signals,
441
+ stop_runner: RunnerRequest,
442
+ stop_side: StopSide,
443
+ log: Log,
444
+ ) -> bool:
445
+ """Run one side to its reported outcome; say if `stop_signal` cut it short.
446
+
447
+ The event loop lives no longer than the side: the poll, the waits and the
448
+ report around it are synchronous, and a loop kept across them would outlive
449
+ the workers it was opened for.
450
+ """
451
+ return asyncio.run(
452
+ _reported_or_signalled(
453
+ dispatch,
454
+ work,
455
+ stop_signal=stop_signal,
456
+ stop_runner=stop_runner,
457
+ stop_side=stop_side,
458
+ log=log,
459
+ )
460
+ )
461
+
462
+
463
+ async def _reported_or_signalled(
464
+ dispatch: AssignmentDispatch,
465
+ work: Coroutine[Any, Any, None],
466
+ *,
467
+ stop_signal: signal.Signals,
468
+ stop_runner: RunnerRequest,
469
+ stop_side: StopSide,
470
+ log: Log,
471
+ ) -> bool:
472
+ """Run `work` as a task that `stop_signal` cancels, and say whether the
473
+ signal arrived.
474
+
475
+ Python's default SIGTERM kills the process outright: no `finally` runs, so
476
+ a `systemctl restart` leaves a container holding the GPU and a run waiting
477
+ on a side nothing reports until the server sweeps it. Cancelling the task
478
+ instead unwinds into the side's own `finally`, which reports it -- and the
479
+ report survives the cancellation because the client is synchronous, so its
480
+ POST is not interruptible by `CancelledError`.
481
+
482
+ The stop request and `stop_side` precede the cancellation, because the
483
+ cancellation stops the side's worker, and the peer reports a failure when
484
+ the worker stops. The daemon sends the stop request and calls `stop_side`
485
+ once, on the first signal.
486
+ """
487
+ loop = asyncio.get_running_loop()
488
+ task = asyncio.ensure_future(work)
489
+ signalled = asyncio.Event()
490
+ side = dispatch.assignment.side
491
+
492
+ def stop() -> None:
493
+ if not signalled.is_set():
494
+ signalled.set()
495
+ log(f"Runner stopping; ending the {side.value}.")
496
+ _send_stop(stop_runner, log=log)
497
+ stop_side(dispatch)
498
+ task.cancel()
499
+
500
+ loop.add_signal_handler(stop_signal, stop)
501
+ waiting_for_signal = asyncio.ensure_future(signalled.wait())
502
+ try:
503
+ await asyncio.wait(
504
+ {task, waiting_for_signal}, return_when=asyncio.FIRST_COMPLETED
505
+ )
506
+ if not signalled.is_set():
507
+ # Raises what the side raised, which is nothing: a side reports its
508
+ # own failures rather than escaping with them.
509
+ task.result()
510
+ return False
511
+ await asyncio.wait({task}, timeout=SHUTDOWN_TIMEOUT_SECONDS)
512
+ if not task.done():
513
+ log(
514
+ f"[red]The {side.value} has not stopped after "
515
+ f"{SHUTDOWN_TIMEOUT_SECONDS:g}s; the runner will keep waiting.[/red]"
516
+ )
517
+ return True
518
+ finally:
519
+ loop.remove_signal_handler(stop_signal)
520
+ waiting_for_signal.cancel()
@@ -0,0 +1,38 @@
1
+ """Which adapter plans the workers of a registered version.
2
+
3
+ A dispatch names the adapter its version was registered with, and this is where
4
+ that name becomes the object that builds the plan. The adapters are stateless,
5
+ so the table holds one instance of each rather than the classes.
6
+
7
+ A `subprocess` version has no adapter here. Only `manifold policy serve` runs
8
+ it, and serve starts the policy process itself.
9
+
10
+ `LoopbackAdapter` is this repo's double for a container and not an adapter a
11
+ version can be registered against, so it is absent.
12
+ """
13
+
14
+ from manifold_cli.models.catalog import Adapter
15
+ from manifold_cli.runner.container import ContainerAdapter
16
+ from manifold_cli.runner.plan import PlanError, WorkerAdapter
17
+
18
+ ADAPTERS: dict[Adapter, WorkerAdapter] = {
19
+ Adapter.CONTAINER: ContainerAdapter(),
20
+ }
21
+
22
+
23
+ def adapter_for(adapter: Adapter) -> WorkerAdapter:
24
+ """The adapter that plans a version registered as `adapter`.
25
+
26
+ Raises PlanError naming the adapter where nothing here runs one, so the
27
+ runner reports the side failed rather than dying on a side already
28
+ assigned to it.
29
+ """
30
+ if adapter is Adapter.SUBPROCESS:
31
+ raise PlanError("Only `manifold policy serve` can run a self-hosted version.")
32
+ worker_adapter = ADAPTERS.get(adapter)
33
+ if worker_adapter is None:
34
+ raise PlanError(
35
+ f"This runner runs no '{adapter.value}' version, so there is nothing "
36
+ "to plan its worker with."
37
+ )
38
+ return worker_adapter