manifold-cli 0.0.39__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- manifold_cli/__init__.py +3 -0
- manifold_cli/__main__.py +4 -0
- manifold_cli/bridge.py +70 -0
- manifold_cli/cli.py +54 -0
- manifold_cli/clients/__init__.py +0 -0
- manifold_cli/clients/auth_flow.py +169 -0
- manifold_cli/clients/credentials.py +53 -0
- manifold_cli/clients/docker_engine.py +43 -0
- manifold_cli/clients/image.py +130 -0
- manifold_cli/clients/manifold.py +1288 -0
- manifold_cli/clients/runner_home.py +165 -0
- manifold_cli/clients/workos.py +99 -0
- manifold_cli/commands/__init__.py +0 -0
- manifold_cli/commands/auth.py +101 -0
- manifold_cli/commands/benchmark.py +539 -0
- manifold_cli/commands/organization.py +154 -0
- manifold_cli/commands/policy.py +749 -0
- manifold_cli/commands/run.py +337 -0
- manifold_cli/commands/run_log.py +291 -0
- manifold_cli/commands/runner.py +852 -0
- manifold_cli/commands/table.py +101 -0
- manifold_cli/commands/token.py +144 -0
- manifold_cli/dependencies.py +116 -0
- manifold_cli/models/__init__.py +0 -0
- manifold_cli/models/auth.py +49 -0
- manifold_cli/models/benchmark.py +87 -0
- manifold_cli/models/catalog.py +189 -0
- manifold_cli/models/credentials.py +19 -0
- manifold_cli/models/errors.py +121 -0
- manifold_cli/models/log.py +114 -0
- manifold_cli/models/observability.py +121 -0
- manifold_cli/models/organization.py +19 -0
- manifold_cli/models/pagination.py +59 -0
- manifold_cli/models/policy.py +49 -0
- manifold_cli/models/run.py +381 -0
- manifold_cli/models/runner.py +186 -0
- manifold_cli/models/runner_home.py +69 -0
- manifold_cli/models/task_queue.py +40 -0
- manifold_cli/models/token.py +39 -0
- manifold_cli/replay/__init__.py +68 -0
- manifold_cli/replay/log.py +814 -0
- manifold_cli/runner/__init__.py +72 -0
- manifold_cli/runner/api_retry.py +36 -0
- manifold_cli/runner/benchmark/__init__.py +800 -0
- manifold_cli/runner/benchmark/live_view.py +248 -0
- manifold_cli/runner/benchmark/preassigned_scheduling.py +325 -0
- manifold_cli/runner/benchmark/recording.py +1208 -0
- manifold_cli/runner/benchmark/scheduling.py +256 -0
- manifold_cli/runner/benchmark/task_queue_scheduling.py +717 -0
- manifold_cli/runner/benchmark/task_server.py +123 -0
- manifold_cli/runner/benchmark/video.py +124 -0
- manifold_cli/runner/benchmark_steps.py +178 -0
- manifold_cli/runner/cancellation.py +22 -0
- manifold_cli/runner/capture.py +282 -0
- manifold_cli/runner/container.py +370 -0
- manifold_cli/runner/current_dispatch.py +25 -0
- manifold_cli/runner/daemon.py +520 -0
- manifold_cli/runner/dispatch.py +38 -0
- manifold_cli/runner/gpu.py +182 -0
- manifold_cli/runner/heartbeat.py +202 -0
- manifold_cli/runner/loopback.py +102 -0
- manifold_cli/runner/loopback_bench.py +183 -0
- manifold_cli/runner/loopback_policy.py +39 -0
- manifold_cli/runner/output.py +218 -0
- manifold_cli/runner/pairing_acceptance.py +57 -0
- manifold_cli/runner/plan.py +369 -0
- manifold_cli/runner/policy.py +350 -0
- manifold_cli/runner/policy_steps.py +164 -0
- manifold_cli/runner/preparation.py +47 -0
- manifold_cli/runner/process.py +219 -0
- manifold_cli/runner/ram.py +17 -0
- manifold_cli/runner/resource_usage.py +265 -0
- manifold_cli/runner/served.py +218 -0
- manifold_cli/runner/service.py +120 -0
- manifold_cli/runner/side_failure.py +68 -0
- manifold_cli/runner/signature.py +98 -0
- manifold_cli/runner/source.py +66 -0
- manifold_cli/runner/stages.py +241 -0
- manifold_cli/runner/step_timing.py +229 -0
- manifold_cli/runner/worker.py +534 -0
- manifold_cli/runner/workload.py +401 -0
- manifold_cli/services/__init__.py +0 -0
- manifold_cli/services/auth.py +89 -0
- manifold_cli/services/benchmark.py +357 -0
- manifold_cli/services/organization.py +22 -0
- manifold_cli/services/policy.py +234 -0
- manifold_cli/services/run.py +304 -0
- manifold_cli/services/runner.py +207 -0
- manifold_cli/services/runner_daemon.py +146 -0
- manifold_cli/services/token.py +52 -0
- manifold_cli/settings.py +30 -0
- manifold_cli/transport/__init__.py +31 -0
- manifold_cli/transport/diagnostics.py +72 -0
- manifold_cli/transport/interfaces.py +69 -0
- manifold_cli/transport/iroh.py +317 -0
- manifold_cli/transport/registry.py +34 -0
- manifold_cli/transport/tcp.py +146 -0
- manifold_cli-0.0.39.dist-info/METADATA +406 -0
- manifold_cli-0.0.39.dist-info/RECORD +102 -0
- manifold_cli-0.0.39.dist-info/WHEEL +4 -0
- manifold_cli-0.0.39.dist-info/entry_points.txt +2 -0
- manifold_cli-0.0.39.dist-info/licenses/LICENSE +373 -0
|
@@ -0,0 +1,520 @@
|
|
|
1
|
+
"""The loop an installed runner runs: take the side it is handed, run it, ask again.
|
|
2
|
+
|
|
3
|
+
This is the whole of what a runner does. It sends a start request, asks whether
|
|
4
|
+
a side is assigned to it, drives that side to its reported outcome, and asks
|
|
5
|
+
again. On the stop signal it sends a stop request. Everything a side needs --
|
|
6
|
+
the plan, the containers, the bridge, the beating, the outcome -- belongs to
|
|
7
|
+
`policy.run_policy_side` and `benchmark.run_benchmark_side`; what is here is
|
|
8
|
+
which of the two a dispatch reaches, and what a daemon does when a poll or a
|
|
9
|
+
box fails it.
|
|
10
|
+
|
|
11
|
+
Nothing is chosen here. A placement pass decides which run a box serves and which
|
|
12
|
+
side of it, so a runner cannot prefer one run over another and cannot be refused
|
|
13
|
+
the side it was handed.
|
|
14
|
+
|
|
15
|
+
The loop is synchronous. The API client is, and so are the waits between polls;
|
|
16
|
+
only a side's workers and their bridges are not, so each side runs under an event
|
|
17
|
+
loop of its own that is torn down before the next poll.
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
import asyncio
|
|
21
|
+
import contextlib
|
|
22
|
+
import shutil
|
|
23
|
+
import signal
|
|
24
|
+
import time
|
|
25
|
+
from collections.abc import Callable, Coroutine, Iterator
|
|
26
|
+
from http import HTTPStatus
|
|
27
|
+
from pathlib import Path
|
|
28
|
+
from typing import Any, Protocol
|
|
29
|
+
|
|
30
|
+
from rich.markup import escape
|
|
31
|
+
from tenacity import RetryCallState, Retrying, retry_if_exception, wait_fixed
|
|
32
|
+
|
|
33
|
+
from manifold_cli.clients.manifold import ManifoldApiClient
|
|
34
|
+
from manifold_cli.models.errors import (
|
|
35
|
+
ManifoldApiError,
|
|
36
|
+
NoRunnerCredentialError,
|
|
37
|
+
RunnerCredentialRejectedError,
|
|
38
|
+
)
|
|
39
|
+
from manifold_cli.models.run import AssignmentDispatch, RunSide
|
|
40
|
+
from manifold_cli.models.runner_home import BoxFacts
|
|
41
|
+
from manifold_cli.runner.current_dispatch import CurrentDispatch
|
|
42
|
+
from manifold_cli.runner.plan import RESULTS_ROOT
|
|
43
|
+
from manifold_cli.runner.policy import run_policy_side
|
|
44
|
+
from manifold_cli.runner.process import Spawner
|
|
45
|
+
from manifold_cli.runner.stages import side_started
|
|
46
|
+
from manifold_cli.runner.workload import Log
|
|
47
|
+
|
|
48
|
+
# How long a poll may be held open. The server holds it for a window of its own
|
|
49
|
+
# and answers with nothing when that is up, so this has to exceed that window or
|
|
50
|
+
# every idle poll reads as a transport failure.
|
|
51
|
+
POLL_TIMEOUT_SECONDS = 90.0
|
|
52
|
+
|
|
53
|
+
# The least a daemon waits after a poll that answered with nothing. The server
|
|
54
|
+
# holds the request open, so this is a floor rather than an interval: it is what
|
|
55
|
+
# keeps a deployment that answers immediately from being asked in a tight loop.
|
|
56
|
+
IDLE_FLOOR_SECONDS = 1.0
|
|
57
|
+
|
|
58
|
+
# How long a daemon waits after a start request or a poll it could not make.
|
|
59
|
+
RETRY_SECONDS = 5.0
|
|
60
|
+
|
|
61
|
+
# How long a side gets to read the bridge its peer publishes. Shorter than
|
|
62
|
+
# the poll above: the side is already this box's, and a worker is waiting on
|
|
63
|
+
# what the read answers.
|
|
64
|
+
BRIDGE_POLL_SECONDS = 2.0
|
|
65
|
+
|
|
66
|
+
# Delay after the stop signal before the daemon logs that the side task is still
|
|
67
|
+
# running. The daemon then continues waiting while the task tears down the side
|
|
68
|
+
# and reports its outcome. Keep this below the service unit's TimeoutStopSec so
|
|
69
|
+
# the warning appears before systemd sends SIGKILL.
|
|
70
|
+
SHUTDOWN_TIMEOUT_SECONDS = 60.0
|
|
71
|
+
|
|
72
|
+
# How long a run's results may remain under the results root before a starting
|
|
73
|
+
# daemon removes them. A daemon that was killed rather than stopped did not run
|
|
74
|
+
# its own cleanup, and nothing else removes the results it left. Two days is
|
|
75
|
+
# long enough for someone to find an overnight failure and short enough that
|
|
76
|
+
# last week's results are not still on a tmpfs.
|
|
77
|
+
ORPHANED_RESULTS_MAX_AGE_SECONDS = 48 * 60 * 60
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
class PolicySide(Protocol):
|
|
81
|
+
"""Runs the policy side of a run: `run_policy_side`."""
|
|
82
|
+
|
|
83
|
+
def __call__(
|
|
84
|
+
self,
|
|
85
|
+
dispatch: AssignmentDispatch,
|
|
86
|
+
*,
|
|
87
|
+
client: ManifoldApiClient,
|
|
88
|
+
spawner: Spawner,
|
|
89
|
+
box: BoxFacts,
|
|
90
|
+
log: Log,
|
|
91
|
+
) -> Coroutine[Any, Any, None]: ...
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
class BenchmarkSide(Protocol):
|
|
95
|
+
"""Runs the benchmark side of a run: `run_benchmark_side`.
|
|
96
|
+
|
|
97
|
+
Takes the bridge poll the policy side has no use for: only this side waits
|
|
98
|
+
on an address the other one publishes.
|
|
99
|
+
"""
|
|
100
|
+
|
|
101
|
+
def __call__(
|
|
102
|
+
self,
|
|
103
|
+
dispatch: AssignmentDispatch,
|
|
104
|
+
*,
|
|
105
|
+
client: ManifoldApiClient,
|
|
106
|
+
spawner: Spawner,
|
|
107
|
+
box: BoxFacts,
|
|
108
|
+
poll_s: float,
|
|
109
|
+
log: Log,
|
|
110
|
+
) -> Coroutine[Any, Any, None]: ...
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
# A callback that the daemon calls with the dispatch after the stop request and
|
|
114
|
+
# before it cancels the side's task.
|
|
115
|
+
StopSide = Callable[[AssignmentDispatch], None]
|
|
116
|
+
|
|
117
|
+
# A request about the runner itself: `ManifoldApiClient.start_runner` or
|
|
118
|
+
# `ManifoldApiClient.stop_runner`.
|
|
119
|
+
RunnerRequest = Callable[[], None]
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def load_benchmark_side(
|
|
123
|
+
dispatch: AssignmentDispatch,
|
|
124
|
+
*,
|
|
125
|
+
client: ManifoldApiClient,
|
|
126
|
+
spawner: Spawner,
|
|
127
|
+
box: BoxFacts,
|
|
128
|
+
poll_s: float,
|
|
129
|
+
log: Log,
|
|
130
|
+
) -> Coroutine[Any, Any, None]:
|
|
131
|
+
"""Import `run_benchmark_side` when a benchmark side arrives, then run it.
|
|
132
|
+
|
|
133
|
+
The benchmark side records replays and live views, which need the
|
|
134
|
+
`[runner]` extra. `manifold policy serve` only ever runs a policy side,
|
|
135
|
+
so importing the benchmark side at the top of this module would make a
|
|
136
|
+
plain install fail before it serves anything.
|
|
137
|
+
"""
|
|
138
|
+
from manifold_cli.runner.benchmark import run_benchmark_side
|
|
139
|
+
|
|
140
|
+
return run_benchmark_side(
|
|
141
|
+
dispatch, client=client, spawner=spawner, box=box, poll_s=poll_s, log=log
|
|
142
|
+
)
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def sweep_orphaned_results(
|
|
146
|
+
*,
|
|
147
|
+
log: Log,
|
|
148
|
+
root: Path = RESULTS_ROOT,
|
|
149
|
+
max_age_s: float = ORPHANED_RESULTS_MAX_AGE_SECONDS,
|
|
150
|
+
now: Callable[[], float] = time.time,
|
|
151
|
+
) -> None:
|
|
152
|
+
"""Remove run directories nothing is coming back for.
|
|
153
|
+
|
|
154
|
+
Run on the way in, before this daemon polls for its first side, so every
|
|
155
|
+
directory under the root belongs to a run some earlier process was serving.
|
|
156
|
+
|
|
157
|
+
Select a directory by its age, not by its presence. A daemon that was
|
|
158
|
+
killed leaves results behind without deciding anything about them. Keep
|
|
159
|
+
them for a window in which someone can look, but not for a window that
|
|
160
|
+
lasts forever on a root that is a box's memory by default.
|
|
161
|
+
|
|
162
|
+
A directory's age is the newest thing in it, so one a leftover container is
|
|
163
|
+
still writing to is not swept out from under it.
|
|
164
|
+
|
|
165
|
+
Reports rather than raises. A daemon that could not tidy up still has work to
|
|
166
|
+
do, and a directory made by an older runner is root's rather than this
|
|
167
|
+
user's -- which is exactly the case an operator needs told.
|
|
168
|
+
"""
|
|
169
|
+
if not root.is_dir():
|
|
170
|
+
return
|
|
171
|
+
for run_dir in sorted(root.iterdir()):
|
|
172
|
+
if not run_dir.is_dir():
|
|
173
|
+
continue
|
|
174
|
+
age_s = now() - _newest_mtime(run_dir)
|
|
175
|
+
if age_s < max_age_s:
|
|
176
|
+
continue
|
|
177
|
+
try:
|
|
178
|
+
shutil.rmtree(run_dir)
|
|
179
|
+
except OSError as exc:
|
|
180
|
+
log(f"Old results were left on the box: {escape(str(exc))}")
|
|
181
|
+
else:
|
|
182
|
+
log(
|
|
183
|
+
f"Removed the results of run [bold]{escape(run_dir.name)}[/bold], "
|
|
184
|
+
f"last written {age_s / 3600:.0f} hours ago."
|
|
185
|
+
)
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
def _newest_mtime(run_dir: Path) -> float:
|
|
189
|
+
"""When anything under `run_dir` was last written, the directory included.
|
|
190
|
+
|
|
191
|
+
The directory's own mtime moves when a shard is added and not when a log
|
|
192
|
+
inside one is, so a walk is what answers this for a run still being written.
|
|
193
|
+
An unreadable entry is skipped: it still counts through its parent, which is
|
|
194
|
+
readable or the run directory would not have been listed.
|
|
195
|
+
"""
|
|
196
|
+
newest = run_dir.stat().st_mtime
|
|
197
|
+
for path in run_dir.rglob("*"):
|
|
198
|
+
try:
|
|
199
|
+
newest = max(newest, path.stat().st_mtime)
|
|
200
|
+
except OSError:
|
|
201
|
+
continue
|
|
202
|
+
return newest
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
def run_daemon(
|
|
206
|
+
*,
|
|
207
|
+
client: ManifoldApiClient,
|
|
208
|
+
spawner: Spawner,
|
|
209
|
+
box: BoxFacts,
|
|
210
|
+
poll_timeout_s: float,
|
|
211
|
+
bridge_poll_s: float,
|
|
212
|
+
current_dispatch: CurrentDispatch,
|
|
213
|
+
stop_signal: signal.Signals,
|
|
214
|
+
start_runner: RunnerRequest,
|
|
215
|
+
stop_runner: RunnerRequest,
|
|
216
|
+
stop_side: StopSide,
|
|
217
|
+
log: Log,
|
|
218
|
+
sleep: Callable[[float], None] = time.sleep,
|
|
219
|
+
policy_side: PolicySide = run_policy_side,
|
|
220
|
+
benchmark_side: BenchmarkSide = load_benchmark_side,
|
|
221
|
+
) -> None:
|
|
222
|
+
"""Serve the sides this box is handed, one at a time, until `stop_signal`
|
|
223
|
+
arrives.
|
|
224
|
+
|
|
225
|
+
A box holds one policy container and one GPU, and the server places at most
|
|
226
|
+
one live side on a runner, so the two agree: the daemon asks again once the
|
|
227
|
+
side it was handed has reported.
|
|
228
|
+
|
|
229
|
+
Nothing here is fatal except a credential this box cannot use. An empty poll
|
|
230
|
+
is the ordinary answer for an idle runner, and a deployment that is
|
|
231
|
+
restarting is logged and polled again -- a daemon that exited on a 502 would
|
|
232
|
+
have to be restarted by hand on every box pointed at it. An unusable
|
|
233
|
+
credential ends the daemon loop.
|
|
234
|
+
|
|
235
|
+
The daemon sets `current_dispatch` to the dispatch while it drives the
|
|
236
|
+
dispatch's side, and to None otherwise.
|
|
237
|
+
|
|
238
|
+
The server places a runner only after its start request, so the daemon
|
|
239
|
+
sends `start_runner` before its first poll, and retries it as it retries a
|
|
240
|
+
poll.
|
|
241
|
+
|
|
242
|
+
When `stop_signal` arrives, the daemon sends `stop_runner` once. If a side
|
|
243
|
+
runs, the daemon then calls `stop_side` with the dispatch and cancels the
|
|
244
|
+
side's task. The daemon then returns.
|
|
245
|
+
"""
|
|
246
|
+
_send_start(start_runner, log=log, sleep=sleep)
|
|
247
|
+
while True:
|
|
248
|
+
try:
|
|
249
|
+
with _stop_signal_raised(stop_signal):
|
|
250
|
+
dispatch = _next_dispatch(
|
|
251
|
+
client, poll_timeout_s=poll_timeout_s, log=log, sleep=sleep
|
|
252
|
+
)
|
|
253
|
+
except _StopSignalled:
|
|
254
|
+
log("Runner stopping.")
|
|
255
|
+
_send_stop(stop_runner, log=log)
|
|
256
|
+
return
|
|
257
|
+
log(side_started(dispatch.assignment.side, dispatch.run.name))
|
|
258
|
+
current_dispatch.set(dispatch)
|
|
259
|
+
try:
|
|
260
|
+
stopped = _drive_side(
|
|
261
|
+
dispatch,
|
|
262
|
+
_side_of(
|
|
263
|
+
dispatch,
|
|
264
|
+
client=client,
|
|
265
|
+
spawner=spawner,
|
|
266
|
+
box=box,
|
|
267
|
+
bridge_poll_s=bridge_poll_s,
|
|
268
|
+
log=log,
|
|
269
|
+
policy_side=policy_side,
|
|
270
|
+
benchmark_side=benchmark_side,
|
|
271
|
+
),
|
|
272
|
+
stop_signal=stop_signal,
|
|
273
|
+
stop_runner=stop_runner,
|
|
274
|
+
stop_side=stop_side,
|
|
275
|
+
log=log,
|
|
276
|
+
)
|
|
277
|
+
finally:
|
|
278
|
+
current_dispatch.set(None)
|
|
279
|
+
if stopped:
|
|
280
|
+
return
|
|
281
|
+
|
|
282
|
+
|
|
283
|
+
def credential_unusable(exc: Exception) -> bool:
|
|
284
|
+
"""Whether this daemon can no longer authenticate, which is terminal.
|
|
285
|
+
|
|
286
|
+
`RunnerAuth` raises its own error rather than refreshing, since a runner
|
|
287
|
+
token is revoked rather than rotated. A client built without it reports the
|
|
288
|
+
same refusal as a 401. A home removed under a running daemon raises a third
|
|
289
|
+
error, and is as terminal as the other two: registering again mints a new
|
|
290
|
+
runner and writes a home this process is not holding.
|
|
291
|
+
|
|
292
|
+
The command that starts the daemon asks the same question of what reaches
|
|
293
|
+
it, so the two answer it here rather than each spelling it out.
|
|
294
|
+
"""
|
|
295
|
+
if isinstance(exc, RunnerCredentialRejectedError | NoRunnerCredentialError):
|
|
296
|
+
return True
|
|
297
|
+
return (
|
|
298
|
+
isinstance(exc, ManifoldApiError) and exc.status_code == HTTPStatus.UNAUTHORIZED
|
|
299
|
+
)
|
|
300
|
+
|
|
301
|
+
|
|
302
|
+
def _send_start(
|
|
303
|
+
start_runner: RunnerRequest, *, log: Log, sleep: Callable[[float], None]
|
|
304
|
+
) -> None:
|
|
305
|
+
"""Send the start request until the server accepts it.
|
|
306
|
+
|
|
307
|
+
An unusable credential is terminal, as it is for a poll. Any other failure
|
|
308
|
+
is logged, and the request is sent again after `RETRY_SECONDS`.
|
|
309
|
+
"""
|
|
310
|
+
retrying = _create_retrying(
|
|
311
|
+
"send the start request to Manifold", log=log, sleep=sleep
|
|
312
|
+
)
|
|
313
|
+
retrying(start_runner)
|
|
314
|
+
|
|
315
|
+
|
|
316
|
+
def _next_dispatch(
|
|
317
|
+
client: ManifoldApiClient,
|
|
318
|
+
*,
|
|
319
|
+
poll_timeout_s: float,
|
|
320
|
+
log: Log,
|
|
321
|
+
sleep: Callable[[float], None],
|
|
322
|
+
) -> AssignmentDispatch:
|
|
323
|
+
"""Poll until a side is assigned to this runner."""
|
|
324
|
+
retrying = _create_retrying("poll Manifold for work", log=log, sleep=sleep)
|
|
325
|
+
while True:
|
|
326
|
+
dispatch = retrying(client.poll_for_assignment, timeout_s=poll_timeout_s)
|
|
327
|
+
if dispatch is not None:
|
|
328
|
+
return dispatch
|
|
329
|
+
sleep(IDLE_FLOOR_SECONDS)
|
|
330
|
+
|
|
331
|
+
|
|
332
|
+
def _create_retrying(
|
|
333
|
+
action: str, *, log: Log, sleep: Callable[[float], None]
|
|
334
|
+
) -> Retrying:
|
|
335
|
+
"""Retry a failed call every `RETRY_SECONDS`, and log each failure with
|
|
336
|
+
`action`.
|
|
337
|
+
|
|
338
|
+
An unusable credential is terminal, so its error is raised as it is.
|
|
339
|
+
"""
|
|
340
|
+
|
|
341
|
+
def log_failure(retry_state: RetryCallState) -> None:
|
|
342
|
+
exc = retry_state.outcome.exception() if retry_state.outcome else None
|
|
343
|
+
log(
|
|
344
|
+
f"[red]Could not {action}; retrying in "
|
|
345
|
+
f"{RETRY_SECONDS:g}s.[/red] {escape(str(exc))}"
|
|
346
|
+
)
|
|
347
|
+
|
|
348
|
+
return Retrying(
|
|
349
|
+
sleep=sleep,
|
|
350
|
+
wait=wait_fixed(RETRY_SECONDS),
|
|
351
|
+
retry=retry_if_exception(_is_retryable_failure),
|
|
352
|
+
before_sleep=log_failure,
|
|
353
|
+
reraise=True,
|
|
354
|
+
)
|
|
355
|
+
|
|
356
|
+
|
|
357
|
+
def _is_retryable_failure(exc: BaseException) -> bool:
|
|
358
|
+
"""Whether a failed call is retried.
|
|
359
|
+
|
|
360
|
+
Tenacity catches a BaseException around each attempt. `_StopSignalled` and
|
|
361
|
+
`KeyboardInterrupt` must end the wait for work, so only an `Exception` is
|
|
362
|
+
retried.
|
|
363
|
+
"""
|
|
364
|
+
return isinstance(exc, Exception) and not credential_unusable(exc)
|
|
365
|
+
|
|
366
|
+
|
|
367
|
+
class _StopSignalled(BaseException):
|
|
368
|
+
"""Raised by the handler that `_stop_signal_raised` installs.
|
|
369
|
+
|
|
370
|
+
A BaseException, so that the retry around a poll does not catch it and poll
|
|
371
|
+
again.
|
|
372
|
+
"""
|
|
373
|
+
|
|
374
|
+
|
|
375
|
+
@contextlib.contextmanager
|
|
376
|
+
def _stop_signal_raised(stop_signal: signal.Signals) -> Iterator[None]:
|
|
377
|
+
"""Raise `_StopSignalled` when `stop_signal` arrives inside the block.
|
|
378
|
+
|
|
379
|
+
The handler raises rather than sets a flag: the poll blocks for up to its
|
|
380
|
+
timeout, and the daemon would read a flag only after the poll returns.
|
|
381
|
+
Restore the previous handler on exit, so that outside the wait for work
|
|
382
|
+
`stop_signal` has the effect it had before.
|
|
383
|
+
"""
|
|
384
|
+
|
|
385
|
+
def raise_stop_signalled(signum: int, frame: object) -> None:
|
|
386
|
+
raise _StopSignalled
|
|
387
|
+
|
|
388
|
+
previous = signal.getsignal(stop_signal)
|
|
389
|
+
try:
|
|
390
|
+
signal.signal(stop_signal, raise_stop_signalled)
|
|
391
|
+
yield
|
|
392
|
+
finally:
|
|
393
|
+
signal.signal(stop_signal, previous)
|
|
394
|
+
|
|
395
|
+
|
|
396
|
+
def _send_stop(stop_runner: RunnerRequest, *, log: Log) -> None:
|
|
397
|
+
"""Send the stop request once.
|
|
398
|
+
|
|
399
|
+
A failure is logged rather than retried. Placement skips a runner that has
|
|
400
|
+
not polled within the server's liveness window, and a running side reports
|
|
401
|
+
its failure when the daemon cancels it.
|
|
402
|
+
"""
|
|
403
|
+
try:
|
|
404
|
+
stop_runner()
|
|
405
|
+
except Exception as exc:
|
|
406
|
+
log(
|
|
407
|
+
"[red]Could not send the stop request to Manifold.[/red] "
|
|
408
|
+
f"{escape(str(exc) or type(exc).__name__)}"
|
|
409
|
+
)
|
|
410
|
+
|
|
411
|
+
|
|
412
|
+
def _side_of(
|
|
413
|
+
dispatch: AssignmentDispatch,
|
|
414
|
+
*,
|
|
415
|
+
client: ManifoldApiClient,
|
|
416
|
+
spawner: Spawner,
|
|
417
|
+
box: BoxFacts,
|
|
418
|
+
bridge_poll_s: float,
|
|
419
|
+
log: Log,
|
|
420
|
+
policy_side: PolicySide,
|
|
421
|
+
benchmark_side: BenchmarkSide,
|
|
422
|
+
) -> Coroutine[Any, Any, None]:
|
|
423
|
+
"""The half of the pairing this dispatch is for, ready to be driven."""
|
|
424
|
+
if dispatch.assignment.side is RunSide.POLICY:
|
|
425
|
+
return policy_side(dispatch, client=client, spawner=spawner, box=box, log=log)
|
|
426
|
+
return benchmark_side(
|
|
427
|
+
dispatch,
|
|
428
|
+
client=client,
|
|
429
|
+
spawner=spawner,
|
|
430
|
+
box=box,
|
|
431
|
+
poll_s=bridge_poll_s,
|
|
432
|
+
log=log,
|
|
433
|
+
)
|
|
434
|
+
|
|
435
|
+
|
|
436
|
+
def _drive_side(
|
|
437
|
+
dispatch: AssignmentDispatch,
|
|
438
|
+
work: Coroutine[Any, Any, None],
|
|
439
|
+
*,
|
|
440
|
+
stop_signal: signal.Signals,
|
|
441
|
+
stop_runner: RunnerRequest,
|
|
442
|
+
stop_side: StopSide,
|
|
443
|
+
log: Log,
|
|
444
|
+
) -> bool:
|
|
445
|
+
"""Run one side to its reported outcome; say if `stop_signal` cut it short.
|
|
446
|
+
|
|
447
|
+
The event loop lives no longer than the side: the poll, the waits and the
|
|
448
|
+
report around it are synchronous, and a loop kept across them would outlive
|
|
449
|
+
the workers it was opened for.
|
|
450
|
+
"""
|
|
451
|
+
return asyncio.run(
|
|
452
|
+
_reported_or_signalled(
|
|
453
|
+
dispatch,
|
|
454
|
+
work,
|
|
455
|
+
stop_signal=stop_signal,
|
|
456
|
+
stop_runner=stop_runner,
|
|
457
|
+
stop_side=stop_side,
|
|
458
|
+
log=log,
|
|
459
|
+
)
|
|
460
|
+
)
|
|
461
|
+
|
|
462
|
+
|
|
463
|
+
async def _reported_or_signalled(
|
|
464
|
+
dispatch: AssignmentDispatch,
|
|
465
|
+
work: Coroutine[Any, Any, None],
|
|
466
|
+
*,
|
|
467
|
+
stop_signal: signal.Signals,
|
|
468
|
+
stop_runner: RunnerRequest,
|
|
469
|
+
stop_side: StopSide,
|
|
470
|
+
log: Log,
|
|
471
|
+
) -> bool:
|
|
472
|
+
"""Run `work` as a task that `stop_signal` cancels, and say whether the
|
|
473
|
+
signal arrived.
|
|
474
|
+
|
|
475
|
+
Python's default SIGTERM kills the process outright: no `finally` runs, so
|
|
476
|
+
a `systemctl restart` leaves a container holding the GPU and a run waiting
|
|
477
|
+
on a side nothing reports until the server sweeps it. Cancelling the task
|
|
478
|
+
instead unwinds into the side's own `finally`, which reports it -- and the
|
|
479
|
+
report survives the cancellation because the client is synchronous, so its
|
|
480
|
+
POST is not interruptible by `CancelledError`.
|
|
481
|
+
|
|
482
|
+
The stop request and `stop_side` precede the cancellation, because the
|
|
483
|
+
cancellation stops the side's worker, and the peer reports a failure when
|
|
484
|
+
the worker stops. The daemon sends the stop request and calls `stop_side`
|
|
485
|
+
once, on the first signal.
|
|
486
|
+
"""
|
|
487
|
+
loop = asyncio.get_running_loop()
|
|
488
|
+
task = asyncio.ensure_future(work)
|
|
489
|
+
signalled = asyncio.Event()
|
|
490
|
+
side = dispatch.assignment.side
|
|
491
|
+
|
|
492
|
+
def stop() -> None:
|
|
493
|
+
if not signalled.is_set():
|
|
494
|
+
signalled.set()
|
|
495
|
+
log(f"Runner stopping; ending the {side.value}.")
|
|
496
|
+
_send_stop(stop_runner, log=log)
|
|
497
|
+
stop_side(dispatch)
|
|
498
|
+
task.cancel()
|
|
499
|
+
|
|
500
|
+
loop.add_signal_handler(stop_signal, stop)
|
|
501
|
+
waiting_for_signal = asyncio.ensure_future(signalled.wait())
|
|
502
|
+
try:
|
|
503
|
+
await asyncio.wait(
|
|
504
|
+
{task, waiting_for_signal}, return_when=asyncio.FIRST_COMPLETED
|
|
505
|
+
)
|
|
506
|
+
if not signalled.is_set():
|
|
507
|
+
# Raises what the side raised, which is nothing: a side reports its
|
|
508
|
+
# own failures rather than escaping with them.
|
|
509
|
+
task.result()
|
|
510
|
+
return False
|
|
511
|
+
await asyncio.wait({task}, timeout=SHUTDOWN_TIMEOUT_SECONDS)
|
|
512
|
+
if not task.done():
|
|
513
|
+
log(
|
|
514
|
+
f"[red]The {side.value} has not stopped after "
|
|
515
|
+
f"{SHUTDOWN_TIMEOUT_SECONDS:g}s; the runner will keep waiting.[/red]"
|
|
516
|
+
)
|
|
517
|
+
return True
|
|
518
|
+
finally:
|
|
519
|
+
loop.remove_signal_handler(stop_signal)
|
|
520
|
+
waiting_for_signal.cancel()
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
"""Which adapter plans the workers of a registered version.
|
|
2
|
+
|
|
3
|
+
A dispatch names the adapter its version was registered with, and this is where
|
|
4
|
+
that name becomes the object that builds the plan. The adapters are stateless,
|
|
5
|
+
so the table holds one instance of each rather than the classes.
|
|
6
|
+
|
|
7
|
+
A `subprocess` version has no adapter here. Only `manifold policy serve` runs
|
|
8
|
+
it, and serve starts the policy process itself.
|
|
9
|
+
|
|
10
|
+
`LoopbackAdapter` is this repo's double for a container and not an adapter a
|
|
11
|
+
version can be registered against, so it is absent.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from manifold_cli.models.catalog import Adapter
|
|
15
|
+
from manifold_cli.runner.container import ContainerAdapter
|
|
16
|
+
from manifold_cli.runner.plan import PlanError, WorkerAdapter
|
|
17
|
+
|
|
18
|
+
ADAPTERS: dict[Adapter, WorkerAdapter] = {
|
|
19
|
+
Adapter.CONTAINER: ContainerAdapter(),
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def adapter_for(adapter: Adapter) -> WorkerAdapter:
|
|
24
|
+
"""The adapter that plans a version registered as `adapter`.
|
|
25
|
+
|
|
26
|
+
Raises PlanError naming the adapter where nothing here runs one, so the
|
|
27
|
+
runner reports the side failed rather than dying on a side already
|
|
28
|
+
assigned to it.
|
|
29
|
+
"""
|
|
30
|
+
if adapter is Adapter.SUBPROCESS:
|
|
31
|
+
raise PlanError("Only `manifold policy serve` can run a self-hosted version.")
|
|
32
|
+
worker_adapter = ADAPTERS.get(adapter)
|
|
33
|
+
if worker_adapter is None:
|
|
34
|
+
raise PlanError(
|
|
35
|
+
f"This runner runs no '{adapter.value}' version, so there is nothing "
|
|
36
|
+
"to plan its worker with."
|
|
37
|
+
)
|
|
38
|
+
return worker_adapter
|