manifold-cli 0.0.39__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (102) hide show
  1. manifold_cli/__init__.py +3 -0
  2. manifold_cli/__main__.py +4 -0
  3. manifold_cli/bridge.py +70 -0
  4. manifold_cli/cli.py +54 -0
  5. manifold_cli/clients/__init__.py +0 -0
  6. manifold_cli/clients/auth_flow.py +169 -0
  7. manifold_cli/clients/credentials.py +53 -0
  8. manifold_cli/clients/docker_engine.py +43 -0
  9. manifold_cli/clients/image.py +130 -0
  10. manifold_cli/clients/manifold.py +1288 -0
  11. manifold_cli/clients/runner_home.py +165 -0
  12. manifold_cli/clients/workos.py +99 -0
  13. manifold_cli/commands/__init__.py +0 -0
  14. manifold_cli/commands/auth.py +101 -0
  15. manifold_cli/commands/benchmark.py +539 -0
  16. manifold_cli/commands/organization.py +154 -0
  17. manifold_cli/commands/policy.py +749 -0
  18. manifold_cli/commands/run.py +337 -0
  19. manifold_cli/commands/run_log.py +291 -0
  20. manifold_cli/commands/runner.py +852 -0
  21. manifold_cli/commands/table.py +101 -0
  22. manifold_cli/commands/token.py +144 -0
  23. manifold_cli/dependencies.py +116 -0
  24. manifold_cli/models/__init__.py +0 -0
  25. manifold_cli/models/auth.py +49 -0
  26. manifold_cli/models/benchmark.py +87 -0
  27. manifold_cli/models/catalog.py +189 -0
  28. manifold_cli/models/credentials.py +19 -0
  29. manifold_cli/models/errors.py +121 -0
  30. manifold_cli/models/log.py +114 -0
  31. manifold_cli/models/observability.py +121 -0
  32. manifold_cli/models/organization.py +19 -0
  33. manifold_cli/models/pagination.py +59 -0
  34. manifold_cli/models/policy.py +49 -0
  35. manifold_cli/models/run.py +381 -0
  36. manifold_cli/models/runner.py +186 -0
  37. manifold_cli/models/runner_home.py +69 -0
  38. manifold_cli/models/task_queue.py +40 -0
  39. manifold_cli/models/token.py +39 -0
  40. manifold_cli/replay/__init__.py +68 -0
  41. manifold_cli/replay/log.py +814 -0
  42. manifold_cli/runner/__init__.py +72 -0
  43. manifold_cli/runner/api_retry.py +36 -0
  44. manifold_cli/runner/benchmark/__init__.py +800 -0
  45. manifold_cli/runner/benchmark/live_view.py +248 -0
  46. manifold_cli/runner/benchmark/preassigned_scheduling.py +325 -0
  47. manifold_cli/runner/benchmark/recording.py +1208 -0
  48. manifold_cli/runner/benchmark/scheduling.py +256 -0
  49. manifold_cli/runner/benchmark/task_queue_scheduling.py +717 -0
  50. manifold_cli/runner/benchmark/task_server.py +123 -0
  51. manifold_cli/runner/benchmark/video.py +124 -0
  52. manifold_cli/runner/benchmark_steps.py +178 -0
  53. manifold_cli/runner/cancellation.py +22 -0
  54. manifold_cli/runner/capture.py +282 -0
  55. manifold_cli/runner/container.py +370 -0
  56. manifold_cli/runner/current_dispatch.py +25 -0
  57. manifold_cli/runner/daemon.py +520 -0
  58. manifold_cli/runner/dispatch.py +38 -0
  59. manifold_cli/runner/gpu.py +182 -0
  60. manifold_cli/runner/heartbeat.py +202 -0
  61. manifold_cli/runner/loopback.py +102 -0
  62. manifold_cli/runner/loopback_bench.py +183 -0
  63. manifold_cli/runner/loopback_policy.py +39 -0
  64. manifold_cli/runner/output.py +218 -0
  65. manifold_cli/runner/pairing_acceptance.py +57 -0
  66. manifold_cli/runner/plan.py +369 -0
  67. manifold_cli/runner/policy.py +350 -0
  68. manifold_cli/runner/policy_steps.py +164 -0
  69. manifold_cli/runner/preparation.py +47 -0
  70. manifold_cli/runner/process.py +219 -0
  71. manifold_cli/runner/ram.py +17 -0
  72. manifold_cli/runner/resource_usage.py +265 -0
  73. manifold_cli/runner/served.py +218 -0
  74. manifold_cli/runner/service.py +120 -0
  75. manifold_cli/runner/side_failure.py +68 -0
  76. manifold_cli/runner/signature.py +98 -0
  77. manifold_cli/runner/source.py +66 -0
  78. manifold_cli/runner/stages.py +241 -0
  79. manifold_cli/runner/step_timing.py +229 -0
  80. manifold_cli/runner/worker.py +534 -0
  81. manifold_cli/runner/workload.py +401 -0
  82. manifold_cli/services/__init__.py +0 -0
  83. manifold_cli/services/auth.py +89 -0
  84. manifold_cli/services/benchmark.py +357 -0
  85. manifold_cli/services/organization.py +22 -0
  86. manifold_cli/services/policy.py +234 -0
  87. manifold_cli/services/run.py +304 -0
  88. manifold_cli/services/runner.py +207 -0
  89. manifold_cli/services/runner_daemon.py +146 -0
  90. manifold_cli/services/token.py +52 -0
  91. manifold_cli/settings.py +30 -0
  92. manifold_cli/transport/__init__.py +31 -0
  93. manifold_cli/transport/diagnostics.py +72 -0
  94. manifold_cli/transport/interfaces.py +69 -0
  95. manifold_cli/transport/iroh.py +317 -0
  96. manifold_cli/transport/registry.py +34 -0
  97. manifold_cli/transport/tcp.py +146 -0
  98. manifold_cli-0.0.39.dist-info/METADATA +406 -0
  99. manifold_cli-0.0.39.dist-info/RECORD +102 -0
  100. manifold_cli-0.0.39.dist-info/WHEEL +4 -0
  101. manifold_cli-0.0.39.dist-info/entry_points.txt +2 -0
  102. manifold_cli-0.0.39.dist-info/licenses/LICENSE +373 -0
@@ -0,0 +1,39 @@
1
+ """The stand-in policy worker: serves the built-in echo policy on a local port.
2
+
3
+ `LoopbackAdapter` spawns this module. It stands in for a policy container on a
4
+ machine with no GPU: it binds the address the policy bridge dials and echoes
5
+ every byte back, which is what `loopback_bench` round-trips.
6
+
7
+ It is a worker process of its own, so it reports on stdout as a container does.
8
+ """
9
+
10
+ import argparse
11
+ import asyncio
12
+ import contextlib
13
+
14
+ from manifold_cli.transport.diagnostics import serve_echo
15
+ from manifold_cli.transport.tcp import TcpTransport
16
+
17
+
18
+ def main() -> None:
19
+ parser = argparse.ArgumentParser(description="Serve the built-in echo policy.")
20
+ parser.add_argument(
21
+ "--listen", required=True, help="Address to serve on, as HOST:PORT."
22
+ )
23
+ args = parser.parse_args()
24
+ with contextlib.suppress(KeyboardInterrupt):
25
+ asyncio.run(_serve(args.listen))
26
+
27
+
28
+ async def _serve(listen: str) -> None:
29
+ listener = await TcpTransport().open_listener(listen)
30
+ _say(f"serving the echo policy on {listener.target}")
31
+ await serve_echo(listener, on_event=_say)
32
+
33
+
34
+ def _say(message: str) -> None:
35
+ print(message, flush=True)
36
+
37
+
38
+ if __name__ == "__main__":
39
+ main()
@@ -0,0 +1,218 @@
1
+ """Batching one side's captured output and posting it while the workers run.
2
+
3
+ One reporter per side rather than one per worker. A benchmark side fans out over
4
+ shards, and a post per shard would multiply the request rate by the fan-out for
5
+ output a reader asks for by run.
6
+
7
+ The reporter holds every stream the side has declared, numbers each stream's
8
+ lines itself, and sends what has accumulated on whichever threshold comes first.
9
+ A post that fails is logged and its lines dropped: a runner that retried would
10
+ spend a side's remaining time resending output rather than driving episodes.
11
+ """
12
+
13
+ import asyncio
14
+ import contextlib
15
+ import logging
16
+ from collections.abc import AsyncIterator, Callable
17
+ from contextlib import asynccontextmanager
18
+ from datetime import UTC, datetime
19
+ from uuid import UUID
20
+
21
+ from rich.text import Text
22
+
23
+ from manifold_cli.clients.manifold import ManifoldApiClient
24
+ from manifold_cli.models.log import (
25
+ LogBatch,
26
+ LogProducer,
27
+ LogSource,
28
+ ReportedLine,
29
+ ReportedStream,
30
+ )
31
+
32
+ logger = logging.getLogger(__name__)
33
+
34
+ # How long lines are held before they are posted, so a reader watching a run
35
+ # sees output while the worker is still writing it.
36
+ FLUSH_INTERVAL_SECONDS = 1.0
37
+
38
+ # How many held lines force a post before the interval is up.
39
+ FLUSH_LINES = 500
40
+
41
+ # How many held characters do the same, for a worker whose lines are long enough
42
+ # that FLUSH_LINES of them would be one large request.
43
+ FLUSH_CHARS = 256 * 1024
44
+
45
+
46
+ class CapturedOutputReporter:
47
+ """The output one side has captured and not yet reported.
48
+
49
+ `declare`, `add` and `close` are called by the pumps reading each worker;
50
+ `flush` is what sends, and `end_open_streams` runs once the side is over, for
51
+ the streams no worker's exit closed. Sequence numbers are assigned here and
52
+ per stream, so a worker's two sources are numbered independently of each
53
+ other and of when any batch was sent.
54
+ """
55
+
56
+ def __init__(self, client: ManifoldApiClient, run_id: UUID):
57
+ self._client = client
58
+ self._run_id = run_id
59
+ self._streams: dict[tuple[LogProducer, int, LogSource], ReportedStream] = {}
60
+ self._sent: dict[tuple[LogProducer, int, LogSource], ReportedStream] = {}
61
+ self._next_seq: dict[tuple[LogProducer, int, LogSource], int] = {}
62
+ self._lines: list[ReportedLine] = []
63
+ self._held_chars = 0
64
+ self._full = asyncio.Event()
65
+
66
+ def declare(self, worker_index: int, source: LogSource) -> None:
67
+ """Record that a worker has a stream, written to or not.
68
+
69
+ This is what lists a container still pulling its image, so it happens
70
+ when the pipe is first read rather than when a line arrives on it.
71
+ """
72
+ self._declare(LogProducer.WORKER, worker_index, source)
73
+
74
+ def add(self, worker_index: int, source: LogSource, text: str) -> None:
75
+ """Hold one worker line for the next post."""
76
+ self._add(LogProducer.WORKER, worker_index, source, text)
77
+
78
+ def runner_log(self, echo: Callable[[str], None]) -> Callable[[str], None]:
79
+ """Tee runner progress to the console and its captured stdout stream."""
80
+
81
+ def write(message: str) -> None:
82
+ echo(message)
83
+ self.capture_runner(message)
84
+
85
+ return write
86
+
87
+ def capture_runner(self, message: str) -> None:
88
+ """Hold one runner progress line after removing its console markup."""
89
+ self._add(
90
+ LogProducer.RUNNER,
91
+ 0,
92
+ LogSource.STDOUT,
93
+ Text.from_markup(message).plain,
94
+ )
95
+
96
+ def _declare(
97
+ self, producer: LogProducer, worker_index: int, source: LogSource
98
+ ) -> None:
99
+ """Record that a process has a stream, written to or not."""
100
+ key = (producer, worker_index, source)
101
+ if key in self._streams:
102
+ return
103
+ self._streams[key] = ReportedStream(
104
+ producer=producer, worker_index=worker_index, source=source
105
+ )
106
+ self._next_seq[key] = 0
107
+
108
+ def _add(
109
+ self,
110
+ producer: LogProducer,
111
+ worker_index: int,
112
+ source: LogSource,
113
+ text: str,
114
+ ) -> None:
115
+ """Hold one line for the next post, at its place in its own stream."""
116
+ key = (producer, worker_index, source)
117
+ self._declare(producer, worker_index, source)
118
+ self._lines.append(
119
+ ReportedLine(
120
+ producer=producer,
121
+ worker_index=worker_index,
122
+ source=source,
123
+ seq=self._next_seq[key],
124
+ ts=datetime.now(tz=UTC),
125
+ text=text,
126
+ )
127
+ )
128
+ self._next_seq[key] += 1
129
+ self._held_chars += len(text)
130
+ if len(self._lines) >= FLUSH_LINES or self._held_chars >= FLUSH_CHARS:
131
+ self._full.set()
132
+
133
+ def close(self, worker_index: int, source: LogSource, *, exit_code: int) -> None:
134
+ """Record that the worker behind a stream has exited.
135
+
136
+ The next post carries it rather than this waiting for the interval: the
137
+ exit is what a reader watching a stream is waiting to see.
138
+ """
139
+ self.declare(worker_index, source)
140
+ key = (LogProducer.WORKER, worker_index, source)
141
+ self._streams[key] = self._streams[key].model_copy(
142
+ update={"ended_at": datetime.now(tz=UTC), "exit_code": exit_code}
143
+ )
144
+ self._full.set()
145
+
146
+ def end_open_streams(self) -> None:
147
+ """End every stream still open, for a side that is over.
148
+
149
+ A worker whose exit was never classified -- one that failed its readiness
150
+ probe, or one whose pairing ended first -- leaves a stream nothing has
151
+ ended. A stream left open has no `ended_at`, and a reader displays it as
152
+ a worker still running on a run that has finished. The end carries no
153
+ exit code, because nothing here has one to report.
154
+ """
155
+ ended_at = datetime.now(tz=UTC)
156
+ for key, stream in self._streams.items():
157
+ if stream.ended_at is None:
158
+ self._streams[key] = stream.model_copy(update={"ended_at": ended_at})
159
+
160
+ async def flush(self) -> None:
161
+ """Post what is held, dropping the lines whether or not that worked.
162
+
163
+ The declarations are not dropped with them. Every batch carries all of
164
+ them, so a post that never arrived leaves the stream list short only
165
+ until the next one -- which is why a failure leaves `_sent` behind and
166
+ the next batch re-declares.
167
+
168
+ A post with nothing new in it is skipped, so a side whose workers are
169
+ quiet is not sending an empty batch every interval.
170
+ """
171
+ if len(self._lines) == 0 and self._streams == self._sent:
172
+ return
173
+ lines, self._lines, self._held_chars = self._lines, [], 0
174
+ streams = dict(self._streams)
175
+ batch = LogBatch(streams=list(streams.values()), lines=lines)
176
+ try:
177
+ await asyncio.to_thread(
178
+ self._client.post_captured_output, self._run_id, batch
179
+ )
180
+ except Exception:
181
+ logger.exception("%d captured lines were not posted", len(lines))
182
+ else:
183
+ self._sent = streams
184
+
185
+ async def run(self) -> None:
186
+ """Flush every interval, and sooner where a batch has filled up."""
187
+ while True:
188
+ with contextlib.suppress(TimeoutError):
189
+ await asyncio.wait_for(
190
+ self._full.wait(), timeout=FLUSH_INTERVAL_SECONDS
191
+ )
192
+ self._full.clear()
193
+ await self.flush()
194
+
195
+
196
+ @asynccontextmanager
197
+ async def reporting_captured_output(
198
+ client: ManifoldApiClient, run_id: UUID
199
+ ) -> AsyncIterator[CapturedOutputReporter]:
200
+ """A reporter for one side, flushing for as long as the body runs.
201
+
202
+ The last flush is on the way out, and shielded: a worker writes its last
203
+ words as it dies, which is after the final interval has passed, and a side
204
+ torn down by a cancelled pairing would otherwise drop them.
205
+
206
+ By the time the body has returned its workers have gone too, so
207
+ `end_open_streams` runs before that last flush.
208
+ """
209
+ reporter = CapturedOutputReporter(client, run_id)
210
+ flushing = asyncio.create_task(reporter.run())
211
+ try:
212
+ yield reporter
213
+ finally:
214
+ flushing.cancel()
215
+ with contextlib.suppress(asyncio.CancelledError):
216
+ await flushing
217
+ reporter.end_open_streams()
218
+ await asyncio.shield(reporter.flush())
@@ -0,0 +1,57 @@
1
+ """Detect when the policy accepts the benchmark's signature.
2
+
3
+ The runner of each side sends its first serving heartbeat at that point, so that
4
+ the policy and the benchmark are compatible for the run before the side enters
5
+ `running`.
6
+ """
7
+
8
+ from collections.abc import Callable
9
+
10
+ from manifold_cli.runner.step_timing import (
11
+ READY_PREFIX,
12
+ FrameScanner,
13
+ PrefixMatched,
14
+ read_exactly_by_reads,
15
+ )
16
+ from manifold_cli.transport import ByteStream, Connector
17
+
18
+
19
+ class PairingAcceptanceConnector:
20
+ """A `Connector` to the policy. Its streams call `on_accepted` when the
21
+ policy accepts the benchmark's signature.
22
+
23
+ The streams of the wrapped connector must read the policy's frames without
24
+ step times, because the scanner follows the frame boundaries.
25
+ """
26
+
27
+ def __init__(self, connector: Connector, on_accepted: Callable[[], None]) -> None:
28
+ self._connector = connector
29
+ self._on_accepted = on_accepted
30
+
31
+ async def connect(self) -> ByteStream:
32
+ stream = await self._connector.connect()
33
+ return _PairingAcceptanceStream(stream, self._on_accepted)
34
+
35
+
36
+ class _PairingAcceptanceStream:
37
+ def __init__(self, stream: ByteStream, on_accepted: Callable[[], None]) -> None:
38
+ self._stream = stream
39
+ self._on_accepted = on_accepted
40
+ self._from_policy = FrameScanner(READY_PREFIX)
41
+
42
+ async def read(self, n: int) -> bytes:
43
+ data = await self._stream.read(n)
44
+ if any(
45
+ isinstance(event, PrefixMatched) for event in self._from_policy.scan(data)
46
+ ):
47
+ self._on_accepted()
48
+ return data
49
+
50
+ async def read_exactly(self, n: int) -> bytes:
51
+ return await read_exactly_by_reads(self.read, n)
52
+
53
+ async def write(self, data: bytes) -> None:
54
+ await self._stream.write(data)
55
+
56
+ async def close(self) -> None:
57
+ await self._stream.close()
@@ -0,0 +1,369 @@
1
+ """What an adapter hands back for one worker, and the contract it satisfies.
2
+
3
+ A `WorkerPlan` is data: the commands to run before the worker, the argv to
4
+ spawn, how to tell it is ready, and the cleanup to run afterwards. An adapter
5
+ in this package builds one from the `config` of a registered version;
6
+ `worker.py` executes it.
7
+
8
+ An adapter plans a policy worker and a benchmark worker through separate entry
9
+ points rather than taking a role argument, because the two plans differ in
10
+ kind: a policy worker binds an address the runner dials, and a benchmark worker
11
+ dials an address the runner listens on.
12
+ """
13
+
14
+ import socket
15
+ from dataclasses import dataclass, field
16
+ from pathlib import Path
17
+ from typing import Any, Protocol, runtime_checkable
18
+ from urllib.parse import urlsplit
19
+ from uuid import UUID
20
+
21
+ from manifold_cli.models.run import EpisodeRollupError
22
+
23
+ # Ephemeral scratch area for benchmark results, one directory per run. A
24
+ # benchmark worker writes its results into a mount rooted here, so they survive
25
+ # the removal of the container that wrote them.
26
+ RESULTS_ROOT = Path("/tmp/manifold/results")
27
+
28
+
29
+ # The host a worker and the bridge beside it share. Both ends of that wire sit
30
+ # on the runner's own box: a policy worker binds a port the bridge dials, and a
31
+ # benchmark worker dials a listener the bridge opened. It is not the host the
32
+ # far side dials, which `outbound_host` resolves.
33
+ WIRE_HOST = "127.0.0.1"
34
+
35
+
36
+ UNASSIGNED_PORT = 0
37
+
38
+
39
+ # Where `outbound_host` routes towards to learn this box's own address. Nothing
40
+ # is sent, so the port only has to be one the API could plausibly answer on.
41
+ ROUTE_PROBE_PORT = 443
42
+
43
+
44
+ class RunnerError(Exception):
45
+ """Base for the runner's own failures."""
46
+
47
+
48
+ class PlanError(RunnerError):
49
+ """A version cannot be turned into a worker plan.
50
+
51
+ Carries a message naming what the config is missing, or what the runner
52
+ needs installed to run it.
53
+ """
54
+
55
+
56
+ @dataclass(frozen=True)
57
+ class WireBinding:
58
+ """The local TCP address one worker binds, or dials.
59
+
60
+ A policy worker publishes its port here and `manifold policy bridge` dials
61
+ it with a `TcpConnector`; a benchmark worker dials the address
62
+ `manifold benchmark bridge` accepts on. Both name a socket on the runner's
63
+ own machine, so a wire binding holds a host and a port and no URL.
64
+
65
+ UNASSIGNED_PORT asks whoever binds the socket to choose the port, which is
66
+ the only safe answer where that binder is not the runner: a port named up
67
+ front is unbound until the worker starts, and a policy worker starts behind
68
+ an image pull. The plan then carries `published_wire` to read back what was
69
+ chosen.
70
+ """
71
+
72
+ host: str
73
+ port: int
74
+
75
+ @property
76
+ def address(self) -> str:
77
+ return f"{self.host}:{self.port}"
78
+
79
+
80
+ @dataclass(frozen=True)
81
+ class ShardAssignment:
82
+ """The partition of a run's episodes one benchmark worker drives.
83
+
84
+ The three travel together and are meaningless apart, so they are one
85
+ argument rather than three: a worker cannot be told its index without also
86
+ being told how many shards it is one of, or how many episodes they divide.
87
+
88
+ `count` may not exceed `total_episodes`. The partition gives shard `i` every
89
+ global index where `idx % count == i`, so a shard with `index < count <=
90
+ total_episodes` always owns at least episode `index`, and the bound is
91
+ exactly the condition that no shard is handed nothing. A worker with nothing
92
+ to drive still pulls its image and pairs, and the empty rollup it leaves is
93
+ indistinguishable from one a broken image wrote.
94
+ """
95
+
96
+ index: int
97
+ count: int
98
+ total_episodes: int
99
+
100
+ def __post_init__(self) -> None:
101
+ if self.count < 1:
102
+ raise ValueError(f"a shard count must be at least 1; got {self.count}")
103
+ if not 0 <= self.index < self.count:
104
+ raise ValueError(
105
+ f"a shard index must be in [0, {self.count}); got {self.index}"
106
+ )
107
+ if self.total_episodes < 1:
108
+ raise ValueError(
109
+ f"a run must have at least 1 episode; got {self.total_episodes}"
110
+ )
111
+ if self.count > self.total_episodes:
112
+ raise ValueError(
113
+ f"{self.count} shards divide {self.total_episodes} episodes, so "
114
+ "some would drive none; claim at most one shard per episode"
115
+ )
116
+
117
+ @property
118
+ def label(self) -> str:
119
+ """What names this shard wherever one worker's own is needed.
120
+
121
+ The container name and the results directory both take it, so the two
122
+ cannot come to disagree about which shard wrote what.
123
+ """
124
+ return f"shard-{self.index}"
125
+
126
+
127
+ @dataclass(frozen=True)
128
+ class TcpProbe:
129
+ """Dial the worker's address until it accepts a connection, or time out."""
130
+
131
+ wire: WireBinding
132
+ timeout_s: float = 10.0
133
+
134
+
135
+ @dataclass(frozen=True)
136
+ class CommandProbe:
137
+ """Run `argv` until it exits zero, or time out.
138
+
139
+ For a worker whose address the runner cannot usefully dial itself.
140
+ `target` names what the command checks, for the message a timeout carries.
141
+ """
142
+
143
+ argv: tuple[str, ...]
144
+ target: str
145
+ timeout_s: float = 10.0
146
+
147
+
148
+ @dataclass(frozen=True)
149
+ class ProcessOnly:
150
+ """No probe: the worker is ready as soon as its process is up.
151
+
152
+ For a worker that dials out instead of binding. Nothing is listening to
153
+ probe, and the bridge accepts the connection once the worker makes it.
154
+ """
155
+
156
+
157
+ Readiness = TcpProbe | CommandProbe | ProcessOnly
158
+
159
+ IMAGE_PULL_PHASE = "image_pull"
160
+
161
+
162
+ @dataclass(frozen=True)
163
+ class SetupCommand:
164
+ """A command that must succeed before the worker is spawned.
165
+
166
+ `docker pull` of a multi-GB image is what this exists for. Pulling inside
167
+ the worker spends the readiness budget on the download, and a probe window
168
+ wide enough to cover a cold pull is no longer a useful timeout. Each
169
+ command runs to completion on its own timeout.
170
+
171
+ `started` and `completed` are the plain text of its stage lines, such as
172
+ "Pulling <image>" and "Pulled <image>". A command without them logs none.
173
+ """
174
+
175
+ argv: tuple[str, ...]
176
+ phase: str = "setup"
177
+ timeout_s: float = 3600.0
178
+ started: str = ""
179
+ completed: str = ""
180
+
181
+
182
+ @dataclass(frozen=True)
183
+ class SidePreparation:
184
+ """Commands and progress text for the work that prepares a side."""
185
+
186
+ commands: tuple[SetupCommand, ...]
187
+ started: str
188
+ completed: str
189
+
190
+
191
+ @dataclass(frozen=True)
192
+ class WorkerPlan:
193
+ """How to spawn one worker, verify it, and clean up after it."""
194
+
195
+ argv: tuple[str, ...]
196
+ env: dict[str, str] = field(default_factory=dict)
197
+ workdir: Path | None = None
198
+ readiness: Readiness = field(default_factory=ProcessOnly)
199
+ setup: tuple[SetupCommand, ...] = ()
200
+ # Commands run once the worker has exited, however it exited. For state a
201
+ # worker cannot reclaim itself, such as a container the docker client left
202
+ # running when it was killed.
203
+ teardown: tuple[tuple[str, ...], ...] = ()
204
+ # Where the spawned worker records the id of what it started, for a teardown
205
+ # that must not stop something this attempt did not start. Absent where the
206
+ # spawn never got that far, which is the case teardown has to skip.
207
+ cidfile: Path | None = None
208
+ # Where a benchmark worker writes its results. None for a policy worker,
209
+ # which produces none.
210
+ results_dir: Path | None = None
211
+ # The GPU index the plan pins the worker to, set only where the plan asks
212
+ # for one card. It describes what is spawned, so a worker that asked for no
213
+ # GPU and a worker handed every card both leave it None.
214
+ gpu_device: int | None = None
215
+ # How to read back the address the worker's binder chose, printed as
216
+ # `host:port` on stdout. Set only where the plan asked for UNASSIGNED_PORT,
217
+ # so a runner that named the port itself already knows the answer. Run once
218
+ # the worker is ready: nothing has bound the port before then.
219
+ published_wire: tuple[str, ...] | None = None
220
+
221
+ @property
222
+ def startup_budget_s(self) -> float:
223
+ """How long this worker may take before it answers on its wire.
224
+
225
+ Its setup commands, then its readiness check, each already carrying the
226
+ budget the adapter gave it. The far side of a pairing waits this out, so
227
+ it is the plan's own claim about itself rather than a wait the runner
228
+ that holds the plan ever spends.
229
+ """
230
+ if isinstance(self.readiness, ProcessOnly):
231
+ ready_s = 0.0
232
+ else:
233
+ ready_s = self.readiness.timeout_s
234
+ return sum(step.timeout_s for step in self.setup) + ready_s
235
+
236
+
237
+ def free_port(host: str) -> int:
238
+ """Return a free TCP port on `host`.
239
+
240
+ The socket closes before the worker binds the port, so another process
241
+ could take it first. That race is unlikely on a machine serving one policy.
242
+ """
243
+ with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as probe:
244
+ probe.bind((host, UNASSIGNED_PORT))
245
+ return int(probe.getsockname()[1])
246
+
247
+
248
+ def published_wire_from(output: str) -> WireBinding:
249
+ """The wire a `WorkerPlan.published_wire` command reported.
250
+
251
+ `docker port` prints one `host:port` per published mapping. A policy
252
+ container publishes one, on the host the plan named, so anything else means
253
+ the container is not the one the plan describes.
254
+
255
+ Raises PlanError for output that names no address.
256
+ """
257
+ lines = [line.strip() for line in output.splitlines() if len(line.strip()) > 0]
258
+ if len(lines) != 1:
259
+ raise PlanError(
260
+ f"Reading a policy container's published port expected one address, "
261
+ f"but docker reported {len(lines)}: {output.strip()!r}"
262
+ )
263
+ host, _, port = lines[0].rpartition(":")
264
+ if len(host) == 0 or not port.isdigit():
265
+ raise PlanError(
266
+ f"A policy container's published port is not an address: {lines[0]!r}"
267
+ )
268
+ return WireBinding(host=host, port=int(port))
269
+
270
+
271
+ def rollup_in(results_dir: Path) -> Path:
272
+ """The one rollup document a benchmark worker left in `results_dir`.
273
+
274
+ The image names the file after the benchmark it ran, which the runner is
275
+ never told, so the plan names a directory and the runner finds the file in it.
276
+ One shard writes one document, and a directory holding none or several is a
277
+ worker that did not do what the plan describes.
278
+
279
+ Raises:
280
+ EpisodeRollupError: The directory does not hold exactly one `*.json`.
281
+ """
282
+ documents = sorted(results_dir.glob("*.json"))
283
+ if len(documents) != 1:
284
+ raise EpisodeRollupError(
285
+ f"expected one rollup under {results_dir}, found {len(documents)}"
286
+ )
287
+ return documents[0]
288
+
289
+
290
+ def outbound_host(api_url: str) -> str:
291
+ """This box's own address, as the interface that reaches Manifold sees it.
292
+
293
+ Connecting a UDP socket sends nothing: it fixes the route, so the kernel
294
+ picks the source address it would use for `api_url` and `getsockname` reads
295
+ it back.
296
+
297
+ That address is what a peer on the same network dials. A peer somewhere the
298
+ route towards Manifold does not lead is the case `--advertise-host` exists
299
+ for, and the operator states it there rather than here.
300
+
301
+ Raises:
302
+ RunnerError: The url names no host, or there is no route to it.
303
+ """
304
+ host = urlsplit(api_url).hostname
305
+ if host is None:
306
+ raise RunnerError(f"The Manifold url {api_url!r} names no host to route to.")
307
+ try:
308
+ with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as probe:
309
+ probe.connect((host, ROUTE_PROBE_PORT))
310
+ return str(probe.getsockname()[0])
311
+ except OSError as exc:
312
+ raise RunnerError(
313
+ f"This box has no route to {host} to read its own address from: {exc}. "
314
+ "Register the runner with --advertise-host to state the address its "
315
+ "peer dials."
316
+ ) from exc
317
+
318
+
319
+ @runtime_checkable
320
+ class WorkerAdapter(Protocol):
321
+ """Plans the workers and the side preparation for a version."""
322
+
323
+ def plan_policy(
324
+ self,
325
+ config: dict[str, Any],
326
+ *,
327
+ run_id: UUID,
328
+ assignment_id: UUID,
329
+ wire: WireBinding,
330
+ cache_dir: Path | None = None,
331
+ gpu_device: int | None = None,
332
+ ) -> WorkerPlan:
333
+ """Plan the policy worker `config` describes, bound to `wire`.
334
+
335
+ Raises PlanError for a config this adapter cannot run.
336
+ """
337
+ ...
338
+
339
+ def plan_side_preparation(self, config: dict[str, Any]) -> SidePreparation | None:
340
+ """Plan work that must complete once before the side's workers start.
341
+
342
+ Return None when the adapter does not plan a preparation for the side.
343
+
344
+ Raises PlanError for a config this adapter cannot run.
345
+ """
346
+ ...
347
+
348
+ def plan_benchmark(
349
+ self,
350
+ config: dict[str, Any],
351
+ *,
352
+ run_id: UUID,
353
+ assignment_id: UUID,
354
+ wire: WireBinding,
355
+ shard: ShardAssignment,
356
+ live_view_wire: WireBinding | None = None,
357
+ results_root: Path = RESULTS_ROOT,
358
+ cache_dir: Path | None = None,
359
+ gpu_device: int | None = None,
360
+ ) -> WorkerPlan:
361
+ """Plan the benchmark worker `config` describes, dialling its wires.
362
+
363
+ `shard` has no default. Every benchmark worker drives one, the caller
364
+ always knows which, and a defaulted `0 of 1` is exactly what a caller
365
+ that forgot to pass one would produce.
366
+
367
+ Raises PlanError for a config this adapter cannot run.
368
+ """
369
+ ...