manifold-cli 0.0.39__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- manifold_cli/__init__.py +3 -0
- manifold_cli/__main__.py +4 -0
- manifold_cli/bridge.py +70 -0
- manifold_cli/cli.py +54 -0
- manifold_cli/clients/__init__.py +0 -0
- manifold_cli/clients/auth_flow.py +169 -0
- manifold_cli/clients/credentials.py +53 -0
- manifold_cli/clients/docker_engine.py +43 -0
- manifold_cli/clients/image.py +130 -0
- manifold_cli/clients/manifold.py +1288 -0
- manifold_cli/clients/runner_home.py +165 -0
- manifold_cli/clients/workos.py +99 -0
- manifold_cli/commands/__init__.py +0 -0
- manifold_cli/commands/auth.py +101 -0
- manifold_cli/commands/benchmark.py +539 -0
- manifold_cli/commands/organization.py +154 -0
- manifold_cli/commands/policy.py +749 -0
- manifold_cli/commands/run.py +337 -0
- manifold_cli/commands/run_log.py +291 -0
- manifold_cli/commands/runner.py +852 -0
- manifold_cli/commands/table.py +101 -0
- manifold_cli/commands/token.py +144 -0
- manifold_cli/dependencies.py +116 -0
- manifold_cli/models/__init__.py +0 -0
- manifold_cli/models/auth.py +49 -0
- manifold_cli/models/benchmark.py +87 -0
- manifold_cli/models/catalog.py +189 -0
- manifold_cli/models/credentials.py +19 -0
- manifold_cli/models/errors.py +121 -0
- manifold_cli/models/log.py +114 -0
- manifold_cli/models/observability.py +121 -0
- manifold_cli/models/organization.py +19 -0
- manifold_cli/models/pagination.py +59 -0
- manifold_cli/models/policy.py +49 -0
- manifold_cli/models/run.py +381 -0
- manifold_cli/models/runner.py +186 -0
- manifold_cli/models/runner_home.py +69 -0
- manifold_cli/models/task_queue.py +40 -0
- manifold_cli/models/token.py +39 -0
- manifold_cli/replay/__init__.py +68 -0
- manifold_cli/replay/log.py +814 -0
- manifold_cli/runner/__init__.py +72 -0
- manifold_cli/runner/api_retry.py +36 -0
- manifold_cli/runner/benchmark/__init__.py +800 -0
- manifold_cli/runner/benchmark/live_view.py +248 -0
- manifold_cli/runner/benchmark/preassigned_scheduling.py +325 -0
- manifold_cli/runner/benchmark/recording.py +1208 -0
- manifold_cli/runner/benchmark/scheduling.py +256 -0
- manifold_cli/runner/benchmark/task_queue_scheduling.py +717 -0
- manifold_cli/runner/benchmark/task_server.py +123 -0
- manifold_cli/runner/benchmark/video.py +124 -0
- manifold_cli/runner/benchmark_steps.py +178 -0
- manifold_cli/runner/cancellation.py +22 -0
- manifold_cli/runner/capture.py +282 -0
- manifold_cli/runner/container.py +370 -0
- manifold_cli/runner/current_dispatch.py +25 -0
- manifold_cli/runner/daemon.py +520 -0
- manifold_cli/runner/dispatch.py +38 -0
- manifold_cli/runner/gpu.py +182 -0
- manifold_cli/runner/heartbeat.py +202 -0
- manifold_cli/runner/loopback.py +102 -0
- manifold_cli/runner/loopback_bench.py +183 -0
- manifold_cli/runner/loopback_policy.py +39 -0
- manifold_cli/runner/output.py +218 -0
- manifold_cli/runner/pairing_acceptance.py +57 -0
- manifold_cli/runner/plan.py +369 -0
- manifold_cli/runner/policy.py +350 -0
- manifold_cli/runner/policy_steps.py +164 -0
- manifold_cli/runner/preparation.py +47 -0
- manifold_cli/runner/process.py +219 -0
- manifold_cli/runner/ram.py +17 -0
- manifold_cli/runner/resource_usage.py +265 -0
- manifold_cli/runner/served.py +218 -0
- manifold_cli/runner/service.py +120 -0
- manifold_cli/runner/side_failure.py +68 -0
- manifold_cli/runner/signature.py +98 -0
- manifold_cli/runner/source.py +66 -0
- manifold_cli/runner/stages.py +241 -0
- manifold_cli/runner/step_timing.py +229 -0
- manifold_cli/runner/worker.py +534 -0
- manifold_cli/runner/workload.py +401 -0
- manifold_cli/services/__init__.py +0 -0
- manifold_cli/services/auth.py +89 -0
- manifold_cli/services/benchmark.py +357 -0
- manifold_cli/services/organization.py +22 -0
- manifold_cli/services/policy.py +234 -0
- manifold_cli/services/run.py +304 -0
- manifold_cli/services/runner.py +207 -0
- manifold_cli/services/runner_daemon.py +146 -0
- manifold_cli/services/token.py +52 -0
- manifold_cli/settings.py +30 -0
- manifold_cli/transport/__init__.py +31 -0
- manifold_cli/transport/diagnostics.py +72 -0
- manifold_cli/transport/interfaces.py +69 -0
- manifold_cli/transport/iroh.py +317 -0
- manifold_cli/transport/registry.py +34 -0
- manifold_cli/transport/tcp.py +146 -0
- manifold_cli-0.0.39.dist-info/METADATA +406 -0
- manifold_cli-0.0.39.dist-info/RECORD +102 -0
- manifold_cli-0.0.39.dist-info/WHEEL +4 -0
- manifold_cli-0.0.39.dist-info/entry_points.txt +2 -0
- manifold_cli-0.0.39.dist-info/licenses/LICENSE +373 -0
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
"""The stand-in policy worker: serves the built-in echo policy on a local port.
|
|
2
|
+
|
|
3
|
+
`LoopbackAdapter` spawns this module. It stands in for a policy container on a
|
|
4
|
+
machine with no GPU: it binds the address the policy bridge dials and echoes
|
|
5
|
+
every byte back, which is what `loopback_bench` round-trips.
|
|
6
|
+
|
|
7
|
+
It is a worker process of its own, so it reports on stdout as a container does.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
import argparse
|
|
11
|
+
import asyncio
|
|
12
|
+
import contextlib
|
|
13
|
+
|
|
14
|
+
from manifold_cli.transport.diagnostics import serve_echo
|
|
15
|
+
from manifold_cli.transport.tcp import TcpTransport
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def main() -> None:
|
|
19
|
+
parser = argparse.ArgumentParser(description="Serve the built-in echo policy.")
|
|
20
|
+
parser.add_argument(
|
|
21
|
+
"--listen", required=True, help="Address to serve on, as HOST:PORT."
|
|
22
|
+
)
|
|
23
|
+
args = parser.parse_args()
|
|
24
|
+
with contextlib.suppress(KeyboardInterrupt):
|
|
25
|
+
asyncio.run(_serve(args.listen))
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
async def _serve(listen: str) -> None:
|
|
29
|
+
listener = await TcpTransport().open_listener(listen)
|
|
30
|
+
_say(f"serving the echo policy on {listener.target}")
|
|
31
|
+
await serve_echo(listener, on_event=_say)
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _say(message: str) -> None:
|
|
35
|
+
print(message, flush=True)
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
if __name__ == "__main__":
|
|
39
|
+
main()
|
|
@@ -0,0 +1,218 @@
|
|
|
1
|
+
"""Batching one side's captured output and posting it while the workers run.
|
|
2
|
+
|
|
3
|
+
One reporter per side rather than one per worker. A benchmark side fans out over
|
|
4
|
+
shards, and a post per shard would multiply the request rate by the fan-out for
|
|
5
|
+
output a reader asks for by run.
|
|
6
|
+
|
|
7
|
+
The reporter holds every stream the side has declared, numbers each stream's
|
|
8
|
+
lines itself, and sends what has accumulated on whichever threshold comes first.
|
|
9
|
+
A post that fails is logged and its lines dropped: a runner that retried would
|
|
10
|
+
spend a side's remaining time resending output rather than driving episodes.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
import asyncio
|
|
14
|
+
import contextlib
|
|
15
|
+
import logging
|
|
16
|
+
from collections.abc import AsyncIterator, Callable
|
|
17
|
+
from contextlib import asynccontextmanager
|
|
18
|
+
from datetime import UTC, datetime
|
|
19
|
+
from uuid import UUID
|
|
20
|
+
|
|
21
|
+
from rich.text import Text
|
|
22
|
+
|
|
23
|
+
from manifold_cli.clients.manifold import ManifoldApiClient
|
|
24
|
+
from manifold_cli.models.log import (
|
|
25
|
+
LogBatch,
|
|
26
|
+
LogProducer,
|
|
27
|
+
LogSource,
|
|
28
|
+
ReportedLine,
|
|
29
|
+
ReportedStream,
|
|
30
|
+
)
|
|
31
|
+
|
|
32
|
+
logger = logging.getLogger(__name__)
|
|
33
|
+
|
|
34
|
+
# How long lines are held before they are posted, so a reader watching a run
|
|
35
|
+
# sees output while the worker is still writing it.
|
|
36
|
+
FLUSH_INTERVAL_SECONDS = 1.0
|
|
37
|
+
|
|
38
|
+
# How many held lines force a post before the interval is up.
|
|
39
|
+
FLUSH_LINES = 500
|
|
40
|
+
|
|
41
|
+
# How many held characters do the same, for a worker whose lines are long enough
|
|
42
|
+
# that FLUSH_LINES of them would be one large request.
|
|
43
|
+
FLUSH_CHARS = 256 * 1024
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
class CapturedOutputReporter:
|
|
47
|
+
"""The output one side has captured and not yet reported.
|
|
48
|
+
|
|
49
|
+
`declare`, `add` and `close` are called by the pumps reading each worker;
|
|
50
|
+
`flush` is what sends, and `end_open_streams` runs once the side is over, for
|
|
51
|
+
the streams no worker's exit closed. Sequence numbers are assigned here and
|
|
52
|
+
per stream, so a worker's two sources are numbered independently of each
|
|
53
|
+
other and of when any batch was sent.
|
|
54
|
+
"""
|
|
55
|
+
|
|
56
|
+
def __init__(self, client: ManifoldApiClient, run_id: UUID):
|
|
57
|
+
self._client = client
|
|
58
|
+
self._run_id = run_id
|
|
59
|
+
self._streams: dict[tuple[LogProducer, int, LogSource], ReportedStream] = {}
|
|
60
|
+
self._sent: dict[tuple[LogProducer, int, LogSource], ReportedStream] = {}
|
|
61
|
+
self._next_seq: dict[tuple[LogProducer, int, LogSource], int] = {}
|
|
62
|
+
self._lines: list[ReportedLine] = []
|
|
63
|
+
self._held_chars = 0
|
|
64
|
+
self._full = asyncio.Event()
|
|
65
|
+
|
|
66
|
+
def declare(self, worker_index: int, source: LogSource) -> None:
|
|
67
|
+
"""Record that a worker has a stream, written to or not.
|
|
68
|
+
|
|
69
|
+
This is what lists a container still pulling its image, so it happens
|
|
70
|
+
when the pipe is first read rather than when a line arrives on it.
|
|
71
|
+
"""
|
|
72
|
+
self._declare(LogProducer.WORKER, worker_index, source)
|
|
73
|
+
|
|
74
|
+
def add(self, worker_index: int, source: LogSource, text: str) -> None:
|
|
75
|
+
"""Hold one worker line for the next post."""
|
|
76
|
+
self._add(LogProducer.WORKER, worker_index, source, text)
|
|
77
|
+
|
|
78
|
+
def runner_log(self, echo: Callable[[str], None]) -> Callable[[str], None]:
|
|
79
|
+
"""Tee runner progress to the console and its captured stdout stream."""
|
|
80
|
+
|
|
81
|
+
def write(message: str) -> None:
|
|
82
|
+
echo(message)
|
|
83
|
+
self.capture_runner(message)
|
|
84
|
+
|
|
85
|
+
return write
|
|
86
|
+
|
|
87
|
+
def capture_runner(self, message: str) -> None:
|
|
88
|
+
"""Hold one runner progress line after removing its console markup."""
|
|
89
|
+
self._add(
|
|
90
|
+
LogProducer.RUNNER,
|
|
91
|
+
0,
|
|
92
|
+
LogSource.STDOUT,
|
|
93
|
+
Text.from_markup(message).plain,
|
|
94
|
+
)
|
|
95
|
+
|
|
96
|
+
def _declare(
|
|
97
|
+
self, producer: LogProducer, worker_index: int, source: LogSource
|
|
98
|
+
) -> None:
|
|
99
|
+
"""Record that a process has a stream, written to or not."""
|
|
100
|
+
key = (producer, worker_index, source)
|
|
101
|
+
if key in self._streams:
|
|
102
|
+
return
|
|
103
|
+
self._streams[key] = ReportedStream(
|
|
104
|
+
producer=producer, worker_index=worker_index, source=source
|
|
105
|
+
)
|
|
106
|
+
self._next_seq[key] = 0
|
|
107
|
+
|
|
108
|
+
def _add(
|
|
109
|
+
self,
|
|
110
|
+
producer: LogProducer,
|
|
111
|
+
worker_index: int,
|
|
112
|
+
source: LogSource,
|
|
113
|
+
text: str,
|
|
114
|
+
) -> None:
|
|
115
|
+
"""Hold one line for the next post, at its place in its own stream."""
|
|
116
|
+
key = (producer, worker_index, source)
|
|
117
|
+
self._declare(producer, worker_index, source)
|
|
118
|
+
self._lines.append(
|
|
119
|
+
ReportedLine(
|
|
120
|
+
producer=producer,
|
|
121
|
+
worker_index=worker_index,
|
|
122
|
+
source=source,
|
|
123
|
+
seq=self._next_seq[key],
|
|
124
|
+
ts=datetime.now(tz=UTC),
|
|
125
|
+
text=text,
|
|
126
|
+
)
|
|
127
|
+
)
|
|
128
|
+
self._next_seq[key] += 1
|
|
129
|
+
self._held_chars += len(text)
|
|
130
|
+
if len(self._lines) >= FLUSH_LINES or self._held_chars >= FLUSH_CHARS:
|
|
131
|
+
self._full.set()
|
|
132
|
+
|
|
133
|
+
def close(self, worker_index: int, source: LogSource, *, exit_code: int) -> None:
|
|
134
|
+
"""Record that the worker behind a stream has exited.
|
|
135
|
+
|
|
136
|
+
The next post carries it rather than this waiting for the interval: the
|
|
137
|
+
exit is what a reader watching a stream is waiting to see.
|
|
138
|
+
"""
|
|
139
|
+
self.declare(worker_index, source)
|
|
140
|
+
key = (LogProducer.WORKER, worker_index, source)
|
|
141
|
+
self._streams[key] = self._streams[key].model_copy(
|
|
142
|
+
update={"ended_at": datetime.now(tz=UTC), "exit_code": exit_code}
|
|
143
|
+
)
|
|
144
|
+
self._full.set()
|
|
145
|
+
|
|
146
|
+
def end_open_streams(self) -> None:
|
|
147
|
+
"""End every stream still open, for a side that is over.
|
|
148
|
+
|
|
149
|
+
A worker whose exit was never classified -- one that failed its readiness
|
|
150
|
+
probe, or one whose pairing ended first -- leaves a stream nothing has
|
|
151
|
+
ended. A stream left open has no `ended_at`, and a reader displays it as
|
|
152
|
+
a worker still running on a run that has finished. The end carries no
|
|
153
|
+
exit code, because nothing here has one to report.
|
|
154
|
+
"""
|
|
155
|
+
ended_at = datetime.now(tz=UTC)
|
|
156
|
+
for key, stream in self._streams.items():
|
|
157
|
+
if stream.ended_at is None:
|
|
158
|
+
self._streams[key] = stream.model_copy(update={"ended_at": ended_at})
|
|
159
|
+
|
|
160
|
+
async def flush(self) -> None:
|
|
161
|
+
"""Post what is held, dropping the lines whether or not that worked.
|
|
162
|
+
|
|
163
|
+
The declarations are not dropped with them. Every batch carries all of
|
|
164
|
+
them, so a post that never arrived leaves the stream list short only
|
|
165
|
+
until the next one -- which is why a failure leaves `_sent` behind and
|
|
166
|
+
the next batch re-declares.
|
|
167
|
+
|
|
168
|
+
A post with nothing new in it is skipped, so a side whose workers are
|
|
169
|
+
quiet is not sending an empty batch every interval.
|
|
170
|
+
"""
|
|
171
|
+
if len(self._lines) == 0 and self._streams == self._sent:
|
|
172
|
+
return
|
|
173
|
+
lines, self._lines, self._held_chars = self._lines, [], 0
|
|
174
|
+
streams = dict(self._streams)
|
|
175
|
+
batch = LogBatch(streams=list(streams.values()), lines=lines)
|
|
176
|
+
try:
|
|
177
|
+
await asyncio.to_thread(
|
|
178
|
+
self._client.post_captured_output, self._run_id, batch
|
|
179
|
+
)
|
|
180
|
+
except Exception:
|
|
181
|
+
logger.exception("%d captured lines were not posted", len(lines))
|
|
182
|
+
else:
|
|
183
|
+
self._sent = streams
|
|
184
|
+
|
|
185
|
+
async def run(self) -> None:
|
|
186
|
+
"""Flush every interval, and sooner where a batch has filled up."""
|
|
187
|
+
while True:
|
|
188
|
+
with contextlib.suppress(TimeoutError):
|
|
189
|
+
await asyncio.wait_for(
|
|
190
|
+
self._full.wait(), timeout=FLUSH_INTERVAL_SECONDS
|
|
191
|
+
)
|
|
192
|
+
self._full.clear()
|
|
193
|
+
await self.flush()
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
@asynccontextmanager
|
|
197
|
+
async def reporting_captured_output(
|
|
198
|
+
client: ManifoldApiClient, run_id: UUID
|
|
199
|
+
) -> AsyncIterator[CapturedOutputReporter]:
|
|
200
|
+
"""A reporter for one side, flushing for as long as the body runs.
|
|
201
|
+
|
|
202
|
+
The last flush is on the way out, and shielded: a worker writes its last
|
|
203
|
+
words as it dies, which is after the final interval has passed, and a side
|
|
204
|
+
torn down by a cancelled pairing would otherwise drop them.
|
|
205
|
+
|
|
206
|
+
By the time the body has returned its workers have gone too, so
|
|
207
|
+
`end_open_streams` runs before that last flush.
|
|
208
|
+
"""
|
|
209
|
+
reporter = CapturedOutputReporter(client, run_id)
|
|
210
|
+
flushing = asyncio.create_task(reporter.run())
|
|
211
|
+
try:
|
|
212
|
+
yield reporter
|
|
213
|
+
finally:
|
|
214
|
+
flushing.cancel()
|
|
215
|
+
with contextlib.suppress(asyncio.CancelledError):
|
|
216
|
+
await flushing
|
|
217
|
+
reporter.end_open_streams()
|
|
218
|
+
await asyncio.shield(reporter.flush())
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
"""Detect when the policy accepts the benchmark's signature.
|
|
2
|
+
|
|
3
|
+
The runner of each side sends its first serving heartbeat at that point, so that
|
|
4
|
+
the policy and the benchmark are compatible for the run before the side enters
|
|
5
|
+
`running`.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from collections.abc import Callable
|
|
9
|
+
|
|
10
|
+
from manifold_cli.runner.step_timing import (
|
|
11
|
+
READY_PREFIX,
|
|
12
|
+
FrameScanner,
|
|
13
|
+
PrefixMatched,
|
|
14
|
+
read_exactly_by_reads,
|
|
15
|
+
)
|
|
16
|
+
from manifold_cli.transport import ByteStream, Connector
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class PairingAcceptanceConnector:
|
|
20
|
+
"""A `Connector` to the policy. Its streams call `on_accepted` when the
|
|
21
|
+
policy accepts the benchmark's signature.
|
|
22
|
+
|
|
23
|
+
The streams of the wrapped connector must read the policy's frames without
|
|
24
|
+
step times, because the scanner follows the frame boundaries.
|
|
25
|
+
"""
|
|
26
|
+
|
|
27
|
+
def __init__(self, connector: Connector, on_accepted: Callable[[], None]) -> None:
|
|
28
|
+
self._connector = connector
|
|
29
|
+
self._on_accepted = on_accepted
|
|
30
|
+
|
|
31
|
+
async def connect(self) -> ByteStream:
|
|
32
|
+
stream = await self._connector.connect()
|
|
33
|
+
return _PairingAcceptanceStream(stream, self._on_accepted)
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
class _PairingAcceptanceStream:
|
|
37
|
+
def __init__(self, stream: ByteStream, on_accepted: Callable[[], None]) -> None:
|
|
38
|
+
self._stream = stream
|
|
39
|
+
self._on_accepted = on_accepted
|
|
40
|
+
self._from_policy = FrameScanner(READY_PREFIX)
|
|
41
|
+
|
|
42
|
+
async def read(self, n: int) -> bytes:
|
|
43
|
+
data = await self._stream.read(n)
|
|
44
|
+
if any(
|
|
45
|
+
isinstance(event, PrefixMatched) for event in self._from_policy.scan(data)
|
|
46
|
+
):
|
|
47
|
+
self._on_accepted()
|
|
48
|
+
return data
|
|
49
|
+
|
|
50
|
+
async def read_exactly(self, n: int) -> bytes:
|
|
51
|
+
return await read_exactly_by_reads(self.read, n)
|
|
52
|
+
|
|
53
|
+
async def write(self, data: bytes) -> None:
|
|
54
|
+
await self._stream.write(data)
|
|
55
|
+
|
|
56
|
+
async def close(self) -> None:
|
|
57
|
+
await self._stream.close()
|
|
@@ -0,0 +1,369 @@
|
|
|
1
|
+
"""What an adapter hands back for one worker, and the contract it satisfies.
|
|
2
|
+
|
|
3
|
+
A `WorkerPlan` is data: the commands to run before the worker, the argv to
|
|
4
|
+
spawn, how to tell it is ready, and the cleanup to run afterwards. An adapter
|
|
5
|
+
in this package builds one from the `config` of a registered version;
|
|
6
|
+
`worker.py` executes it.
|
|
7
|
+
|
|
8
|
+
An adapter plans a policy worker and a benchmark worker through separate entry
|
|
9
|
+
points rather than taking a role argument, because the two plans differ in
|
|
10
|
+
kind: a policy worker binds an address the runner dials, and a benchmark worker
|
|
11
|
+
dials an address the runner listens on.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
import socket
|
|
15
|
+
from dataclasses import dataclass, field
|
|
16
|
+
from pathlib import Path
|
|
17
|
+
from typing import Any, Protocol, runtime_checkable
|
|
18
|
+
from urllib.parse import urlsplit
|
|
19
|
+
from uuid import UUID
|
|
20
|
+
|
|
21
|
+
from manifold_cli.models.run import EpisodeRollupError
|
|
22
|
+
|
|
23
|
+
# Ephemeral scratch area for benchmark results, one directory per run. A
|
|
24
|
+
# benchmark worker writes its results into a mount rooted here, so they survive
|
|
25
|
+
# the removal of the container that wrote them.
|
|
26
|
+
RESULTS_ROOT = Path("/tmp/manifold/results")
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
# The host a worker and the bridge beside it share. Both ends of that wire sit
|
|
30
|
+
# on the runner's own box: a policy worker binds a port the bridge dials, and a
|
|
31
|
+
# benchmark worker dials a listener the bridge opened. It is not the host the
|
|
32
|
+
# far side dials, which `outbound_host` resolves.
|
|
33
|
+
WIRE_HOST = "127.0.0.1"
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
UNASSIGNED_PORT = 0
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
# Where `outbound_host` routes towards to learn this box's own address. Nothing
|
|
40
|
+
# is sent, so the port only has to be one the API could plausibly answer on.
|
|
41
|
+
ROUTE_PROBE_PORT = 443
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
class RunnerError(Exception):
|
|
45
|
+
"""Base for the runner's own failures."""
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
class PlanError(RunnerError):
|
|
49
|
+
"""A version cannot be turned into a worker plan.
|
|
50
|
+
|
|
51
|
+
Carries a message naming what the config is missing, or what the runner
|
|
52
|
+
needs installed to run it.
|
|
53
|
+
"""
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
@dataclass(frozen=True)
|
|
57
|
+
class WireBinding:
|
|
58
|
+
"""The local TCP address one worker binds, or dials.
|
|
59
|
+
|
|
60
|
+
A policy worker publishes its port here and `manifold policy bridge` dials
|
|
61
|
+
it with a `TcpConnector`; a benchmark worker dials the address
|
|
62
|
+
`manifold benchmark bridge` accepts on. Both name a socket on the runner's
|
|
63
|
+
own machine, so a wire binding holds a host and a port and no URL.
|
|
64
|
+
|
|
65
|
+
UNASSIGNED_PORT asks whoever binds the socket to choose the port, which is
|
|
66
|
+
the only safe answer where that binder is not the runner: a port named up
|
|
67
|
+
front is unbound until the worker starts, and a policy worker starts behind
|
|
68
|
+
an image pull. The plan then carries `published_wire` to read back what was
|
|
69
|
+
chosen.
|
|
70
|
+
"""
|
|
71
|
+
|
|
72
|
+
host: str
|
|
73
|
+
port: int
|
|
74
|
+
|
|
75
|
+
@property
|
|
76
|
+
def address(self) -> str:
|
|
77
|
+
return f"{self.host}:{self.port}"
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
@dataclass(frozen=True)
|
|
81
|
+
class ShardAssignment:
|
|
82
|
+
"""The partition of a run's episodes one benchmark worker drives.
|
|
83
|
+
|
|
84
|
+
The three travel together and are meaningless apart, so they are one
|
|
85
|
+
argument rather than three: a worker cannot be told its index without also
|
|
86
|
+
being told how many shards it is one of, or how many episodes they divide.
|
|
87
|
+
|
|
88
|
+
`count` may not exceed `total_episodes`. The partition gives shard `i` every
|
|
89
|
+
global index where `idx % count == i`, so a shard with `index < count <=
|
|
90
|
+
total_episodes` always owns at least episode `index`, and the bound is
|
|
91
|
+
exactly the condition that no shard is handed nothing. A worker with nothing
|
|
92
|
+
to drive still pulls its image and pairs, and the empty rollup it leaves is
|
|
93
|
+
indistinguishable from one a broken image wrote.
|
|
94
|
+
"""
|
|
95
|
+
|
|
96
|
+
index: int
|
|
97
|
+
count: int
|
|
98
|
+
total_episodes: int
|
|
99
|
+
|
|
100
|
+
def __post_init__(self) -> None:
|
|
101
|
+
if self.count < 1:
|
|
102
|
+
raise ValueError(f"a shard count must be at least 1; got {self.count}")
|
|
103
|
+
if not 0 <= self.index < self.count:
|
|
104
|
+
raise ValueError(
|
|
105
|
+
f"a shard index must be in [0, {self.count}); got {self.index}"
|
|
106
|
+
)
|
|
107
|
+
if self.total_episodes < 1:
|
|
108
|
+
raise ValueError(
|
|
109
|
+
f"a run must have at least 1 episode; got {self.total_episodes}"
|
|
110
|
+
)
|
|
111
|
+
if self.count > self.total_episodes:
|
|
112
|
+
raise ValueError(
|
|
113
|
+
f"{self.count} shards divide {self.total_episodes} episodes, so "
|
|
114
|
+
"some would drive none; claim at most one shard per episode"
|
|
115
|
+
)
|
|
116
|
+
|
|
117
|
+
@property
|
|
118
|
+
def label(self) -> str:
|
|
119
|
+
"""What names this shard wherever one worker's own is needed.
|
|
120
|
+
|
|
121
|
+
The container name and the results directory both take it, so the two
|
|
122
|
+
cannot come to disagree about which shard wrote what.
|
|
123
|
+
"""
|
|
124
|
+
return f"shard-{self.index}"
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
@dataclass(frozen=True)
|
|
128
|
+
class TcpProbe:
|
|
129
|
+
"""Dial the worker's address until it accepts a connection, or time out."""
|
|
130
|
+
|
|
131
|
+
wire: WireBinding
|
|
132
|
+
timeout_s: float = 10.0
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
@dataclass(frozen=True)
|
|
136
|
+
class CommandProbe:
|
|
137
|
+
"""Run `argv` until it exits zero, or time out.
|
|
138
|
+
|
|
139
|
+
For a worker whose address the runner cannot usefully dial itself.
|
|
140
|
+
`target` names what the command checks, for the message a timeout carries.
|
|
141
|
+
"""
|
|
142
|
+
|
|
143
|
+
argv: tuple[str, ...]
|
|
144
|
+
target: str
|
|
145
|
+
timeout_s: float = 10.0
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
@dataclass(frozen=True)
|
|
149
|
+
class ProcessOnly:
|
|
150
|
+
"""No probe: the worker is ready as soon as its process is up.
|
|
151
|
+
|
|
152
|
+
For a worker that dials out instead of binding. Nothing is listening to
|
|
153
|
+
probe, and the bridge accepts the connection once the worker makes it.
|
|
154
|
+
"""
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
Readiness = TcpProbe | CommandProbe | ProcessOnly
|
|
158
|
+
|
|
159
|
+
IMAGE_PULL_PHASE = "image_pull"
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
@dataclass(frozen=True)
|
|
163
|
+
class SetupCommand:
|
|
164
|
+
"""A command that must succeed before the worker is spawned.
|
|
165
|
+
|
|
166
|
+
`docker pull` of a multi-GB image is what this exists for. Pulling inside
|
|
167
|
+
the worker spends the readiness budget on the download, and a probe window
|
|
168
|
+
wide enough to cover a cold pull is no longer a useful timeout. Each
|
|
169
|
+
command runs to completion on its own timeout.
|
|
170
|
+
|
|
171
|
+
`started` and `completed` are the plain text of its stage lines, such as
|
|
172
|
+
"Pulling <image>" and "Pulled <image>". A command without them logs none.
|
|
173
|
+
"""
|
|
174
|
+
|
|
175
|
+
argv: tuple[str, ...]
|
|
176
|
+
phase: str = "setup"
|
|
177
|
+
timeout_s: float = 3600.0
|
|
178
|
+
started: str = ""
|
|
179
|
+
completed: str = ""
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
@dataclass(frozen=True)
|
|
183
|
+
class SidePreparation:
|
|
184
|
+
"""Commands and progress text for the work that prepares a side."""
|
|
185
|
+
|
|
186
|
+
commands: tuple[SetupCommand, ...]
|
|
187
|
+
started: str
|
|
188
|
+
completed: str
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
@dataclass(frozen=True)
|
|
192
|
+
class WorkerPlan:
|
|
193
|
+
"""How to spawn one worker, verify it, and clean up after it."""
|
|
194
|
+
|
|
195
|
+
argv: tuple[str, ...]
|
|
196
|
+
env: dict[str, str] = field(default_factory=dict)
|
|
197
|
+
workdir: Path | None = None
|
|
198
|
+
readiness: Readiness = field(default_factory=ProcessOnly)
|
|
199
|
+
setup: tuple[SetupCommand, ...] = ()
|
|
200
|
+
# Commands run once the worker has exited, however it exited. For state a
|
|
201
|
+
# worker cannot reclaim itself, such as a container the docker client left
|
|
202
|
+
# running when it was killed.
|
|
203
|
+
teardown: tuple[tuple[str, ...], ...] = ()
|
|
204
|
+
# Where the spawned worker records the id of what it started, for a teardown
|
|
205
|
+
# that must not stop something this attempt did not start. Absent where the
|
|
206
|
+
# spawn never got that far, which is the case teardown has to skip.
|
|
207
|
+
cidfile: Path | None = None
|
|
208
|
+
# Where a benchmark worker writes its results. None for a policy worker,
|
|
209
|
+
# which produces none.
|
|
210
|
+
results_dir: Path | None = None
|
|
211
|
+
# The GPU index the plan pins the worker to, set only where the plan asks
|
|
212
|
+
# for one card. It describes what is spawned, so a worker that asked for no
|
|
213
|
+
# GPU and a worker handed every card both leave it None.
|
|
214
|
+
gpu_device: int | None = None
|
|
215
|
+
# How to read back the address the worker's binder chose, printed as
|
|
216
|
+
# `host:port` on stdout. Set only where the plan asked for UNASSIGNED_PORT,
|
|
217
|
+
# so a runner that named the port itself already knows the answer. Run once
|
|
218
|
+
# the worker is ready: nothing has bound the port before then.
|
|
219
|
+
published_wire: tuple[str, ...] | None = None
|
|
220
|
+
|
|
221
|
+
@property
|
|
222
|
+
def startup_budget_s(self) -> float:
|
|
223
|
+
"""How long this worker may take before it answers on its wire.
|
|
224
|
+
|
|
225
|
+
Its setup commands, then its readiness check, each already carrying the
|
|
226
|
+
budget the adapter gave it. The far side of a pairing waits this out, so
|
|
227
|
+
it is the plan's own claim about itself rather than a wait the runner
|
|
228
|
+
that holds the plan ever spends.
|
|
229
|
+
"""
|
|
230
|
+
if isinstance(self.readiness, ProcessOnly):
|
|
231
|
+
ready_s = 0.0
|
|
232
|
+
else:
|
|
233
|
+
ready_s = self.readiness.timeout_s
|
|
234
|
+
return sum(step.timeout_s for step in self.setup) + ready_s
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
def free_port(host: str) -> int:
|
|
238
|
+
"""Return a free TCP port on `host`.
|
|
239
|
+
|
|
240
|
+
The socket closes before the worker binds the port, so another process
|
|
241
|
+
could take it first. That race is unlikely on a machine serving one policy.
|
|
242
|
+
"""
|
|
243
|
+
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as probe:
|
|
244
|
+
probe.bind((host, UNASSIGNED_PORT))
|
|
245
|
+
return int(probe.getsockname()[1])
|
|
246
|
+
|
|
247
|
+
|
|
248
|
+
def published_wire_from(output: str) -> WireBinding:
|
|
249
|
+
"""The wire a `WorkerPlan.published_wire` command reported.
|
|
250
|
+
|
|
251
|
+
`docker port` prints one `host:port` per published mapping. A policy
|
|
252
|
+
container publishes one, on the host the plan named, so anything else means
|
|
253
|
+
the container is not the one the plan describes.
|
|
254
|
+
|
|
255
|
+
Raises PlanError for output that names no address.
|
|
256
|
+
"""
|
|
257
|
+
lines = [line.strip() for line in output.splitlines() if len(line.strip()) > 0]
|
|
258
|
+
if len(lines) != 1:
|
|
259
|
+
raise PlanError(
|
|
260
|
+
f"Reading a policy container's published port expected one address, "
|
|
261
|
+
f"but docker reported {len(lines)}: {output.strip()!r}"
|
|
262
|
+
)
|
|
263
|
+
host, _, port = lines[0].rpartition(":")
|
|
264
|
+
if len(host) == 0 or not port.isdigit():
|
|
265
|
+
raise PlanError(
|
|
266
|
+
f"A policy container's published port is not an address: {lines[0]!r}"
|
|
267
|
+
)
|
|
268
|
+
return WireBinding(host=host, port=int(port))
|
|
269
|
+
|
|
270
|
+
|
|
271
|
+
def rollup_in(results_dir: Path) -> Path:
|
|
272
|
+
"""The one rollup document a benchmark worker left in `results_dir`.
|
|
273
|
+
|
|
274
|
+
The image names the file after the benchmark it ran, which the runner is
|
|
275
|
+
never told, so the plan names a directory and the runner finds the file in it.
|
|
276
|
+
One shard writes one document, and a directory holding none or several is a
|
|
277
|
+
worker that did not do what the plan describes.
|
|
278
|
+
|
|
279
|
+
Raises:
|
|
280
|
+
EpisodeRollupError: The directory does not hold exactly one `*.json`.
|
|
281
|
+
"""
|
|
282
|
+
documents = sorted(results_dir.glob("*.json"))
|
|
283
|
+
if len(documents) != 1:
|
|
284
|
+
raise EpisodeRollupError(
|
|
285
|
+
f"expected one rollup under {results_dir}, found {len(documents)}"
|
|
286
|
+
)
|
|
287
|
+
return documents[0]
|
|
288
|
+
|
|
289
|
+
|
|
290
|
+
def outbound_host(api_url: str) -> str:
|
|
291
|
+
"""This box's own address, as the interface that reaches Manifold sees it.
|
|
292
|
+
|
|
293
|
+
Connecting a UDP socket sends nothing: it fixes the route, so the kernel
|
|
294
|
+
picks the source address it would use for `api_url` and `getsockname` reads
|
|
295
|
+
it back.
|
|
296
|
+
|
|
297
|
+
That address is what a peer on the same network dials. A peer somewhere the
|
|
298
|
+
route towards Manifold does not lead is the case `--advertise-host` exists
|
|
299
|
+
for, and the operator states it there rather than here.
|
|
300
|
+
|
|
301
|
+
Raises:
|
|
302
|
+
RunnerError: The url names no host, or there is no route to it.
|
|
303
|
+
"""
|
|
304
|
+
host = urlsplit(api_url).hostname
|
|
305
|
+
if host is None:
|
|
306
|
+
raise RunnerError(f"The Manifold url {api_url!r} names no host to route to.")
|
|
307
|
+
try:
|
|
308
|
+
with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as probe:
|
|
309
|
+
probe.connect((host, ROUTE_PROBE_PORT))
|
|
310
|
+
return str(probe.getsockname()[0])
|
|
311
|
+
except OSError as exc:
|
|
312
|
+
raise RunnerError(
|
|
313
|
+
f"This box has no route to {host} to read its own address from: {exc}. "
|
|
314
|
+
"Register the runner with --advertise-host to state the address its "
|
|
315
|
+
"peer dials."
|
|
316
|
+
) from exc
|
|
317
|
+
|
|
318
|
+
|
|
319
|
+
@runtime_checkable
|
|
320
|
+
class WorkerAdapter(Protocol):
|
|
321
|
+
"""Plans the workers and the side preparation for a version."""
|
|
322
|
+
|
|
323
|
+
def plan_policy(
|
|
324
|
+
self,
|
|
325
|
+
config: dict[str, Any],
|
|
326
|
+
*,
|
|
327
|
+
run_id: UUID,
|
|
328
|
+
assignment_id: UUID,
|
|
329
|
+
wire: WireBinding,
|
|
330
|
+
cache_dir: Path | None = None,
|
|
331
|
+
gpu_device: int | None = None,
|
|
332
|
+
) -> WorkerPlan:
|
|
333
|
+
"""Plan the policy worker `config` describes, bound to `wire`.
|
|
334
|
+
|
|
335
|
+
Raises PlanError for a config this adapter cannot run.
|
|
336
|
+
"""
|
|
337
|
+
...
|
|
338
|
+
|
|
339
|
+
def plan_side_preparation(self, config: dict[str, Any]) -> SidePreparation | None:
|
|
340
|
+
"""Plan work that must complete once before the side's workers start.
|
|
341
|
+
|
|
342
|
+
Return None when the adapter does not plan a preparation for the side.
|
|
343
|
+
|
|
344
|
+
Raises PlanError for a config this adapter cannot run.
|
|
345
|
+
"""
|
|
346
|
+
...
|
|
347
|
+
|
|
348
|
+
def plan_benchmark(
|
|
349
|
+
self,
|
|
350
|
+
config: dict[str, Any],
|
|
351
|
+
*,
|
|
352
|
+
run_id: UUID,
|
|
353
|
+
assignment_id: UUID,
|
|
354
|
+
wire: WireBinding,
|
|
355
|
+
shard: ShardAssignment,
|
|
356
|
+
live_view_wire: WireBinding | None = None,
|
|
357
|
+
results_root: Path = RESULTS_ROOT,
|
|
358
|
+
cache_dir: Path | None = None,
|
|
359
|
+
gpu_device: int | None = None,
|
|
360
|
+
) -> WorkerPlan:
|
|
361
|
+
"""Plan the benchmark worker `config` describes, dialling its wires.
|
|
362
|
+
|
|
363
|
+
`shard` has no default. Every benchmark worker drives one, the caller
|
|
364
|
+
always knows which, and a defaulted `0 of 1` is exactly what a caller
|
|
365
|
+
that forgot to pass one would produce.
|
|
366
|
+
|
|
367
|
+
Raises PlanError for a config this adapter cannot run.
|
|
368
|
+
"""
|
|
369
|
+
...
|