manifold-cli 0.0.39__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- manifold_cli/__init__.py +3 -0
- manifold_cli/__main__.py +4 -0
- manifold_cli/bridge.py +70 -0
- manifold_cli/cli.py +54 -0
- manifold_cli/clients/__init__.py +0 -0
- manifold_cli/clients/auth_flow.py +169 -0
- manifold_cli/clients/credentials.py +53 -0
- manifold_cli/clients/docker_engine.py +43 -0
- manifold_cli/clients/image.py +130 -0
- manifold_cli/clients/manifold.py +1288 -0
- manifold_cli/clients/runner_home.py +165 -0
- manifold_cli/clients/workos.py +99 -0
- manifold_cli/commands/__init__.py +0 -0
- manifold_cli/commands/auth.py +101 -0
- manifold_cli/commands/benchmark.py +539 -0
- manifold_cli/commands/organization.py +154 -0
- manifold_cli/commands/policy.py +749 -0
- manifold_cli/commands/run.py +337 -0
- manifold_cli/commands/run_log.py +291 -0
- manifold_cli/commands/runner.py +852 -0
- manifold_cli/commands/table.py +101 -0
- manifold_cli/commands/token.py +144 -0
- manifold_cli/dependencies.py +116 -0
- manifold_cli/models/__init__.py +0 -0
- manifold_cli/models/auth.py +49 -0
- manifold_cli/models/benchmark.py +87 -0
- manifold_cli/models/catalog.py +189 -0
- manifold_cli/models/credentials.py +19 -0
- manifold_cli/models/errors.py +121 -0
- manifold_cli/models/log.py +114 -0
- manifold_cli/models/observability.py +121 -0
- manifold_cli/models/organization.py +19 -0
- manifold_cli/models/pagination.py +59 -0
- manifold_cli/models/policy.py +49 -0
- manifold_cli/models/run.py +381 -0
- manifold_cli/models/runner.py +186 -0
- manifold_cli/models/runner_home.py +69 -0
- manifold_cli/models/task_queue.py +40 -0
- manifold_cli/models/token.py +39 -0
- manifold_cli/replay/__init__.py +68 -0
- manifold_cli/replay/log.py +814 -0
- manifold_cli/runner/__init__.py +72 -0
- manifold_cli/runner/api_retry.py +36 -0
- manifold_cli/runner/benchmark/__init__.py +800 -0
- manifold_cli/runner/benchmark/live_view.py +248 -0
- manifold_cli/runner/benchmark/preassigned_scheduling.py +325 -0
- manifold_cli/runner/benchmark/recording.py +1208 -0
- manifold_cli/runner/benchmark/scheduling.py +256 -0
- manifold_cli/runner/benchmark/task_queue_scheduling.py +717 -0
- manifold_cli/runner/benchmark/task_server.py +123 -0
- manifold_cli/runner/benchmark/video.py +124 -0
- manifold_cli/runner/benchmark_steps.py +178 -0
- manifold_cli/runner/cancellation.py +22 -0
- manifold_cli/runner/capture.py +282 -0
- manifold_cli/runner/container.py +370 -0
- manifold_cli/runner/current_dispatch.py +25 -0
- manifold_cli/runner/daemon.py +520 -0
- manifold_cli/runner/dispatch.py +38 -0
- manifold_cli/runner/gpu.py +182 -0
- manifold_cli/runner/heartbeat.py +202 -0
- manifold_cli/runner/loopback.py +102 -0
- manifold_cli/runner/loopback_bench.py +183 -0
- manifold_cli/runner/loopback_policy.py +39 -0
- manifold_cli/runner/output.py +218 -0
- manifold_cli/runner/pairing_acceptance.py +57 -0
- manifold_cli/runner/plan.py +369 -0
- manifold_cli/runner/policy.py +350 -0
- manifold_cli/runner/policy_steps.py +164 -0
- manifold_cli/runner/preparation.py +47 -0
- manifold_cli/runner/process.py +219 -0
- manifold_cli/runner/ram.py +17 -0
- manifold_cli/runner/resource_usage.py +265 -0
- manifold_cli/runner/served.py +218 -0
- manifold_cli/runner/service.py +120 -0
- manifold_cli/runner/side_failure.py +68 -0
- manifold_cli/runner/signature.py +98 -0
- manifold_cli/runner/source.py +66 -0
- manifold_cli/runner/stages.py +241 -0
- manifold_cli/runner/step_timing.py +229 -0
- manifold_cli/runner/worker.py +534 -0
- manifold_cli/runner/workload.py +401 -0
- manifold_cli/services/__init__.py +0 -0
- manifold_cli/services/auth.py +89 -0
- manifold_cli/services/benchmark.py +357 -0
- manifold_cli/services/organization.py +22 -0
- manifold_cli/services/policy.py +234 -0
- manifold_cli/services/run.py +304 -0
- manifold_cli/services/runner.py +207 -0
- manifold_cli/services/runner_daemon.py +146 -0
- manifold_cli/services/token.py +52 -0
- manifold_cli/settings.py +30 -0
- manifold_cli/transport/__init__.py +31 -0
- manifold_cli/transport/diagnostics.py +72 -0
- manifold_cli/transport/interfaces.py +69 -0
- manifold_cli/transport/iroh.py +317 -0
- manifold_cli/transport/registry.py +34 -0
- manifold_cli/transport/tcp.py +146 -0
- manifold_cli-0.0.39.dist-info/METADATA +406 -0
- manifold_cli-0.0.39.dist-info/RECORD +102 -0
- manifold_cli-0.0.39.dist-info/WHEEL +4 -0
- manifold_cli-0.0.39.dist-info/entry_points.txt +2 -0
- manifold_cli-0.0.39.dist-info/licenses/LICENSE +373 -0
|
@@ -0,0 +1,357 @@
|
|
|
1
|
+
"""Benchmark use cases: list the benchmarks and versions visible to the caller,
|
|
2
|
+
register a container image as a version along with its spec file
|
|
3
|
+
(IMAGE_SPEC_PATH), and publish its rubric file (IMAGE_RUBRIC_PATH).
|
|
4
|
+
|
|
5
|
+
Mirrors PolicyService. The benchmark list is keyset-paginated and read to the
|
|
6
|
+
end. Reading a version list takes the slug plus the organization that owns the
|
|
7
|
+
benchmark, and the command layer settles that owner before calling here. A bare
|
|
8
|
+
slug takes the same own-then-public route as it does for a policy.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
import json
|
|
12
|
+
from typing import Any, Protocol
|
|
13
|
+
|
|
14
|
+
from manifold_cli.models.benchmark import (
|
|
15
|
+
Benchmark,
|
|
16
|
+
BenchmarkNotFoundError,
|
|
17
|
+
BenchmarkVersion,
|
|
18
|
+
InvalidImageRubricError,
|
|
19
|
+
InvalidImageSpecError,
|
|
20
|
+
RubricContent,
|
|
21
|
+
)
|
|
22
|
+
from manifold_cli.models.catalog import (
|
|
23
|
+
Adapter,
|
|
24
|
+
Visibility,
|
|
25
|
+
in_owner_order,
|
|
26
|
+
newest_first,
|
|
27
|
+
version_from_image,
|
|
28
|
+
)
|
|
29
|
+
from manifold_cli.models.pagination import Page, collect
|
|
30
|
+
from manifold_cli.models.run import SchedulingType
|
|
31
|
+
|
|
32
|
+
# The path inside a benchmark image where its rubric file sits.
|
|
33
|
+
IMAGE_RUBRIC_PATH = "/opt/manifold/rubric.json"
|
|
34
|
+
|
|
35
|
+
# The path inside a benchmark image where the build writes the spec: the
|
|
36
|
+
# observations, actions and embodiment the benchmark sends to a policy.
|
|
37
|
+
IMAGE_SPEC_PATH = "/opt/manifold/spec.json"
|
|
38
|
+
|
|
39
|
+
# Registration stores the image's spec under this key in the version's `config`.
|
|
40
|
+
# Runners never read it.
|
|
41
|
+
SPEC_CONFIG_KEY = "benchmark"
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
class BenchmarkApiClient(Protocol):
|
|
45
|
+
"""The benchmark endpoints BenchmarkService drives on the API client."""
|
|
46
|
+
|
|
47
|
+
def list_benchmarks(
|
|
48
|
+
self, *, limit: int | None = None, cursor: str | None = None
|
|
49
|
+
) -> Page[Benchmark]: ...
|
|
50
|
+
|
|
51
|
+
def list_benchmark_versions(
|
|
52
|
+
self, slug: str, organization: str
|
|
53
|
+
) -> list[BenchmarkVersion]:
|
|
54
|
+
"""Raises BenchmarkNotFoundError when the organization has no such benchmark."""
|
|
55
|
+
...
|
|
56
|
+
|
|
57
|
+
def upsert_benchmark(
|
|
58
|
+
self,
|
|
59
|
+
slug: str,
|
|
60
|
+
organization: str,
|
|
61
|
+
*,
|
|
62
|
+
display_name: str,
|
|
63
|
+
visibility: Visibility,
|
|
64
|
+
) -> None:
|
|
65
|
+
"""Raises CatalogRegisterError when the server refuses the write."""
|
|
66
|
+
...
|
|
67
|
+
|
|
68
|
+
def register_benchmark_version(
|
|
69
|
+
self,
|
|
70
|
+
slug: str,
|
|
71
|
+
version: str,
|
|
72
|
+
organization: str,
|
|
73
|
+
*,
|
|
74
|
+
adapter: Adapter,
|
|
75
|
+
config: dict[str, Any],
|
|
76
|
+
minimum_gpu_memory_gb: int,
|
|
77
|
+
minimum_driver_major: int | None,
|
|
78
|
+
maximum_driver_major: int | None,
|
|
79
|
+
minimum_gpu_count: int | None = None,
|
|
80
|
+
scheduling_type: SchedulingType,
|
|
81
|
+
allow_preemptible: bool,
|
|
82
|
+
episodes: int,
|
|
83
|
+
shards: int,
|
|
84
|
+
shards_per_gpu: int | None,
|
|
85
|
+
) -> BenchmarkVersion:
|
|
86
|
+
"""Raises BenchmarkNotFoundError, or CatalogRegisterError as above."""
|
|
87
|
+
...
|
|
88
|
+
|
|
89
|
+
def publish_benchmark_rubric(
|
|
90
|
+
self,
|
|
91
|
+
slug: str,
|
|
92
|
+
version: str,
|
|
93
|
+
organization: str,
|
|
94
|
+
*,
|
|
95
|
+
rubric: RubricContent,
|
|
96
|
+
) -> None:
|
|
97
|
+
"""Raises BenchmarkNotFoundError, or CatalogRegisterError as above."""
|
|
98
|
+
...
|
|
99
|
+
|
|
100
|
+
def delete_benchmark(self, slug: str, organization: str) -> None:
|
|
101
|
+
"""Raises BenchmarkNotFoundError when the organization has no such benchmark."""
|
|
102
|
+
...
|
|
103
|
+
|
|
104
|
+
def delete_benchmark_version(
|
|
105
|
+
self, slug: str, version: str, organization: str
|
|
106
|
+
) -> None:
|
|
107
|
+
"""Raises BenchmarkNotFoundError when the benchmark or the version is not
|
|
108
|
+
found.
|
|
109
|
+
"""
|
|
110
|
+
...
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
class ImageFileReader(Protocol):
|
|
114
|
+
"""How the service reads a file out of a container image."""
|
|
115
|
+
|
|
116
|
+
def read_file(self, image: str, path: str) -> bytes | None:
|
|
117
|
+
"""The file's bytes, None when the image does not carry it.
|
|
118
|
+
|
|
119
|
+
Raises ImageInspectError when the image could not be read at all.
|
|
120
|
+
"""
|
|
121
|
+
...
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
class BenchmarkService:
|
|
125
|
+
def __init__(
|
|
126
|
+
self, manifold_client: BenchmarkApiClient, image_client: ImageFileReader
|
|
127
|
+
):
|
|
128
|
+
self._manifold_client = manifold_client
|
|
129
|
+
self._image_client = image_client
|
|
130
|
+
|
|
131
|
+
def list_benchmarks(self) -> list[Benchmark]:
|
|
132
|
+
"""Every benchmark the caller can see, grouped by owning organization."""
|
|
133
|
+
items, _ = collect(
|
|
134
|
+
lambda limit, cursor: self._manifold_client.list_benchmarks(
|
|
135
|
+
limit=limit, cursor=cursor
|
|
136
|
+
)
|
|
137
|
+
)
|
|
138
|
+
return in_owner_order(items)
|
|
139
|
+
|
|
140
|
+
def register_container_version(
|
|
141
|
+
self,
|
|
142
|
+
slug: str,
|
|
143
|
+
organization: str,
|
|
144
|
+
*,
|
|
145
|
+
image: str,
|
|
146
|
+
episodes: int,
|
|
147
|
+
minimum_gpu_memory_gb: int,
|
|
148
|
+
minimum_driver_major: int | None = None,
|
|
149
|
+
maximum_driver_major: int | None = None,
|
|
150
|
+
minimum_gpu_count: int | None = None,
|
|
151
|
+
scheduling_type: SchedulingType,
|
|
152
|
+
allow_preemptible: bool,
|
|
153
|
+
shards: int = 1,
|
|
154
|
+
shards_per_gpu: int | None,
|
|
155
|
+
display_name: str | None = None,
|
|
156
|
+
visibility: Visibility | None = None,
|
|
157
|
+
spec: dict[str, Any] | None = None,
|
|
158
|
+
) -> BenchmarkVersion:
|
|
159
|
+
"""Register `image` as a version of the organization's benchmark `slug`.
|
|
160
|
+
|
|
161
|
+
Mirrors PolicyService.register_container_version, including what an
|
|
162
|
+
unset display name or visibility carries over from the existing
|
|
163
|
+
benchmark. `episodes` and `shards` belong to the version rather than
|
|
164
|
+
the benchmark: every run of this image executes that many episodes,
|
|
165
|
+
divided across `shards` parallel partitions unless `run submit --shards`
|
|
166
|
+
names another count. `shards_per_gpu`, when set, replaces `shards` with
|
|
167
|
+
that many shards per GPU of each benchmark runner. `spec` is stored
|
|
168
|
+
unchanged in the version's `config` for `version_spec` to read.
|
|
169
|
+
|
|
170
|
+
Raises InvalidImageError for an image that names no version,
|
|
171
|
+
BenchmarkNotFoundError when the benchmark is gone by the time the
|
|
172
|
+
version is written, and CatalogRegisterError when the server refuses
|
|
173
|
+
the write.
|
|
174
|
+
"""
|
|
175
|
+
version = version_from_image(image)
|
|
176
|
+
existing = self._own_benchmark(slug, organization)
|
|
177
|
+
if display_name is None:
|
|
178
|
+
display_name = existing.display_name if existing is not None else slug
|
|
179
|
+
if visibility is None:
|
|
180
|
+
visibility = existing.visibility if existing is not None else Visibility.ORG
|
|
181
|
+
self._manifold_client.upsert_benchmark(
|
|
182
|
+
slug,
|
|
183
|
+
organization,
|
|
184
|
+
display_name=display_name,
|
|
185
|
+
visibility=visibility,
|
|
186
|
+
)
|
|
187
|
+
config: dict[str, Any] = {"image": image}
|
|
188
|
+
if minimum_gpu_memory_gb > 0:
|
|
189
|
+
config["gpu"] = True
|
|
190
|
+
if spec is not None:
|
|
191
|
+
config[SPEC_CONFIG_KEY] = spec
|
|
192
|
+
return self._manifold_client.register_benchmark_version(
|
|
193
|
+
slug,
|
|
194
|
+
version,
|
|
195
|
+
organization,
|
|
196
|
+
adapter=Adapter.CONTAINER,
|
|
197
|
+
config=config,
|
|
198
|
+
minimum_gpu_memory_gb=minimum_gpu_memory_gb,
|
|
199
|
+
minimum_driver_major=minimum_driver_major,
|
|
200
|
+
maximum_driver_major=maximum_driver_major,
|
|
201
|
+
minimum_gpu_count=minimum_gpu_count,
|
|
202
|
+
scheduling_type=scheduling_type,
|
|
203
|
+
allow_preemptible=allow_preemptible,
|
|
204
|
+
episodes=episodes,
|
|
205
|
+
shards=shards,
|
|
206
|
+
shards_per_gpu=shards_per_gpu,
|
|
207
|
+
)
|
|
208
|
+
|
|
209
|
+
def read_image_spec(self, image: str) -> dict[str, Any] | None:
|
|
210
|
+
"""The spec read from `image`, or None when the image has no spec file.
|
|
211
|
+
|
|
212
|
+
Raises InvalidImageSpecError when the file is not a JSON object, and
|
|
213
|
+
ImageInspectError when the image cannot be read.
|
|
214
|
+
"""
|
|
215
|
+
found = self._image_client.read_file(image, IMAGE_SPEC_PATH)
|
|
216
|
+
if found is None:
|
|
217
|
+
return None
|
|
218
|
+
try:
|
|
219
|
+
spec = json.loads(found)
|
|
220
|
+
except ValueError as exc:
|
|
221
|
+
raise InvalidImageSpecError(
|
|
222
|
+
f"'{image}' has a spec at {IMAGE_SPEC_PATH} that is not JSON: {exc}"
|
|
223
|
+
) from exc
|
|
224
|
+
if not isinstance(spec, dict):
|
|
225
|
+
raise InvalidImageSpecError(
|
|
226
|
+
f"'{image}' has a spec at {IMAGE_SPEC_PATH} that is not an object."
|
|
227
|
+
)
|
|
228
|
+
return spec
|
|
229
|
+
|
|
230
|
+
def version_spec(
|
|
231
|
+
self, slug: str, organization: str, version: str, *, or_public: bool = False
|
|
232
|
+
) -> dict[str, Any] | None:
|
|
233
|
+
"""The spec saved when `version` of benchmark `slug` was registered, or
|
|
234
|
+
None if it was registered without one.
|
|
235
|
+
|
|
236
|
+
With `or_public`, a public benchmark named `slug` is used when the
|
|
237
|
+
organization has none of its own.
|
|
238
|
+
|
|
239
|
+
Raises BenchmarkNotFoundError when no such benchmark or version exists.
|
|
240
|
+
"""
|
|
241
|
+
versions = (
|
|
242
|
+
self.list_own_or_public_versions(slug, organization)
|
|
243
|
+
if or_public
|
|
244
|
+
else self.list_versions(slug, organization)
|
|
245
|
+
)
|
|
246
|
+
for found in versions:
|
|
247
|
+
if found.version == version:
|
|
248
|
+
spec = found.config.get(SPEC_CONFIG_KEY)
|
|
249
|
+
return spec if isinstance(spec, dict) else None
|
|
250
|
+
raise BenchmarkNotFoundError(f"'{slug}' has no version '{version}'")
|
|
251
|
+
|
|
252
|
+
def publish_image_rubric(
|
|
253
|
+
self, slug: str, version: str, organization: str, *, image: str
|
|
254
|
+
) -> int | None:
|
|
255
|
+
"""Publish the rubric `image` carries; how many tasks it held.
|
|
256
|
+
|
|
257
|
+
None means the image carries no rubric, which is most of them: the
|
|
258
|
+
caller registers and says nothing further.
|
|
259
|
+
|
|
260
|
+
Raises InvalidImageRubricError when the image carries a rubric that is
|
|
261
|
+
not a readable one, and ImageInspectError when the image could not be
|
|
262
|
+
read; the two are different answers and the caller reports them so.
|
|
263
|
+
CatalogRegisterError and BenchmarkNotFoundError propagate as they do
|
|
264
|
+
from a registration.
|
|
265
|
+
"""
|
|
266
|
+
found = self._image_client.read_file(image, IMAGE_RUBRIC_PATH)
|
|
267
|
+
if found is None:
|
|
268
|
+
return None
|
|
269
|
+
rubric = _rubric_content(found, image)
|
|
270
|
+
self._manifold_client.publish_benchmark_rubric(
|
|
271
|
+
slug, version, organization, rubric=rubric
|
|
272
|
+
)
|
|
273
|
+
return len(rubric.tasks)
|
|
274
|
+
|
|
275
|
+
def list_versions(self, slug: str, organization: str) -> list[BenchmarkVersion]:
|
|
276
|
+
"""The versions registered under the organization's benchmark, newest first."""
|
|
277
|
+
return newest_first(
|
|
278
|
+
self._manifold_client.list_benchmark_versions(slug, organization)
|
|
279
|
+
)
|
|
280
|
+
|
|
281
|
+
def list_own_or_public_versions(
|
|
282
|
+
self, slug: str, organization: str
|
|
283
|
+
) -> list[BenchmarkVersion]:
|
|
284
|
+
"""The versions of a bare slug, resolved as `run submit` resolves one.
|
|
285
|
+
|
|
286
|
+
The organization's own benchmark wins; failing that, a public benchmark
|
|
287
|
+
of that slug belonging to anyone. The two commands have to agree: a slug
|
|
288
|
+
that submits must not read as unregistered here.
|
|
289
|
+
"""
|
|
290
|
+
try:
|
|
291
|
+
return self.list_versions(slug, organization)
|
|
292
|
+
except BenchmarkNotFoundError:
|
|
293
|
+
owner = self._public_owner(slug)
|
|
294
|
+
if owner is None:
|
|
295
|
+
raise
|
|
296
|
+
return self.list_versions(slug, owner)
|
|
297
|
+
|
|
298
|
+
def delete(self, slug: str, organization: str, version: str | None) -> None:
|
|
299
|
+
"""Delete the organization's benchmark `slug`, or only its `version`."""
|
|
300
|
+
if version is None:
|
|
301
|
+
self._manifold_client.delete_benchmark(slug, organization)
|
|
302
|
+
else:
|
|
303
|
+
self._manifold_client.delete_benchmark_version(slug, version, organization)
|
|
304
|
+
|
|
305
|
+
def _own_benchmark(self, slug: str, organization: str) -> Benchmark | None:
|
|
306
|
+
"""The organization's own benchmark of that slug, if it has one.
|
|
307
|
+
|
|
308
|
+
The API serves no single benchmark, so this scans the listing as
|
|
309
|
+
`_public_owner` does.
|
|
310
|
+
"""
|
|
311
|
+
for benchmark in self.list_benchmarks():
|
|
312
|
+
if benchmark.slug == slug and benchmark.owner_org_slug == organization:
|
|
313
|
+
return benchmark
|
|
314
|
+
return None
|
|
315
|
+
|
|
316
|
+
def _public_owner(self, slug: str) -> str | None:
|
|
317
|
+
"""Which organization publishes `slug`, if any.
|
|
318
|
+
|
|
319
|
+
The API serves at most one public benchmark per slug, so the search can
|
|
320
|
+
stop at the first match.
|
|
321
|
+
"""
|
|
322
|
+
for benchmark in self.list_benchmarks():
|
|
323
|
+
if benchmark.slug == slug and benchmark.visibility is Visibility.PUBLIC:
|
|
324
|
+
return benchmark.owner_org_slug
|
|
325
|
+
return None
|
|
326
|
+
|
|
327
|
+
|
|
328
|
+
def _rubric_content(found: bytes, image: str) -> RubricContent:
|
|
329
|
+
"""Return the groups and tasks listed in a rubric file unchanged.
|
|
330
|
+
|
|
331
|
+
The file must be JSON, and neither its `groups` list nor its `tasks` list may
|
|
332
|
+
be empty. The server checks each group and task when the rubric is published.
|
|
333
|
+
"""
|
|
334
|
+
try:
|
|
335
|
+
document = json.loads(found)
|
|
336
|
+
except ValueError as exc:
|
|
337
|
+
raise InvalidImageRubricError(
|
|
338
|
+
f"'{image}' carries a rubric at {IMAGE_RUBRIC_PATH} that is not JSON: {exc}"
|
|
339
|
+
) from exc
|
|
340
|
+
groups = document.get("groups") if isinstance(document, dict) else None
|
|
341
|
+
tasks = document.get("tasks") if isinstance(document, dict) else None
|
|
342
|
+
if not isinstance(tasks, list) or not tasks:
|
|
343
|
+
raise InvalidImageRubricError(
|
|
344
|
+
f"'{image}' carries a rubric at {IMAGE_RUBRIC_PATH} with no tasks."
|
|
345
|
+
)
|
|
346
|
+
if not isinstance(groups, list) or not groups:
|
|
347
|
+
raise InvalidImageRubricError(
|
|
348
|
+
f"'{image}' carries a rubric at {IMAGE_RUBRIC_PATH} with no groups. "
|
|
349
|
+
"Rebuild the image with a generator that writes groups."
|
|
350
|
+
)
|
|
351
|
+
grading = document.get("grading", {})
|
|
352
|
+
if not isinstance(grading, dict):
|
|
353
|
+
raise InvalidImageRubricError(
|
|
354
|
+
f"'{image}' carries a rubric at {IMAGE_RUBRIC_PATH} whose grading is "
|
|
355
|
+
"not an object."
|
|
356
|
+
)
|
|
357
|
+
return RubricContent(groups=groups, tasks=tasks, grading=grading)
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
"""Organization use cases.
|
|
2
|
+
|
|
3
|
+
WorkOS owns membership, so the CLI only lists what the caller belongs to.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from typing import Protocol
|
|
7
|
+
|
|
8
|
+
from manifold_cli.models.organization import Organization
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class OrganizationApiClient(Protocol):
|
|
12
|
+
"""The organization endpoint OrganizationService drives on the API client."""
|
|
13
|
+
|
|
14
|
+
def list_organizations(self) -> list[Organization]: ...
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class OrganizationService:
|
|
18
|
+
def __init__(self, manifold_client: OrganizationApiClient):
|
|
19
|
+
self._manifold_client = manifold_client
|
|
20
|
+
|
|
21
|
+
def list(self) -> list[Organization]:
|
|
22
|
+
return self._manifold_client.list_organizations()
|
|
@@ -0,0 +1,234 @@
|
|
|
1
|
+
"""Policy use cases — list the policies and versions visible to the caller, and
|
|
2
|
+
register a container image or a served policy as a version.
|
|
3
|
+
|
|
4
|
+
The policy list is keyset-paginated. Following the cursor to the end lives here
|
|
5
|
+
because how much of a list a command shows is a use-case decision, not a
|
|
6
|
+
transport one.
|
|
7
|
+
|
|
8
|
+
Version lists are not paginated.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from typing import Any, Protocol
|
|
12
|
+
|
|
13
|
+
from manifold_cli.models.catalog import (
|
|
14
|
+
Adapter,
|
|
15
|
+
Visibility,
|
|
16
|
+
in_owner_order,
|
|
17
|
+
newest_first,
|
|
18
|
+
version_from_image,
|
|
19
|
+
)
|
|
20
|
+
from manifold_cli.models.pagination import Page, collect
|
|
21
|
+
from manifold_cli.models.policy import Policy, PolicyNotFoundError, PolicyVersion
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class PolicyApiClient(Protocol):
|
|
25
|
+
"""The policy endpoints PolicyService drives on the API client."""
|
|
26
|
+
|
|
27
|
+
def list_policies(
|
|
28
|
+
self, *, limit: int | None = None, cursor: str | None = None
|
|
29
|
+
) -> Page[Policy]: ...
|
|
30
|
+
|
|
31
|
+
def list_policy_versions(self, slug: str, organization: str) -> list[PolicyVersion]:
|
|
32
|
+
"""Raises PolicyNotFoundError when the organization has no such policy."""
|
|
33
|
+
...
|
|
34
|
+
|
|
35
|
+
def upsert_policy(
|
|
36
|
+
self,
|
|
37
|
+
slug: str,
|
|
38
|
+
organization: str,
|
|
39
|
+
*,
|
|
40
|
+
display_name: str,
|
|
41
|
+
visibility: Visibility,
|
|
42
|
+
) -> None:
|
|
43
|
+
"""Raises CatalogRegisterError when the server refuses the write."""
|
|
44
|
+
...
|
|
45
|
+
|
|
46
|
+
def register_policy_version(
|
|
47
|
+
self,
|
|
48
|
+
slug: str,
|
|
49
|
+
version: str,
|
|
50
|
+
organization: str,
|
|
51
|
+
*,
|
|
52
|
+
adapter: Adapter,
|
|
53
|
+
config: dict[str, Any],
|
|
54
|
+
minimum_gpu_memory_gb: int,
|
|
55
|
+
minimum_driver_major: int | None,
|
|
56
|
+
maximum_driver_major: int | None,
|
|
57
|
+
minimum_gpu_count: int | None = None,
|
|
58
|
+
) -> PolicyVersion:
|
|
59
|
+
"""Raises PolicyNotFoundError, or CatalogRegisterError as above."""
|
|
60
|
+
...
|
|
61
|
+
|
|
62
|
+
def delete_policy(self, slug: str, organization: str) -> None:
|
|
63
|
+
"""Raises PolicyNotFoundError when the organization has no such policy."""
|
|
64
|
+
...
|
|
65
|
+
|
|
66
|
+
def delete_policy_version(self, slug: str, version: str, organization: str) -> None:
|
|
67
|
+
"""Raises PolicyNotFoundError when the policy or the version is not found."""
|
|
68
|
+
...
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
class PolicyService:
|
|
72
|
+
def __init__(self, manifold_client: PolicyApiClient):
|
|
73
|
+
self._manifold_client = manifold_client
|
|
74
|
+
|
|
75
|
+
def list_policies(self) -> list[Policy]:
|
|
76
|
+
"""Every policy the caller can see, grouped by owning organization."""
|
|
77
|
+
items, _ = collect(
|
|
78
|
+
lambda limit, cursor: self._manifold_client.list_policies(
|
|
79
|
+
limit=limit, cursor=cursor
|
|
80
|
+
)
|
|
81
|
+
)
|
|
82
|
+
return in_owner_order(items)
|
|
83
|
+
|
|
84
|
+
def register_container_version(
|
|
85
|
+
self,
|
|
86
|
+
slug: str,
|
|
87
|
+
organization: str,
|
|
88
|
+
*,
|
|
89
|
+
image: str,
|
|
90
|
+
environment: dict[str, str],
|
|
91
|
+
minimum_gpu_memory_gb: int,
|
|
92
|
+
minimum_driver_major: int | None = None,
|
|
93
|
+
maximum_driver_major: int | None = None,
|
|
94
|
+
minimum_gpu_count: int | None = None,
|
|
95
|
+
display_name: str | None = None,
|
|
96
|
+
visibility: Visibility | None = None,
|
|
97
|
+
) -> PolicyVersion:
|
|
98
|
+
"""Register `image` as a version of the organization's policy `slug`.
|
|
99
|
+
|
|
100
|
+
The policy is created if the organization has none of that slug. Its
|
|
101
|
+
version is the image's tag, so registering a second image is what adds
|
|
102
|
+
a second version.
|
|
103
|
+
|
|
104
|
+
A display name or visibility left unset is carried over from the
|
|
105
|
+
existing policy: the entity route is a PUT that replaces the row, so
|
|
106
|
+
sending a default would unpublish a public policy the next time an
|
|
107
|
+
image is registered under it. A new policy takes its slug as the
|
|
108
|
+
display name and is visible to its organization.
|
|
109
|
+
|
|
110
|
+
The entity is written first, so a refused version leaves the policy
|
|
111
|
+
created or renamed behind it.
|
|
112
|
+
|
|
113
|
+
Raises InvalidImageError for an image that names no version,
|
|
114
|
+
PolicyNotFoundError when the policy is gone by the time the version is
|
|
115
|
+
written, and CatalogRegisterError when the server refuses the write.
|
|
116
|
+
"""
|
|
117
|
+
version = version_from_image(image)
|
|
118
|
+
existing = self._own_policy(slug, organization)
|
|
119
|
+
if display_name is None:
|
|
120
|
+
display_name = existing.display_name if existing is not None else slug
|
|
121
|
+
if visibility is None:
|
|
122
|
+
visibility = existing.visibility if existing is not None else Visibility.ORG
|
|
123
|
+
self._manifold_client.upsert_policy(
|
|
124
|
+
slug,
|
|
125
|
+
organization,
|
|
126
|
+
display_name=display_name,
|
|
127
|
+
visibility=visibility,
|
|
128
|
+
)
|
|
129
|
+
config: dict[str, Any] = {"image": image}
|
|
130
|
+
if len(environment) > 0:
|
|
131
|
+
config["env"] = environment
|
|
132
|
+
if minimum_gpu_memory_gb > 0:
|
|
133
|
+
config["gpu"] = True
|
|
134
|
+
return self._manifold_client.register_policy_version(
|
|
135
|
+
slug,
|
|
136
|
+
version,
|
|
137
|
+
organization,
|
|
138
|
+
adapter=Adapter.CONTAINER,
|
|
139
|
+
config=config,
|
|
140
|
+
minimum_gpu_memory_gb=minimum_gpu_memory_gb,
|
|
141
|
+
minimum_driver_major=minimum_driver_major,
|
|
142
|
+
maximum_driver_major=maximum_driver_major,
|
|
143
|
+
minimum_gpu_count=minimum_gpu_count,
|
|
144
|
+
)
|
|
145
|
+
|
|
146
|
+
def register_served_version(
|
|
147
|
+
self,
|
|
148
|
+
slug: str,
|
|
149
|
+
organization: str,
|
|
150
|
+
*,
|
|
151
|
+
version: str,
|
|
152
|
+
) -> PolicyVersion:
|
|
153
|
+
"""Register `version` of the organization's policy `slug` for serving.
|
|
154
|
+
|
|
155
|
+
Creates the policy if it does not exist. An existing policy keeps its
|
|
156
|
+
display name and visibility. The config is `{}`, because serve starts
|
|
157
|
+
the policy process itself. Serving the same version again sends the
|
|
158
|
+
same settings, so the server accepts it.
|
|
159
|
+
|
|
160
|
+
Raises PolicyNotFoundError when the policy is deleted before the
|
|
161
|
+
version is written, and CatalogRegisterError when the server refuses
|
|
162
|
+
the write. The server refuses a version that exists with other
|
|
163
|
+
settings, such as a container version with the same name.
|
|
164
|
+
"""
|
|
165
|
+
existing = self._own_policy(slug, organization)
|
|
166
|
+
self._manifold_client.upsert_policy(
|
|
167
|
+
slug,
|
|
168
|
+
organization,
|
|
169
|
+
display_name=existing.display_name if existing is not None else slug,
|
|
170
|
+
visibility=existing.visibility if existing is not None else Visibility.ORG,
|
|
171
|
+
)
|
|
172
|
+
return self._manifold_client.register_policy_version(
|
|
173
|
+
slug,
|
|
174
|
+
version,
|
|
175
|
+
organization,
|
|
176
|
+
adapter=Adapter.SUBPROCESS,
|
|
177
|
+
config={},
|
|
178
|
+
# Placement does not check hardware for a serving runner.
|
|
179
|
+
minimum_gpu_memory_gb=0,
|
|
180
|
+
minimum_driver_major=None,
|
|
181
|
+
maximum_driver_major=None,
|
|
182
|
+
)
|
|
183
|
+
|
|
184
|
+
def list_versions(self, slug: str, organization: str) -> list[PolicyVersion]:
|
|
185
|
+
"""The versions registered under the organization's policy, newest first."""
|
|
186
|
+
return newest_first(
|
|
187
|
+
self._manifold_client.list_policy_versions(slug, organization)
|
|
188
|
+
)
|
|
189
|
+
|
|
190
|
+
def list_own_or_public_versions(
|
|
191
|
+
self, slug: str, organization: str
|
|
192
|
+
) -> list[PolicyVersion]:
|
|
193
|
+
"""The versions of a bare slug, resolved as `run submit` resolves one.
|
|
194
|
+
|
|
195
|
+
The organization's own policy wins; failing that, a public policy of
|
|
196
|
+
that slug belonging to anyone. The two commands have to agree: a slug
|
|
197
|
+
that submits must not read as unregistered here.
|
|
198
|
+
"""
|
|
199
|
+
try:
|
|
200
|
+
return self.list_versions(slug, organization)
|
|
201
|
+
except PolicyNotFoundError:
|
|
202
|
+
owner = self._public_owner(slug)
|
|
203
|
+
if owner is None:
|
|
204
|
+
raise
|
|
205
|
+
return self.list_versions(slug, owner)
|
|
206
|
+
|
|
207
|
+
def delete(self, slug: str, organization: str, version: str | None) -> None:
|
|
208
|
+
"""Delete the organization's policy `slug`, or only its `version`."""
|
|
209
|
+
if version is None:
|
|
210
|
+
self._manifold_client.delete_policy(slug, organization)
|
|
211
|
+
else:
|
|
212
|
+
self._manifold_client.delete_policy_version(slug, version, organization)
|
|
213
|
+
|
|
214
|
+
def _own_policy(self, slug: str, organization: str) -> Policy | None:
|
|
215
|
+
"""The organization's own policy of that slug, if it has one.
|
|
216
|
+
|
|
217
|
+
The API serves no single policy, so this scans the listing as
|
|
218
|
+
`_public_owner` does.
|
|
219
|
+
"""
|
|
220
|
+
for policy in self.list_policies():
|
|
221
|
+
if policy.slug == slug and policy.owner_org_slug == organization:
|
|
222
|
+
return policy
|
|
223
|
+
return None
|
|
224
|
+
|
|
225
|
+
def _public_owner(self, slug: str) -> str | None:
|
|
226
|
+
"""Which organization publishes `slug`, if any.
|
|
227
|
+
|
|
228
|
+
The API serves at most one public policy per slug, so the first match is
|
|
229
|
+
the only one.
|
|
230
|
+
"""
|
|
231
|
+
for policy in self.list_policies():
|
|
232
|
+
if policy.slug == slug and policy.visibility is Visibility.PUBLIC:
|
|
233
|
+
return policy.owner_org_slug
|
|
234
|
+
return None
|