launchhelm 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- launchhelm/__init__.py +35 -0
- launchhelm/_version.py +5 -0
- launchhelm/client.py +1660 -0
- launchhelm/commands.py +231 -0
- launchhelm/env.py +70 -0
- launchhelm/errors.py +62 -0
- launchhelm/models.py +6546 -0
- launchhelm/py.typed +0 -0
- launchhelm/rich.py +807 -0
- launchhelm/runtime.py +1348 -0
- launchhelm/session.py +2824 -0
- launchhelm/store.py +893 -0
- launchhelm/system.py +213 -0
- launchhelm/torch/__init__.py +41 -0
- launchhelm/torch/_checkpoint.py +181 -0
- launchhelm/torch/_controls.py +306 -0
- launchhelm/torch/_dist.py +210 -0
- launchhelm/torch/_trainer.py +1167 -0
- launchhelm/torch/hf.py +172 -0
- launchhelm/torch/lightning.py +179 -0
- launchhelm/transport.py +197 -0
- launchhelm/values.py +325 -0
- launchhelm-0.1.0.dist-info/METADATA +193 -0
- launchhelm-0.1.0.dist-info/RECORD +26 -0
- launchhelm-0.1.0.dist-info/WHEEL +4 -0
- launchhelm-0.1.0.dist-info/licenses/LICENSE +201 -0
launchhelm/session.py
ADDED
|
@@ -0,0 +1,2824 @@
|
|
|
1
|
+
"""Logging session: project, run, participant, and scalar metrics.
|
|
2
|
+
|
|
3
|
+
Contract: ``init`` is the public entry. Overflow is required and has no
|
|
4
|
+
default. ``log`` admits scalars into the runtime queue. ``finish`` waits
|
|
5
|
+
for a query-visible receipt. A session that never registers a capability
|
|
6
|
+
or services a safe point does not claim commands.
|
|
7
|
+
|
|
8
|
+
Failure: ``TypeError`` for a missing overflow, a nested metric, or a
|
|
9
|
+
changed metric type. ``ValueError`` for an invalid name, axis, or config
|
|
10
|
+
label. ``LaunchHelmError`` when project lookup is ambiguous or the server
|
|
11
|
+
rejects a document.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
import atexit
|
|
17
|
+
import json
|
|
18
|
+
import math
|
|
19
|
+
import os
|
|
20
|
+
import platform
|
|
21
|
+
import re
|
|
22
|
+
import shlex
|
|
23
|
+
import shutil
|
|
24
|
+
import socket
|
|
25
|
+
import sqlite3
|
|
26
|
+
import subprocess
|
|
27
|
+
import sys
|
|
28
|
+
import threading
|
|
29
|
+
from collections.abc import Callable, Mapping, Sequence
|
|
30
|
+
from concurrent.futures import ThreadPoolExecutor
|
|
31
|
+
from decimal import Decimal
|
|
32
|
+
from hashlib import sha256 as _sha256
|
|
33
|
+
from pathlib import Path
|
|
34
|
+
from typing import Any, Literal, TextIO, cast
|
|
35
|
+
from uuid import uuid4
|
|
36
|
+
|
|
37
|
+
from .client import Client, SessionClient
|
|
38
|
+
from .commands import CommandFunction, adapt, command_capability
|
|
39
|
+
from .env import load_env
|
|
40
|
+
from .errors import LaunchHelmError
|
|
41
|
+
from .models import Capability, Document, Scope, opaque_schema
|
|
42
|
+
from .rich import (
|
|
43
|
+
_SUFFIX_MEDIA,
|
|
44
|
+
MEDIA_OBJECT_BYTES,
|
|
45
|
+
PENDING_PREFIX,
|
|
46
|
+
Audio,
|
|
47
|
+
Histogram,
|
|
48
|
+
Image,
|
|
49
|
+
MediaGrid,
|
|
50
|
+
PendingBlob,
|
|
51
|
+
RichValue,
|
|
52
|
+
Text,
|
|
53
|
+
Video,
|
|
54
|
+
_check_media_type,
|
|
55
|
+
_NativeMedia,
|
|
56
|
+
bin_values,
|
|
57
|
+
compile_columns,
|
|
58
|
+
compile_rows,
|
|
59
|
+
infer_columns,
|
|
60
|
+
is_media_item,
|
|
61
|
+
pending_item,
|
|
62
|
+
)
|
|
63
|
+
from .rich import (
|
|
64
|
+
media_url as _media_url_item,
|
|
65
|
+
)
|
|
66
|
+
from .runtime import CapabilityHandler, Runtime
|
|
67
|
+
from .values import JSONObject, JSONValue, scalar
|
|
68
|
+
|
|
69
|
+
OverflowPolicy = Literal["drop_new", "block", "raise"]
|
|
70
|
+
DEFAULT_URL = "https://launchhelm.com"
|
|
71
|
+
_AXIS = re.compile(r"^[a-z][a-z0-9_.:-]{0,127}$")
|
|
72
|
+
_COORD = re.compile(r"^-?(0|[1-9][0-9]*)(\.[0-9]+)?$")
|
|
73
|
+
_MAX_LABELS = 64
|
|
74
|
+
_MAX_LABEL = 1024
|
|
75
|
+
_JSON_LABEL = "json:"
|
|
76
|
+
_MAX_PAGES = 100
|
|
77
|
+
_STREAM_CREATES = 8
|
|
78
|
+
_HISTOGRAM_STATS = frozenset({"underflow", "overflow", "total", "min", "max", "mean", "std"})
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
class _BuiltTable(RichValue):
|
|
82
|
+
"""A table value from pre-compiled columns and rows."""
|
|
83
|
+
|
|
84
|
+
renderer = "table"
|
|
85
|
+
|
|
86
|
+
def __init__(
|
|
87
|
+
self,
|
|
88
|
+
columns: list[tuple[str, str]],
|
|
89
|
+
rows: list[list[JSONValue]],
|
|
90
|
+
blobs: list[PendingBlob],
|
|
91
|
+
) -> None:
|
|
92
|
+
self._columns = [{"name": name, "type": ctype} for name, ctype in columns]
|
|
93
|
+
self._rows = rows
|
|
94
|
+
self._blobs = blobs
|
|
95
|
+
|
|
96
|
+
def value(self) -> JSONObject:
|
|
97
|
+
return {
|
|
98
|
+
"columns": cast(JSONValue, self._columns),
|
|
99
|
+
"rows": cast(JSONValue, self._rows),
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
def blobs(self) -> list[PendingBlob]:
|
|
103
|
+
return list(self._blobs)
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def _as_rich_value(value: object) -> RichValue | None:
|
|
107
|
+
"""Map a ``log`` metric value to its rich wrapper, or ``None``.
|
|
108
|
+
|
|
109
|
+
``RichValue`` instances pass through; a non-empty sequence of media
|
|
110
|
+
values or media item mappings becomes a ``MediaGrid``. Plain strings
|
|
111
|
+
and objects keep their scalar stream types.
|
|
112
|
+
"""
|
|
113
|
+
if isinstance(value, RichValue):
|
|
114
|
+
return value
|
|
115
|
+
if isinstance(value, Sequence) and not isinstance(value, str | bytes | bytearray):
|
|
116
|
+
items = list(value)
|
|
117
|
+
if items and all(
|
|
118
|
+
isinstance(item, _NativeMedia | RichValue) or is_media_item(item) for item in items
|
|
119
|
+
):
|
|
120
|
+
if any(
|
|
121
|
+
isinstance(item, RichValue) and not isinstance(item, _NativeMedia) for item in items
|
|
122
|
+
):
|
|
123
|
+
return None
|
|
124
|
+
return MediaGrid(items)
|
|
125
|
+
return None
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def _pending_hashes(node: object) -> list[str]:
|
|
129
|
+
"""Collect ``pending:<hash>`` artifact ids from a built value."""
|
|
130
|
+
found: list[str] = []
|
|
131
|
+
if isinstance(node, Mapping):
|
|
132
|
+
for key, item in node.items():
|
|
133
|
+
if key == "id" and isinstance(item, str) and item.startswith(PENDING_PREFIX):
|
|
134
|
+
found.append(item[len(PENDING_PREFIX) :])
|
|
135
|
+
else:
|
|
136
|
+
found.extend(_pending_hashes(item))
|
|
137
|
+
elif isinstance(node, Sequence) and not isinstance(node, str | bytes | bytearray):
|
|
138
|
+
for item in node:
|
|
139
|
+
found.extend(_pending_hashes(item))
|
|
140
|
+
return found
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def _stream_renderer(document: Document) -> str:
|
|
144
|
+
presentation = document.spec.get("presentation")
|
|
145
|
+
if isinstance(presentation, Mapping):
|
|
146
|
+
renderer = presentation.get("renderer")
|
|
147
|
+
if isinstance(renderer, str):
|
|
148
|
+
return renderer
|
|
149
|
+
return ""
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
class _LineCapture:
|
|
153
|
+
"""Write through to the original stream and record each complete line.
|
|
154
|
+
|
|
155
|
+
A failed ``console`` record never propagates through ``write`` — the
|
|
156
|
+
stream keeps working (tracebacks stay intact) and the first failure is
|
|
157
|
+
reported by ``Run.finish`` instead.
|
|
158
|
+
"""
|
|
159
|
+
|
|
160
|
+
def __init__(self, original: TextIO, run: Run, stream: str) -> None:
|
|
161
|
+
self.original = original
|
|
162
|
+
self._run = run
|
|
163
|
+
self._stream = stream
|
|
164
|
+
self._buffer = ""
|
|
165
|
+
self._inside = False
|
|
166
|
+
self._failed = False
|
|
167
|
+
|
|
168
|
+
def write(self, text: str) -> int:
|
|
169
|
+
self.original.write(text)
|
|
170
|
+
if self._inside or self._failed:
|
|
171
|
+
return len(text)
|
|
172
|
+
self._buffer += text
|
|
173
|
+
while "\n" in self._buffer:
|
|
174
|
+
line, self._buffer = self._buffer.split("\n", 1)
|
|
175
|
+
self._inside = True
|
|
176
|
+
try:
|
|
177
|
+
self._run.console(line, stream=self._stream)
|
|
178
|
+
except Exception as error: # a dead tee must not break the stream
|
|
179
|
+
self._failed = True
|
|
180
|
+
self._run._console_failure(error)
|
|
181
|
+
finally:
|
|
182
|
+
self._inside = False
|
|
183
|
+
return len(text)
|
|
184
|
+
|
|
185
|
+
def flush(self) -> None:
|
|
186
|
+
self.original.flush()
|
|
187
|
+
|
|
188
|
+
def flush_line(self) -> None:
|
|
189
|
+
if not self._buffer:
|
|
190
|
+
return
|
|
191
|
+
line = self._buffer
|
|
192
|
+
self._buffer = ""
|
|
193
|
+
self._run.console(line, stream=self._stream)
|
|
194
|
+
|
|
195
|
+
def __getattr__(self, name: str) -> Any: # noqa: ANN401 - delegates arbitrary stream attributes
|
|
196
|
+
return getattr(self.original, name)
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
class Run:
|
|
200
|
+
"""One attached participant and the streams it has declared.
|
|
201
|
+
|
|
202
|
+
Contract: ``log`` returns after memory admission. ``flush`` and
|
|
203
|
+
``finish`` use the runtime durability rules. The step counter is
|
|
204
|
+
local to this process.
|
|
205
|
+
|
|
206
|
+
Failure: see ``log`` and ``finish``.
|
|
207
|
+
"""
|
|
208
|
+
|
|
209
|
+
def __init__(
|
|
210
|
+
self,
|
|
211
|
+
runtime: Runtime,
|
|
212
|
+
*,
|
|
213
|
+
client: SessionClient,
|
|
214
|
+
name: str,
|
|
215
|
+
owned_client: SessionClient | None,
|
|
216
|
+
metadata_client: SessionClient | None = None,
|
|
217
|
+
owned_metadata: SessionClient | None = None,
|
|
218
|
+
checkpoint: Path,
|
|
219
|
+
restored: Mapping[str, JSONValue] | None = None,
|
|
220
|
+
next_step: int = 0,
|
|
221
|
+
reuse_streams: bool = False,
|
|
222
|
+
config: Mapping[str, JSONValue] | None = None,
|
|
223
|
+
summary: Mapping[str, JSONValue] | None = None,
|
|
224
|
+
summary_changed: bool = False,
|
|
225
|
+
group: str = "",
|
|
226
|
+
tags: tuple[str, ...] = (),
|
|
227
|
+
notes: str = "",
|
|
228
|
+
git: str = "",
|
|
229
|
+
host: str = "",
|
|
230
|
+
python: str = "",
|
|
231
|
+
argv: str = "",
|
|
232
|
+
) -> None:
|
|
233
|
+
self.runtime = runtime
|
|
234
|
+
self.name = name
|
|
235
|
+
self.project_id = runtime.scope.project_id
|
|
236
|
+
self.run_id = runtime.scope.run_id
|
|
237
|
+
self.restored: Mapping[str, JSONValue] = dict(restored or {})
|
|
238
|
+
self.config: Mapping[str, JSONValue] = dict(config or {})
|
|
239
|
+
self.summary_values: dict[str, JSONValue] = dict(summary or {})
|
|
240
|
+
self.group = group
|
|
241
|
+
self.tags = tags
|
|
242
|
+
self.notes = notes
|
|
243
|
+
self.git = git
|
|
244
|
+
self.host = host
|
|
245
|
+
self.python = python
|
|
246
|
+
self.argv = argv
|
|
247
|
+
self._checkpoint = checkpoint
|
|
248
|
+
self._client = client
|
|
249
|
+
self._owned_client = owned_client
|
|
250
|
+
self._metadata = metadata_client if metadata_client is not None else client
|
|
251
|
+
self._owned_metadata = owned_metadata
|
|
252
|
+
self._streams: dict[str, tuple[Document, str, tuple[str, ...]]] = {}
|
|
253
|
+
self._next_step = next_step
|
|
254
|
+
self._closed = False
|
|
255
|
+
self._reuse_streams = reuse_streams
|
|
256
|
+
self._summary: dict[str, JSONValue] = dict(self.summary_values)
|
|
257
|
+
self._summary_changed = summary_changed
|
|
258
|
+
self._reductions: dict[str, str] = {}
|
|
259
|
+
self._reduced: dict[str, tuple[int, float]] = {}
|
|
260
|
+
self._summary_locked: set[str] = set()
|
|
261
|
+
self._seeded_mean: set[str] = set()
|
|
262
|
+
self.last_applied: dict[str, str] | None = None
|
|
263
|
+
self.last_result: JSONValue | None = None
|
|
264
|
+
self._console_line = 0
|
|
265
|
+
self._console_lock = threading.Lock()
|
|
266
|
+
self._console_stdout: _LineCapture | None = None
|
|
267
|
+
self._console_stderr: _LineCapture | None = None
|
|
268
|
+
self._console_error: Exception | None = None
|
|
269
|
+
self._media_blobs: dict[str, PendingBlob] = {}
|
|
270
|
+
|
|
271
|
+
def __enter__(self) -> Run:
|
|
272
|
+
"""Return ``self``. ``__exit__`` calls ``finish``."""
|
|
273
|
+
return self
|
|
274
|
+
|
|
275
|
+
def __exit__(self, *args: object) -> None:
|
|
276
|
+
"""Finish the run. Does not suppress the block exception."""
|
|
277
|
+
self.finish()
|
|
278
|
+
|
|
279
|
+
def define(
|
|
280
|
+
self,
|
|
281
|
+
name: str,
|
|
282
|
+
kind: str = "number",
|
|
283
|
+
*axes: str,
|
|
284
|
+
unit: str = "",
|
|
285
|
+
summary: str = "",
|
|
286
|
+
) -> None:
|
|
287
|
+
"""Create the stream for ``name`` before the first point.
|
|
288
|
+
|
|
289
|
+
Contract: ``kind`` is ``number``, ``string``, or ``boolean``. ``step``
|
|
290
|
+
is always an axis. Extra names in ``axes`` are locked with it. A later
|
|
291
|
+
``log`` that changes the kind or the axes fails before the step
|
|
292
|
+
counter moves. ``summary`` is ``min``, ``max``, or ``mean`` for a
|
|
293
|
+
number metric. ``finish`` stores that reduction unless ``summary()``
|
|
294
|
+
set the same name. ``min`` and ``max`` continue from the value already
|
|
295
|
+
stored on a resumed run. ``mean`` continues from the stored count.
|
|
296
|
+
|
|
297
|
+
Failure: ``TypeError`` for an empty name. ``ValueError`` for a kind
|
|
298
|
+
or axis the protocol rejects, or when the stream already has a
|
|
299
|
+
different kind or axes.
|
|
300
|
+
"""
|
|
301
|
+
if not isinstance(name, str) or not name or len(name) > 256:
|
|
302
|
+
raise TypeError("metric name must be 1 to 256 characters")
|
|
303
|
+
if kind not in {"number", "string", "boolean"}:
|
|
304
|
+
raise ValueError("metric kind must be number, string, or boolean")
|
|
305
|
+
if not isinstance(unit, str) or len(unit) > 64:
|
|
306
|
+
raise ValueError("unit must be at most 64 characters")
|
|
307
|
+
if summary and summary not in {"min", "max", "mean"}:
|
|
308
|
+
raise ValueError("summary must be min, max, or mean")
|
|
309
|
+
if summary and kind != "number":
|
|
310
|
+
raise ValueError("summary applies to a number metric")
|
|
311
|
+
if summary:
|
|
312
|
+
self._reductions[name] = summary
|
|
313
|
+
if summary == "mean":
|
|
314
|
+
saved = self._saved_mean(name)
|
|
315
|
+
if saved is not None:
|
|
316
|
+
self._reduced[name] = saved
|
|
317
|
+
self._seeded_mean.add(name)
|
|
318
|
+
locked = ["step"]
|
|
319
|
+
for axis in axes:
|
|
320
|
+
if axis == "step" or not isinstance(axis, str) or not _AXIS.fullmatch(axis):
|
|
321
|
+
raise ValueError(f"invalid axis {axis!r}")
|
|
322
|
+
if axis not in locked:
|
|
323
|
+
locked.append(axis)
|
|
324
|
+
self._stream(name, kind, tuple(locked), unit=unit)
|
|
325
|
+
|
|
326
|
+
def log( # noqa: PLR0912 - rich/scalar dispatch keeps the wire contract in one place
|
|
327
|
+
self,
|
|
328
|
+
metrics: Mapping[str, JSONValue | RichValue | Sequence[RichValue]],
|
|
329
|
+
*,
|
|
330
|
+
step: int | None = None,
|
|
331
|
+
**axes: int | float | str,
|
|
332
|
+
) -> bool:
|
|
333
|
+
"""Admit one point per metric at the same coordinates.
|
|
334
|
+
|
|
335
|
+
Contract: scalar values keep the existing stream kinds; a
|
|
336
|
+
``RichValue`` (``Text``/``Table``/``Histogram``/``Image`` etc.) logs
|
|
337
|
+
its canonical object on a rendered stream and a non-empty sequence
|
|
338
|
+
of media values logs a media grid. An omitted ``step`` uses the
|
|
339
|
+
local counter, starting at 0.
|
|
340
|
+
An explicit ``step`` sets the counter. Returns ``False`` when any
|
|
341
|
+
point is dropped under ``drop_new``. Nested values are rejected.
|
|
342
|
+
Missing stream names in one call are created in one batch. A client
|
|
343
|
+
without ``create_documents`` creates them concurrently, up to eight
|
|
344
|
+
at a time. Each name is still its own stream.
|
|
345
|
+
|
|
346
|
+
Failure: ``TypeError`` for an empty name, a non-scalar, a boolean
|
|
347
|
+
step, or a metric whose JSON type changed. ``ValueError`` for an
|
|
348
|
+
axis or coordinate the protocol rejects, or a new axis on an
|
|
349
|
+
existing metric. Propagates runtime admission errors.
|
|
350
|
+
"""
|
|
351
|
+
if self._closed:
|
|
352
|
+
raise RuntimeError("run is closed")
|
|
353
|
+
if not metrics:
|
|
354
|
+
return True
|
|
355
|
+
prepared: list[tuple[str, str, JSONValue, RichValue | None]] = []
|
|
356
|
+
for name, value in metrics.items():
|
|
357
|
+
if not isinstance(name, str) or not name or len(name) > 256:
|
|
358
|
+
raise TypeError("metric name must be 1 to 256 characters")
|
|
359
|
+
rich_value = _as_rich_value(value)
|
|
360
|
+
if rich_value is not None:
|
|
361
|
+
prepared.append((name, "object", rich_value.value(), rich_value))
|
|
362
|
+
continue
|
|
363
|
+
kind, encoded = _scalar(cast(JSONValue, value))
|
|
364
|
+
prepared.append((name, kind, encoded, None))
|
|
365
|
+
_, axis_names = self._coordinates(step, axes, advance=False)
|
|
366
|
+
for name, kind, _encoded, rich_value in prepared:
|
|
367
|
+
known = self._streams.get(name)
|
|
368
|
+
if known is None:
|
|
369
|
+
continue
|
|
370
|
+
document, known_kind, known_axes = known
|
|
371
|
+
if rich_value is not None:
|
|
372
|
+
if known_kind != "object":
|
|
373
|
+
raise TypeError(f"metric {name!r} is {known_kind}, not a rich value")
|
|
374
|
+
stored = _stream_renderer(document)
|
|
375
|
+
if stored != rich_value.renderer:
|
|
376
|
+
raise ValueError(
|
|
377
|
+
f"metric {name!r} renders as {stored}, not {rich_value.renderer}"
|
|
378
|
+
)
|
|
379
|
+
elif known_kind != kind:
|
|
380
|
+
raise TypeError(f"metric {name!r} is {known_kind}, not {kind}")
|
|
381
|
+
if known_axes != axis_names:
|
|
382
|
+
raise ValueError(f"metric {name!r} axes are fixed")
|
|
383
|
+
coordinates, axis_names = self._coordinates(step, axes, advance=True)
|
|
384
|
+
missing = [
|
|
385
|
+
(name, kind)
|
|
386
|
+
for name, kind, _encoded, rich_value in prepared
|
|
387
|
+
if rich_value is None and name not in self._streams
|
|
388
|
+
]
|
|
389
|
+
self._open_streams(missing, axis_names)
|
|
390
|
+
admitted = True
|
|
391
|
+
for name, kind, encoded, rich_value in prepared:
|
|
392
|
+
if rich_value is not None:
|
|
393
|
+
document = self._rich_stream(name, rich_value.renderer, axis_names)
|
|
394
|
+
if not self._log_rich(document, rich_value, coordinates=coordinates):
|
|
395
|
+
admitted = False
|
|
396
|
+
continue
|
|
397
|
+
document = self._stream(name, kind, axis_names)
|
|
398
|
+
if not self.runtime.log(document, encoded, coordinates=coordinates):
|
|
399
|
+
admitted = False
|
|
400
|
+
continue
|
|
401
|
+
self._note_reduction(name, encoded)
|
|
402
|
+
return admitted
|
|
403
|
+
|
|
404
|
+
def console(self, text: str, *, stream: str = "stdout") -> None:
|
|
405
|
+
"""Record one output line without advancing the step counter.
|
|
406
|
+
|
|
407
|
+
Contract: ``text`` is one line, at most 1024 characters, stored on
|
|
408
|
+
the string metric ``console``. Coordinates are the current step, a
|
|
409
|
+
monotonic ``line``, and ``stream`` (``stdout`` or ``stderr``). A
|
|
410
|
+
later ``log`` still starts at the same step.
|
|
411
|
+
|
|
412
|
+
Failure: ``TypeError`` when ``text`` is not a string. ``ValueError``
|
|
413
|
+
for a stream other than ``stdout`` or ``stderr``, or a line longer
|
|
414
|
+
than 1024 characters.
|
|
415
|
+
"""
|
|
416
|
+
if not isinstance(text, str):
|
|
417
|
+
raise TypeError("console line must be a string")
|
|
418
|
+
if stream not in {"stdout", "stderr"}:
|
|
419
|
+
raise ValueError("console stream must be stdout or stderr")
|
|
420
|
+
if len(text) > 1024:
|
|
421
|
+
raise ValueError("console line must be at most 1024 characters")
|
|
422
|
+
with self._console_lock:
|
|
423
|
+
line = self._console_line
|
|
424
|
+
self._console_line += 1
|
|
425
|
+
coordinates, axis_names = self._coordinates(
|
|
426
|
+
None,
|
|
427
|
+
{"line": line, "stream": 0 if stream == "stdout" else 1},
|
|
428
|
+
advance=False,
|
|
429
|
+
)
|
|
430
|
+
self._open_streams([("console", "string")], axis_names)
|
|
431
|
+
document = self._stream("console", "string", axis_names)
|
|
432
|
+
if not self.runtime.log(document, text, coordinates=coordinates):
|
|
433
|
+
raise LaunchHelmError("protocol", "console line was dropped")
|
|
434
|
+
|
|
435
|
+
def capture_console(self) -> None:
|
|
436
|
+
"""Tee stdout and stderr into ``console`` until ``finish``.
|
|
437
|
+
|
|
438
|
+
Contract: each complete line is recorded and still written to the
|
|
439
|
+
original stream. A line emitted while a console point is being
|
|
440
|
+
recorded is not captured again. ``finish`` restores the original
|
|
441
|
+
streams and records a trailing partial line.
|
|
442
|
+
|
|
443
|
+
Failure: same as ``console``.
|
|
444
|
+
"""
|
|
445
|
+
if self._console_stdout is not None:
|
|
446
|
+
return
|
|
447
|
+
self._console_stdout = _LineCapture(sys.stdout, self, "stdout")
|
|
448
|
+
self._console_stderr = _LineCapture(sys.stderr, self, "stderr")
|
|
449
|
+
sys.stdout = self._console_stdout
|
|
450
|
+
sys.stderr = self._console_stderr
|
|
451
|
+
atexit.register(self._restore_console)
|
|
452
|
+
|
|
453
|
+
def _restore_console(self) -> None:
|
|
454
|
+
"""Return the original streams without recording; safe at interpreter exit."""
|
|
455
|
+
stdout = self._console_stdout
|
|
456
|
+
stderr = self._console_stderr
|
|
457
|
+
if stdout is not None and sys.stdout is stdout:
|
|
458
|
+
sys.stdout = stdout.original
|
|
459
|
+
if stderr is not None and sys.stderr is stderr:
|
|
460
|
+
sys.stderr = stderr.original
|
|
461
|
+
self._console_stdout = None
|
|
462
|
+
self._console_stderr = None
|
|
463
|
+
|
|
464
|
+
def _console_failure(self, error: Exception) -> None:
|
|
465
|
+
if self._console_error is None:
|
|
466
|
+
self._console_error = error
|
|
467
|
+
|
|
468
|
+
def _stop_console(self) -> None:
|
|
469
|
+
if self._console_stdout is None:
|
|
470
|
+
return
|
|
471
|
+
stdout = self._console_stdout
|
|
472
|
+
stderr = self._console_stderr
|
|
473
|
+
self._restore_console()
|
|
474
|
+
errors: list[Exception] = []
|
|
475
|
+
try:
|
|
476
|
+
stdout.flush_line()
|
|
477
|
+
except Exception as error: # restore already done; collect
|
|
478
|
+
errors.append(error)
|
|
479
|
+
if stderr is not None:
|
|
480
|
+
try:
|
|
481
|
+
stderr.flush_line()
|
|
482
|
+
except Exception as error:
|
|
483
|
+
errors.append(error)
|
|
484
|
+
if self._console_error is not None:
|
|
485
|
+
raise self._console_error
|
|
486
|
+
if errors:
|
|
487
|
+
raise errors[0]
|
|
488
|
+
|
|
489
|
+
def flush(self, *, delivered: bool = True, timeout: float = 30.0) -> None:
|
|
490
|
+
"""Wait for local durability, or for a query-visible receipt."""
|
|
491
|
+
self.runtime.flush(timeout=timeout, delivered=delivered)
|
|
492
|
+
|
|
493
|
+
def finish(self, *, close_run: bool = True, timeout: float = 30.0) -> None:
|
|
494
|
+
"""End the session: flush, record the summary, and close the runtime.
|
|
495
|
+
|
|
496
|
+
Contract: idempotent. Console capture stops first so buffered lines
|
|
497
|
+
reach the spool, admission is then fenced so the flush drains a
|
|
498
|
+
fixed set, and admitted points are flushed until query-visible —
|
|
499
|
+
telemetry delivery is never skipped because a later step fails. The
|
|
500
|
+
summary and stored reductions are written to the run with the
|
|
501
|
+
original credential, not the adopted worker key. The
|
|
502
|
+
participant session is then ended on the server: with ``close_run``
|
|
503
|
+
(the default) the run closes once no participant still holds a live
|
|
504
|
+
session; ``close_run=False`` leaves the run open for a deliberate
|
|
505
|
+
resume. The heartbeat loop stops when the session ends. Every step
|
|
506
|
+
is attempted even when an earlier one fails. Does not execute queued
|
|
507
|
+
commands. Closes every client this session constructed.
|
|
508
|
+
|
|
509
|
+
Failure: the first step's error is raised after all steps were
|
|
510
|
+
attempted.
|
|
511
|
+
"""
|
|
512
|
+
if self._closed:
|
|
513
|
+
return
|
|
514
|
+
self._closed = True
|
|
515
|
+
errors: list[BaseException] = []
|
|
516
|
+
for step in (
|
|
517
|
+
self._stop_console,
|
|
518
|
+
self.runtime.stop_accepting,
|
|
519
|
+
lambda: self.runtime.flush(timeout=timeout, delivered=True),
|
|
520
|
+
self._write_summary,
|
|
521
|
+
lambda: self._end_session(close_run),
|
|
522
|
+
lambda: self.runtime.close(timeout=timeout),
|
|
523
|
+
):
|
|
524
|
+
try:
|
|
525
|
+
step()
|
|
526
|
+
except BaseException as error: # report the first
|
|
527
|
+
errors.append(error)
|
|
528
|
+
for extra in (self._owned_metadata, self._owned_client):
|
|
529
|
+
if extra is not None:
|
|
530
|
+
try:
|
|
531
|
+
extra.close()
|
|
532
|
+
except BaseException as error:
|
|
533
|
+
errors.append(error)
|
|
534
|
+
if errors:
|
|
535
|
+
raise errors[0]
|
|
536
|
+
|
|
537
|
+
def _end_session(self, close_run: bool) -> None:
|
|
538
|
+
"""End the participant session, closing the run when asked.
|
|
539
|
+
|
|
540
|
+
``close_run=True`` asks the server to close the run once no
|
|
541
|
+
participant still holds a live session (``when_idle``); ``False``
|
|
542
|
+
always leaves it open (``never``). A client without ``end_session``
|
|
543
|
+
falls back to closing the run document directly.
|
|
544
|
+
"""
|
|
545
|
+
policy = "when_idle" if close_run else "never"
|
|
546
|
+
ended = self.runtime.end_session(close_run=policy)
|
|
547
|
+
if ended is None and close_run:
|
|
548
|
+
self._close_run()
|
|
549
|
+
|
|
550
|
+
def on(self, safe_point: str, capability: Capability, handler: CapabilityHandler) -> Document:
|
|
551
|
+
"""Register ``handler`` for ``safe_point`` and start background claim.
|
|
552
|
+
|
|
553
|
+
Contract: claim runs on the control thread. The training step does
|
|
554
|
+
not wait on the network. ``sync`` applies a command only when one
|
|
555
|
+
is already waiting locally.
|
|
556
|
+
|
|
557
|
+
Failure: ``ValueError`` when ``safe_point`` is not declared on the
|
|
558
|
+
capability. ``LaunchHelmError`` from registration.
|
|
559
|
+
"""
|
|
560
|
+
if safe_point not in capability.safe_points:
|
|
561
|
+
raise ValueError(f"capability does not declare safe point {safe_point!r}")
|
|
562
|
+
return self.register(capability, handler)
|
|
563
|
+
|
|
564
|
+
def command(
|
|
565
|
+
self, safe_point: str, *, name: str = ""
|
|
566
|
+
) -> Callable[[CommandFunction], CommandFunction]:
|
|
567
|
+
"""Register a typed function as a command at ``safe_point``.
|
|
568
|
+
|
|
569
|
+
Contract: annotations ``int``, ``float``, ``str``, and ``bool``, a list
|
|
570
|
+
of those, and an optional annotation become the input form. A
|
|
571
|
+
``context: CommandContext`` parameter is filled by
|
|
572
|
+
the session and is not a form field. The function name is the
|
|
573
|
+
capability name unless ``name`` is set. Several commands may share
|
|
574
|
+
one safe point. ``sync`` still runs at most one waiting command.
|
|
575
|
+
|
|
576
|
+
Failure: ``TypeError`` for an unsupported annotation.
|
|
577
|
+
``LaunchHelmError`` from registration.
|
|
578
|
+
"""
|
|
579
|
+
|
|
580
|
+
def decorate(function: CommandFunction) -> CommandFunction:
|
|
581
|
+
capability = command_capability(function, safe_point, name=name)
|
|
582
|
+
self.on(safe_point, capability, adapt(function))
|
|
583
|
+
return function
|
|
584
|
+
|
|
585
|
+
return decorate
|
|
586
|
+
|
|
587
|
+
def sync(self, safe_point: str, *, step: int | None = None, **axes: int | float | str) -> int:
|
|
588
|
+
"""Apply at most one command already claimed for ``safe_point``.
|
|
589
|
+
|
|
590
|
+
Contract: returns ``1`` when a handler ran and ``0`` when the local
|
|
591
|
+
queue is empty. The handler's return value is ``last_result``, or
|
|
592
|
+
``None`` when nothing ran. The first call for a point not declared
|
|
593
|
+
at connect advertises it on one heartbeat so a parked claim can
|
|
594
|
+
observe it; later calls do not call the network and do not
|
|
595
|
+
wait for a command to arrive.
|
|
596
|
+
|
|
597
|
+
Failure: same as ``safe_point``.
|
|
598
|
+
"""
|
|
599
|
+
return self.safe_point(safe_point, step=step, **axes)
|
|
600
|
+
|
|
601
|
+
def register(self, capability: Capability, handler: CapabilityHandler) -> Document:
|
|
602
|
+
"""Publish a capability. Starts command claim on this runtime."""
|
|
603
|
+
return self.runtime.register(capability, handler)
|
|
604
|
+
|
|
605
|
+
def pending(self, safe_point: str) -> bool:
|
|
606
|
+
"""Return ``True`` when a command for ``safe_point`` waits locally.
|
|
607
|
+
|
|
608
|
+
Contract: reads the local claimed-command queue only; no network.
|
|
609
|
+
Distributed adapters use this to broadcast one flag before every
|
|
610
|
+
rank enters the collective handler.
|
|
611
|
+
|
|
612
|
+
Failure: none.
|
|
613
|
+
"""
|
|
614
|
+
return self.runtime.pending(safe_point)
|
|
615
|
+
|
|
616
|
+
def safe_point(self, name: str, *, step: int | None = None, **axes: int | float | str) -> int:
|
|
617
|
+
"""Service one command at ``name``. Starts command claim."""
|
|
618
|
+
coordinates, _ = self._coordinates(step, axes, advance=False)
|
|
619
|
+
applied = self.runtime.safe_point(name, coordinates=coordinates)
|
|
620
|
+
if applied:
|
|
621
|
+
self.last_applied = {item["axis"]: item["value"] for item in coordinates}
|
|
622
|
+
self.last_result = self.runtime.last_result
|
|
623
|
+
else:
|
|
624
|
+
self.last_result = None
|
|
625
|
+
return applied
|
|
626
|
+
|
|
627
|
+
def summary(self, metrics: Mapping[str, JSONValue]) -> None:
|
|
628
|
+
"""Remember scalar end-of-run values. ``finish`` stores them on the run.
|
|
629
|
+
|
|
630
|
+
Contract: values are booleans, strings, or finite numbers. They are
|
|
631
|
+
one run label, ``launchhelm.summary``, not a time series. A later
|
|
632
|
+
call merges keys into the map. ``None`` removes that key. The encoded
|
|
633
|
+
label is at most 1024 characters.
|
|
634
|
+
|
|
635
|
+
Failure: ``RuntimeError`` after ``finish``. ``TypeError`` for a
|
|
636
|
+
nested value. ``ValueError`` for an empty name, a non-finite number,
|
|
637
|
+
or a map that does not fit in the label.
|
|
638
|
+
"""
|
|
639
|
+
if self._closed:
|
|
640
|
+
raise RuntimeError("run is closed")
|
|
641
|
+
stored: dict[str, JSONValue] = dict(self._summary)
|
|
642
|
+
for key, value in metrics.items():
|
|
643
|
+
if not isinstance(key, str) or not key or len(key) > 256:
|
|
644
|
+
raise ValueError("summary name must be 1 to 256 characters")
|
|
645
|
+
if value is None:
|
|
646
|
+
stored.pop(key, None)
|
|
647
|
+
self._summary_locked.discard(key)
|
|
648
|
+
continue
|
|
649
|
+
self._summary_locked.add(key)
|
|
650
|
+
if isinstance(value, bool | str | int):
|
|
651
|
+
stored[key] = value
|
|
652
|
+
elif isinstance(value, float):
|
|
653
|
+
if not math.isfinite(value):
|
|
654
|
+
raise ValueError("summary cannot be non-finite")
|
|
655
|
+
stored[key] = value
|
|
656
|
+
else:
|
|
657
|
+
raise TypeError("summary values must be scalars")
|
|
658
|
+
encoded = json.dumps(stored, allow_nan=False, separators=(",", ":"), sort_keys=True)
|
|
659
|
+
if len(encoded) > 1024:
|
|
660
|
+
raise ValueError("summary is longer than 1024 characters")
|
|
661
|
+
self._summary = stored
|
|
662
|
+
self._summary_changed = True
|
|
663
|
+
self.summary_values = dict(stored)
|
|
664
|
+
|
|
665
|
+
def _saved_mean(self, name: str) -> tuple[int, float] | None:
|
|
666
|
+
if not self.run_id:
|
|
667
|
+
return None
|
|
668
|
+
try:
|
|
669
|
+
found = self._run_document()
|
|
670
|
+
except LaunchHelmError:
|
|
671
|
+
return None
|
|
672
|
+
item = _reduction_item(found, name)
|
|
673
|
+
if item is None:
|
|
674
|
+
return None
|
|
675
|
+
count = item.get("count")
|
|
676
|
+
held = item.get("held")
|
|
677
|
+
if isinstance(count, bool) or not isinstance(count, int) or count < 1:
|
|
678
|
+
return None
|
|
679
|
+
if (
|
|
680
|
+
isinstance(held, bool)
|
|
681
|
+
or not isinstance(held, int | float)
|
|
682
|
+
or not math.isfinite(float(held))
|
|
683
|
+
):
|
|
684
|
+
return None
|
|
685
|
+
return count, float(held)
|
|
686
|
+
|
|
687
|
+
def _note_reduction(self, name: str, encoded: JSONValue) -> None:
|
|
688
|
+
policy = self._reductions.get(name)
|
|
689
|
+
if policy is None or isinstance(encoded, bool) or not isinstance(encoded, int | float):
|
|
690
|
+
return
|
|
691
|
+
if isinstance(encoded, float) and not math.isfinite(encoded):
|
|
692
|
+
return
|
|
693
|
+
value = float(encoded)
|
|
694
|
+
count, held = self._reduced.get(name, (0, 0.0))
|
|
695
|
+
if policy == "min":
|
|
696
|
+
held = value if count == 0 else min(held, value)
|
|
697
|
+
elif policy == "max":
|
|
698
|
+
held = value if count == 0 else max(held, value)
|
|
699
|
+
else:
|
|
700
|
+
held += value
|
|
701
|
+
self._reduced[name] = (count + 1, held)
|
|
702
|
+
|
|
703
|
+
def _apply_reductions(self) -> None:
|
|
704
|
+
changed = False
|
|
705
|
+
for name, policy in self._reductions.items():
|
|
706
|
+
if name in self._summary_locked:
|
|
707
|
+
continue
|
|
708
|
+
stats = self._reduced.get(name)
|
|
709
|
+
if stats is None or stats[0] == 0:
|
|
710
|
+
continue
|
|
711
|
+
count, held = stats
|
|
712
|
+
value = held if policy != "mean" else held / count
|
|
713
|
+
stored = self._summary.get(name)
|
|
714
|
+
if (
|
|
715
|
+
policy in {"min", "max"}
|
|
716
|
+
and isinstance(stored, int | float)
|
|
717
|
+
and not isinstance(stored, bool)
|
|
718
|
+
and math.isfinite(float(stored))
|
|
719
|
+
):
|
|
720
|
+
value = min(value, float(stored)) if policy == "min" else max(value, float(stored))
|
|
721
|
+
elif policy == "mean" and name in self._summary and name not in self._seeded_mean:
|
|
722
|
+
continue
|
|
723
|
+
self._summary[name] = value
|
|
724
|
+
changed = True
|
|
725
|
+
if changed:
|
|
726
|
+
self._summary_changed = True
|
|
727
|
+
self.summary_values = dict(self._summary)
|
|
728
|
+
|
|
729
|
+
def _write_summary(self) -> None:
|
|
730
|
+
self._apply_reductions()
|
|
731
|
+
if not self._summary and not self._summary_changed:
|
|
732
|
+
return
|
|
733
|
+
summary = json.dumps(self._summary, allow_nan=False, separators=(",", ":"), sort_keys=True)
|
|
734
|
+
reduction = {
|
|
735
|
+
name: {"count": count, "held": held}
|
|
736
|
+
for name, policy in self._reductions.items()
|
|
737
|
+
if policy == "mean"
|
|
738
|
+
for count, held in [self._reduced.get(name, (0, 0.0))]
|
|
739
|
+
if count
|
|
740
|
+
}
|
|
741
|
+
|
|
742
|
+
def apply(found: Document) -> JSONObject:
|
|
743
|
+
stored = found.spec.get("labels")
|
|
744
|
+
labels: dict[str, JSONValue] = dict(stored) if isinstance(stored, dict) else {}
|
|
745
|
+
labels["launchhelm.summary"] = summary
|
|
746
|
+
if reduction:
|
|
747
|
+
labels["launchhelm.reduction"] = json.dumps(
|
|
748
|
+
reduction, allow_nan=False, separators=(",", ":"), sort_keys=True
|
|
749
|
+
)
|
|
750
|
+
spec = dict(found.spec)
|
|
751
|
+
spec["labels"] = labels
|
|
752
|
+
return spec
|
|
753
|
+
|
|
754
|
+
self._update_run(apply)
|
|
755
|
+
|
|
756
|
+
def _update_run(
|
|
757
|
+
self, mutate: Callable[[Document], JSONObject], *, retries: int = 5
|
|
758
|
+
) -> Document:
|
|
759
|
+
"""Re-read the run and apply ``mutate`` to its spec.
|
|
760
|
+
|
|
761
|
+
The write runs under the original credential and retries a bounded
|
|
762
|
+
number of revision conflicts so concurrent participants do not lose
|
|
763
|
+
their keys.
|
|
764
|
+
"""
|
|
765
|
+
for attempt in range(retries):
|
|
766
|
+
found = self._run_document(self._metadata)
|
|
767
|
+
try:
|
|
768
|
+
return self._metadata.update_document(found, mutate(found))
|
|
769
|
+
except LaunchHelmError as error:
|
|
770
|
+
if error.code != "conflict" or attempt == retries - 1:
|
|
771
|
+
raise
|
|
772
|
+
raise LaunchHelmError("conflict", "run document is contended")
|
|
773
|
+
|
|
774
|
+
def _close_run(self) -> None:
|
|
775
|
+
if not self.run_id:
|
|
776
|
+
return
|
|
777
|
+
|
|
778
|
+
def close(found: Document) -> JSONObject:
|
|
779
|
+
spec = dict(found.spec)
|
|
780
|
+
spec["state"] = "closed"
|
|
781
|
+
return spec
|
|
782
|
+
|
|
783
|
+
found = self._run_document(self._metadata)
|
|
784
|
+
if found.spec.get("state") == "closed":
|
|
785
|
+
return
|
|
786
|
+
self._metadata.update_document(found, close(found))
|
|
787
|
+
|
|
788
|
+
def _run_document(self, client: SessionClient | None = None) -> Document:
|
|
789
|
+
if not self.run_id:
|
|
790
|
+
raise LaunchHelmError("protocol", "run document is missing")
|
|
791
|
+
reader = client if client is not None else self._client
|
|
792
|
+
found = reader.get_document(self.run_id)
|
|
793
|
+
if found.kind != "run" or found.id != self.run_id:
|
|
794
|
+
raise LaunchHelmError("protocol", "run document is missing")
|
|
795
|
+
return found
|
|
796
|
+
|
|
797
|
+
def about(self, run: str = "") -> dict[str, JSONValue]:
|
|
798
|
+
"""Return the stored name, state, group, notes, tags, and summary for ``run``.
|
|
799
|
+
|
|
800
|
+
Contract: ``run`` is a title or id in this project. ``summary`` is the
|
|
801
|
+
parsed ``launchhelm.summary`` object, or ``{}`` when that label is
|
|
802
|
+
absent. ``config`` is the label map with the reserved ``launchhelm.*``
|
|
803
|
+
keys removed. A ``parameters`` spec object carries the full config when
|
|
804
|
+
values do not fit in labels; older runs may hold a ``launchhelm.config``
|
|
805
|
+
artifact reference instead.
|
|
806
|
+
``git``, ``host``, and ``argv`` are the recorded commit,
|
|
807
|
+
machine, and command. ``tags`` is a list of strings. This does not open the run.
|
|
808
|
+
|
|
809
|
+
Failure: ``ValueError`` when ``run`` is empty and this session has no run.
|
|
810
|
+
``LaunchHelmError``
|
|
811
|
+
when ``run`` is not one run in this project or a stored label is malformed.
|
|
812
|
+
"""
|
|
813
|
+
if not isinstance(run, str):
|
|
814
|
+
raise ValueError("run name is required")
|
|
815
|
+
if not run:
|
|
816
|
+
run = self.run_id or self.name
|
|
817
|
+
if not run:
|
|
818
|
+
raise ValueError("run name is required")
|
|
819
|
+
found = [
|
|
820
|
+
document
|
|
821
|
+
for document in self._project_runs()
|
|
822
|
+
if document.id == run or document.spec.get("name") == run
|
|
823
|
+
]
|
|
824
|
+
unique = {document.id: document for document in found}
|
|
825
|
+
if len(unique) != 1:
|
|
826
|
+
raise LaunchHelmError("config", f"run {run!r} is not one run in this project")
|
|
827
|
+
document = next(iter(unique.values()))
|
|
828
|
+
raw_summary = _metadata_value(document, "launchhelm.summary")
|
|
829
|
+
summary: JSONValue = {}
|
|
830
|
+
if raw_summary:
|
|
831
|
+
parsed = json.loads(raw_summary)
|
|
832
|
+
if not isinstance(parsed, dict):
|
|
833
|
+
raise LaunchHelmError("protocol", "run summary is not an object")
|
|
834
|
+
summary = parsed
|
|
835
|
+
name = document.spec.get("name")
|
|
836
|
+
state = document.spec.get("state")
|
|
837
|
+
return {
|
|
838
|
+
"name": name if isinstance(name, str) else "",
|
|
839
|
+
"state": state if isinstance(state, str) else "",
|
|
840
|
+
"group": _metadata_value(document, "launchhelm.group"),
|
|
841
|
+
"notes": _metadata_value(document, "launchhelm.notes"),
|
|
842
|
+
"tags": list(_metadata_tags(document)),
|
|
843
|
+
"summary": summary,
|
|
844
|
+
"config": _resume_config(self._client, document),
|
|
845
|
+
"git": _metadata_value(document, "launchhelm.git"),
|
|
846
|
+
"host": _metadata_value(document, "launchhelm.host"),
|
|
847
|
+
"argv": _metadata_value(document, "launchhelm.argv"),
|
|
848
|
+
}
|
|
849
|
+
|
|
850
|
+
def runs(self) -> list[str]:
|
|
851
|
+
"""Return the titles of runs in this project that share this group.
|
|
852
|
+
|
|
853
|
+
Contract: titles are unique and sorted. An empty group returns every
|
|
854
|
+
run in the project. A set group returns only runs whose
|
|
855
|
+
``launchhelm.group`` label matches. This does not open those runs.
|
|
856
|
+
|
|
857
|
+
Failure: ``LaunchHelmError`` when a run page is malformed.
|
|
858
|
+
"""
|
|
859
|
+
return [name for name, _state in self.statuses()]
|
|
860
|
+
|
|
861
|
+
def statuses(self) -> list[tuple[str, str]]:
|
|
862
|
+
"""Return ``(title, state)`` for runs in this group, sorted by title.
|
|
863
|
+
|
|
864
|
+
Contract: the same group filter as ``runs``. ``state`` is the run
|
|
865
|
+
document state, or ``""`` when it is missing. A closed sibling stays
|
|
866
|
+
in the list so a sweep can skip it.
|
|
867
|
+
|
|
868
|
+
Failure: ``LaunchHelmError`` when a run page is malformed.
|
|
869
|
+
"""
|
|
870
|
+
found: dict[str, str] = {}
|
|
871
|
+
for document in self._project_runs():
|
|
872
|
+
if self.group and _metadata_value(document, "launchhelm.group") != self.group:
|
|
873
|
+
continue
|
|
874
|
+
name = document.spec.get("name")
|
|
875
|
+
if isinstance(name, str) and name:
|
|
876
|
+
state = document.spec.get("state")
|
|
877
|
+
found[name] = state if isinstance(state, str) else ""
|
|
878
|
+
return sorted(found.items())
|
|
879
|
+
|
|
880
|
+
def _project_runs(self) -> list[Document]:
|
|
881
|
+
found: list[Document] = []
|
|
882
|
+
cursor = ""
|
|
883
|
+
for _ in range(_MAX_PAGES):
|
|
884
|
+
page = self._client.list_documents(
|
|
885
|
+
kind="run", project_id=self.project_id, cursor=cursor, limit=100
|
|
886
|
+
)
|
|
887
|
+
items = page.get("items") or []
|
|
888
|
+
if not isinstance(items, list):
|
|
889
|
+
raise LaunchHelmError("protocol", "run list is not a page")
|
|
890
|
+
for item in items:
|
|
891
|
+
if not isinstance(item, dict):
|
|
892
|
+
raise LaunchHelmError("protocol", "run list entry is not a document")
|
|
893
|
+
document = Document.parse(item)
|
|
894
|
+
if document.kind == "run" and document.scope.project_id == self.project_id:
|
|
895
|
+
found.append(document)
|
|
896
|
+
next_cursor = page.get("next_cursor") or ""
|
|
897
|
+
if not isinstance(next_cursor, str) or not next_cursor or next_cursor == cursor:
|
|
898
|
+
break
|
|
899
|
+
cursor = next_cursor
|
|
900
|
+
return found
|
|
901
|
+
|
|
902
|
+
def describe(
|
|
903
|
+
self,
|
|
904
|
+
*,
|
|
905
|
+
group: str | None = None,
|
|
906
|
+
tags: Sequence[str] | None = None,
|
|
907
|
+
notes: str | None = None,
|
|
908
|
+
name: str | None = None,
|
|
909
|
+
) -> None:
|
|
910
|
+
"""Update the run group, tags, notes, or title without touching config.
|
|
911
|
+
|
|
912
|
+
Contract: an omitted argument keeps the current value. An empty
|
|
913
|
+
group, tag list, or note removes that label. The title cannot be
|
|
914
|
+
empty. Config and the end-of-run summary stay on the run.
|
|
915
|
+
|
|
916
|
+
Failure: ``RuntimeError`` after ``finish``. ``ValueError`` for a
|
|
917
|
+
group, note, tag, or title that does not fit. ``TypeError`` when
|
|
918
|
+
tags are not strings.
|
|
919
|
+
"""
|
|
920
|
+
if self._closed:
|
|
921
|
+
raise RuntimeError("run is closed")
|
|
922
|
+
if name is not None and (not isinstance(name, str) or not name or len(name) > 256):
|
|
923
|
+
raise ValueError("run name must be 1 to 256 characters")
|
|
924
|
+
next_group = self.group if group is None else group
|
|
925
|
+
next_tags = self.tags if tags is None else tags
|
|
926
|
+
next_notes = self.notes if notes is None else notes
|
|
927
|
+
staged: JSONObject = {}
|
|
928
|
+
_metadata(staged, next_group, next_tags, next_notes)
|
|
929
|
+
self.group = next_group
|
|
930
|
+
self.tags = tuple(next_tags)
|
|
931
|
+
self.notes = next_notes
|
|
932
|
+
if name is not None:
|
|
933
|
+
self.name = name
|
|
934
|
+
self._write_metadata()
|
|
935
|
+
|
|
936
|
+
def _write_metadata(self) -> None:
|
|
937
|
+
def apply(found: Document) -> JSONObject:
|
|
938
|
+
stored = found.spec.get("labels")
|
|
939
|
+
labels: dict[str, JSONValue] = dict(stored) if isinstance(stored, dict) else {}
|
|
940
|
+
for key, value in (
|
|
941
|
+
("launchhelm.group", self.group),
|
|
942
|
+
("launchhelm.notes", self.notes),
|
|
943
|
+
):
|
|
944
|
+
if value:
|
|
945
|
+
labels[key] = value
|
|
946
|
+
else:
|
|
947
|
+
labels.pop(key, None)
|
|
948
|
+
if self.tags:
|
|
949
|
+
labels["launchhelm.tags"] = json.dumps(list(self.tags), separators=(",", ":"))
|
|
950
|
+
else:
|
|
951
|
+
labels.pop("launchhelm.tags", None)
|
|
952
|
+
spec = dict(found.spec)
|
|
953
|
+
spec["labels"] = labels
|
|
954
|
+
if self.name:
|
|
955
|
+
spec["name"] = self.name
|
|
956
|
+
return spec
|
|
957
|
+
|
|
958
|
+
self._update_run(apply)
|
|
959
|
+
|
|
960
|
+
def keep(self, state: Mapping[str, JSONValue]) -> None:
|
|
961
|
+
"""Save trainer state so a later ``init(..., resume=)`` can continue.
|
|
962
|
+
|
|
963
|
+
Contract: ``state`` is JSON data owned by the trainer, such as the
|
|
964
|
+
learning rate, schedule boundaries, and layer weights. The file also
|
|
965
|
+
records the next log step and the run id. A crash after ``keep`` resumes from this
|
|
966
|
+
snapshot. Logs already admitted stay in the local spool until
|
|
967
|
+
``finish`` uploads them.
|
|
968
|
+
|
|
969
|
+
Failure: ``TypeError`` when ``state`` is not JSON. ``OSError`` when
|
|
970
|
+
the checkpoint cannot be replaced.
|
|
971
|
+
"""
|
|
972
|
+
payload = {
|
|
973
|
+
"step": self._next_step,
|
|
974
|
+
"participant_id": self.runtime.participant.id,
|
|
975
|
+
"state": state,
|
|
976
|
+
}
|
|
977
|
+
if self.run_id:
|
|
978
|
+
payload["run_id"] = self.run_id
|
|
979
|
+
if self._summary_changed:
|
|
980
|
+
payload["summary"] = self._summary
|
|
981
|
+
encoded = json.dumps(payload, allow_nan=False, separators=(",", ":"))
|
|
982
|
+
temporary = self._checkpoint.with_suffix(".tmp")
|
|
983
|
+
temporary.write_text(encoded, encoding="utf-8")
|
|
984
|
+
temporary.replace(self._checkpoint)
|
|
985
|
+
|
|
986
|
+
def artifact(
|
|
987
|
+
self,
|
|
988
|
+
path: str | Path,
|
|
989
|
+
*,
|
|
990
|
+
step: int | None = None,
|
|
991
|
+
name: str | None = None,
|
|
992
|
+
media_type: str = "application/octet-stream",
|
|
993
|
+
idempotency_key: str = "",
|
|
994
|
+
**axes: int | float | str,
|
|
995
|
+
) -> Document:
|
|
996
|
+
"""Upload one file as an artifact at these coordinates.
|
|
997
|
+
|
|
998
|
+
Contract: ``log`` stays scalar-only. A checkpoint, image, or histogram
|
|
999
|
+
file uses this call. An omitted ``step`` is the current counter and
|
|
1000
|
+
does not advance it. Pass the same ``step`` just logged when the file
|
|
1001
|
+
belongs to that point. ``name`` also records the artifact id as a
|
|
1002
|
+
string metric at those coordinates and does not advance the counter.
|
|
1003
|
+
The upload runs on this thread and does not
|
|
1004
|
+
wait for telemetry already in the queue.
|
|
1005
|
+
|
|
1006
|
+
Failure: ``LaunchHelmError`` when the client cannot upload or the
|
|
1007
|
+
result is not an artifact document. Coordinate errors match ``log``.
|
|
1008
|
+
"""
|
|
1009
|
+
return self._artifact(
|
|
1010
|
+
path,
|
|
1011
|
+
step=step,
|
|
1012
|
+
name=name,
|
|
1013
|
+
media_type=media_type,
|
|
1014
|
+
idempotency_key=idempotency_key,
|
|
1015
|
+
axes=axes,
|
|
1016
|
+
)
|
|
1017
|
+
|
|
1018
|
+
def _artifact(
|
|
1019
|
+
self,
|
|
1020
|
+
path: str | Path,
|
|
1021
|
+
*,
|
|
1022
|
+
step: int | None,
|
|
1023
|
+
name: str | None,
|
|
1024
|
+
media_type: str,
|
|
1025
|
+
idempotency_key: str,
|
|
1026
|
+
axes: Mapping[str, int | float | str],
|
|
1027
|
+
) -> Document:
|
|
1028
|
+
upload = getattr(self.runtime.client, "upload_file", None)
|
|
1029
|
+
if not callable(upload):
|
|
1030
|
+
raise LaunchHelmError("protocol", "session client cannot upload artifacts")
|
|
1031
|
+
if name is not None and (not isinstance(name, str) or not name or len(name) > 256):
|
|
1032
|
+
raise TypeError("metric name must be 1 to 256 characters")
|
|
1033
|
+
coordinates, axis_names = self._coordinates(step, axes, advance=False)
|
|
1034
|
+
if name is not None:
|
|
1035
|
+
known = self._streams.get(name)
|
|
1036
|
+
if known is not None and known[1] != "string":
|
|
1037
|
+
raise TypeError(f"metric {name!r} is {known[1]}, not string")
|
|
1038
|
+
if known is not None and known[2] != axis_names:
|
|
1039
|
+
raise ValueError(f"metric {name!r} axes are fixed")
|
|
1040
|
+
points: list[JSONValue] = [dict(item) for item in coordinates]
|
|
1041
|
+
provenance: JSONObject = {"inputs": [], "coordinates": points, "labels": {}}
|
|
1042
|
+
document = upload(
|
|
1043
|
+
path,
|
|
1044
|
+
self.runtime.scope,
|
|
1045
|
+
idempotency_key=idempotency_key or str(uuid4()),
|
|
1046
|
+
media_type=media_type,
|
|
1047
|
+
provenance=provenance,
|
|
1048
|
+
)
|
|
1049
|
+
if not isinstance(document, Document):
|
|
1050
|
+
raise LaunchHelmError("protocol", "upload did not return an artifact document")
|
|
1051
|
+
if name is not None:
|
|
1052
|
+
self._open_streams([(name, "string")], axis_names)
|
|
1053
|
+
stream = self._stream(name, "string", axis_names)
|
|
1054
|
+
if not self.runtime.log(stream, document.id, coordinates=coordinates):
|
|
1055
|
+
raise LaunchHelmError("protocol", "artifact reference was dropped")
|
|
1056
|
+
return document
|
|
1057
|
+
|
|
1058
|
+
def fetch(self, name: str, *, run: str | None = None) -> bytes: # noqa: PLR0912 - media/general artifact dispatch stays readable as one flow
|
|
1059
|
+
"""Download the newest artifact recorded under ``name``.
|
|
1060
|
+
|
|
1061
|
+
Contract: ``artifact(..., name=)`` stores the artifact id as a string
|
|
1062
|
+
metric. This returns the file for the newest point of that metric.
|
|
1063
|
+
``run`` is another run in this project, the same way ``latest``
|
|
1064
|
+
selects one. A later upload of the same name is the file a resumed
|
|
1065
|
+
run receives.
|
|
1066
|
+
|
|
1067
|
+
Failure: ``ValueError`` for an empty name. ``LaunchHelmError`` when
|
|
1068
|
+
the metric has no point, the value is not an artifact id, or the
|
|
1069
|
+
client cannot download the file.
|
|
1070
|
+
"""
|
|
1071
|
+
if not isinstance(name, str) or not name:
|
|
1072
|
+
raise ValueError("metric name is required")
|
|
1073
|
+
point = self.latest(name, run=run)
|
|
1074
|
+
if point is None:
|
|
1075
|
+
raise LaunchHelmError("protocol", "artifact metric has no point")
|
|
1076
|
+
value = point.get("value")
|
|
1077
|
+
if isinstance(value, Mapping):
|
|
1078
|
+
items = value.get("items")
|
|
1079
|
+
if isinstance(items, list):
|
|
1080
|
+
artifact_id = ""
|
|
1081
|
+
for item in items:
|
|
1082
|
+
if not isinstance(item, Mapping):
|
|
1083
|
+
continue
|
|
1084
|
+
source = item.get("source")
|
|
1085
|
+
if isinstance(source, Mapping):
|
|
1086
|
+
artifact = source.get("artifact")
|
|
1087
|
+
if isinstance(artifact, Mapping):
|
|
1088
|
+
candidate = artifact.get("id")
|
|
1089
|
+
if isinstance(candidate, str):
|
|
1090
|
+
artifact_id = candidate
|
|
1091
|
+
break
|
|
1092
|
+
if artifact_id:
|
|
1093
|
+
download = getattr(self._client, "download_artifact", None)
|
|
1094
|
+
if not callable(download):
|
|
1095
|
+
raise LaunchHelmError(
|
|
1096
|
+
"protocol", "session client cannot download artifacts"
|
|
1097
|
+
)
|
|
1098
|
+
body = download(artifact_id)
|
|
1099
|
+
if not isinstance(body, bytes):
|
|
1100
|
+
raise LaunchHelmError("protocol", "artifact download is not bytes")
|
|
1101
|
+
return body
|
|
1102
|
+
raise LaunchHelmError("protocol", "media stream has no native item")
|
|
1103
|
+
if not isinstance(value, str) or not value:
|
|
1104
|
+
raise LaunchHelmError("protocol", "artifact metric is not an id")
|
|
1105
|
+
artifact_id = value
|
|
1106
|
+
download = getattr(self._client, "download_artifact", None)
|
|
1107
|
+
if not callable(download):
|
|
1108
|
+
raise LaunchHelmError("protocol", "session client cannot download artifacts")
|
|
1109
|
+
body = download(artifact_id)
|
|
1110
|
+
if not isinstance(body, bytes):
|
|
1111
|
+
raise LaunchHelmError("protocol", "artifact download is not bytes")
|
|
1112
|
+
return body
|
|
1113
|
+
|
|
1114
|
+
def checkpoint(
|
|
1115
|
+
self,
|
|
1116
|
+
path: str | Path,
|
|
1117
|
+
*,
|
|
1118
|
+
step: int | None = None,
|
|
1119
|
+
name: str = "checkpoint",
|
|
1120
|
+
**axes: int | float | str,
|
|
1121
|
+
) -> Document:
|
|
1122
|
+
"""Upload a checkpoint file and record its id as a string metric.
|
|
1123
|
+
|
|
1124
|
+
Contract: this is ``artifact`` with the metric name ``checkpoint``.
|
|
1125
|
+
``fetch(name)`` returns the newest file. An omitted ``step`` is the
|
|
1126
|
+
current counter and does not advance it.
|
|
1127
|
+
|
|
1128
|
+
Failure: same as ``artifact``.
|
|
1129
|
+
"""
|
|
1130
|
+
return self._artifact(
|
|
1131
|
+
path,
|
|
1132
|
+
step=step,
|
|
1133
|
+
name=name,
|
|
1134
|
+
media_type="application/octet-stream",
|
|
1135
|
+
idempotency_key="",
|
|
1136
|
+
axes=axes,
|
|
1137
|
+
)
|
|
1138
|
+
|
|
1139
|
+
def text(
|
|
1140
|
+
self,
|
|
1141
|
+
name: str,
|
|
1142
|
+
text: str,
|
|
1143
|
+
*,
|
|
1144
|
+
format: str | None = None,
|
|
1145
|
+
caption: str | None = None,
|
|
1146
|
+
step: int | None = None,
|
|
1147
|
+
**axes: int | float | str,
|
|
1148
|
+
) -> bool:
|
|
1149
|
+
"""Log one ``rich_text`` point on the ``text`` renderer stream.
|
|
1150
|
+
|
|
1151
|
+
Contract: creates or reuses the stream ``name`` with presentation
|
|
1152
|
+
``text``. An omitted ``step`` is the current counter and does not
|
|
1153
|
+
advance it. Returns ``False`` when the point is dropped under
|
|
1154
|
+
``drop_new``.
|
|
1155
|
+
|
|
1156
|
+
Failure: ``TypeError`` for a bad name. ``ValueError`` for an
|
|
1157
|
+
over-long text, unknown format, or a stream conflict.
|
|
1158
|
+
"""
|
|
1159
|
+
return self._rich_log(Text(text, format=format, caption=caption), name, step, axes)
|
|
1160
|
+
|
|
1161
|
+
def image(
|
|
1162
|
+
self,
|
|
1163
|
+
name: str,
|
|
1164
|
+
source: str | Path | bytes | object,
|
|
1165
|
+
*,
|
|
1166
|
+
media_type: str | None = None,
|
|
1167
|
+
caption: str | None = None,
|
|
1168
|
+
step: int | None = None,
|
|
1169
|
+
**axes: int | float | str,
|
|
1170
|
+
) -> bool:
|
|
1171
|
+
"""Log one image item on the ``media`` renderer stream ``name``.
|
|
1172
|
+
|
|
1173
|
+
Contract: ``source`` is a path, bytes with ``media_type``, a PIL
|
|
1174
|
+
image when Pillow is installed, or a NumPy array / torch tensor
|
|
1175
|
+
(HW or HWC with 1/3/4 channels; uint8 or floats in [0, 1]) encoded
|
|
1176
|
+
to PNG. The upload runs on a background thread and the point is
|
|
1177
|
+
admitted once the ``media`` artifact exists; ``flush()`` waits
|
|
1178
|
+
for that admission. An omitted ``step`` does not advance the
|
|
1179
|
+
counter.
|
|
1180
|
+
|
|
1181
|
+
Failure: ``TypeError`` for a bad name or source kind.
|
|
1182
|
+
``ValueError`` for an uninferable or unallowlisted media type,
|
|
1183
|
+
an empty or over-16 MiB payload, or a stream conflict.
|
|
1184
|
+
"""
|
|
1185
|
+
return self._rich_log(
|
|
1186
|
+
Image(source, media_type=media_type, caption=caption), name, step, axes
|
|
1187
|
+
)
|
|
1188
|
+
|
|
1189
|
+
def audio(
|
|
1190
|
+
self,
|
|
1191
|
+
name: str,
|
|
1192
|
+
source: str | Path | bytes,
|
|
1193
|
+
*,
|
|
1194
|
+
media_type: str | None = None,
|
|
1195
|
+
caption: str | None = None,
|
|
1196
|
+
step: int | None = None,
|
|
1197
|
+
**axes: int | float | str,
|
|
1198
|
+
) -> bool:
|
|
1199
|
+
"""Log one audio clip on the ``media`` renderer stream ``name``.
|
|
1200
|
+
|
|
1201
|
+
Contract: matches ``image`` for path/bytes sources. The media
|
|
1202
|
+
type comes from the suffix (``wav``, ``mp3``, ``ogg``, ``flac``,
|
|
1203
|
+
``weba``) or ``media_type`` and must be allowlisted.
|
|
1204
|
+
"""
|
|
1205
|
+
return self._rich_log(
|
|
1206
|
+
Audio(source, media_type=media_type, caption=caption), name, step, axes
|
|
1207
|
+
)
|
|
1208
|
+
|
|
1209
|
+
def video(
|
|
1210
|
+
self,
|
|
1211
|
+
name: str,
|
|
1212
|
+
source: str | Path | bytes,
|
|
1213
|
+
*,
|
|
1214
|
+
media_type: str | None = None,
|
|
1215
|
+
caption: str | None = None,
|
|
1216
|
+
step: int | None = None,
|
|
1217
|
+
**axes: int | float | str,
|
|
1218
|
+
) -> bool:
|
|
1219
|
+
"""Log one video on the ``media`` renderer stream ``name``.
|
|
1220
|
+
|
|
1221
|
+
Contract: matches ``image`` for path/bytes sources; the suffixes
|
|
1222
|
+
``mp4`` and ``webm`` infer video types.
|
|
1223
|
+
"""
|
|
1224
|
+
return self._rich_log(
|
|
1225
|
+
Video(source, media_type=media_type, caption=caption), name, step, axes
|
|
1226
|
+
)
|
|
1227
|
+
|
|
1228
|
+
def media(
|
|
1229
|
+
self,
|
|
1230
|
+
name: str,
|
|
1231
|
+
items: Sequence[object],
|
|
1232
|
+
*,
|
|
1233
|
+
step: int | None = None,
|
|
1234
|
+
**axes: int | float | str,
|
|
1235
|
+
) -> bool:
|
|
1236
|
+
"""Log a media grid on the ``media`` renderer stream ``name``.
|
|
1237
|
+
|
|
1238
|
+
Contract: ``items`` mix entries from ``upload_media``,
|
|
1239
|
+
``media_url``, and ``Image``/``Audio``/``Video`` values — up to
|
|
1240
|
+
64 items. Native items upload in the background; external items
|
|
1241
|
+
admit at once.
|
|
1242
|
+
"""
|
|
1243
|
+
return self._rich_log(MediaGrid(list(items)), name, step, axes)
|
|
1244
|
+
|
|
1245
|
+
def upload_media(
|
|
1246
|
+
self,
|
|
1247
|
+
source: str | Path | bytes,
|
|
1248
|
+
*,
|
|
1249
|
+
media_type: str | None = None,
|
|
1250
|
+
caption: str | None = None,
|
|
1251
|
+
) -> JSONObject:
|
|
1252
|
+
"""Stage local media bytes for a later ``media``/``log_table`` call.
|
|
1253
|
+
|
|
1254
|
+
Contract: returns a media item whose artifact reference is a
|
|
1255
|
+
pending marker; the bytes are registered with this run and the
|
|
1256
|
+
background uploader creates the ``media`` artifact (idempotent
|
|
1257
|
+
on content hash and run — identical bytes share one artifact).
|
|
1258
|
+
The point that uses the item is admitted after the upload.
|
|
1259
|
+
Synchronous-from-caller shape is NOT the contract here — unlike
|
|
1260
|
+
the TypeScript/Rust/C++ SDKs, the upload always happens off the
|
|
1261
|
+
training thread.
|
|
1262
|
+
|
|
1263
|
+
Failure: ``ValueError`` for an uninferable/unallowlisted media
|
|
1264
|
+
type, empty bytes, or over-16 MiB payloads.
|
|
1265
|
+
"""
|
|
1266
|
+
if isinstance(source, str | Path):
|
|
1267
|
+
path = Path(source)
|
|
1268
|
+
resolved = media_type or _SUFFIX_MEDIA.get(path.suffix.lower())
|
|
1269
|
+
if resolved is None:
|
|
1270
|
+
raise ValueError(f"cannot infer media type from {path.suffix!r}; pass media_type")
|
|
1271
|
+
data = path.read_bytes()
|
|
1272
|
+
else:
|
|
1273
|
+
resolved = media_type
|
|
1274
|
+
data = bytes(source)
|
|
1275
|
+
if resolved is None:
|
|
1276
|
+
raise ValueError("bytes sources need an explicit media_type")
|
|
1277
|
+
_check_media_type(resolved)
|
|
1278
|
+
if not data or len(data) > MEDIA_OBJECT_BYTES:
|
|
1279
|
+
raise ValueError("media must be 1 byte to 16 MiB")
|
|
1280
|
+
blob = PendingBlob(_sha256(data).hexdigest(), resolved, data)
|
|
1281
|
+
self._media_blobs[blob.hash] = blob
|
|
1282
|
+
return pending_item(blob, caption)
|
|
1283
|
+
|
|
1284
|
+
def media_url(self, href: str, media_type: str, *, caption: str | None = None) -> JSONObject:
|
|
1285
|
+
"""Build an external media item; needs no upload."""
|
|
1286
|
+
return _media_url_item(href, media_type, caption)
|
|
1287
|
+
|
|
1288
|
+
def rows(
|
|
1289
|
+
self,
|
|
1290
|
+
name: str,
|
|
1291
|
+
records: Sequence[Mapping[str, JSONValue]],
|
|
1292
|
+
*,
|
|
1293
|
+
step: int | None = None,
|
|
1294
|
+
**axes: int | float | str,
|
|
1295
|
+
) -> bool:
|
|
1296
|
+
"""Log scalar records as a ``rich_table`` point on ``name``.
|
|
1297
|
+
|
|
1298
|
+
Contract: columns are the sorted key set with inferred types —
|
|
1299
|
+
``integer`` only when every non-null value is an int, ``number``
|
|
1300
|
+
for mixed numerics, ``boolean``/``string``/``media`` for those
|
|
1301
|
+
cell kinds; mixed unrelated types are rejected. A missing key
|
|
1302
|
+
logs null. An omitted ``step`` does not advance the counter.
|
|
1303
|
+
"""
|
|
1304
|
+
if not isinstance(name, str) or not name or len(name) > 256:
|
|
1305
|
+
raise TypeError("metric name must be 1 to 256 characters")
|
|
1306
|
+
columns, body, blobs = infer_columns(records)
|
|
1307
|
+
return self._log_table_value(name, columns, body, blobs, step, axes)
|
|
1308
|
+
|
|
1309
|
+
def log_table(
|
|
1310
|
+
self,
|
|
1311
|
+
name: str,
|
|
1312
|
+
columns: Sequence[object],
|
|
1313
|
+
rows: Sequence[Sequence[object]],
|
|
1314
|
+
*,
|
|
1315
|
+
step: int | None = None,
|
|
1316
|
+
**axes: int | float | str,
|
|
1317
|
+
) -> bool:
|
|
1318
|
+
"""Log a ``rich_table`` point with explicit columns on ``name``.
|
|
1319
|
+
|
|
1320
|
+
Contract: named ``log_table`` because ``table`` is the existing
|
|
1321
|
+
read method. ``columns`` are ``{"name", "type"}`` mappings or
|
|
1322
|
+
``(name, type)`` pairs; a ``media`` column takes media items from
|
|
1323
|
+
``upload_media``/``media_url`` or ``Image``/``Audio``/``Video``
|
|
1324
|
+
values. Cells may be null.
|
|
1325
|
+
"""
|
|
1326
|
+
if not isinstance(name, str) or not name or len(name) > 256:
|
|
1327
|
+
raise TypeError("metric name must be 1 to 256 characters")
|
|
1328
|
+
compiled = compile_columns(columns)
|
|
1329
|
+
body, blobs = compile_rows(compiled, rows)
|
|
1330
|
+
return self._log_table_value(name, compiled, body, blobs, step, axes)
|
|
1331
|
+
|
|
1332
|
+
def histogram(
|
|
1333
|
+
self,
|
|
1334
|
+
name: str,
|
|
1335
|
+
values: Sequence[int | float],
|
|
1336
|
+
*,
|
|
1337
|
+
bins: int = 64,
|
|
1338
|
+
step: int | None = None,
|
|
1339
|
+
**axes: int | float | str,
|
|
1340
|
+
) -> bool:
|
|
1341
|
+
"""Log a binned ``rich_histogram`` point on ``name``.
|
|
1342
|
+
|
|
1343
|
+
Contract: equal-width bins from the minimum to the maximum with
|
|
1344
|
+
the maximum in the last bin; equal samples use one bin; ``bins``
|
|
1345
|
+
is 1 to 512 (default 64). Only finite values are accepted. An
|
|
1346
|
+
omitted ``step`` does not advance the counter.
|
|
1347
|
+
"""
|
|
1348
|
+
if not isinstance(name, str) or not name or len(name) > 256:
|
|
1349
|
+
raise TypeError("metric name must be 1 to 256 characters")
|
|
1350
|
+
edges, counts = bin_values(values, bins=bins)
|
|
1351
|
+
return self._rich_log(Histogram.from_counts(edges, counts), name, step, axes)
|
|
1352
|
+
|
|
1353
|
+
def histogram_counts(
|
|
1354
|
+
self,
|
|
1355
|
+
name: str,
|
|
1356
|
+
edges: Sequence[int | float],
|
|
1357
|
+
counts: Sequence[int | float],
|
|
1358
|
+
*,
|
|
1359
|
+
step: int | None = None,
|
|
1360
|
+
**axes: int | float | str,
|
|
1361
|
+
) -> bool:
|
|
1362
|
+
"""Log a ``rich_histogram`` point from explicit bins on ``name``.
|
|
1363
|
+
|
|
1364
|
+
Contract: ``edges`` are finite and strictly increasing and
|
|
1365
|
+
``counts`` holds one non-negative finite entry per bin. Optional
|
|
1366
|
+
statistics pass as keyword arguments (``underflow``, ``overflow``,
|
|
1367
|
+
``total``, ``min``, ``max``, ``mean``, ``std``) and must be
|
|
1368
|
+
finite — the named trio non-negative.
|
|
1369
|
+
"""
|
|
1370
|
+
if not isinstance(name, str) or not name or len(name) > 256:
|
|
1371
|
+
raise TypeError("metric name must be 1 to 256 characters")
|
|
1372
|
+
stats = {
|
|
1373
|
+
key: value
|
|
1374
|
+
for key, value in axes.items()
|
|
1375
|
+
if key in _HISTOGRAM_STATS and isinstance(value, int | float)
|
|
1376
|
+
}
|
|
1377
|
+
point_axes = {key: value for key, value in axes.items() if key not in _HISTOGRAM_STATS}
|
|
1378
|
+
return self._rich_log(Histogram.from_counts(edges, counts, **stats), name, step, point_axes)
|
|
1379
|
+
|
|
1380
|
+
def _rich_log(
|
|
1381
|
+
self,
|
|
1382
|
+
value: RichValue,
|
|
1383
|
+
name: str,
|
|
1384
|
+
step: int | None,
|
|
1385
|
+
axes: Mapping[str, int | float | str],
|
|
1386
|
+
) -> bool:
|
|
1387
|
+
if not isinstance(name, str) or not name or len(name) > 256:
|
|
1388
|
+
raise TypeError("metric name must be 1 to 256 characters")
|
|
1389
|
+
if self._closed:
|
|
1390
|
+
raise RuntimeError("run is closed")
|
|
1391
|
+
coordinates, axis_names = self._coordinates(step, axes, advance=False)
|
|
1392
|
+
document = self._rich_stream(name, value.renderer, axis_names)
|
|
1393
|
+
return self._log_rich(document, value, coordinates=coordinates)
|
|
1394
|
+
|
|
1395
|
+
def _log_table_value(
|
|
1396
|
+
self,
|
|
1397
|
+
name: str,
|
|
1398
|
+
columns: list[tuple[str, str]],
|
|
1399
|
+
rows: list[list[JSONValue]],
|
|
1400
|
+
blobs: list[PendingBlob],
|
|
1401
|
+
step: int | None,
|
|
1402
|
+
axes: Mapping[str, int | float | str],
|
|
1403
|
+
) -> bool:
|
|
1404
|
+
table = _BuiltTable(columns, rows, blobs)
|
|
1405
|
+
return self._rich_log(table, name, step, axes)
|
|
1406
|
+
|
|
1407
|
+
def _log_rich(
|
|
1408
|
+
self,
|
|
1409
|
+
document: Document,
|
|
1410
|
+
value: RichValue,
|
|
1411
|
+
*,
|
|
1412
|
+
coordinates: list[dict[str, str]],
|
|
1413
|
+
) -> bool:
|
|
1414
|
+
point_value = value.value()
|
|
1415
|
+
blobs = self._pending_blobs(value, point_value)
|
|
1416
|
+
if blobs:
|
|
1417
|
+
return self.runtime.log_media(
|
|
1418
|
+
document, point_value, coordinates=coordinates, blobs=blobs
|
|
1419
|
+
)
|
|
1420
|
+
return self.runtime.log(document, point_value, coordinates=coordinates)
|
|
1421
|
+
|
|
1422
|
+
def _pending_blobs(self, value: RichValue, point_value: JSONObject) -> list[PendingBlob]:
|
|
1423
|
+
"""Resolve every ``pending:<hash>`` artifact id inside a value."""
|
|
1424
|
+
by_hash = {blob.hash: blob for blob in value.blobs()}
|
|
1425
|
+
for hash in _pending_hashes(point_value):
|
|
1426
|
+
if hash not in by_hash:
|
|
1427
|
+
blob = self._media_blobs.get(hash)
|
|
1428
|
+
if blob is None:
|
|
1429
|
+
raise ValueError("media item references an unknown pending upload")
|
|
1430
|
+
by_hash[hash] = blob
|
|
1431
|
+
return list(by_hash.values())
|
|
1432
|
+
|
|
1433
|
+
def _rich_stream(self, name: str, renderer: str, axes: tuple[str, ...]) -> Document:
|
|
1434
|
+
existing = self._streams.get(name)
|
|
1435
|
+
if existing is not None:
|
|
1436
|
+
document, declared, declared_axes = existing
|
|
1437
|
+
if declared != "object":
|
|
1438
|
+
raise TypeError(f"metric {name!r} is {declared}, not a rich value")
|
|
1439
|
+
stored = _stream_renderer(document)
|
|
1440
|
+
if stored != renderer:
|
|
1441
|
+
raise ValueError(f"metric {name!r} renders as {stored}, not {renderer}")
|
|
1442
|
+
if declared_axes != axes:
|
|
1443
|
+
raise ValueError(f"metric {name!r} axes are fixed")
|
|
1444
|
+
return document
|
|
1445
|
+
if self._reuse_streams:
|
|
1446
|
+
found = self._find_stream(name)
|
|
1447
|
+
if found is not None:
|
|
1448
|
+
value_schema = found.spec.get("value_schema")
|
|
1449
|
+
found_type = value_schema.get("type") if isinstance(value_schema, dict) else None
|
|
1450
|
+
if found_type != "object":
|
|
1451
|
+
raise TypeError(f"metric {name!r} is {found_type}, not a rich value")
|
|
1452
|
+
stored = _stream_renderer(found)
|
|
1453
|
+
if stored != renderer:
|
|
1454
|
+
raise ValueError(f"metric {name!r} renders as {stored}, not {renderer}")
|
|
1455
|
+
raw_axes = found.spec.get("axes")
|
|
1456
|
+
if (
|
|
1457
|
+
not isinstance(raw_axes, list)
|
|
1458
|
+
or any(not isinstance(item, str) for item in raw_axes)
|
|
1459
|
+
or tuple(raw_axes) != axes
|
|
1460
|
+
):
|
|
1461
|
+
raise ValueError(f"metric {name!r} axes are fixed")
|
|
1462
|
+
self._streams[name] = (found, "object", axes)
|
|
1463
|
+
return found
|
|
1464
|
+
spec: JSONObject = {
|
|
1465
|
+
"name": name,
|
|
1466
|
+
"participant_id": self.runtime.participant.id,
|
|
1467
|
+
"value_schema": {"type": "object"},
|
|
1468
|
+
"axes": list(axes),
|
|
1469
|
+
"labels": {},
|
|
1470
|
+
"presentation": {"renderer": renderer},
|
|
1471
|
+
}
|
|
1472
|
+
document = self._client.create_document("stream", self.runtime.scope, spec)
|
|
1473
|
+
self._streams[name] = (document, "object", axes)
|
|
1474
|
+
return document
|
|
1475
|
+
|
|
1476
|
+
def latest(
|
|
1477
|
+
self, metric: str, *, run: str | None = None, **axes: int | float | str
|
|
1478
|
+
) -> dict[str, JSONValue] | None:
|
|
1479
|
+
"""Return the newest query-visible point for ``metric`` in this run.
|
|
1480
|
+
|
|
1481
|
+
Contract: streams that share the name are compared by ``observed_at``.
|
|
1482
|
+
``run`` is a title or id in this project; the point then comes from
|
|
1483
|
+
that run. Keyword axes keep only points whose coordinates match those decimals,
|
|
1484
|
+
then the newest ``observed_at`` among them is returned. The result is
|
|
1485
|
+
``{"value", "coordinates", "observed_at"}``. ``coordinates`` maps axis
|
|
1486
|
+
name to decimal string. Returns ``None`` when the run has no matching
|
|
1487
|
+
point. This does not read ``resume.json``.
|
|
1488
|
+
|
|
1489
|
+
Failure: ``ValueError`` for an empty name or an invalid axis.
|
|
1490
|
+
``LaunchHelmError`` when the client cannot query or a page is malformed.
|
|
1491
|
+
"""
|
|
1492
|
+
if not isinstance(metric, str) or not metric:
|
|
1493
|
+
raise ValueError("metric name is required")
|
|
1494
|
+
if axes or run is not None:
|
|
1495
|
+
filters: dict[str, Any] = dict(axes)
|
|
1496
|
+
series = self.history(metric, run=run, **filters)
|
|
1497
|
+
if not series:
|
|
1498
|
+
return None
|
|
1499
|
+
return max(series, key=lambda point: str(point["observed_at"]))
|
|
1500
|
+
query = getattr(self.runtime.client, "query", None)
|
|
1501
|
+
if not callable(query):
|
|
1502
|
+
raise LaunchHelmError("protocol", "session client cannot query telemetry")
|
|
1503
|
+
newest: dict[str, JSONValue] | None = None
|
|
1504
|
+
newest_at = ""
|
|
1505
|
+
for document in self._streams_named(metric):
|
|
1506
|
+
point = _newest_point(query, document.id)
|
|
1507
|
+
if point is None:
|
|
1508
|
+
continue
|
|
1509
|
+
observed = point["observed_at"]
|
|
1510
|
+
if newest is not None and str(observed) <= newest_at:
|
|
1511
|
+
continue
|
|
1512
|
+
newest = point
|
|
1513
|
+
newest_at = str(observed)
|
|
1514
|
+
return newest
|
|
1515
|
+
|
|
1516
|
+
def history(
|
|
1517
|
+
self,
|
|
1518
|
+
metric: str,
|
|
1519
|
+
*,
|
|
1520
|
+
limit: int = 1000,
|
|
1521
|
+
step_from: int | None = None,
|
|
1522
|
+
step_to: int | None = None,
|
|
1523
|
+
run: str | None = None,
|
|
1524
|
+
**axes: int | float | str,
|
|
1525
|
+
) -> list[dict[str, JSONValue]]:
|
|
1526
|
+
"""Return query-visible points for ``metric``, oldest step first.
|
|
1527
|
+
|
|
1528
|
+
Contract: each item has the same ``value``, ``coordinates``, and
|
|
1529
|
+
``observed_at`` fields as ``latest``. Streams that share the name are
|
|
1530
|
+
merged, including an earlier participant on this run. ``run`` is a
|
|
1531
|
+
title or id in this project; the series then comes from that run
|
|
1532
|
+
instead of this one. Keyword axes
|
|
1533
|
+
keep only points whose coordinates match those decimals. ``step_from``
|
|
1534
|
+
and ``step_to`` are an inclusive step window. Order is the
|
|
1535
|
+
integer ``step`` coordinate, then ``observed_at``. Each server page is
|
|
1536
|
+
1000 points. A full page is followed from its newest ``observed_at``,
|
|
1537
|
+
so a step past the first page is still visible. A full page whose
|
|
1538
|
+
points share one ``observed_at`` continues after that page's
|
|
1539
|
+
``epoch`` and ``sequence``. Axis and step filters
|
|
1540
|
+
run after those pages. Without a step window, the newest ``limit``
|
|
1541
|
+
points are returned, oldest step first within that tail, so the end
|
|
1542
|
+
of a long run stays visible. A step window returns the oldest
|
|
1543
|
+
``limit`` matches inside it. ``limit`` is from 1 to 10000 and defaults to
|
|
1544
|
+
1000. This does
|
|
1545
|
+
not read ``resume.json``.
|
|
1546
|
+
|
|
1547
|
+
Failure: ``ValueError`` for an empty name, a limit outside that range,
|
|
1548
|
+
or an invalid axis. ``LaunchHelmError`` when the client cannot query,
|
|
1549
|
+
``run`` does not match one run in this project, or a point is malformed.
|
|
1550
|
+
"""
|
|
1551
|
+
if not isinstance(metric, str) or not metric:
|
|
1552
|
+
raise ValueError("metric name is required")
|
|
1553
|
+
if isinstance(limit, bool) or not isinstance(limit, int) or limit < 1 or limit > 10000:
|
|
1554
|
+
raise ValueError("history limit must be from 1 to 10000")
|
|
1555
|
+
for bound in (step_from, step_to):
|
|
1556
|
+
if bound is not None and (isinstance(bound, bool) or not isinstance(bound, int)):
|
|
1557
|
+
raise TypeError("step must be an integer")
|
|
1558
|
+
if step_from is not None and step_to is not None and step_from > step_to:
|
|
1559
|
+
raise ValueError("step_from is after step_to")
|
|
1560
|
+
wanted = {name: _coordinate(value) for name, value in axes.items()}
|
|
1561
|
+
for name in wanted:
|
|
1562
|
+
if name == "step" or not _AXIS.fullmatch(name):
|
|
1563
|
+
raise ValueError(f"invalid axis {name!r}")
|
|
1564
|
+
query = getattr(self.runtime.client, "query", None)
|
|
1565
|
+
if not callable(query):
|
|
1566
|
+
raise LaunchHelmError("protocol", "session client cannot query telemetry")
|
|
1567
|
+
series: list[dict[str, JSONValue]] = []
|
|
1568
|
+
run_id = self._run_named(run) if run is not None else None
|
|
1569
|
+
for document in self._streams_named(metric, run_id=run_id):
|
|
1570
|
+
for viewed in self._history_pages(query, document.id):
|
|
1571
|
+
coordinates = viewed["coordinates"]
|
|
1572
|
+
if (
|
|
1573
|
+
isinstance(coordinates, dict)
|
|
1574
|
+
and all(coordinates.get(name) == value for name, value in wanted.items())
|
|
1575
|
+
and _step_in(viewed, step_from, step_to)
|
|
1576
|
+
):
|
|
1577
|
+
series.append(viewed)
|
|
1578
|
+
series = _newer_coordinates(series)
|
|
1579
|
+
series.sort(key=_history_key)
|
|
1580
|
+
if step_from is None and step_to is None:
|
|
1581
|
+
return series[-limit:]
|
|
1582
|
+
return series[:limit]
|
|
1583
|
+
|
|
1584
|
+
def table(
|
|
1585
|
+
self,
|
|
1586
|
+
*metrics: str,
|
|
1587
|
+
run: str | None = None,
|
|
1588
|
+
axis: str = "step",
|
|
1589
|
+
limit: int = 1000,
|
|
1590
|
+
) -> list[dict[str, JSONValue]]:
|
|
1591
|
+
"""Join query-visible series on one coordinate.
|
|
1592
|
+
|
|
1593
|
+
Contract: each row has that coordinate and the metric names that logged
|
|
1594
|
+
a value there. A coordinate that omitted a metric does not invent a
|
|
1595
|
+
value. Rows are oldest coordinate first. ``axis`` defaults to ``step``.
|
|
1596
|
+
``run`` is the same selector as ``history``.
|
|
1597
|
+
|
|
1598
|
+
Failure: ``ValueError`` when no metric is named or a name is empty.
|
|
1599
|
+
``LaunchHelmError`` when ``history`` cannot read a series.
|
|
1600
|
+
"""
|
|
1601
|
+
if not metrics:
|
|
1602
|
+
raise ValueError("metric name is required")
|
|
1603
|
+
if not isinstance(axis, str) or not axis:
|
|
1604
|
+
raise ValueError("axis name is required")
|
|
1605
|
+
rows: dict[str, dict[str, JSONValue]] = {}
|
|
1606
|
+
for name in metrics:
|
|
1607
|
+
if not isinstance(name, str) or not name:
|
|
1608
|
+
raise ValueError("metric name is required")
|
|
1609
|
+
for point in self.history(name, run=run, limit=limit):
|
|
1610
|
+
coordinates = point.get("coordinates")
|
|
1611
|
+
key = coordinates.get(axis) if isinstance(coordinates, dict) else None
|
|
1612
|
+
if not isinstance(key, str) or not key:
|
|
1613
|
+
continue
|
|
1614
|
+
row = rows.setdefault(key, {axis: key})
|
|
1615
|
+
row[name] = point["value"]
|
|
1616
|
+
|
|
1617
|
+
def order(key: str) -> tuple[int, int | str]:
|
|
1618
|
+
try:
|
|
1619
|
+
return (0, int(key))
|
|
1620
|
+
except ValueError:
|
|
1621
|
+
return (1, key)
|
|
1622
|
+
|
|
1623
|
+
return [rows[key] for key in sorted(rows, key=order)]
|
|
1624
|
+
|
|
1625
|
+
def compare(
|
|
1626
|
+
self, metric: str, *runs: str, axis: str = "step", limit: int = 1000
|
|
1627
|
+
) -> list[dict[str, JSONValue]]:
|
|
1628
|
+
"""Join one metric across runs on the ``step`` coordinate.
|
|
1629
|
+
|
|
1630
|
+
Contract: each row has ``step`` and a column for every named run that
|
|
1631
|
+
logged the metric there. A run that skipped the step has no column.
|
|
1632
|
+
``runs`` entries are titles or ids, the same selector as ``history``.
|
|
1633
|
+
An empty name is this run, and its column is this run's title.
|
|
1634
|
+
``axis`` defaults to ``step``. Rows are oldest coordinate first.
|
|
1635
|
+
|
|
1636
|
+
Failure: ``ValueError`` when fewer than two runs are named.
|
|
1637
|
+
``LaunchHelmError`` when a run is not one run in this project.
|
|
1638
|
+
"""
|
|
1639
|
+
if len(runs) < 2:
|
|
1640
|
+
raise ValueError("compare needs two runs")
|
|
1641
|
+
if not isinstance(axis, str) or not axis:
|
|
1642
|
+
raise ValueError("axis name is required")
|
|
1643
|
+
rows: dict[str, dict[str, JSONValue]] = {}
|
|
1644
|
+
for name in runs:
|
|
1645
|
+
if not isinstance(name, str):
|
|
1646
|
+
raise ValueError("run name is required")
|
|
1647
|
+
label = name or self.name
|
|
1648
|
+
if not label:
|
|
1649
|
+
raise ValueError("run name is required")
|
|
1650
|
+
for point in self.history(metric, run=name or None, limit=limit):
|
|
1651
|
+
coordinates = point.get("coordinates")
|
|
1652
|
+
key = coordinates.get(axis) if isinstance(coordinates, dict) else None
|
|
1653
|
+
if not isinstance(key, str) or not key:
|
|
1654
|
+
continue
|
|
1655
|
+
row = rows.setdefault(key, {axis: key})
|
|
1656
|
+
row[label] = point["value"]
|
|
1657
|
+
|
|
1658
|
+
def order(step: str) -> tuple[int, int | str]:
|
|
1659
|
+
try:
|
|
1660
|
+
return (0, int(step))
|
|
1661
|
+
except ValueError:
|
|
1662
|
+
return (1, step)
|
|
1663
|
+
|
|
1664
|
+
return [rows[step] for step in sorted(rows, key=order)]
|
|
1665
|
+
|
|
1666
|
+
def _history_pages(
|
|
1667
|
+
self, query: Callable[..., JSONObject], stream_id: str
|
|
1668
|
+
) -> list[dict[str, JSONValue]]:
|
|
1669
|
+
start = "1970-01-01T00:00:00Z"
|
|
1670
|
+
end = "2261-01-01T00:00:00Z"
|
|
1671
|
+
after_epoch = ""
|
|
1672
|
+
after_sequence = ""
|
|
1673
|
+
seen: set[tuple[str, tuple[tuple[str, JSONValue], ...], str, str, str]] = set()
|
|
1674
|
+
collected: list[dict[str, JSONValue]] = []
|
|
1675
|
+
for _ in range(_MAX_PAGES):
|
|
1676
|
+
page = query(
|
|
1677
|
+
stream_id,
|
|
1678
|
+
start=start,
|
|
1679
|
+
end=end,
|
|
1680
|
+
limit=1000,
|
|
1681
|
+
order="asc",
|
|
1682
|
+
after_epoch=after_epoch,
|
|
1683
|
+
after_sequence=after_sequence,
|
|
1684
|
+
)
|
|
1685
|
+
items = page.get("items") if isinstance(page, dict) else None
|
|
1686
|
+
if not isinstance(items, list):
|
|
1687
|
+
raise LaunchHelmError("protocol", "telemetry page is not a list")
|
|
1688
|
+
if not items:
|
|
1689
|
+
break
|
|
1690
|
+
added = 0
|
|
1691
|
+
last_at = start
|
|
1692
|
+
last_epoch = after_epoch
|
|
1693
|
+
last_sequence = after_sequence
|
|
1694
|
+
for point in items:
|
|
1695
|
+
viewed = _view_point(point)
|
|
1696
|
+
coordinates = viewed["coordinates"]
|
|
1697
|
+
pairs = tuple(sorted(coordinates.items())) if isinstance(coordinates, dict) else ()
|
|
1698
|
+
identity = (
|
|
1699
|
+
str(viewed["observed_at"]),
|
|
1700
|
+
pairs,
|
|
1701
|
+
str(viewed["value"]),
|
|
1702
|
+
str(viewed.get("epoch") or ""),
|
|
1703
|
+
str(viewed.get("sequence") or ""),
|
|
1704
|
+
)
|
|
1705
|
+
if identity in seen:
|
|
1706
|
+
continue
|
|
1707
|
+
seen.add(identity)
|
|
1708
|
+
collected.append(viewed)
|
|
1709
|
+
added += 1
|
|
1710
|
+
observed = viewed["observed_at"]
|
|
1711
|
+
if isinstance(observed, str):
|
|
1712
|
+
last_at = observed
|
|
1713
|
+
if isinstance(point, dict):
|
|
1714
|
+
epoch = point.get("epoch")
|
|
1715
|
+
sequence = point.get("sequence")
|
|
1716
|
+
if isinstance(epoch, str) and epoch:
|
|
1717
|
+
last_epoch = epoch
|
|
1718
|
+
if isinstance(sequence, str) and sequence:
|
|
1719
|
+
last_sequence = sequence
|
|
1720
|
+
full = len(items) >= 1000 or (isinstance(page, dict) and page.get("complete") is False)
|
|
1721
|
+
cursor = (
|
|
1722
|
+
bool(last_epoch)
|
|
1723
|
+
and bool(last_sequence)
|
|
1724
|
+
and (last_sequence != after_sequence or last_epoch != after_epoch)
|
|
1725
|
+
)
|
|
1726
|
+
advanced = last_at > start or cursor
|
|
1727
|
+
if not full or added == 0 or not advanced:
|
|
1728
|
+
break
|
|
1729
|
+
start = last_at
|
|
1730
|
+
after_epoch = last_epoch if cursor else ""
|
|
1731
|
+
after_sequence = last_sequence if cursor else ""
|
|
1732
|
+
return collected
|
|
1733
|
+
|
|
1734
|
+
def metrics(self, *, run: str | None = None) -> list[str]:
|
|
1735
|
+
"""Return the metric names already stored on this run, in order.
|
|
1736
|
+
|
|
1737
|
+
Contract: each name is a stream on the run, including a stream from
|
|
1738
|
+
an earlier participant. ``run`` is a title or id in this project; the
|
|
1739
|
+
names then come from that run. The list is unique and sorted. This
|
|
1740
|
+
does not read ``resume.json``.
|
|
1741
|
+
|
|
1742
|
+
Failure: ``LaunchHelmError`` when a stream page is malformed or
|
|
1743
|
+
``run`` does not match one run in this project.
|
|
1744
|
+
"""
|
|
1745
|
+
names: set[str] = set()
|
|
1746
|
+
cursor = ""
|
|
1747
|
+
run_id = self._run_named(run) if run is not None else self.runtime.scope.run_id
|
|
1748
|
+
for _ in range(_MAX_PAGES):
|
|
1749
|
+
page = self._client.list_documents(
|
|
1750
|
+
kind="stream",
|
|
1751
|
+
project_id=self.project_id,
|
|
1752
|
+
run_id=run_id,
|
|
1753
|
+
cursor=cursor,
|
|
1754
|
+
limit=100,
|
|
1755
|
+
)
|
|
1756
|
+
items = page.get("items") or []
|
|
1757
|
+
if not isinstance(items, list):
|
|
1758
|
+
raise LaunchHelmError("protocol", "stream list is not a page")
|
|
1759
|
+
for item in items:
|
|
1760
|
+
if not isinstance(item, dict):
|
|
1761
|
+
raise LaunchHelmError("protocol", "stream list entry is not a document")
|
|
1762
|
+
document = Document.parse(item)
|
|
1763
|
+
name = document.spec.get("name")
|
|
1764
|
+
if document.kind == "stream" and isinstance(name, str) and name:
|
|
1765
|
+
names.add(name)
|
|
1766
|
+
next_cursor = page.get("next_cursor") or ""
|
|
1767
|
+
if not isinstance(next_cursor, str) or not next_cursor or next_cursor == cursor:
|
|
1768
|
+
break
|
|
1769
|
+
cursor = next_cursor
|
|
1770
|
+
return sorted(names)
|
|
1771
|
+
|
|
1772
|
+
def metric(self, name: str, *, run: str | None = None) -> dict[str, JSONValue] | None:
|
|
1773
|
+
"""Return the stored kind and axes for ``name``.
|
|
1774
|
+
|
|
1775
|
+
Contract: ``{"kind", "axes", "unit"}`` from the stream on this run.
|
|
1776
|
+
``run`` is a title or id in this project; the description then comes
|
|
1777
|
+
from that run. ``kind`` is ``number``, ``string``, or ``boolean``.
|
|
1778
|
+
``axes`` is the coordinate names, including ``step``. Returns ``None``
|
|
1779
|
+
when the run has no stream of that name. This does not read ``resume.json``.
|
|
1780
|
+
|
|
1781
|
+
Failure: ``ValueError`` when ``name`` is empty. ``LaunchHelmError``
|
|
1782
|
+
when a stream page is malformed or ``run`` does not match one run.
|
|
1783
|
+
"""
|
|
1784
|
+
if not isinstance(name, str) or not name:
|
|
1785
|
+
raise ValueError("metric name is required")
|
|
1786
|
+
run_id = self._run_named(run) if run is not None else None
|
|
1787
|
+
for document in self._streams_named(name, run_id=run_id):
|
|
1788
|
+
schema = document.spec.get("value_schema")
|
|
1789
|
+
kind = schema.get("type") if isinstance(schema, dict) else None
|
|
1790
|
+
axes = document.spec.get("axes")
|
|
1791
|
+
if not isinstance(kind, str) or not isinstance(axes, list):
|
|
1792
|
+
continue
|
|
1793
|
+
stored_unit = document.spec.get("unit")
|
|
1794
|
+
return {
|
|
1795
|
+
"kind": kind,
|
|
1796
|
+
"axes": [item for item in axes if isinstance(item, str)],
|
|
1797
|
+
"unit": stored_unit if isinstance(stored_unit, str) else "",
|
|
1798
|
+
}
|
|
1799
|
+
return None
|
|
1800
|
+
|
|
1801
|
+
def _run_named(self, run: str) -> str:
|
|
1802
|
+
if not isinstance(run, str) or not run:
|
|
1803
|
+
raise ValueError("run name is required")
|
|
1804
|
+
matches: list[str] = []
|
|
1805
|
+
cursor = ""
|
|
1806
|
+
for _ in range(_MAX_PAGES):
|
|
1807
|
+
page = self._client.list_documents(
|
|
1808
|
+
kind="run", project_id=self.project_id, cursor=cursor, limit=100
|
|
1809
|
+
)
|
|
1810
|
+
items = page.get("items") or []
|
|
1811
|
+
if not isinstance(items, list):
|
|
1812
|
+
raise LaunchHelmError("protocol", "run list is not a page")
|
|
1813
|
+
for item in items:
|
|
1814
|
+
if not isinstance(item, dict):
|
|
1815
|
+
raise LaunchHelmError("protocol", "run list entry is not a document")
|
|
1816
|
+
document = Document.parse(item)
|
|
1817
|
+
if document.kind != "run" or document.scope.project_id != self.project_id:
|
|
1818
|
+
continue
|
|
1819
|
+
if document.id == run or document.spec.get("name") == run:
|
|
1820
|
+
matches.append(document.id)
|
|
1821
|
+
next_cursor = page.get("next_cursor") or ""
|
|
1822
|
+
if not isinstance(next_cursor, str) or not next_cursor or next_cursor == cursor:
|
|
1823
|
+
break
|
|
1824
|
+
cursor = next_cursor
|
|
1825
|
+
unique = list(dict.fromkeys(matches))
|
|
1826
|
+
if len(unique) != 1:
|
|
1827
|
+
raise LaunchHelmError("config", f"run {run!r} is not one run in this project")
|
|
1828
|
+
return unique[0]
|
|
1829
|
+
|
|
1830
|
+
def _streams_named(self, metric: str, *, run_id: str | None = None) -> list[Document]:
|
|
1831
|
+
found: list[Document] = []
|
|
1832
|
+
cursor = ""
|
|
1833
|
+
run_id = self.runtime.scope.run_id if run_id is None else run_id
|
|
1834
|
+
for _ in range(_MAX_PAGES):
|
|
1835
|
+
page = self._client.list_documents(
|
|
1836
|
+
kind="stream",
|
|
1837
|
+
project_id=self.project_id,
|
|
1838
|
+
run_id=run_id,
|
|
1839
|
+
cursor=cursor,
|
|
1840
|
+
limit=100,
|
|
1841
|
+
)
|
|
1842
|
+
items = page.get("items") or []
|
|
1843
|
+
if not isinstance(items, list):
|
|
1844
|
+
raise LaunchHelmError("protocol", "stream list is not a page")
|
|
1845
|
+
for item in items:
|
|
1846
|
+
if not isinstance(item, dict):
|
|
1847
|
+
raise LaunchHelmError("protocol", "stream list entry is not a document")
|
|
1848
|
+
document = Document.parse(item)
|
|
1849
|
+
if document.kind == "stream" and document.spec.get("name") == metric:
|
|
1850
|
+
found.append(document)
|
|
1851
|
+
next_cursor = page.get("next_cursor") or ""
|
|
1852
|
+
if not isinstance(next_cursor, str) or not next_cursor or next_cursor == cursor:
|
|
1853
|
+
break
|
|
1854
|
+
cursor = next_cursor
|
|
1855
|
+
return found
|
|
1856
|
+
|
|
1857
|
+
def _coordinates(
|
|
1858
|
+
self, step: int | None, axes: Mapping[str, int | float | str], *, advance: bool
|
|
1859
|
+
) -> tuple[list[dict[str, str]], tuple[str, ...]]:
|
|
1860
|
+
if step is None:
|
|
1861
|
+
current = self._next_step
|
|
1862
|
+
else:
|
|
1863
|
+
if isinstance(step, bool) or not isinstance(step, int):
|
|
1864
|
+
raise TypeError("step must be an integer")
|
|
1865
|
+
current = step
|
|
1866
|
+
values: dict[str, str] = {"step": str(current)}
|
|
1867
|
+
for name, value in axes.items():
|
|
1868
|
+
if name == "step" or not _AXIS.fullmatch(name):
|
|
1869
|
+
raise ValueError(f"invalid axis {name!r}")
|
|
1870
|
+
values[name] = _coordinate(value)
|
|
1871
|
+
if advance:
|
|
1872
|
+
self._next_step = max(self._next_step, current + 1)
|
|
1873
|
+
names = tuple(values)
|
|
1874
|
+
return [{"axis": name, "value": values[name]} for name in names], names
|
|
1875
|
+
|
|
1876
|
+
def _open_streams(self, missing: list[tuple[str, str]], axes: tuple[str, ...]) -> None:
|
|
1877
|
+
if len(missing) < 2:
|
|
1878
|
+
return
|
|
1879
|
+
create_many = getattr(self._client, "create_documents", None)
|
|
1880
|
+
if create_many is not None:
|
|
1881
|
+
documents = create_many(
|
|
1882
|
+
[
|
|
1883
|
+
(
|
|
1884
|
+
"stream",
|
|
1885
|
+
self.runtime.scope,
|
|
1886
|
+
{
|
|
1887
|
+
"name": name,
|
|
1888
|
+
"participant_id": self.runtime.participant.id,
|
|
1889
|
+
"value_schema": {"type": kind},
|
|
1890
|
+
"axes": list(axes),
|
|
1891
|
+
"labels": {},
|
|
1892
|
+
},
|
|
1893
|
+
)
|
|
1894
|
+
for name, kind in missing
|
|
1895
|
+
]
|
|
1896
|
+
)
|
|
1897
|
+
if len(documents) != len(missing):
|
|
1898
|
+
raise LaunchHelmError("protocol", "stream batch length does not match")
|
|
1899
|
+
for (name, kind), document in zip(missing, documents, strict=True):
|
|
1900
|
+
self._streams[name] = (document, kind, axes)
|
|
1901
|
+
return
|
|
1902
|
+
|
|
1903
|
+
def create(item: tuple[str, str]) -> tuple[str, str, Document]:
|
|
1904
|
+
name, kind = item
|
|
1905
|
+
document = self._client.create_document(
|
|
1906
|
+
"stream",
|
|
1907
|
+
self.runtime.scope,
|
|
1908
|
+
{
|
|
1909
|
+
"name": name,
|
|
1910
|
+
"participant_id": self.runtime.participant.id,
|
|
1911
|
+
"value_schema": {"type": kind},
|
|
1912
|
+
"axes": list(axes),
|
|
1913
|
+
"labels": {},
|
|
1914
|
+
},
|
|
1915
|
+
)
|
|
1916
|
+
return name, kind, document
|
|
1917
|
+
|
|
1918
|
+
workers = min(_STREAM_CREATES, len(missing))
|
|
1919
|
+
with ThreadPoolExecutor(max_workers=workers) as pool:
|
|
1920
|
+
opened = list(pool.map(create, missing))
|
|
1921
|
+
for name, kind, document in opened:
|
|
1922
|
+
self._streams[name] = (document, kind, axes)
|
|
1923
|
+
|
|
1924
|
+
def _stream(self, name: str, kind: str, axes: tuple[str, ...], unit: str = "") -> Document:
|
|
1925
|
+
existing = self._streams.get(name)
|
|
1926
|
+
if existing is not None:
|
|
1927
|
+
document, declared, declared_axes = existing
|
|
1928
|
+
stored_unit = (
|
|
1929
|
+
document.spec.get("unit") if isinstance(document.spec.get("unit"), str) else ""
|
|
1930
|
+
)
|
|
1931
|
+
if unit and stored_unit and stored_unit != unit:
|
|
1932
|
+
raise ValueError(f"metric {name!r} unit is fixed")
|
|
1933
|
+
if declared != kind:
|
|
1934
|
+
raise TypeError(f"metric {name!r} is {declared}, not {kind}")
|
|
1935
|
+
if declared_axes != axes:
|
|
1936
|
+
raise ValueError(f"metric {name!r} axes are fixed")
|
|
1937
|
+
return document
|
|
1938
|
+
if self._reuse_streams:
|
|
1939
|
+
found = self._find_stream(name)
|
|
1940
|
+
if found is not None:
|
|
1941
|
+
value_schema = found.spec.get("value_schema")
|
|
1942
|
+
found_type = value_schema.get("type") if isinstance(value_schema, dict) else None
|
|
1943
|
+
raw_axes = found.spec.get("axes")
|
|
1944
|
+
if not isinstance(found_type, str) or found_type != kind:
|
|
1945
|
+
raise TypeError(f"metric {name!r} is {found_type}, not {kind}")
|
|
1946
|
+
if (
|
|
1947
|
+
not isinstance(raw_axes, list)
|
|
1948
|
+
or any(not isinstance(item, str) for item in raw_axes)
|
|
1949
|
+
or tuple(raw_axes) != axes
|
|
1950
|
+
):
|
|
1951
|
+
raise ValueError(f"metric {name!r} axes are fixed")
|
|
1952
|
+
self._streams[name] = (found, kind, axes)
|
|
1953
|
+
return found
|
|
1954
|
+
schema: JSONObject = {"type": kind}
|
|
1955
|
+
labels: JSONObject = {}
|
|
1956
|
+
spec: JSONObject = {
|
|
1957
|
+
"name": name,
|
|
1958
|
+
"participant_id": self.runtime.participant.id,
|
|
1959
|
+
"value_schema": schema,
|
|
1960
|
+
"axes": list(axes),
|
|
1961
|
+
"labels": labels,
|
|
1962
|
+
}
|
|
1963
|
+
if unit:
|
|
1964
|
+
spec["unit"] = unit
|
|
1965
|
+
document = self._client.create_document(
|
|
1966
|
+
"stream",
|
|
1967
|
+
self.runtime.scope,
|
|
1968
|
+
spec,
|
|
1969
|
+
)
|
|
1970
|
+
self._streams[name] = (document, kind, axes)
|
|
1971
|
+
return document
|
|
1972
|
+
|
|
1973
|
+
def _find_stream(self, name: str) -> Document | None:
|
|
1974
|
+
participant_id = self.runtime.participant.id
|
|
1975
|
+
run_id = self.runtime.scope.run_id
|
|
1976
|
+
cursor = ""
|
|
1977
|
+
for _ in range(_MAX_PAGES):
|
|
1978
|
+
page = self._client.list_documents(
|
|
1979
|
+
kind="stream",
|
|
1980
|
+
project_id=self.project_id,
|
|
1981
|
+
run_id=run_id,
|
|
1982
|
+
cursor=cursor,
|
|
1983
|
+
limit=100,
|
|
1984
|
+
)
|
|
1985
|
+
items = page.get("items") or []
|
|
1986
|
+
if not isinstance(items, list):
|
|
1987
|
+
raise LaunchHelmError("protocol", "stream list is not a page")
|
|
1988
|
+
for item in items:
|
|
1989
|
+
if not isinstance(item, dict):
|
|
1990
|
+
raise LaunchHelmError("protocol", "stream list entry is not a document")
|
|
1991
|
+
document = Document.parse(item)
|
|
1992
|
+
if document.kind != "stream" or document.spec.get("name") != name:
|
|
1993
|
+
continue
|
|
1994
|
+
if document.spec.get("participant_id") == participant_id:
|
|
1995
|
+
return document
|
|
1996
|
+
next_cursor = page.get("next_cursor") or ""
|
|
1997
|
+
if not isinstance(next_cursor, str) or not next_cursor or next_cursor == cursor:
|
|
1998
|
+
break
|
|
1999
|
+
cursor = next_cursor
|
|
2000
|
+
return None
|
|
2001
|
+
|
|
2002
|
+
|
|
2003
|
+
def init(
|
|
2004
|
+
project: str,
|
|
2005
|
+
*,
|
|
2006
|
+
overflow: OverflowPolicy = "block",
|
|
2007
|
+
name: str = "",
|
|
2008
|
+
config: Mapping[str, JSONValue] | None = None,
|
|
2009
|
+
group: str = "",
|
|
2010
|
+
tags: Sequence[str] = (),
|
|
2011
|
+
notes: str = "",
|
|
2012
|
+
organization: str = "",
|
|
2013
|
+
client: SessionClient | None = None,
|
|
2014
|
+
state_dir: str | Path | None = None,
|
|
2015
|
+
media_spool_bytes: int = 256 << 20,
|
|
2016
|
+
resume: str = "",
|
|
2017
|
+
project_id: str = "",
|
|
2018
|
+
participant_name: str = "",
|
|
2019
|
+
participant_role: str = "logger",
|
|
2020
|
+
participant_labels: Mapping[str, str] | None = None,
|
|
2021
|
+
) -> Run:
|
|
2022
|
+
"""Open a logging run.
|
|
2023
|
+
|
|
2024
|
+
Contract: the default overflow is ``block``. Admitted points are written
|
|
2025
|
+
to the local spool and are not dropped while the run is open. An existing
|
|
2026
|
+
project with this exact name is reused. A new run is created unless
|
|
2027
|
+
``resume`` names an existing run id or run title in that project. The
|
|
2028
|
+
``keep`` writes trainer state, the next log step, and the participant id
|
|
2029
|
+
to ``resume.json``. ``init`` with ``resume`` loads that snapshot and
|
|
2030
|
+
reopens the same participant, so later points use the same streams and
|
|
2031
|
+
the existing spool can finish uploading. A resume with no checkpoint
|
|
2032
|
+
reopens a participant directory that still has unacknowledged points,
|
|
2033
|
+
so those points upload. Otherwise it creates a new participant.
|
|
2034
|
+
When ``client`` is omitted, the API origin is ``LAUNCHHELM_URL`` or
|
|
2035
|
+
``https://launchhelm.com``. ``LAUNCHHELM_API_KEY`` is the credential.
|
|
2036
|
+
``LAUNCHHELM_ORGANIZATION`` is required only when the key belongs to more
|
|
2037
|
+
than one organization. ``LAUNCHHELM_PROJECT`` or ``project_id`` opens that
|
|
2038
|
+
project directly. A project-scoped key cannot scan the organization
|
|
2039
|
+
catalog, so the credential the app issues sets ``LAUNCHHELM_PROJECT``.
|
|
2040
|
+
Unset keys are filled from ``.env`` files from the working directory up
|
|
2041
|
+
to the filesystem root. Shell variables win over files. ``participant_name``
|
|
2042
|
+
names the new participant instead of the ``logger-<pid>`` default,
|
|
2043
|
+
``participant_role`` overrides its role, and ``participant_labels``
|
|
2044
|
+
merges labels into its spec; all apply only when a participant is
|
|
2045
|
+
created, so a resumed run keeps its stored identity.
|
|
2046
|
+
|
|
2047
|
+
Failure: ``ValueError`` for an empty project name, a bad overflow, or a
|
|
2048
|
+
config that cannot be a label. ``LaunchHelmError`` when the key is
|
|
2049
|
+
missing, the organization is ambiguous, or two projects share the name.
|
|
2050
|
+
"""
|
|
2051
|
+
load_env()
|
|
2052
|
+
if overflow not in {"drop_new", "block", "raise"}:
|
|
2053
|
+
raise ValueError("overflow must be drop_new, block, or raise")
|
|
2054
|
+
if not project or len(project) > 256:
|
|
2055
|
+
raise ValueError("project name must be 1 to 256 characters")
|
|
2056
|
+
owns_client = client is None
|
|
2057
|
+
if client is None:
|
|
2058
|
+
client = _init_client(organization)
|
|
2059
|
+
owned_metadata: SessionClient | None = None
|
|
2060
|
+
try:
|
|
2061
|
+
run_name, supplied, labels = _run_labels(name, config, group, tags, notes)
|
|
2062
|
+
found = _init_project(client, project, project_id, owns_client, resume)
|
|
2063
|
+
opened, run_name = _open_init_run(
|
|
2064
|
+
client, found, resume, run_name, labels, supplied, state_dir
|
|
2065
|
+
)
|
|
2066
|
+
directory = Path.cwd() / ".launchhelm" / opened.id if state_dir is None else Path(state_dir)
|
|
2067
|
+
directory.mkdir(parents=True, exist_ok=True)
|
|
2068
|
+
checkpoint = directory / "resume.json"
|
|
2069
|
+
participant, next_step, restored, saved_summary, reused = _init_participant(
|
|
2070
|
+
client,
|
|
2071
|
+
found,
|
|
2072
|
+
opened,
|
|
2073
|
+
resume,
|
|
2074
|
+
directory,
|
|
2075
|
+
checkpoint,
|
|
2076
|
+
participant_name,
|
|
2077
|
+
participant_role,
|
|
2078
|
+
participant_labels,
|
|
2079
|
+
)
|
|
2080
|
+
# The worker key only publishes this participant's catalog; run-level
|
|
2081
|
+
# writes (summary, metadata, state) keep the original credential on a
|
|
2082
|
+
# client held in memory only.
|
|
2083
|
+
bound = _bind_worker(client, participant)
|
|
2084
|
+
metadata: SessionClient = client
|
|
2085
|
+
if bound:
|
|
2086
|
+
with_key = getattr(client, "with_key", None)
|
|
2087
|
+
if callable(with_key):
|
|
2088
|
+
derived = with_key()
|
|
2089
|
+
if derived is not client:
|
|
2090
|
+
owned_metadata = derived
|
|
2091
|
+
metadata = derived
|
|
2092
|
+
runtime = Runtime(
|
|
2093
|
+
client,
|
|
2094
|
+
participant.id,
|
|
2095
|
+
state_dir=directory / "participants" / participant.id,
|
|
2096
|
+
overflow=overflow,
|
|
2097
|
+
media_spool_bytes=media_spool_bytes,
|
|
2098
|
+
serve_commands=False,
|
|
2099
|
+
)
|
|
2100
|
+
except BaseException:
|
|
2101
|
+
if owned_metadata is not None:
|
|
2102
|
+
owned_metadata.close()
|
|
2103
|
+
if owns_client:
|
|
2104
|
+
client.close()
|
|
2105
|
+
raise
|
|
2106
|
+
return Run(
|
|
2107
|
+
runtime,
|
|
2108
|
+
client=client,
|
|
2109
|
+
name=run_name,
|
|
2110
|
+
owned_client=client if owns_client else None,
|
|
2111
|
+
metadata_client=metadata,
|
|
2112
|
+
owned_metadata=owned_metadata,
|
|
2113
|
+
checkpoint=checkpoint,
|
|
2114
|
+
restored=restored,
|
|
2115
|
+
next_step=next_step,
|
|
2116
|
+
reuse_streams=reused,
|
|
2117
|
+
config=_resume_config(client, opened) if resume else supplied,
|
|
2118
|
+
summary=(
|
|
2119
|
+
saved_summary
|
|
2120
|
+
if saved_summary is not None
|
|
2121
|
+
else (_summary_from_run(opened) if resume else None)
|
|
2122
|
+
),
|
|
2123
|
+
summary_changed=saved_summary is not None,
|
|
2124
|
+
group=_metadata_value(opened, "launchhelm.group"),
|
|
2125
|
+
tags=_metadata_tags(opened),
|
|
2126
|
+
notes=_metadata_value(opened, "launchhelm.notes"),
|
|
2127
|
+
git=_metadata_value(opened, "launchhelm.git"),
|
|
2128
|
+
host=_metadata_value(opened, "launchhelm.host"),
|
|
2129
|
+
python=_metadata_value(opened, "launchhelm.python"),
|
|
2130
|
+
argv=_metadata_value(opened, "launchhelm.argv"),
|
|
2131
|
+
)
|
|
2132
|
+
|
|
2133
|
+
|
|
2134
|
+
def _init_client(organization: str) -> Client:
|
|
2135
|
+
url = _api_url()
|
|
2136
|
+
key = _env("LAUNCHHELM_API_KEY")
|
|
2137
|
+
selected = organization or os.environ.get("LAUNCHHELM_ORGANIZATION", "")
|
|
2138
|
+
if not selected:
|
|
2139
|
+
selected = _only_organization(url, key)
|
|
2140
|
+
return Client(url, key, selected)
|
|
2141
|
+
|
|
2142
|
+
|
|
2143
|
+
def _run_labels(
|
|
2144
|
+
name: str,
|
|
2145
|
+
config: Mapping[str, JSONValue] | None,
|
|
2146
|
+
group: str,
|
|
2147
|
+
tags: Sequence[str],
|
|
2148
|
+
notes: str,
|
|
2149
|
+
) -> tuple[str, dict[str, JSONValue], JSONObject]:
|
|
2150
|
+
run_name = name or f"run-{uuid4().hex[:12]}"
|
|
2151
|
+
if len(run_name) > 256:
|
|
2152
|
+
raise ValueError("run name must be at most 256 characters")
|
|
2153
|
+
supplied = dict(config or {})
|
|
2154
|
+
labels = _labels(supplied)
|
|
2155
|
+
_metadata(labels, group, tags, notes)
|
|
2156
|
+
_provenance(labels)
|
|
2157
|
+
return run_name, supplied, labels
|
|
2158
|
+
|
|
2159
|
+
|
|
2160
|
+
def _init_project(
|
|
2161
|
+
client: SessionClient, project: str, project_id: str, owns_client: bool, resume: str
|
|
2162
|
+
) -> Document:
|
|
2163
|
+
selected = project_id
|
|
2164
|
+
if not selected and owns_client:
|
|
2165
|
+
selected = os.environ.get("LAUNCHHELM_PROJECT", "")
|
|
2166
|
+
found = _open_project(client, project, selected)
|
|
2167
|
+
if found is None:
|
|
2168
|
+
if resume:
|
|
2169
|
+
raise LaunchHelmError("not_found", f"project {project!r} does not exist")
|
|
2170
|
+
found = client.create_document(
|
|
2171
|
+
"project",
|
|
2172
|
+
Scope(client.organization_id),
|
|
2173
|
+
{"name": project, "labels": {}},
|
|
2174
|
+
)
|
|
2175
|
+
return found
|
|
2176
|
+
|
|
2177
|
+
|
|
2178
|
+
def _open_init_run(
|
|
2179
|
+
client: SessionClient,
|
|
2180
|
+
found: Document,
|
|
2181
|
+
resume: str,
|
|
2182
|
+
run_name: str,
|
|
2183
|
+
labels: JSONObject,
|
|
2184
|
+
supplied: dict[str, JSONValue],
|
|
2185
|
+
state_dir: str | Path | None,
|
|
2186
|
+
) -> tuple[Document, str]:
|
|
2187
|
+
saved_run = ""
|
|
2188
|
+
if resume and state_dir is not None:
|
|
2189
|
+
preview = Path(state_dir) / "resume.json"
|
|
2190
|
+
if preview.is_file():
|
|
2191
|
+
saved_run = _read_checkpoint(preview)[4]
|
|
2192
|
+
if resume and saved_run:
|
|
2193
|
+
opened = client.get_document(saved_run)
|
|
2194
|
+
if opened.kind != "run" or opened.scope.project_id != found.id:
|
|
2195
|
+
raise LaunchHelmError("not_found", f"open run {saved_run!r} was not found")
|
|
2196
|
+
if opened.spec.get("state") != "open":
|
|
2197
|
+
raise LaunchHelmError("conflict", f"run {saved_run!r} is not open")
|
|
2198
|
+
return opened, str(opened.spec.get("name") or resume)
|
|
2199
|
+
if resume:
|
|
2200
|
+
opened = _run(client, found.id, resume)
|
|
2201
|
+
return opened, str(opened.spec.get("name") or resume)
|
|
2202
|
+
opened = client.create_document(
|
|
2203
|
+
"run",
|
|
2204
|
+
Scope(client.organization_id, found.id),
|
|
2205
|
+
{
|
|
2206
|
+
"name": run_name,
|
|
2207
|
+
"state": "open",
|
|
2208
|
+
"configuration": opaque_schema(),
|
|
2209
|
+
"labels": labels,
|
|
2210
|
+
**({"parameters": supplied} if supplied else {}),
|
|
2211
|
+
},
|
|
2212
|
+
)
|
|
2213
|
+
return opened, run_name
|
|
2214
|
+
|
|
2215
|
+
|
|
2216
|
+
def _init_participant(
|
|
2217
|
+
client: SessionClient,
|
|
2218
|
+
found: Document,
|
|
2219
|
+
opened: Document,
|
|
2220
|
+
resume: str,
|
|
2221
|
+
directory: Path,
|
|
2222
|
+
checkpoint: Path,
|
|
2223
|
+
participant_name: str,
|
|
2224
|
+
participant_role: str,
|
|
2225
|
+
participant_labels: Mapping[str, str] | None,
|
|
2226
|
+
) -> tuple[Document, int, dict[str, JSONValue], dict[str, JSONValue] | None, bool]:
|
|
2227
|
+
next_step, restored, saved_participant, saved_summary, _saved_run = _read_checkpoint(checkpoint)
|
|
2228
|
+
if resume and not checkpoint.is_file():
|
|
2229
|
+
next_step, restored = _server_cursor(client, opened.scope.project_id, opened.id)
|
|
2230
|
+
if not saved_participant:
|
|
2231
|
+
saved_participant = _pending_participant(directory)
|
|
2232
|
+
participant = _saved_participant(client, opened, saved_participant) if resume else None
|
|
2233
|
+
reused = participant is not None
|
|
2234
|
+
if participant is None:
|
|
2235
|
+
labels: JSONObject = {}
|
|
2236
|
+
if participant_labels:
|
|
2237
|
+
labels.update(participant_labels)
|
|
2238
|
+
participant = client.create_document(
|
|
2239
|
+
"participant",
|
|
2240
|
+
Scope(client.organization_id, found.id, opened.id),
|
|
2241
|
+
{
|
|
2242
|
+
"name": participant_name or f"logger-{os.getpid()}",
|
|
2243
|
+
"role": participant_role,
|
|
2244
|
+
"labels": labels,
|
|
2245
|
+
},
|
|
2246
|
+
)
|
|
2247
|
+
return participant, next_step, restored, saved_summary, reused
|
|
2248
|
+
|
|
2249
|
+
|
|
2250
|
+
def _bind_worker(client: SessionClient, participant: Document) -> bool:
|
|
2251
|
+
"""Mint the participant-bound worker key and adopt it on ``client``.
|
|
2252
|
+
|
|
2253
|
+
Returns ``True`` when a worker credential was adopted; the caller then
|
|
2254
|
+
keeps a second client on the original credential for run-level writes.
|
|
2255
|
+
"""
|
|
2256
|
+
create = getattr(client, "create_key", None)
|
|
2257
|
+
if create is None:
|
|
2258
|
+
return False
|
|
2259
|
+
created = create(participant)
|
|
2260
|
+
token = created.get("token") if isinstance(created, dict) else ""
|
|
2261
|
+
adopt = getattr(client, "adopt", None)
|
|
2262
|
+
if isinstance(token, str) and token and adopt is not None:
|
|
2263
|
+
adopt(token)
|
|
2264
|
+
return True
|
|
2265
|
+
return False
|
|
2266
|
+
|
|
2267
|
+
|
|
2268
|
+
def _only_organization(url: str, key: str) -> str:
|
|
2269
|
+
probe = Client(url, key, "00000000-0000-4000-8000-000000000000")
|
|
2270
|
+
try:
|
|
2271
|
+
found = probe.list_my_organizations()
|
|
2272
|
+
finally:
|
|
2273
|
+
probe.close()
|
|
2274
|
+
return _select_organization("", found)
|
|
2275
|
+
|
|
2276
|
+
|
|
2277
|
+
def _select_organization(explicit: str, found: list[tuple[str, str]]) -> str:
|
|
2278
|
+
if explicit:
|
|
2279
|
+
return explicit
|
|
2280
|
+
if len(found) == 1:
|
|
2281
|
+
return found[0][0]
|
|
2282
|
+
if not found:
|
|
2283
|
+
raise LaunchHelmError("config", "this key is not a member of an organization")
|
|
2284
|
+
shown = ", ".join(name or identity for identity, name in found[:8])
|
|
2285
|
+
raise LaunchHelmError(
|
|
2286
|
+
"config",
|
|
2287
|
+
f"set LAUNCHHELM_ORGANIZATION; this key is in more than one organization ({shown})",
|
|
2288
|
+
)
|
|
2289
|
+
|
|
2290
|
+
|
|
2291
|
+
def _api_url() -> str:
|
|
2292
|
+
return os.environ.get("LAUNCHHELM_URL") or DEFAULT_URL
|
|
2293
|
+
|
|
2294
|
+
|
|
2295
|
+
def _env(name: str) -> str:
|
|
2296
|
+
value = os.environ.get(name, "")
|
|
2297
|
+
if not value:
|
|
2298
|
+
raise LaunchHelmError("config", f"{name} is required")
|
|
2299
|
+
return value
|
|
2300
|
+
|
|
2301
|
+
|
|
2302
|
+
def _metadata(labels: JSONObject, group: str, tags: Sequence[str], notes: str) -> None:
|
|
2303
|
+
if not isinstance(group, str) or len(group) > 256:
|
|
2304
|
+
raise ValueError("group must be at most 256 characters")
|
|
2305
|
+
if not isinstance(notes, str) or len(notes) > _MAX_LABEL:
|
|
2306
|
+
raise ValueError("notes must be at most 1024 characters")
|
|
2307
|
+
if not isinstance(tags, Sequence) or isinstance(tags, str):
|
|
2308
|
+
raise TypeError("tags must be a sequence of strings")
|
|
2309
|
+
cleaned = []
|
|
2310
|
+
for tag in tags:
|
|
2311
|
+
if not isinstance(tag, str) or not tag or len(tag) > 64:
|
|
2312
|
+
raise ValueError("each tag must be 1 to 64 characters")
|
|
2313
|
+
cleaned.append(tag)
|
|
2314
|
+
encoded = json.dumps(cleaned, separators=(",", ":"))
|
|
2315
|
+
if len(encoded) > _MAX_LABEL:
|
|
2316
|
+
raise ValueError("tags must fit in 1024 characters")
|
|
2317
|
+
if group:
|
|
2318
|
+
labels["launchhelm.group"] = group
|
|
2319
|
+
if cleaned:
|
|
2320
|
+
labels["launchhelm.tags"] = encoded
|
|
2321
|
+
if notes:
|
|
2322
|
+
labels["launchhelm.notes"] = notes
|
|
2323
|
+
|
|
2324
|
+
|
|
2325
|
+
def _provenance(labels: JSONObject) -> None:
|
|
2326
|
+
"""Record the code and machine that opened the run.
|
|
2327
|
+
|
|
2328
|
+
Contract: ``launchhelm.python``, ``launchhelm.host``, and ``launchhelm.argv``
|
|
2329
|
+
are always set. ``argv`` is the process command, at most 1024 characters.
|
|
2330
|
+
``launchhelm.git`` is the current commit when this directory is a git
|
|
2331
|
+
checkout. A missing git command or a non-repository leaves that label
|
|
2332
|
+
unset. These labels are not config.
|
|
2333
|
+
|
|
2334
|
+
Failure: none. Capture never blocks ``init``.
|
|
2335
|
+
"""
|
|
2336
|
+
labels["launchhelm.python"] = platform.python_version()
|
|
2337
|
+
command = shlex.join(sys.argv)
|
|
2338
|
+
if command:
|
|
2339
|
+
labels["launchhelm.argv"] = command[:_MAX_LABEL]
|
|
2340
|
+
host = socket.gethostname()
|
|
2341
|
+
if host:
|
|
2342
|
+
labels["launchhelm.host"] = host[:256]
|
|
2343
|
+
git = shutil.which("git")
|
|
2344
|
+
if git is None:
|
|
2345
|
+
return
|
|
2346
|
+
try:
|
|
2347
|
+
completed = subprocess.run( # noqa: S603 - fixed argv, resolved binary
|
|
2348
|
+
[git, "rev-parse", "HEAD"],
|
|
2349
|
+
check=False,
|
|
2350
|
+
capture_output=True,
|
|
2351
|
+
text=True,
|
|
2352
|
+
timeout=2,
|
|
2353
|
+
)
|
|
2354
|
+
except (OSError, subprocess.TimeoutExpired):
|
|
2355
|
+
return
|
|
2356
|
+
revision = completed.stdout.strip()
|
|
2357
|
+
if completed.returncode == 0 and re.fullmatch(r"[0-9a-fA-F]{7,64}", revision):
|
|
2358
|
+
labels["launchhelm.git"] = revision.lower()
|
|
2359
|
+
|
|
2360
|
+
|
|
2361
|
+
def _metadata_value(document: Document, key: str) -> str:
|
|
2362
|
+
labels = document.spec.get("labels")
|
|
2363
|
+
if not isinstance(labels, dict):
|
|
2364
|
+
return ""
|
|
2365
|
+
value = labels.get(key)
|
|
2366
|
+
return value if isinstance(value, str) else ""
|
|
2367
|
+
|
|
2368
|
+
|
|
2369
|
+
def _metadata_tags(document: Document) -> tuple[str, ...]:
|
|
2370
|
+
raw = _metadata_value(document, "launchhelm.tags")
|
|
2371
|
+
if not raw:
|
|
2372
|
+
return ()
|
|
2373
|
+
parsed = json.loads(raw)
|
|
2374
|
+
if not isinstance(parsed, list) or any(not isinstance(item, str) for item in parsed):
|
|
2375
|
+
raise LaunchHelmError("protocol", "run tags are not a list of strings")
|
|
2376
|
+
return tuple(parsed)
|
|
2377
|
+
|
|
2378
|
+
|
|
2379
|
+
def _labels(config: Mapping[str, JSONValue]) -> JSONObject:
|
|
2380
|
+
if len(config) > _MAX_LABELS:
|
|
2381
|
+
raise ValueError("config has more than 64 labels")
|
|
2382
|
+
labels: JSONObject = {}
|
|
2383
|
+
for key, value in config.items():
|
|
2384
|
+
if not isinstance(key, str) or not key:
|
|
2385
|
+
raise ValueError("config label name is empty")
|
|
2386
|
+
if isinstance(value, bool):
|
|
2387
|
+
text = "true" if value else "false"
|
|
2388
|
+
elif isinstance(value, int):
|
|
2389
|
+
text = str(value)
|
|
2390
|
+
elif isinstance(value, float):
|
|
2391
|
+
if not math.isfinite(value):
|
|
2392
|
+
raise ValueError("config label cannot be non-finite")
|
|
2393
|
+
text = format(value, ".16g")
|
|
2394
|
+
elif isinstance(value, str):
|
|
2395
|
+
text = value
|
|
2396
|
+
elif isinstance(value, dict | list):
|
|
2397
|
+
encoded = _JSON_LABEL + json.dumps(value, allow_nan=False, separators=(",", ":"))
|
|
2398
|
+
if len(encoded) > _MAX_LABEL:
|
|
2399
|
+
continue
|
|
2400
|
+
text = encoded
|
|
2401
|
+
else:
|
|
2402
|
+
raise ValueError("config labels must be scalars, objects, or lists")
|
|
2403
|
+
if len(text) > _MAX_LABEL:
|
|
2404
|
+
continue
|
|
2405
|
+
labels[key] = text
|
|
2406
|
+
return labels
|
|
2407
|
+
|
|
2408
|
+
|
|
2409
|
+
def _reduction_item(run: Document, name: str) -> dict[str, JSONValue] | None:
|
|
2410
|
+
labels = run.spec.get("labels")
|
|
2411
|
+
raw = labels.get("launchhelm.reduction") if isinstance(labels, dict) else None
|
|
2412
|
+
if not isinstance(raw, str) or not raw:
|
|
2413
|
+
return None
|
|
2414
|
+
parsed = json.loads(raw)
|
|
2415
|
+
item = parsed.get(name) if isinstance(parsed, dict) else None
|
|
2416
|
+
return item if isinstance(item, dict) else None
|
|
2417
|
+
|
|
2418
|
+
|
|
2419
|
+
def _summary_from_run(run: Document) -> dict[str, JSONValue]:
|
|
2420
|
+
labels = run.spec.get("labels")
|
|
2421
|
+
if not isinstance(labels, dict):
|
|
2422
|
+
return {}
|
|
2423
|
+
raw = labels.get("launchhelm.summary")
|
|
2424
|
+
if not isinstance(raw, str) or not raw:
|
|
2425
|
+
return {}
|
|
2426
|
+
parsed = json.loads(raw)
|
|
2427
|
+
if not isinstance(parsed, dict):
|
|
2428
|
+
raise LaunchHelmError("config", "summary label is not an object")
|
|
2429
|
+
return parsed
|
|
2430
|
+
|
|
2431
|
+
|
|
2432
|
+
def _resume_config(client: SessionClient, run: Document) -> dict[str, JSONValue]:
|
|
2433
|
+
parameters = run.spec.get("parameters")
|
|
2434
|
+
if isinstance(parameters, dict):
|
|
2435
|
+
return dict(parameters)
|
|
2436
|
+
labels = run.spec.get("labels")
|
|
2437
|
+
if isinstance(labels, dict):
|
|
2438
|
+
artifact_id = labels.get("launchhelm.config")
|
|
2439
|
+
if isinstance(artifact_id, str) and artifact_id:
|
|
2440
|
+
download = getattr(client, "download_artifact", None)
|
|
2441
|
+
if not callable(download):
|
|
2442
|
+
raise LaunchHelmError("protocol", "session client cannot download config")
|
|
2443
|
+
body = download(artifact_id)
|
|
2444
|
+
if not isinstance(body, bytes | str):
|
|
2445
|
+
raise LaunchHelmError("protocol", "config artifact is not text")
|
|
2446
|
+
parsed = json.loads(body)
|
|
2447
|
+
if not isinstance(parsed, dict):
|
|
2448
|
+
raise LaunchHelmError("config", "config artifact is not an object")
|
|
2449
|
+
return parsed
|
|
2450
|
+
return _config_from_labels(labels)
|
|
2451
|
+
|
|
2452
|
+
|
|
2453
|
+
def _config_from_labels(labels: object) -> dict[str, JSONValue]:
|
|
2454
|
+
if not isinstance(labels, dict):
|
|
2455
|
+
return {}
|
|
2456
|
+
config: dict[str, JSONValue] = {}
|
|
2457
|
+
for key, value in labels.items():
|
|
2458
|
+
if not isinstance(key, str) or not isinstance(value, str):
|
|
2459
|
+
continue
|
|
2460
|
+
if key in {
|
|
2461
|
+
"launchhelm.summary",
|
|
2462
|
+
"launchhelm.config",
|
|
2463
|
+
"launchhelm.group",
|
|
2464
|
+
"launchhelm.tags",
|
|
2465
|
+
"launchhelm.notes",
|
|
2466
|
+
"launchhelm.git",
|
|
2467
|
+
"launchhelm.host",
|
|
2468
|
+
"launchhelm.python",
|
|
2469
|
+
"launchhelm.argv",
|
|
2470
|
+
}:
|
|
2471
|
+
continue
|
|
2472
|
+
if value.startswith(_JSON_LABEL):
|
|
2473
|
+
parsed = json.loads(value[len(_JSON_LABEL) :])
|
|
2474
|
+
if not isinstance(parsed, dict | list):
|
|
2475
|
+
raise LaunchHelmError("config", f"config label {key!r} is not an object or list")
|
|
2476
|
+
config[key] = parsed
|
|
2477
|
+
else:
|
|
2478
|
+
config[key] = value
|
|
2479
|
+
return config
|
|
2480
|
+
|
|
2481
|
+
|
|
2482
|
+
def _open_project(client: SessionClient, name: str, project_id: str) -> Document | None:
|
|
2483
|
+
if not project_id:
|
|
2484
|
+
return _project(client, name)
|
|
2485
|
+
document = client.get_document(project_id)
|
|
2486
|
+
if document.kind != "project":
|
|
2487
|
+
raise LaunchHelmError("not_found", f"project {project_id!r} does not exist")
|
|
2488
|
+
if document.spec.get("name") != name:
|
|
2489
|
+
raise LaunchHelmError("conflict", f"project {project_id!r} is not named {name!r}")
|
|
2490
|
+
return document
|
|
2491
|
+
|
|
2492
|
+
|
|
2493
|
+
def _project(client: SessionClient, name: str) -> Document | None:
|
|
2494
|
+
cursor = ""
|
|
2495
|
+
matches: list[Document] = []
|
|
2496
|
+
for _ in range(_MAX_PAGES):
|
|
2497
|
+
page = client.list_documents(kind="project", cursor=cursor, limit=100)
|
|
2498
|
+
items = page.get("items") or []
|
|
2499
|
+
if not isinstance(items, list):
|
|
2500
|
+
raise LaunchHelmError("protocol", "project list is not a page")
|
|
2501
|
+
for item in items:
|
|
2502
|
+
if not isinstance(item, dict):
|
|
2503
|
+
raise LaunchHelmError("protocol", "project list entry is not a document")
|
|
2504
|
+
document = Document.parse(item)
|
|
2505
|
+
if document.kind == "project" and document.spec.get("name") == name:
|
|
2506
|
+
matches.append(document)
|
|
2507
|
+
next_cursor = page.get("next_cursor") or ""
|
|
2508
|
+
if not isinstance(next_cursor, str) or not next_cursor or next_cursor == cursor:
|
|
2509
|
+
break
|
|
2510
|
+
cursor = next_cursor
|
|
2511
|
+
if len(matches) > 1:
|
|
2512
|
+
raise LaunchHelmError("conflict", f"more than one project is named {name!r}")
|
|
2513
|
+
return matches[0] if matches else None
|
|
2514
|
+
|
|
2515
|
+
|
|
2516
|
+
def _scalar(value: JSONValue) -> tuple[str, JSONValue]:
|
|
2517
|
+
if isinstance(value, bool):
|
|
2518
|
+
return "boolean", value
|
|
2519
|
+
if isinstance(value, str):
|
|
2520
|
+
return "string", value
|
|
2521
|
+
if isinstance(value, int | float):
|
|
2522
|
+
return "number", scalar(value)
|
|
2523
|
+
raise TypeError("metric value must be a boolean, string, or number")
|
|
2524
|
+
|
|
2525
|
+
|
|
2526
|
+
def _coordinate(value: int | float | str) -> str:
|
|
2527
|
+
if isinstance(value, bool) or not isinstance(value, int | float | str):
|
|
2528
|
+
raise TypeError("coordinate must be an integer, finite float, or decimal string")
|
|
2529
|
+
if isinstance(value, float):
|
|
2530
|
+
if not math.isfinite(value):
|
|
2531
|
+
raise ValueError("coordinate cannot be non-finite")
|
|
2532
|
+
text = format(Decimal(str(value)), "f")
|
|
2533
|
+
if "." in text:
|
|
2534
|
+
text = text.rstrip("0").rstrip(".")
|
|
2535
|
+
if text.startswith("."):
|
|
2536
|
+
text = "0" + text
|
|
2537
|
+
elif text.startswith("-."):
|
|
2538
|
+
text = "-0" + text[1:]
|
|
2539
|
+
if text in {"", "-"}:
|
|
2540
|
+
text = "0"
|
|
2541
|
+
elif isinstance(value, int):
|
|
2542
|
+
text = str(value)
|
|
2543
|
+
else:
|
|
2544
|
+
text = value
|
|
2545
|
+
if not _COORD.fullmatch(text):
|
|
2546
|
+
raise ValueError(f"coordinate {text!r} is not a decimal")
|
|
2547
|
+
return text
|
|
2548
|
+
|
|
2549
|
+
|
|
2550
|
+
def _run(client: SessionClient, project_id: str, token: str) -> Document:
|
|
2551
|
+
cursor = ""
|
|
2552
|
+
matches: list[Document] = []
|
|
2553
|
+
for _ in range(_MAX_PAGES):
|
|
2554
|
+
page = client.list_documents(kind="run", project_id=project_id, cursor=cursor, limit=100)
|
|
2555
|
+
items = page.get("items") or []
|
|
2556
|
+
if not isinstance(items, list):
|
|
2557
|
+
raise LaunchHelmError("protocol", "run list is not a page")
|
|
2558
|
+
for item in items:
|
|
2559
|
+
if not isinstance(item, dict):
|
|
2560
|
+
raise LaunchHelmError("protocol", "run list entry is not a document")
|
|
2561
|
+
document = Document.parse(item)
|
|
2562
|
+
if document.kind != "run":
|
|
2563
|
+
continue
|
|
2564
|
+
if document.id == token or document.spec.get("name") == token:
|
|
2565
|
+
matches.append(document)
|
|
2566
|
+
next_cursor = page.get("next_cursor") or ""
|
|
2567
|
+
if not isinstance(next_cursor, str) or not next_cursor or next_cursor == cursor:
|
|
2568
|
+
break
|
|
2569
|
+
cursor = next_cursor
|
|
2570
|
+
if not matches:
|
|
2571
|
+
raise LaunchHelmError("not_found", f"open run {token!r} was not found")
|
|
2572
|
+
if len(matches) > 1:
|
|
2573
|
+
raise LaunchHelmError("conflict", f"more than one run is named {token!r}")
|
|
2574
|
+
opened = matches[0]
|
|
2575
|
+
if opened.spec.get("state") != "open":
|
|
2576
|
+
raise LaunchHelmError("conflict", f"run {token!r} is not open")
|
|
2577
|
+
return opened
|
|
2578
|
+
|
|
2579
|
+
|
|
2580
|
+
def _pending_participant(directory: Path) -> str:
|
|
2581
|
+
root = directory / "participants"
|
|
2582
|
+
if not root.is_dir():
|
|
2583
|
+
return ""
|
|
2584
|
+
chosen = ""
|
|
2585
|
+
newest = -1.0
|
|
2586
|
+
for child in root.iterdir():
|
|
2587
|
+
database = child / "state.sqlite3"
|
|
2588
|
+
if not database.is_file():
|
|
2589
|
+
continue
|
|
2590
|
+
try:
|
|
2591
|
+
connection = sqlite3.connect(f"file:{database}?mode=ro", uri=True)
|
|
2592
|
+
try:
|
|
2593
|
+
row = connection.execute("SELECT COUNT(*) FROM points").fetchone()
|
|
2594
|
+
pending = int(row[0]) if row else 0
|
|
2595
|
+
try:
|
|
2596
|
+
row = connection.execute("SELECT COUNT(*) FROM media_points").fetchone()
|
|
2597
|
+
pending += int(row[0]) if row else 0
|
|
2598
|
+
except sqlite3.Error:
|
|
2599
|
+
# Spool files written before media support lack the table.
|
|
2600
|
+
pass
|
|
2601
|
+
finally:
|
|
2602
|
+
connection.close()
|
|
2603
|
+
except sqlite3.Error:
|
|
2604
|
+
continue
|
|
2605
|
+
if pending == 0:
|
|
2606
|
+
continue
|
|
2607
|
+
modified = database.stat().st_mtime
|
|
2608
|
+
if modified >= newest:
|
|
2609
|
+
newest = modified
|
|
2610
|
+
chosen = child.name
|
|
2611
|
+
return chosen
|
|
2612
|
+
|
|
2613
|
+
|
|
2614
|
+
def _saved_participant(
|
|
2615
|
+
client: SessionClient, run: Document, participant_id: str
|
|
2616
|
+
) -> Document | None:
|
|
2617
|
+
if not participant_id:
|
|
2618
|
+
return None
|
|
2619
|
+
try:
|
|
2620
|
+
document = client.get_document(participant_id)
|
|
2621
|
+
except (LaunchHelmError, KeyError):
|
|
2622
|
+
return None
|
|
2623
|
+
if document.kind != "participant" or document.scope.run_id != run.id:
|
|
2624
|
+
return None
|
|
2625
|
+
return document
|
|
2626
|
+
|
|
2627
|
+
|
|
2628
|
+
def _view_point(point: JSONValue) -> dict[str, JSONValue]:
|
|
2629
|
+
if not isinstance(point, dict):
|
|
2630
|
+
raise LaunchHelmError("protocol", "telemetry point is not an object")
|
|
2631
|
+
observed = point.get("observed_at")
|
|
2632
|
+
if not isinstance(observed, str):
|
|
2633
|
+
raise LaunchHelmError("protocol", "telemetry point has no observed_at")
|
|
2634
|
+
coordinates = point.get("coordinates")
|
|
2635
|
+
if not isinstance(coordinates, list):
|
|
2636
|
+
raise LaunchHelmError("protocol", "telemetry point has no coordinates")
|
|
2637
|
+
axes: dict[str, JSONValue] = {}
|
|
2638
|
+
for item in coordinates:
|
|
2639
|
+
if not isinstance(item, dict):
|
|
2640
|
+
raise LaunchHelmError("protocol", "telemetry coordinate is not an object")
|
|
2641
|
+
axis = item.get("axis")
|
|
2642
|
+
value = item.get("value")
|
|
2643
|
+
if not isinstance(axis, str) or not isinstance(value, str):
|
|
2644
|
+
raise LaunchHelmError("protocol", "telemetry coordinate is invalid")
|
|
2645
|
+
axes[axis] = value
|
|
2646
|
+
viewed: dict[str, JSONValue] = {
|
|
2647
|
+
"value": point.get("value"),
|
|
2648
|
+
"coordinates": axes,
|
|
2649
|
+
"observed_at": observed,
|
|
2650
|
+
}
|
|
2651
|
+
for key in ("epoch", "sequence"):
|
|
2652
|
+
raw = point.get(key)
|
|
2653
|
+
if isinstance(raw, str) and raw:
|
|
2654
|
+
viewed[key] = raw
|
|
2655
|
+
return viewed
|
|
2656
|
+
|
|
2657
|
+
|
|
2658
|
+
def _newer_coordinates(
|
|
2659
|
+
series: list[dict[str, JSONValue]],
|
|
2660
|
+
) -> list[dict[str, JSONValue]]:
|
|
2661
|
+
chosen: dict[tuple[tuple[str, JSONValue], ...], dict[str, JSONValue]] = {}
|
|
2662
|
+
order: list[tuple[tuple[str, JSONValue], ...]] = []
|
|
2663
|
+
for point in series:
|
|
2664
|
+
coordinates = point["coordinates"]
|
|
2665
|
+
key = tuple(sorted(coordinates.items())) if isinstance(coordinates, dict) else ()
|
|
2666
|
+
sequence = point.get("sequence")
|
|
2667
|
+
if isinstance(sequence, str) and sequence:
|
|
2668
|
+
key = (*key, ("sequence", sequence))
|
|
2669
|
+
previous = chosen.get(key)
|
|
2670
|
+
if previous is None:
|
|
2671
|
+
order.append(key)
|
|
2672
|
+
chosen[key] = point
|
|
2673
|
+
elif str(point["observed_at"]) >= str(previous["observed_at"]):
|
|
2674
|
+
chosen[key] = point
|
|
2675
|
+
return [chosen[key] for key in order]
|
|
2676
|
+
|
|
2677
|
+
|
|
2678
|
+
def _step_in(point: dict[str, JSONValue], start: int | None, end: int | None) -> bool:
|
|
2679
|
+
if start is None and end is None:
|
|
2680
|
+
return True
|
|
2681
|
+
coordinates = point["coordinates"]
|
|
2682
|
+
step = coordinates.get("step") if isinstance(coordinates, dict) else None
|
|
2683
|
+
if not isinstance(step, str) or not step.lstrip("-").isdigit():
|
|
2684
|
+
return False
|
|
2685
|
+
value = int(step)
|
|
2686
|
+
if start is not None and value < start:
|
|
2687
|
+
return False
|
|
2688
|
+
return not (end is not None and value > end)
|
|
2689
|
+
|
|
2690
|
+
|
|
2691
|
+
def _newest_point(query: Callable[..., JSONValue], stream_id: str) -> dict[str, JSONValue] | None:
|
|
2692
|
+
page = query(
|
|
2693
|
+
stream_id,
|
|
2694
|
+
start="1970-01-01T00:00:00Z",
|
|
2695
|
+
end="2261-01-01T00:00:00Z",
|
|
2696
|
+
limit=1,
|
|
2697
|
+
order="desc",
|
|
2698
|
+
)
|
|
2699
|
+
items = page.get("items") if isinstance(page, dict) else None
|
|
2700
|
+
if not isinstance(items, list) or not items:
|
|
2701
|
+
return None
|
|
2702
|
+
point = items[0]
|
|
2703
|
+
if not isinstance(point, dict):
|
|
2704
|
+
raise LaunchHelmError("protocol", "telemetry point is not an object")
|
|
2705
|
+
observed = point.get("observed_at")
|
|
2706
|
+
if not isinstance(observed, str):
|
|
2707
|
+
raise LaunchHelmError("protocol", "telemetry point has no observed_at")
|
|
2708
|
+
coordinates = point.get("coordinates")
|
|
2709
|
+
if not isinstance(coordinates, list):
|
|
2710
|
+
raise LaunchHelmError("protocol", "telemetry point has no coordinates")
|
|
2711
|
+
axes: dict[str, JSONValue] = {}
|
|
2712
|
+
for item in coordinates:
|
|
2713
|
+
if not isinstance(item, dict):
|
|
2714
|
+
raise LaunchHelmError("protocol", "telemetry coordinate is not an object")
|
|
2715
|
+
axis = item.get("axis")
|
|
2716
|
+
value = item.get("value")
|
|
2717
|
+
if not isinstance(axis, str) or not isinstance(value, str):
|
|
2718
|
+
raise LaunchHelmError("protocol", "telemetry coordinate is invalid")
|
|
2719
|
+
axes[axis] = value
|
|
2720
|
+
return {"value": point.get("value"), "coordinates": axes, "observed_at": observed}
|
|
2721
|
+
|
|
2722
|
+
|
|
2723
|
+
def _history_key(point: dict[str, JSONValue]) -> tuple[int, str]:
|
|
2724
|
+
coordinates = point["coordinates"]
|
|
2725
|
+
step = coordinates.get("step") if isinstance(coordinates, dict) else None
|
|
2726
|
+
rank = 2**63 - 1
|
|
2727
|
+
if isinstance(step, str) and step.lstrip("-").isdigit():
|
|
2728
|
+
rank = int(step)
|
|
2729
|
+
observed = point["observed_at"]
|
|
2730
|
+
return rank, observed if isinstance(observed, str) else ""
|
|
2731
|
+
|
|
2732
|
+
|
|
2733
|
+
def _server_cursor(
|
|
2734
|
+
client: SessionClient, project_id: str, run_id: str
|
|
2735
|
+
) -> tuple[int, dict[str, JSONValue]]:
|
|
2736
|
+
query = getattr(client, "query", None)
|
|
2737
|
+
if not callable(query):
|
|
2738
|
+
return 0, {}
|
|
2739
|
+
highest = -1
|
|
2740
|
+
newest_at = ""
|
|
2741
|
+
coordinates: dict[str, JSONValue] = {}
|
|
2742
|
+
cursor = ""
|
|
2743
|
+
for _ in range(_MAX_PAGES):
|
|
2744
|
+
page = client.list_documents(
|
|
2745
|
+
kind="stream",
|
|
2746
|
+
project_id=project_id,
|
|
2747
|
+
run_id=run_id,
|
|
2748
|
+
cursor=cursor,
|
|
2749
|
+
limit=100,
|
|
2750
|
+
)
|
|
2751
|
+
items = page.get("items") or []
|
|
2752
|
+
if not isinstance(items, list):
|
|
2753
|
+
return 0, {}
|
|
2754
|
+
for item in items:
|
|
2755
|
+
if not isinstance(item, dict):
|
|
2756
|
+
continue
|
|
2757
|
+
document = Document.parse(item)
|
|
2758
|
+
if document.kind != "stream":
|
|
2759
|
+
continue
|
|
2760
|
+
try:
|
|
2761
|
+
points = query(
|
|
2762
|
+
document.id,
|
|
2763
|
+
start="1970-01-01T00:00:00Z",
|
|
2764
|
+
end="2261-01-01T00:00:00Z",
|
|
2765
|
+
limit=1,
|
|
2766
|
+
order="desc",
|
|
2767
|
+
)
|
|
2768
|
+
except (LaunchHelmError, TypeError, ValueError):
|
|
2769
|
+
continue
|
|
2770
|
+
listed = points.get("items") if isinstance(points, dict) else None
|
|
2771
|
+
if not isinstance(listed, list) or not listed:
|
|
2772
|
+
continue
|
|
2773
|
+
viewed = _view_point(listed[0])
|
|
2774
|
+
axes = viewed["coordinates"]
|
|
2775
|
+
step = axes.get("step") if isinstance(axes, dict) else None
|
|
2776
|
+
if isinstance(step, str) and step.lstrip("-").isdigit():
|
|
2777
|
+
highest = max(highest, int(step))
|
|
2778
|
+
observed = viewed["observed_at"]
|
|
2779
|
+
if isinstance(observed, str) and observed > newest_at and isinstance(axes, dict):
|
|
2780
|
+
newest_at = observed
|
|
2781
|
+
coordinates = dict(axes)
|
|
2782
|
+
next_cursor = page.get("next_cursor") or ""
|
|
2783
|
+
if not isinstance(next_cursor, str) or not next_cursor or next_cursor == cursor:
|
|
2784
|
+
break
|
|
2785
|
+
cursor = next_cursor
|
|
2786
|
+
return 0 if highest < 0 else highest + 1, coordinates
|
|
2787
|
+
|
|
2788
|
+
|
|
2789
|
+
def _read_checkpoint(
|
|
2790
|
+
path: Path,
|
|
2791
|
+
) -> tuple[int, dict[str, JSONValue], str, dict[str, JSONValue] | None, str]:
|
|
2792
|
+
if not path.is_file():
|
|
2793
|
+
return 0, {}, "", None, ""
|
|
2794
|
+
body = json.loads(path.read_text(encoding="utf-8"))
|
|
2795
|
+
if not isinstance(body, dict):
|
|
2796
|
+
raise LaunchHelmError("config", "resume checkpoint is not an object")
|
|
2797
|
+
step = body.get("step", 0)
|
|
2798
|
+
state = body.get("state", {})
|
|
2799
|
+
participant_id = body.get("participant_id", "")
|
|
2800
|
+
if isinstance(step, bool) or not isinstance(step, int) or step < 0:
|
|
2801
|
+
raise LaunchHelmError("config", "resume checkpoint step is invalid")
|
|
2802
|
+
if not isinstance(state, dict):
|
|
2803
|
+
raise LaunchHelmError("config", "resume checkpoint state is invalid")
|
|
2804
|
+
if not isinstance(participant_id, str):
|
|
2805
|
+
raise LaunchHelmError("config", "resume checkpoint participant is invalid")
|
|
2806
|
+
run_id = body.get("run_id", "")
|
|
2807
|
+
if not isinstance(run_id, str):
|
|
2808
|
+
raise LaunchHelmError("config", "resume checkpoint run is invalid")
|
|
2809
|
+
restored: dict[str, JSONValue] = {}
|
|
2810
|
+
for key, value in state.items():
|
|
2811
|
+
if not isinstance(key, str):
|
|
2812
|
+
raise LaunchHelmError("config", "resume checkpoint state key is invalid")
|
|
2813
|
+
restored[key] = value
|
|
2814
|
+
saved_summary: dict[str, JSONValue] | None = None
|
|
2815
|
+
if "summary" in body:
|
|
2816
|
+
summary = body["summary"]
|
|
2817
|
+
if not isinstance(summary, dict):
|
|
2818
|
+
raise LaunchHelmError("config", "resume checkpoint summary is invalid")
|
|
2819
|
+
saved_summary = {}
|
|
2820
|
+
for key, value in summary.items():
|
|
2821
|
+
if not isinstance(key, str):
|
|
2822
|
+
raise LaunchHelmError("config", "resume checkpoint summary key is invalid")
|
|
2823
|
+
saved_summary[key] = value
|
|
2824
|
+
return step, restored, participant_id, saved_summary, run_id
|