attestation-schema 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- attestation_schema/__init__.py +158 -0
- attestation_schema/canonical.py +110 -0
- attestation_schema/history.py +366 -0
- attestation_schema/jobs.py +135 -0
- attestation_schema/layout.py +137 -0
- attestation_schema/py.typed +0 -0
- attestation_schema/record.py +843 -0
- attestation_schema/schemas/history-1.schema.json +101 -0
- attestation_schema/schemas/run-1.schema.json +393 -0
- attestation_schema/schemas/state-1.schema.json +90 -0
- attestation_schema/state.py +590 -0
- attestation_schema-0.1.0.dist-info/METADATA +105 -0
- attestation_schema-0.1.0.dist-info/RECORD +15 -0
- attestation_schema-0.1.0.dist-info/WHEEL +4 -0
- attestation_schema-0.1.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
"""attestation-schema: the core run record (`attestation.run/1`, frozen 2026-11-19).
|
|
2
|
+
|
|
3
|
+
Standard library only, and imports nothing from `attestation`: producers
|
|
4
|
+
depend on this package to write records without depending on the observer.
|
|
5
|
+
Curves live in a sidecar (`history.py`), bound to the record by sha256; a
|
|
6
|
+
job's state lives in `state.json` (`state.py`).
|
|
7
|
+
Spec: docs/superpowers/specs/2026-11-19-core-run-record-design.md.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from attestation_schema.history import (
|
|
11
|
+
HISTORY_SCHEMA,
|
|
12
|
+
HistoryWriter,
|
|
13
|
+
Point,
|
|
14
|
+
Series,
|
|
15
|
+
read_history,
|
|
16
|
+
write_history,
|
|
17
|
+
)
|
|
18
|
+
from attestation_schema.jobs import (
|
|
19
|
+
TERMINAL,
|
|
20
|
+
TRANSITIONS,
|
|
21
|
+
JobBackend,
|
|
22
|
+
JobError,
|
|
23
|
+
JobExistsError,
|
|
24
|
+
JobHandle,
|
|
25
|
+
JobNotFoundError,
|
|
26
|
+
JobNotTerminalError,
|
|
27
|
+
JobSpec,
|
|
28
|
+
JobState,
|
|
29
|
+
JobStatus,
|
|
30
|
+
can_transition,
|
|
31
|
+
)
|
|
32
|
+
from attestation_schema.record import (
|
|
33
|
+
HISTORY_PATH,
|
|
34
|
+
NON_FINITE,
|
|
35
|
+
READERS,
|
|
36
|
+
SCHEMA,
|
|
37
|
+
SCHEMA_STATUS,
|
|
38
|
+
SOURCE_KINDS,
|
|
39
|
+
CodeIdentity,
|
|
40
|
+
DataDigest,
|
|
41
|
+
Dependency,
|
|
42
|
+
HistoryDigestError,
|
|
43
|
+
HistoryError,
|
|
44
|
+
HistoryFormatError,
|
|
45
|
+
HistoryMissingError,
|
|
46
|
+
HistoryRef,
|
|
47
|
+
InvalidFieldError,
|
|
48
|
+
Metric,
|
|
49
|
+
MissingFieldError,
|
|
50
|
+
Producer,
|
|
51
|
+
RecordError,
|
|
52
|
+
RunRecord,
|
|
53
|
+
UnknownFieldError,
|
|
54
|
+
UnsupportedSchemaError,
|
|
55
|
+
dumps,
|
|
56
|
+
format_timestamp,
|
|
57
|
+
from_dict,
|
|
58
|
+
loads,
|
|
59
|
+
parse_strict,
|
|
60
|
+
read,
|
|
61
|
+
record_digest,
|
|
62
|
+
record_path,
|
|
63
|
+
spell_value,
|
|
64
|
+
write,
|
|
65
|
+
)
|
|
66
|
+
from attestation_schema.state import (
|
|
67
|
+
STATE_SCHEMA,
|
|
68
|
+
UNKNOWN_CODE,
|
|
69
|
+
IllegalTransitionError,
|
|
70
|
+
StateFile,
|
|
71
|
+
StateFormatError,
|
|
72
|
+
StateLockUnavailableError,
|
|
73
|
+
Transition,
|
|
74
|
+
dumps_state,
|
|
75
|
+
handle_in,
|
|
76
|
+
init_state,
|
|
77
|
+
loads_state,
|
|
78
|
+
read_state,
|
|
79
|
+
settle,
|
|
80
|
+
spec_conflicts,
|
|
81
|
+
state_from_dict,
|
|
82
|
+
state_lock,
|
|
83
|
+
state_path,
|
|
84
|
+
terminal_record,
|
|
85
|
+
transition,
|
|
86
|
+
)
|
|
87
|
+
|
|
88
|
+
__all__ = [
|
|
89
|
+
"HISTORY_PATH",
|
|
90
|
+
"HISTORY_SCHEMA",
|
|
91
|
+
"NON_FINITE",
|
|
92
|
+
"READERS",
|
|
93
|
+
"SCHEMA",
|
|
94
|
+
"SCHEMA_STATUS",
|
|
95
|
+
"TERMINAL",
|
|
96
|
+
"TRANSITIONS",
|
|
97
|
+
"CodeIdentity",
|
|
98
|
+
"DataDigest",
|
|
99
|
+
"Dependency",
|
|
100
|
+
"HistoryDigestError",
|
|
101
|
+
"HistoryError",
|
|
102
|
+
"HistoryFormatError",
|
|
103
|
+
"HistoryMissingError",
|
|
104
|
+
"HistoryRef",
|
|
105
|
+
"HistoryWriter",
|
|
106
|
+
"InvalidFieldError",
|
|
107
|
+
"JobBackend",
|
|
108
|
+
"JobError",
|
|
109
|
+
"JobExistsError",
|
|
110
|
+
"JobHandle",
|
|
111
|
+
"JobNotFoundError",
|
|
112
|
+
"JobNotTerminalError",
|
|
113
|
+
"JobSpec",
|
|
114
|
+
"JobState",
|
|
115
|
+
"JobStatus",
|
|
116
|
+
"Metric",
|
|
117
|
+
"MissingFieldError",
|
|
118
|
+
"Point",
|
|
119
|
+
"Producer",
|
|
120
|
+
"RecordError",
|
|
121
|
+
"RunRecord",
|
|
122
|
+
"SOURCE_KINDS",
|
|
123
|
+
"STATE_SCHEMA",
|
|
124
|
+
"UNKNOWN_CODE",
|
|
125
|
+
"IllegalTransitionError",
|
|
126
|
+
"StateFile",
|
|
127
|
+
"StateFormatError",
|
|
128
|
+
"StateLockUnavailableError",
|
|
129
|
+
"Transition",
|
|
130
|
+
"dumps_state",
|
|
131
|
+
"handle_in",
|
|
132
|
+
"init_state",
|
|
133
|
+
"loads_state",
|
|
134
|
+
"parse_strict",
|
|
135
|
+
"read_state",
|
|
136
|
+
"settle",
|
|
137
|
+
"spec_conflicts",
|
|
138
|
+
"state_from_dict",
|
|
139
|
+
"state_lock",
|
|
140
|
+
"state_path",
|
|
141
|
+
"terminal_record",
|
|
142
|
+
"transition",
|
|
143
|
+
"Series",
|
|
144
|
+
"UnknownFieldError",
|
|
145
|
+
"UnsupportedSchemaError",
|
|
146
|
+
"can_transition",
|
|
147
|
+
"dumps",
|
|
148
|
+
"format_timestamp",
|
|
149
|
+
"from_dict",
|
|
150
|
+
"loads",
|
|
151
|
+
"read",
|
|
152
|
+
"read_history",
|
|
153
|
+
"record_digest",
|
|
154
|
+
"record_path",
|
|
155
|
+
"spell_value",
|
|
156
|
+
"write",
|
|
157
|
+
"write_history",
|
|
158
|
+
]
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
"""RFC 8785 JSON Canonicalization Scheme (JCS), standard library only.
|
|
2
|
+
|
|
3
|
+
The one implementation: `attestation.run_record.canonical` re-exports it
|
|
4
|
+
(attestation depends on this package at runtime), and this package may not
|
|
5
|
+
import `attestation` (tests/test_boundaries.py).
|
|
6
|
+
|
|
7
|
+
A run record is signed over its exact bytes (DSSE), so verification never
|
|
8
|
+
needs canonical form. Canonical form buys STABILITY: the same record content
|
|
9
|
+
always serializes to the same bytes, so a record can be named by its digest
|
|
10
|
+
and two builds compared byte for byte -- by this code or by a verifier
|
|
11
|
+
written in another language.
|
|
12
|
+
|
|
13
|
+
`json.dumps(sort_keys=True, separators=(",", ":"))` is close and not equal:
|
|
14
|
+
it orders keys by code point where JCS orders by UTF-16 code unit (they
|
|
15
|
+
differ for characters above U+FFFF against U+E000-U+FFFF), and it prints
|
|
16
|
+
floats Python's way (`1e-07`, `1.0`, `1e+16`) where JCS prints them the
|
|
17
|
+
ECMAScript way (`1e-7`, `1`, `10000000000000000`).
|
|
18
|
+
|
|
19
|
+
Refused rather than rounded: non-finite floats (JSON has no spelling for
|
|
20
|
+
them) and integers outside +/-2^53, which I-JSON -- and every JavaScript
|
|
21
|
+
verifier -- would silently round to a different number.
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
from __future__ import annotations
|
|
25
|
+
|
|
26
|
+
import json
|
|
27
|
+
import math
|
|
28
|
+
from decimal import Decimal
|
|
29
|
+
|
|
30
|
+
MAX_SAFE_INTEGER = 2**53 - 1
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _number(value: float) -> str:
|
|
34
|
+
"""ECMAScript Number::toString for a finite double (ES2020 7.1.12.1)."""
|
|
35
|
+
if not math.isfinite(value):
|
|
36
|
+
raise ValueError(f"JCS cannot represent {value!r}")
|
|
37
|
+
if value == 0:
|
|
38
|
+
return "0" # also -0.0: ES6 prints "0"
|
|
39
|
+
sign = "-" if value < 0 else ""
|
|
40
|
+
# repr() is the shortest round-tripping decimal, which is what ES6 uses too;
|
|
41
|
+
# only the layout differs, so take its digits and re-lay them out.
|
|
42
|
+
_, digit_tuple, exp = Decimal(repr(abs(value))).as_tuple()
|
|
43
|
+
digits = "".join(map(str, digit_tuple)).rstrip("0")
|
|
44
|
+
exp = int(exp) + (len(digit_tuple) - len(digits))
|
|
45
|
+
k = len(digits)
|
|
46
|
+
n = k + exp # position of the decimal point relative to the digits
|
|
47
|
+
if k <= n <= 21:
|
|
48
|
+
return sign + digits + "0" * (n - k)
|
|
49
|
+
if 0 < n <= 21:
|
|
50
|
+
return sign + digits[:n] + "." + digits[n:]
|
|
51
|
+
if -6 < n <= 0:
|
|
52
|
+
return sign + "0." + "0" * (-n) + digits
|
|
53
|
+
e = n - 1
|
|
54
|
+
mantissa = digits[0] + ("." + digits[1:] if k > 1 else "")
|
|
55
|
+
return f"{sign}{mantissa}e{'+' if e > 0 else '-'}{abs(e)}"
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def _string(value: str) -> str:
|
|
59
|
+
# ensure_ascii=False matches ES6 JSON.stringify: only `"`, `\` and C0
|
|
60
|
+
# controls are escaped, with lowercase hex. A lone surrogate is caught
|
|
61
|
+
# by the UTF-8 encode in canonicalize().
|
|
62
|
+
return json.dumps(value, ensure_ascii=False)
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def _utf16_key(key: str) -> bytes:
|
|
66
|
+
return key.encode("utf-16-be", "surrogatepass")
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _serialize(value, out: list[str]) -> None:
|
|
70
|
+
if value is None or isinstance(value, bool):
|
|
71
|
+
out.append({None: "null", True: "true", False: "false"}[value])
|
|
72
|
+
elif isinstance(value, int):
|
|
73
|
+
if abs(value) > MAX_SAFE_INTEGER:
|
|
74
|
+
raise ValueError(f"JCS integer out of the IEEE-754 safe range: {value}")
|
|
75
|
+
out.append(str(value))
|
|
76
|
+
elif isinstance(value, float):
|
|
77
|
+
out.append(_number(value))
|
|
78
|
+
elif isinstance(value, str):
|
|
79
|
+
out.append(_string(value))
|
|
80
|
+
elif isinstance(value, list | tuple):
|
|
81
|
+
_array(value, out)
|
|
82
|
+
elif isinstance(value, dict):
|
|
83
|
+
_object(value, out)
|
|
84
|
+
else:
|
|
85
|
+
raise TypeError(f"JCS cannot serialize {type(value).__name__}")
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def _array(value, out: list[str]) -> None:
|
|
89
|
+
out.append("[")
|
|
90
|
+
for i, item in enumerate(value):
|
|
91
|
+
out.append("," if i else "")
|
|
92
|
+
_serialize(item, out)
|
|
93
|
+
out.append("]")
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def _object(value: dict, out: list[str]) -> None:
|
|
97
|
+
if not all(isinstance(k, str) for k in value):
|
|
98
|
+
raise TypeError("JCS object keys must be strings")
|
|
99
|
+
out.append("{")
|
|
100
|
+
for i, key in enumerate(sorted(value, key=_utf16_key)):
|
|
101
|
+
out.append(("," if i else "") + _string(key) + ":")
|
|
102
|
+
_serialize(value[key], out)
|
|
103
|
+
out.append("}")
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def canonicalize(value) -> bytes:
|
|
107
|
+
"""`value` as RFC 8785 canonical JSON, UTF-8 encoded."""
|
|
108
|
+
out: list[str] = []
|
|
109
|
+
_serialize(value, out)
|
|
110
|
+
return "".join(out).encode("utf-8")
|
|
@@ -0,0 +1,366 @@
|
|
|
1
|
+
"""The history sidecar: a run's logged curves, outside the core record.
|
|
2
|
+
|
|
3
|
+
<run_dir>/attestation/history.jsonl
|
|
4
|
+
|
|
5
|
+
`run.json` never embeds a curve point. Its `history` key is `null` (no curve
|
|
6
|
+
was recorded) or a `HistoryRef` -- `{path, sha256, series, points}` -- that
|
|
7
|
+
binds this file by digest. Reading a record never opens this file;
|
|
8
|
+
`read_history` does, on request, and verifies the digest first.
|
|
9
|
+
|
|
10
|
+
The format (`attestation.history/1`) is JSON Lines, one object per line, each
|
|
11
|
+
line RFC 8785 canonical JSON followed by `\\n` (the last line too: a file
|
|
12
|
+
without a final newline was cut short):
|
|
13
|
+
|
|
14
|
+
{"schema":"attestation.history/1"} line 1, always
|
|
15
|
+
{"labels":{},"name":"loss","series":0,"split":"train"} declares series 0
|
|
16
|
+
{"at":"2026-11-19T10:00:01.500000Z","series":0,"step":0,"value":0.9} a point
|
|
17
|
+
...
|
|
18
|
+
|
|
19
|
+
A series is declared once, immediately before its first point, with ids
|
|
20
|
+
0, 1, 2... in order of declaration, and is identified by `(name, split,
|
|
21
|
+
labels)`; every point names its series by id. Points of different series may
|
|
22
|
+
interleave -- a training loop logs `train/loss` and `eval/loss` alternately,
|
|
23
|
+
and requiring each series to be contiguous would make a long-running producer
|
|
24
|
+
buffer every curve until the end. One point per line, rather than a chunk of
|
|
25
|
+
points per line, because a chunk size is a free parameter: the same points
|
|
26
|
+
chunked twice would be two different files with two different digests. The
|
|
27
|
+
bytes are a pure function of the sequence of `HistoryWriter` calls.
|
|
28
|
+
|
|
29
|
+
`value` is spelled as in `metrics`: a finite JSON number, or `"NaN"`,
|
|
30
|
+
`"Infinity"`, `"-Infinity"`. Spec:
|
|
31
|
+
docs/superpowers/specs/2026-11-19-core-run-record-design.md.
|
|
32
|
+
"""
|
|
33
|
+
|
|
34
|
+
from __future__ import annotations
|
|
35
|
+
|
|
36
|
+
import hashlib
|
|
37
|
+
import json
|
|
38
|
+
import os
|
|
39
|
+
from collections.abc import Iterable
|
|
40
|
+
from dataclasses import dataclass
|
|
41
|
+
from pathlib import Path
|
|
42
|
+
from typing import IO, Any
|
|
43
|
+
|
|
44
|
+
from attestation_schema.canonical import canonicalize
|
|
45
|
+
from attestation_schema.layout import RUN_JSON, LayoutError
|
|
46
|
+
from attestation_schema.record import (
|
|
47
|
+
HISTORY_PATH,
|
|
48
|
+
HistoryError,
|
|
49
|
+
HistoryFormatError,
|
|
50
|
+
HistoryRef,
|
|
51
|
+
InvalidFieldError,
|
|
52
|
+
RecordError,
|
|
53
|
+
RunRecord,
|
|
54
|
+
_at,
|
|
55
|
+
_duplicate_free,
|
|
56
|
+
_integer,
|
|
57
|
+
_label_key,
|
|
58
|
+
_labels,
|
|
59
|
+
_metric_value,
|
|
60
|
+
_no_constant,
|
|
61
|
+
_object,
|
|
62
|
+
_opt_string,
|
|
63
|
+
_strict_float,
|
|
64
|
+
_strict_int,
|
|
65
|
+
_string,
|
|
66
|
+
_timestamp,
|
|
67
|
+
check_history_digest,
|
|
68
|
+
read,
|
|
69
|
+
spell_value,
|
|
70
|
+
)
|
|
71
|
+
|
|
72
|
+
HISTORY_SCHEMA = "attestation.history/1"
|
|
73
|
+
_HEADER = {"schema": HISTORY_SCHEMA}
|
|
74
|
+
_SERIES_KEYS = ("series", "name", "split", "labels")
|
|
75
|
+
_POINT_KEYS = ("series", "step", "value", "at")
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
@dataclass(frozen=True)
|
|
79
|
+
class Point:
|
|
80
|
+
"""One logged value of a series, in the order the producer logged it."""
|
|
81
|
+
|
|
82
|
+
step: int
|
|
83
|
+
value: int | float
|
|
84
|
+
at: str | None
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
@dataclass(frozen=True)
|
|
88
|
+
class Series:
|
|
89
|
+
"""A metric's logged history (a curve). Steps may repeat and need not
|
|
90
|
+
increase: it is a log, not a map. Final values live in `metrics`."""
|
|
91
|
+
|
|
92
|
+
name: str
|
|
93
|
+
split: str | None
|
|
94
|
+
labels: dict[str, str]
|
|
95
|
+
points: list[Point]
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def _line(obj: dict[str, Any]) -> bytes:
|
|
99
|
+
return canonicalize(obj) + b"\n"
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
class HistoryWriter:
|
|
103
|
+
"""Stream a run's curves into `attestation/history.jsonl`, then `close()`
|
|
104
|
+
to get the `HistoryRef` the record carries.
|
|
105
|
+
|
|
106
|
+
with HistoryWriter(run_dir) as hist:
|
|
107
|
+
loss = hist.series("loss", split="train")
|
|
108
|
+
for step in ...:
|
|
109
|
+
hist.point(loss, step, value, at=format_timestamp(now))
|
|
110
|
+
record = RunRecord(..., history=hist.ref, ...)
|
|
111
|
+
|
|
112
|
+
Lines go to a temporary file in `attestation/` as they are logged (call
|
|
113
|
+
`flush()` to push them to the OS); `close()` fsyncs it and moves it into
|
|
114
|
+
place, so `history.jsonl` exists only once finalized, and a producer that
|
|
115
|
+
dies mid-run leaves no sidecar a reader could mistake for a whole one.
|
|
116
|
+
Write order: history (any time during the run) -> `digests.json` ->
|
|
117
|
+
`run.json`; the writer refuses (`LayoutError`, as `write_manifest` does) to
|
|
118
|
+
start or finish once `run.json` exists.
|
|
119
|
+
|
|
120
|
+
Every value is validated as the reader will validate it, so a history the
|
|
121
|
+
reader would refuse cannot be written. A series declared but never given
|
|
122
|
+
a point writes nothing; a history with no points closes to `None`, which is
|
|
123
|
+
what the record says when no curve was recorded.
|
|
124
|
+
"""
|
|
125
|
+
|
|
126
|
+
def __init__(self, run_dir: str | os.PathLike[str], *, overwrite: bool = False):
|
|
127
|
+
self.run_dir = Path(run_dir)
|
|
128
|
+
self.target = self.run_dir / HISTORY_PATH
|
|
129
|
+
self._refuse_after_record()
|
|
130
|
+
if self.target.exists() and not overwrite:
|
|
131
|
+
raise FileExistsError(f"{self.target} exists; pass overwrite=True to replace it")
|
|
132
|
+
self._overwrite = overwrite
|
|
133
|
+
self.target.parent.mkdir(parents=True, exist_ok=True)
|
|
134
|
+
self._tmp = self.target.with_name(f".{self.target.name}.{os.getpid()}.{id(self)}.tmp")
|
|
135
|
+
self._fh: IO[bytes] | None = open(self._tmp, "xb")
|
|
136
|
+
self._hash = hashlib.sha256()
|
|
137
|
+
self._declared: list[tuple[str, str | None, dict[str, str]]] = []
|
|
138
|
+
self._keys: set[tuple[Any, ...]] = set()
|
|
139
|
+
self._ids: dict[int, int] = {} # handle -> id in the file (first point order)
|
|
140
|
+
self._points = 0
|
|
141
|
+
self.ref: HistoryRef | None = None
|
|
142
|
+
self._emit(_HEADER)
|
|
143
|
+
|
|
144
|
+
def _refuse_after_record(self) -> None:
|
|
145
|
+
record = self.run_dir / RUN_JSON
|
|
146
|
+
if record.exists():
|
|
147
|
+
raise LayoutError(f"{record} exists; the history is finalized before the record")
|
|
148
|
+
|
|
149
|
+
def _emit(self, obj: dict[str, Any]) -> None:
|
|
150
|
+
if self._fh is None:
|
|
151
|
+
raise ValueError("this HistoryWriter is closed")
|
|
152
|
+
line = _line(obj)
|
|
153
|
+
self._fh.write(line)
|
|
154
|
+
self._hash.update(line)
|
|
155
|
+
|
|
156
|
+
def series(
|
|
157
|
+
self, name: str, split: str | None = None, labels: dict[str, str] | None = None
|
|
158
|
+
) -> int:
|
|
159
|
+
"""Declare a series; returns the handle `point()` takes. One series
|
|
160
|
+
per `(name, split, labels)`."""
|
|
161
|
+
labels = dict(_labels({} if labels is None else labels, "labels"))
|
|
162
|
+
_string(name, "name")
|
|
163
|
+
_opt_string(split, "split")
|
|
164
|
+
key = (name, split, _label_key(labels))
|
|
165
|
+
if key in self._keys:
|
|
166
|
+
raise _invalid("series", f"duplicate series {key!r}")
|
|
167
|
+
self._keys.add(key)
|
|
168
|
+
self._declared.append((name, split, labels))
|
|
169
|
+
return len(self._declared) - 1
|
|
170
|
+
|
|
171
|
+
def point(self, series: int, step: int, value: int | float, at: str | None = None) -> None:
|
|
172
|
+
"""Append one logged value to `series` (a handle from `series()`)."""
|
|
173
|
+
if isinstance(series, bool) or not isinstance(series, int):
|
|
174
|
+
raise _invalid("series", f"expected a handle from series(), got {series!r}")
|
|
175
|
+
if not 0 <= series < len(self._declared):
|
|
176
|
+
raise _invalid("series", f"no series was declared with handle {series}")
|
|
177
|
+
_integer(step, "step", minimum=0)
|
|
178
|
+
wire = spell_value(value)
|
|
179
|
+
_metric_value(wire, "value")
|
|
180
|
+
_timestamp(at, "at")
|
|
181
|
+
if series not in self._ids:
|
|
182
|
+
self._ids[series] = len(self._ids)
|
|
183
|
+
name, split, labels = self._declared[series]
|
|
184
|
+
sid = self._ids[series]
|
|
185
|
+
self._emit({"series": sid, "name": name, "split": split, "labels": labels})
|
|
186
|
+
self._emit({"series": self._ids[series], "step": step, "value": wire, "at": at})
|
|
187
|
+
self._points += 1
|
|
188
|
+
|
|
189
|
+
def flush(self) -> None:
|
|
190
|
+
"""Push buffered lines to the OS (the file is still the temporary one)."""
|
|
191
|
+
if self._fh is not None:
|
|
192
|
+
self._fh.flush()
|
|
193
|
+
|
|
194
|
+
def close(self) -> HistoryRef | None:
|
|
195
|
+
"""Finalize: the sidecar's `HistoryRef`, or `None` if nothing was logged."""
|
|
196
|
+
if self._fh is None:
|
|
197
|
+
return self.ref
|
|
198
|
+
self._refuse_after_record()
|
|
199
|
+
fh, self._fh = self._fh, None
|
|
200
|
+
try:
|
|
201
|
+
fh.flush()
|
|
202
|
+
os.fsync(fh.fileno())
|
|
203
|
+
finally:
|
|
204
|
+
fh.close()
|
|
205
|
+
try:
|
|
206
|
+
if not self._points:
|
|
207
|
+
return None
|
|
208
|
+
if self._overwrite:
|
|
209
|
+
os.replace(self._tmp, self.target)
|
|
210
|
+
else:
|
|
211
|
+
os.link(self._tmp, self.target) # atomic no-clobber
|
|
212
|
+
self.ref = HistoryRef(
|
|
213
|
+
path=HISTORY_PATH,
|
|
214
|
+
sha256=self._hash.hexdigest(),
|
|
215
|
+
series=len(self._ids),
|
|
216
|
+
points=self._points,
|
|
217
|
+
)
|
|
218
|
+
return self.ref
|
|
219
|
+
finally:
|
|
220
|
+
self._tmp.unlink(missing_ok=True)
|
|
221
|
+
|
|
222
|
+
def abort(self) -> None:
|
|
223
|
+
"""Discard everything logged; no sidecar is left behind."""
|
|
224
|
+
if self._fh is not None:
|
|
225
|
+
fh, self._fh = self._fh, None
|
|
226
|
+
fh.close()
|
|
227
|
+
self._tmp.unlink(missing_ok=True)
|
|
228
|
+
|
|
229
|
+
def __enter__(self) -> HistoryWriter:
|
|
230
|
+
return self
|
|
231
|
+
|
|
232
|
+
def __exit__(self, exc_type: object, exc: object, tb: object) -> None:
|
|
233
|
+
if exc_type is None:
|
|
234
|
+
self.close()
|
|
235
|
+
else:
|
|
236
|
+
self.abort()
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
def _invalid(path: str, message: str) -> RecordError:
|
|
240
|
+
return InvalidFieldError(path, message)
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
def write_history(
|
|
244
|
+
run_dir: str | os.PathLike[str], series: Iterable[Series], *, overwrite: bool = False
|
|
245
|
+
) -> HistoryRef | None:
|
|
246
|
+
"""Write whole curves at once (each series in order, its points in order);
|
|
247
|
+
return the `HistoryRef` for the record, or `None` if there are none."""
|
|
248
|
+
with HistoryWriter(run_dir, overwrite=overwrite) as writer:
|
|
249
|
+
for i, s in enumerate(series):
|
|
250
|
+
if not s.points:
|
|
251
|
+
raise _invalid(f"series[{i}].points", "a series has at least one point")
|
|
252
|
+
handle = writer.series(s.name, s.split, s.labels)
|
|
253
|
+
for p in s.points:
|
|
254
|
+
writer.point(handle, p.step, p.value, p.at)
|
|
255
|
+
return writer.ref
|
|
256
|
+
|
|
257
|
+
|
|
258
|
+
def read_history(
|
|
259
|
+
run_dir: str | os.PathLike[str], ref: HistoryRef | RunRecord | None = None
|
|
260
|
+
) -> list[Series] | None:
|
|
261
|
+
"""The run's curves, or `None` when its record says no history was recorded.
|
|
262
|
+
|
|
263
|
+
`ref` is the record (or its `history`); omitted, the run directory's
|
|
264
|
+
`run.json` is read for it. The sidecar's sha256 is checked against the
|
|
265
|
+
reference before a line is parsed. Raises `HistoryMissingError`,
|
|
266
|
+
`HistoryDigestError`, or `HistoryFormatError` (a malformed or truncated
|
|
267
|
+
line, or counts that disagree with the reference) -- all `HistoryError`,
|
|
268
|
+
so all `RecordError`.
|
|
269
|
+
"""
|
|
270
|
+
if ref is None:
|
|
271
|
+
ref = read(run_dir)
|
|
272
|
+
if isinstance(ref, RunRecord):
|
|
273
|
+
ref = ref.history
|
|
274
|
+
if ref is None:
|
|
275
|
+
return None
|
|
276
|
+
path = check_history_digest(run_dir, ref)
|
|
277
|
+
with path.open("rb") as fh:
|
|
278
|
+
series = _parse(fh)
|
|
279
|
+
total = sum(len(s.points) for s in series)
|
|
280
|
+
if (len(series), total) != (ref.series, ref.points):
|
|
281
|
+
raise HistoryFormatError(
|
|
282
|
+
HISTORY_PATH,
|
|
283
|
+
f"holds {len(series)} series / {total} points; "
|
|
284
|
+
f"the record says {ref.series} / {ref.points}",
|
|
285
|
+
)
|
|
286
|
+
return series
|
|
287
|
+
|
|
288
|
+
|
|
289
|
+
def _loads_line(raw: bytes, where: str) -> Any:
|
|
290
|
+
if not raw.endswith(b"\n"):
|
|
291
|
+
raise HistoryFormatError(where, "no final newline: the file was cut short")
|
|
292
|
+
try:
|
|
293
|
+
return json.loads(
|
|
294
|
+
raw.decode("utf-8"),
|
|
295
|
+
object_pairs_hook=_duplicate_free,
|
|
296
|
+
parse_constant=_no_constant,
|
|
297
|
+
parse_float=_strict_float,
|
|
298
|
+
parse_int=_strict_int,
|
|
299
|
+
)
|
|
300
|
+
except UnicodeDecodeError as exc:
|
|
301
|
+
raise HistoryFormatError(where, f"not UTF-8: {exc}") from exc
|
|
302
|
+
except json.JSONDecodeError as exc:
|
|
303
|
+
raise HistoryFormatError(where, f"not JSON: {exc}") from exc
|
|
304
|
+
|
|
305
|
+
|
|
306
|
+
def _parse(lines: Iterable[bytes]) -> list[Series]:
|
|
307
|
+
declared: list[Series] = []
|
|
308
|
+
keys: set[tuple[Any, ...]] = set()
|
|
309
|
+
lineno = 0
|
|
310
|
+
for lineno, raw in enumerate(lines, start=1):
|
|
311
|
+
where = f"{HISTORY_PATH}:{lineno}"
|
|
312
|
+
try:
|
|
313
|
+
obj = _loads_line(raw, where)
|
|
314
|
+
if lineno == 1:
|
|
315
|
+
if obj != _HEADER:
|
|
316
|
+
raise HistoryFormatError(where, f"first line must be {_HEADER}, got {obj!r}")
|
|
317
|
+
continue
|
|
318
|
+
if not isinstance(obj, dict):
|
|
319
|
+
raise HistoryFormatError(where, f"expected an object, got {type(obj).__name__}")
|
|
320
|
+
if "name" in obj:
|
|
321
|
+
_series_line(obj, declared, keys)
|
|
322
|
+
else:
|
|
323
|
+
_point_line(obj, declared)
|
|
324
|
+
except HistoryError:
|
|
325
|
+
raise
|
|
326
|
+
except RecordError as exc:
|
|
327
|
+
raise HistoryFormatError(_at(where, exc.path) if exc.path else where, str(exc)) from exc
|
|
328
|
+
if lineno == 0:
|
|
329
|
+
raise HistoryFormatError(HISTORY_PATH, "empty file: no header line")
|
|
330
|
+
if declared and not declared[-1].points:
|
|
331
|
+
raise HistoryFormatError(HISTORY_PATH, f"series {len(declared) - 1} has no point")
|
|
332
|
+
return declared
|
|
333
|
+
|
|
334
|
+
|
|
335
|
+
def _series_line(obj: dict[str, Any], declared: list[Series], keys: set[Any]) -> None:
|
|
336
|
+
d = _object(obj, "", _SERIES_KEYS)
|
|
337
|
+
sid = _integer(d["series"], "series", minimum=0)
|
|
338
|
+
if sid != len(declared):
|
|
339
|
+
raise _invalid("series", f"series ids are declared in order; expected {len(declared)}")
|
|
340
|
+
if declared and not declared[-1].points:
|
|
341
|
+
raise _invalid("series", f"series {sid - 1} was declared and given no point")
|
|
342
|
+
s = Series(
|
|
343
|
+
name=_string(d["name"], "name"),
|
|
344
|
+
split=_opt_string(d["split"], "split"),
|
|
345
|
+
labels=_labels(d["labels"], "labels"),
|
|
346
|
+
points=[],
|
|
347
|
+
)
|
|
348
|
+
key = (s.name, s.split, _label_key(s.labels))
|
|
349
|
+
if key in keys:
|
|
350
|
+
raise _invalid("name", f"duplicate series {key!r}")
|
|
351
|
+
keys.add(key)
|
|
352
|
+
declared.append(s)
|
|
353
|
+
|
|
354
|
+
|
|
355
|
+
def _point_line(obj: dict[str, Any], declared: list[Series]) -> None:
|
|
356
|
+
d = _object(obj, "", _POINT_KEYS)
|
|
357
|
+
sid = _integer(d["series"], "series", minimum=0)
|
|
358
|
+
if sid >= len(declared):
|
|
359
|
+
raise _invalid("series", f"point for series {sid}, which is not declared yet")
|
|
360
|
+
declared[sid].points.append(
|
|
361
|
+
Point(
|
|
362
|
+
step=_integer(d["step"], "step", minimum=0),
|
|
363
|
+
value=_metric_value(d["value"], "value"),
|
|
364
|
+
at=_timestamp(d["at"], "at"),
|
|
365
|
+
)
|
|
366
|
+
)
|