attestation-schema 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- attestation_schema-0.1.0/.gitignore +48 -0
- attestation_schema-0.1.0/LICENSE +21 -0
- attestation_schema-0.1.0/PKG-INFO +105 -0
- attestation_schema-0.1.0/README.md +86 -0
- attestation_schema-0.1.0/pyproject.toml +38 -0
- attestation_schema-0.1.0/src/attestation_schema/__init__.py +158 -0
- attestation_schema-0.1.0/src/attestation_schema/canonical.py +110 -0
- attestation_schema-0.1.0/src/attestation_schema/history.py +366 -0
- attestation_schema-0.1.0/src/attestation_schema/jobs.py +135 -0
- attestation_schema-0.1.0/src/attestation_schema/layout.py +137 -0
- attestation_schema-0.1.0/src/attestation_schema/py.typed +0 -0
- attestation_schema-0.1.0/src/attestation_schema/record.py +843 -0
- attestation_schema-0.1.0/src/attestation_schema/schemas/history-1.schema.json +101 -0
- attestation_schema-0.1.0/src/attestation_schema/schemas/run-1.schema.json +393 -0
- attestation_schema-0.1.0/src/attestation_schema/schemas/state-1.schema.json +90 -0
- attestation_schema-0.1.0/src/attestation_schema/state.py +590 -0
- attestation_schema-0.1.0/tests/data/smeltery-local-failed.state.json +1 -0
- attestation_schema-0.1.0/tests/data/smeltery-temporal-cancelled.state.json +1 -0
- attestation_schema-0.1.0/tests/test_attestation_schema.py +1119 -0
- attestation_schema-0.1.0/tests/test_frozen.py +113 -0
- attestation_schema-0.1.0/tests/test_state.py +493 -0
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
.venv/
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.db
|
|
4
|
+
*.db-wal
|
|
5
|
+
*.db-shm
|
|
6
|
+
# Retag/migration snapshots: `*.db` does not match a .bak-retag-<stamp>
|
|
7
|
+
# suffix, so an 8MB copy of the database was one `git add .` from the
|
|
8
|
+
# history it can never be removed from.
|
|
9
|
+
*.db.bak*
|
|
10
|
+
.ruff_cache/
|
|
11
|
+
.pytest_cache/
|
|
12
|
+
.env
|
|
13
|
+
# Credential-bearing env files (neon.env holds a live Postgres password and S3
|
|
14
|
+
# secret keys). `.env` alone does not match `neon.env` or `*.env`.
|
|
15
|
+
*.env
|
|
16
|
+
|
|
17
|
+
# Local demo recordings: a 5.5MB .webm plus stills, never committed. Kept out
|
|
18
|
+
# of the repo so a clone stays small -- a binary that large is permanent
|
|
19
|
+
# history once added. Host it externally and link from the README if it needs to
|
|
20
|
+
# be shared. demos/*/record.sh and record.py write here by default.
|
|
21
|
+
demo/
|
|
22
|
+
|
|
23
|
+
# demos/: recording scripts are committed, their .cast/.gif/.webm output and
|
|
24
|
+
# the scratch databases seed_*_db.py builds are not (same reasoning as
|
|
25
|
+
# demo/ above -- regenerable, and a real .cast is a transcript that can
|
|
26
|
+
# carry whatever path ran it). demos/ lives at the repo root, not under
|
|
27
|
+
# examples/, on purpose -- see CLAUDE.md's Golden paths note.
|
|
28
|
+
demos/**/*.cast
|
|
29
|
+
demos/**/*.gif
|
|
30
|
+
demos/**/*.webm
|
|
31
|
+
demos/**/*.db
|
|
32
|
+
|
|
33
|
+
# Build artifacts: `uv build` / CI wheel smoke test write here.
|
|
34
|
+
dist/
|
|
35
|
+
build/
|
|
36
|
+
.mutate-backup/
|
|
37
|
+
|
|
38
|
+
# `mkdocs build` output. Built, not deployed; CI's `docs` job uploads it as
|
|
39
|
+
# an artifact instead of committing it. Anchored to the root: `docs/site/`
|
|
40
|
+
# holds a committed, generated page (the design-records index) and must not
|
|
41
|
+
# be swept up by an unanchored `site/`.
|
|
42
|
+
/site/
|
|
43
|
+
|
|
44
|
+
# Process logs from the superpowers plugin (design/plan transcripts) and a uv
|
|
45
|
+
# cache left by a run with HOME pointed at a fake path: both sat untracked-but-
|
|
46
|
+
# unignored, one `git add .` from becoming permanent history.
|
|
47
|
+
.superpowers/
|
|
48
|
+
nonexistent-home/
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Matthew Goldey
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: attestation-schema
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: The attestation core run record (attestation.run/1, frozen): schema, canonical writer, strict reader, run-directory layout, job state file and job-backend interface. Standard library only.
|
|
5
|
+
Project-URL: Repository, https://github.com/mgoldey/attestation
|
|
6
|
+
Author: Matthew Goldey
|
|
7
|
+
License-Expression: MIT
|
|
8
|
+
License-File: LICENSE
|
|
9
|
+
Keywords: experiment-tracking,provenance,reproducibility,run-record
|
|
10
|
+
Classifier: Development Status :: 4 - Beta
|
|
11
|
+
Classifier: Intended Audience :: Science/Research
|
|
12
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
16
|
+
Classifier: Typing :: Typed
|
|
17
|
+
Requires-Python: >=3.11
|
|
18
|
+
Description-Content-Type: text/markdown
|
|
19
|
+
|
|
20
|
+
# attestation-schema
|
|
21
|
+
|
|
22
|
+
The core run record that experiment producers write and
|
|
23
|
+
[attestation](https://github.com/mgoldey/attestation) reads:
|
|
24
|
+
`attestation.run/1`, **frozen 2026-11-19**: a record valid today stays valid,
|
|
25
|
+
and any change -- an additive optional field included -- is a new version
|
|
26
|
+
(`attestation.run/2`) that readers dispatch on. Standard library only.
|
|
27
|
+
|
|
28
|
+
```python
|
|
29
|
+
from attestation_schema import (
|
|
30
|
+
RunRecord, Producer, CodeIdentity, Metric, HistoryWriter, write, read, read_history,
|
|
31
|
+
)
|
|
32
|
+
|
|
33
|
+
run_dir = "runs/run-3f2a9c01b7de"
|
|
34
|
+
with HistoryWriter(run_dir) as hist: # streams attestation/history.jsonl
|
|
35
|
+
loss = hist.series("loss", split="eval")
|
|
36
|
+
for step, value in [(0, 0.9), (500, 0.4), (1000, 0.25)]:
|
|
37
|
+
hist.point(loss, step, value) # at=format_timestamp(...) if known
|
|
38
|
+
|
|
39
|
+
record = RunRecord(
|
|
40
|
+
run_id="run-3f2a9c01b7de", family="lr-sweep", arm="lr=1e-3", status="complete",
|
|
41
|
+
started_at="2026-11-19T10:00:00Z", finished_at="2026-11-19T10:42:13Z",
|
|
42
|
+
producer=Producer(name="train.py", version=None),
|
|
43
|
+
code=CodeIdentity(git_sha=None, dirty=None, repo=None, version=None,
|
|
44
|
+
provenance="unknown", evidence=None),
|
|
45
|
+
config={"lr": 0.001}, data=[],
|
|
46
|
+
metrics=[Metric(name="loss", value=0.25, direction="min", step=1000, split="eval",
|
|
47
|
+
unit=None, labels={})],
|
|
48
|
+
history=hist.ref, # {path, sha256, series, points}; None = no curve recorded
|
|
49
|
+
seeds={"python": 0},
|
|
50
|
+
dependencies=None, # None = could not observe the environment; [] = observed, none
|
|
51
|
+
ext={},
|
|
52
|
+
)
|
|
53
|
+
write(record, run_dir) # -> runs/run-3f2a9c01b7de/attestation/run.json
|
|
54
|
+
assert read(run_dir) == record # never opens history.jsonl
|
|
55
|
+
curves = read_history(run_dir) # verifies the sha256, then parses
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
- `dumps`/`write` are the only sanctioned writers: validate, then RFC 8785
|
|
59
|
+
canonical JSON, so the same record is always the same bytes.
|
|
60
|
+
- `loads`/`read` refuse anything non-conforming with a `RecordError` naming
|
|
61
|
+
the path (`metrics[2].direction`); nothing is coerced or defaulted. They
|
|
62
|
+
dispatch on `schema`, so later versions can still read older records.
|
|
63
|
+
- **Curves never go in `run.json`.** `metrics` holds final values; logged
|
|
64
|
+
curves go to the sidecar `attestation/history.jsonl` (`attestation.history/1`:
|
|
65
|
+
JSON Lines, one canonical line per series declaration or point, streamable
|
|
66
|
+
and deterministic), written with `HistoryWriter` (incrementally, finalized
|
|
67
|
+
atomically on close) or `write_history(run_dir, series)`, and bound to the
|
|
68
|
+
record by sha256. `read_history` raises `HistoryMissingError`,
|
|
69
|
+
`HistoryDigestError` or `HistoryFormatError` (all `RecordError`). Write
|
|
70
|
+
order: outputs and history -> `layout.write_manifest` -> `write`.
|
|
71
|
+
- `ext.<namespace>` carries producer data the core does not model, untouched.
|
|
72
|
+
- `attestation_schema.layout` fixes the run directory (`attestation/run.json`,
|
|
73
|
+
`attestation/state.json`, `attestation/digests.json`,
|
|
74
|
+
`attestation/history.jsonl`, `steps/NN-<name>/`, `logs/`);
|
|
75
|
+
`attestation_schema.jobs` is the job-backend `Protocol`, and
|
|
76
|
+
`attestation_schema.state` is what every backend shares: `state.json`
|
|
77
|
+
(`attestation.state/1`: `read_state`, `init_state`, `transition`), its
|
|
78
|
+
cross-process lock (`state_lock`, POSIX `flock`; refused on Windows), and
|
|
79
|
+
`settle` -- end a job exactly once, an existing record deciding, writing
|
|
80
|
+
`terminal_record` for a job that died without one. A job's record carries
|
|
81
|
+
its `JobSpec`'s `run_id`, `family`, `arm`, `config` and `ext` verbatim.
|
|
82
|
+
- `attestation_schema/schemas/run-1.schema.json` (the record),
|
|
83
|
+
`history-1.schema.json` (one sidecar line) and `state-1.schema.json`
|
|
84
|
+
(`state.json`) are JSON Schemas for other languages: necessary, not
|
|
85
|
+
sufficient — the Python reader is the reference. They are frozen and
|
|
86
|
+
pinned by digest.
|
|
87
|
+
- Package version 0.1.0: the formats are frozen; the Python API may still
|
|
88
|
+
change in a 0.x minor release until the job-state API has a consumer.
|
|
89
|
+
|
|
90
|
+
## Releasing
|
|
91
|
+
|
|
92
|
+
Tag `schema-v<version>` (the version in this package's `pyproject.toml`, which
|
|
93
|
+
the workflow checks), independent of attestation's `v<version>` tags:
|
|
94
|
+
|
|
95
|
+
```
|
|
96
|
+
git tag schema-v0.1.0 && git push origin schema-v0.1.0
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
`.github/workflows/release.yml` builds the sdist and wheel, runs this
|
|
100
|
+
package's tests on CPython 3.11 against the wheel, and publishes to PyPI
|
|
101
|
+
through trusted publishing (environment `pypi`, no token). Release it before
|
|
102
|
+
an attestation release whose `attestation-schema>=X` floor is new.
|
|
103
|
+
|
|
104
|
+
Design: `docs/superpowers/specs/2026-11-19-core-run-record-design.md` in the
|
|
105
|
+
attestation repository.
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
# attestation-schema
|
|
2
|
+
|
|
3
|
+
The core run record that experiment producers write and
|
|
4
|
+
[attestation](https://github.com/mgoldey/attestation) reads:
|
|
5
|
+
`attestation.run/1`, **frozen 2026-11-19**: a record valid today stays valid,
|
|
6
|
+
and any change -- an additive optional field included -- is a new version
|
|
7
|
+
(`attestation.run/2`) that readers dispatch on. Standard library only.
|
|
8
|
+
|
|
9
|
+
```python
|
|
10
|
+
from attestation_schema import (
|
|
11
|
+
RunRecord, Producer, CodeIdentity, Metric, HistoryWriter, write, read, read_history,
|
|
12
|
+
)
|
|
13
|
+
|
|
14
|
+
run_dir = "runs/run-3f2a9c01b7de"
|
|
15
|
+
with HistoryWriter(run_dir) as hist: # streams attestation/history.jsonl
|
|
16
|
+
loss = hist.series("loss", split="eval")
|
|
17
|
+
for step, value in [(0, 0.9), (500, 0.4), (1000, 0.25)]:
|
|
18
|
+
hist.point(loss, step, value) # at=format_timestamp(...) if known
|
|
19
|
+
|
|
20
|
+
record = RunRecord(
|
|
21
|
+
run_id="run-3f2a9c01b7de", family="lr-sweep", arm="lr=1e-3", status="complete",
|
|
22
|
+
started_at="2026-11-19T10:00:00Z", finished_at="2026-11-19T10:42:13Z",
|
|
23
|
+
producer=Producer(name="train.py", version=None),
|
|
24
|
+
code=CodeIdentity(git_sha=None, dirty=None, repo=None, version=None,
|
|
25
|
+
provenance="unknown", evidence=None),
|
|
26
|
+
config={"lr": 0.001}, data=[],
|
|
27
|
+
metrics=[Metric(name="loss", value=0.25, direction="min", step=1000, split="eval",
|
|
28
|
+
unit=None, labels={})],
|
|
29
|
+
history=hist.ref, # {path, sha256, series, points}; None = no curve recorded
|
|
30
|
+
seeds={"python": 0},
|
|
31
|
+
dependencies=None, # None = could not observe the environment; [] = observed, none
|
|
32
|
+
ext={},
|
|
33
|
+
)
|
|
34
|
+
write(record, run_dir) # -> runs/run-3f2a9c01b7de/attestation/run.json
|
|
35
|
+
assert read(run_dir) == record # never opens history.jsonl
|
|
36
|
+
curves = read_history(run_dir) # verifies the sha256, then parses
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
- `dumps`/`write` are the only sanctioned writers: validate, then RFC 8785
|
|
40
|
+
canonical JSON, so the same record is always the same bytes.
|
|
41
|
+
- `loads`/`read` refuse anything non-conforming with a `RecordError` naming
|
|
42
|
+
the path (`metrics[2].direction`); nothing is coerced or defaulted. They
|
|
43
|
+
dispatch on `schema`, so later versions can still read older records.
|
|
44
|
+
- **Curves never go in `run.json`.** `metrics` holds final values; logged
|
|
45
|
+
curves go to the sidecar `attestation/history.jsonl` (`attestation.history/1`:
|
|
46
|
+
JSON Lines, one canonical line per series declaration or point, streamable
|
|
47
|
+
and deterministic), written with `HistoryWriter` (incrementally, finalized
|
|
48
|
+
atomically on close) or `write_history(run_dir, series)`, and bound to the
|
|
49
|
+
record by sha256. `read_history` raises `HistoryMissingError`,
|
|
50
|
+
`HistoryDigestError` or `HistoryFormatError` (all `RecordError`). Write
|
|
51
|
+
order: outputs and history -> `layout.write_manifest` -> `write`.
|
|
52
|
+
- `ext.<namespace>` carries producer data the core does not model, untouched.
|
|
53
|
+
- `attestation_schema.layout` fixes the run directory (`attestation/run.json`,
|
|
54
|
+
`attestation/state.json`, `attestation/digests.json`,
|
|
55
|
+
`attestation/history.jsonl`, `steps/NN-<name>/`, `logs/`);
|
|
56
|
+
`attestation_schema.jobs` is the job-backend `Protocol`, and
|
|
57
|
+
`attestation_schema.state` is what every backend shares: `state.json`
|
|
58
|
+
(`attestation.state/1`: `read_state`, `init_state`, `transition`), its
|
|
59
|
+
cross-process lock (`state_lock`, POSIX `flock`; refused on Windows), and
|
|
60
|
+
`settle` -- end a job exactly once, an existing record deciding, writing
|
|
61
|
+
`terminal_record` for a job that died without one. A job's record carries
|
|
62
|
+
its `JobSpec`'s `run_id`, `family`, `arm`, `config` and `ext` verbatim.
|
|
63
|
+
- `attestation_schema/schemas/run-1.schema.json` (the record),
|
|
64
|
+
`history-1.schema.json` (one sidecar line) and `state-1.schema.json`
|
|
65
|
+
(`state.json`) are JSON Schemas for other languages: necessary, not
|
|
66
|
+
sufficient — the Python reader is the reference. They are frozen and
|
|
67
|
+
pinned by digest.
|
|
68
|
+
- Package version 0.1.0: the formats are frozen; the Python API may still
|
|
69
|
+
change in a 0.x minor release until the job-state API has a consumer.
|
|
70
|
+
|
|
71
|
+
## Releasing
|
|
72
|
+
|
|
73
|
+
Tag `schema-v<version>` (the version in this package's `pyproject.toml`, which
|
|
74
|
+
the workflow checks), independent of attestation's `v<version>` tags:
|
|
75
|
+
|
|
76
|
+
```
|
|
77
|
+
git tag schema-v0.1.0 && git push origin schema-v0.1.0
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
`.github/workflows/release.yml` builds the sdist and wheel, runs this
|
|
81
|
+
package's tests on CPython 3.11 against the wheel, and publishes to PyPI
|
|
82
|
+
through trusted publishing (environment `pypi`, no token). Release it before
|
|
83
|
+
an attestation release whose `attestation-schema>=X` floor is new.
|
|
84
|
+
|
|
85
|
+
Design: `docs/superpowers/specs/2026-11-19-core-run-record-design.md` in the
|
|
86
|
+
attestation repository.
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "attestation-schema"
|
|
3
|
+
# 0.1.0, not 1.0.0: the record formats are frozen (their identity is the
|
|
4
|
+
# `schema` string, pinned by tests/test_frozen.py, not this number), but the
|
|
5
|
+
# Python state API (`state.py`) has no consumer yet. 1.0.0 once smeltery runs
|
|
6
|
+
# its job backends on it (spec, "v1 freeze": package version).
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "The attestation core run record (attestation.run/1, frozen): schema, canonical writer, strict reader, run-directory layout, job state file and job-backend interface. Standard library only."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.11"
|
|
11
|
+
license = "MIT"
|
|
12
|
+
license-files = ["LICENSE"]
|
|
13
|
+
authors = [{ name = "Matthew Goldey" }]
|
|
14
|
+
keywords = ["reproducibility", "provenance", "run-record", "experiment-tracking"]
|
|
15
|
+
classifiers = [
|
|
16
|
+
"Development Status :: 4 - Beta",
|
|
17
|
+
"Intended Audience :: Science/Research",
|
|
18
|
+
"License :: OSI Approved :: MIT License",
|
|
19
|
+
"Programming Language :: Python :: 3.11",
|
|
20
|
+
"Programming Language :: Python :: 3.12",
|
|
21
|
+
"Programming Language :: Python :: 3.13",
|
|
22
|
+
"Typing :: Typed",
|
|
23
|
+
]
|
|
24
|
+
# Deliberately empty: a producer (smeltery, a training script) depends on this
|
|
25
|
+
# package to WRITE a record, and a dependency it must install for that is a
|
|
26
|
+
# dependency that gets skipped. See
|
|
27
|
+
# docs/superpowers/specs/2026-11-19-core-run-record-design.md.
|
|
28
|
+
dependencies = []
|
|
29
|
+
|
|
30
|
+
[project.urls]
|
|
31
|
+
Repository = "https://github.com/mgoldey/attestation"
|
|
32
|
+
|
|
33
|
+
[build-system]
|
|
34
|
+
requires = ["hatchling"]
|
|
35
|
+
build-backend = "hatchling.build"
|
|
36
|
+
|
|
37
|
+
[tool.hatch.build.targets.wheel]
|
|
38
|
+
packages = ["src/attestation_schema"]
|
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
"""attestation-schema: the core run record (`attestation.run/1`, frozen 2026-11-19).
|
|
2
|
+
|
|
3
|
+
Standard library only, and imports nothing from `attestation`: producers
|
|
4
|
+
depend on this package to write records without depending on the observer.
|
|
5
|
+
Curves live in a sidecar (`history.py`), bound to the record by sha256; a
|
|
6
|
+
job's state lives in `state.json` (`state.py`).
|
|
7
|
+
Spec: docs/superpowers/specs/2026-11-19-core-run-record-design.md.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from attestation_schema.history import (
|
|
11
|
+
HISTORY_SCHEMA,
|
|
12
|
+
HistoryWriter,
|
|
13
|
+
Point,
|
|
14
|
+
Series,
|
|
15
|
+
read_history,
|
|
16
|
+
write_history,
|
|
17
|
+
)
|
|
18
|
+
from attestation_schema.jobs import (
|
|
19
|
+
TERMINAL,
|
|
20
|
+
TRANSITIONS,
|
|
21
|
+
JobBackend,
|
|
22
|
+
JobError,
|
|
23
|
+
JobExistsError,
|
|
24
|
+
JobHandle,
|
|
25
|
+
JobNotFoundError,
|
|
26
|
+
JobNotTerminalError,
|
|
27
|
+
JobSpec,
|
|
28
|
+
JobState,
|
|
29
|
+
JobStatus,
|
|
30
|
+
can_transition,
|
|
31
|
+
)
|
|
32
|
+
from attestation_schema.record import (
|
|
33
|
+
HISTORY_PATH,
|
|
34
|
+
NON_FINITE,
|
|
35
|
+
READERS,
|
|
36
|
+
SCHEMA,
|
|
37
|
+
SCHEMA_STATUS,
|
|
38
|
+
SOURCE_KINDS,
|
|
39
|
+
CodeIdentity,
|
|
40
|
+
DataDigest,
|
|
41
|
+
Dependency,
|
|
42
|
+
HistoryDigestError,
|
|
43
|
+
HistoryError,
|
|
44
|
+
HistoryFormatError,
|
|
45
|
+
HistoryMissingError,
|
|
46
|
+
HistoryRef,
|
|
47
|
+
InvalidFieldError,
|
|
48
|
+
Metric,
|
|
49
|
+
MissingFieldError,
|
|
50
|
+
Producer,
|
|
51
|
+
RecordError,
|
|
52
|
+
RunRecord,
|
|
53
|
+
UnknownFieldError,
|
|
54
|
+
UnsupportedSchemaError,
|
|
55
|
+
dumps,
|
|
56
|
+
format_timestamp,
|
|
57
|
+
from_dict,
|
|
58
|
+
loads,
|
|
59
|
+
parse_strict,
|
|
60
|
+
read,
|
|
61
|
+
record_digest,
|
|
62
|
+
record_path,
|
|
63
|
+
spell_value,
|
|
64
|
+
write,
|
|
65
|
+
)
|
|
66
|
+
from attestation_schema.state import (
|
|
67
|
+
STATE_SCHEMA,
|
|
68
|
+
UNKNOWN_CODE,
|
|
69
|
+
IllegalTransitionError,
|
|
70
|
+
StateFile,
|
|
71
|
+
StateFormatError,
|
|
72
|
+
StateLockUnavailableError,
|
|
73
|
+
Transition,
|
|
74
|
+
dumps_state,
|
|
75
|
+
handle_in,
|
|
76
|
+
init_state,
|
|
77
|
+
loads_state,
|
|
78
|
+
read_state,
|
|
79
|
+
settle,
|
|
80
|
+
spec_conflicts,
|
|
81
|
+
state_from_dict,
|
|
82
|
+
state_lock,
|
|
83
|
+
state_path,
|
|
84
|
+
terminal_record,
|
|
85
|
+
transition,
|
|
86
|
+
)
|
|
87
|
+
|
|
88
|
+
__all__ = [
|
|
89
|
+
"HISTORY_PATH",
|
|
90
|
+
"HISTORY_SCHEMA",
|
|
91
|
+
"NON_FINITE",
|
|
92
|
+
"READERS",
|
|
93
|
+
"SCHEMA",
|
|
94
|
+
"SCHEMA_STATUS",
|
|
95
|
+
"TERMINAL",
|
|
96
|
+
"TRANSITIONS",
|
|
97
|
+
"CodeIdentity",
|
|
98
|
+
"DataDigest",
|
|
99
|
+
"Dependency",
|
|
100
|
+
"HistoryDigestError",
|
|
101
|
+
"HistoryError",
|
|
102
|
+
"HistoryFormatError",
|
|
103
|
+
"HistoryMissingError",
|
|
104
|
+
"HistoryRef",
|
|
105
|
+
"HistoryWriter",
|
|
106
|
+
"InvalidFieldError",
|
|
107
|
+
"JobBackend",
|
|
108
|
+
"JobError",
|
|
109
|
+
"JobExistsError",
|
|
110
|
+
"JobHandle",
|
|
111
|
+
"JobNotFoundError",
|
|
112
|
+
"JobNotTerminalError",
|
|
113
|
+
"JobSpec",
|
|
114
|
+
"JobState",
|
|
115
|
+
"JobStatus",
|
|
116
|
+
"Metric",
|
|
117
|
+
"MissingFieldError",
|
|
118
|
+
"Point",
|
|
119
|
+
"Producer",
|
|
120
|
+
"RecordError",
|
|
121
|
+
"RunRecord",
|
|
122
|
+
"SOURCE_KINDS",
|
|
123
|
+
"STATE_SCHEMA",
|
|
124
|
+
"UNKNOWN_CODE",
|
|
125
|
+
"IllegalTransitionError",
|
|
126
|
+
"StateFile",
|
|
127
|
+
"StateFormatError",
|
|
128
|
+
"StateLockUnavailableError",
|
|
129
|
+
"Transition",
|
|
130
|
+
"dumps_state",
|
|
131
|
+
"handle_in",
|
|
132
|
+
"init_state",
|
|
133
|
+
"loads_state",
|
|
134
|
+
"parse_strict",
|
|
135
|
+
"read_state",
|
|
136
|
+
"settle",
|
|
137
|
+
"spec_conflicts",
|
|
138
|
+
"state_from_dict",
|
|
139
|
+
"state_lock",
|
|
140
|
+
"state_path",
|
|
141
|
+
"terminal_record",
|
|
142
|
+
"transition",
|
|
143
|
+
"Series",
|
|
144
|
+
"UnknownFieldError",
|
|
145
|
+
"UnsupportedSchemaError",
|
|
146
|
+
"can_transition",
|
|
147
|
+
"dumps",
|
|
148
|
+
"format_timestamp",
|
|
149
|
+
"from_dict",
|
|
150
|
+
"loads",
|
|
151
|
+
"read",
|
|
152
|
+
"read_history",
|
|
153
|
+
"record_digest",
|
|
154
|
+
"record_path",
|
|
155
|
+
"spell_value",
|
|
156
|
+
"write",
|
|
157
|
+
"write_history",
|
|
158
|
+
]
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
"""RFC 8785 JSON Canonicalization Scheme (JCS), standard library only.
|
|
2
|
+
|
|
3
|
+
The one implementation: `attestation.run_record.canonical` re-exports it
|
|
4
|
+
(attestation depends on this package at runtime), and this package may not
|
|
5
|
+
import `attestation` (tests/test_boundaries.py).
|
|
6
|
+
|
|
7
|
+
A run record is signed over its exact bytes (DSSE), so verification never
|
|
8
|
+
needs canonical form. Canonical form buys STABILITY: the same record content
|
|
9
|
+
always serializes to the same bytes, so a record can be named by its digest
|
|
10
|
+
and two builds compared byte for byte -- by this code or by a verifier
|
|
11
|
+
written in another language.
|
|
12
|
+
|
|
13
|
+
`json.dumps(sort_keys=True, separators=(",", ":"))` is close and not equal:
|
|
14
|
+
it orders keys by code point where JCS orders by UTF-16 code unit (they
|
|
15
|
+
differ for characters above U+FFFF against U+E000-U+FFFF), and it prints
|
|
16
|
+
floats Python's way (`1e-07`, `1.0`, `1e+16`) where JCS prints them the
|
|
17
|
+
ECMAScript way (`1e-7`, `1`, `10000000000000000`).
|
|
18
|
+
|
|
19
|
+
Refused rather than rounded: non-finite floats (JSON has no spelling for
|
|
20
|
+
them) and integers outside +/-2^53, which I-JSON -- and every JavaScript
|
|
21
|
+
verifier -- would silently round to a different number.
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
from __future__ import annotations
|
|
25
|
+
|
|
26
|
+
import json
|
|
27
|
+
import math
|
|
28
|
+
from decimal import Decimal
|
|
29
|
+
|
|
30
|
+
MAX_SAFE_INTEGER = 2**53 - 1
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _number(value: float) -> str:
|
|
34
|
+
"""ECMAScript Number::toString for a finite double (ES2020 7.1.12.1)."""
|
|
35
|
+
if not math.isfinite(value):
|
|
36
|
+
raise ValueError(f"JCS cannot represent {value!r}")
|
|
37
|
+
if value == 0:
|
|
38
|
+
return "0" # also -0.0: ES6 prints "0"
|
|
39
|
+
sign = "-" if value < 0 else ""
|
|
40
|
+
# repr() is the shortest round-tripping decimal, which is what ES6 uses too;
|
|
41
|
+
# only the layout differs, so take its digits and re-lay them out.
|
|
42
|
+
_, digit_tuple, exp = Decimal(repr(abs(value))).as_tuple()
|
|
43
|
+
digits = "".join(map(str, digit_tuple)).rstrip("0")
|
|
44
|
+
exp = int(exp) + (len(digit_tuple) - len(digits))
|
|
45
|
+
k = len(digits)
|
|
46
|
+
n = k + exp # position of the decimal point relative to the digits
|
|
47
|
+
if k <= n <= 21:
|
|
48
|
+
return sign + digits + "0" * (n - k)
|
|
49
|
+
if 0 < n <= 21:
|
|
50
|
+
return sign + digits[:n] + "." + digits[n:]
|
|
51
|
+
if -6 < n <= 0:
|
|
52
|
+
return sign + "0." + "0" * (-n) + digits
|
|
53
|
+
e = n - 1
|
|
54
|
+
mantissa = digits[0] + ("." + digits[1:] if k > 1 else "")
|
|
55
|
+
return f"{sign}{mantissa}e{'+' if e > 0 else '-'}{abs(e)}"
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def _string(value: str) -> str:
|
|
59
|
+
# ensure_ascii=False matches ES6 JSON.stringify: only `"`, `\` and C0
|
|
60
|
+
# controls are escaped, with lowercase hex. A lone surrogate is caught
|
|
61
|
+
# by the UTF-8 encode in canonicalize().
|
|
62
|
+
return json.dumps(value, ensure_ascii=False)
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def _utf16_key(key: str) -> bytes:
|
|
66
|
+
return key.encode("utf-16-be", "surrogatepass")
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _serialize(value, out: list[str]) -> None:
|
|
70
|
+
if value is None or isinstance(value, bool):
|
|
71
|
+
out.append({None: "null", True: "true", False: "false"}[value])
|
|
72
|
+
elif isinstance(value, int):
|
|
73
|
+
if abs(value) > MAX_SAFE_INTEGER:
|
|
74
|
+
raise ValueError(f"JCS integer out of the IEEE-754 safe range: {value}")
|
|
75
|
+
out.append(str(value))
|
|
76
|
+
elif isinstance(value, float):
|
|
77
|
+
out.append(_number(value))
|
|
78
|
+
elif isinstance(value, str):
|
|
79
|
+
out.append(_string(value))
|
|
80
|
+
elif isinstance(value, list | tuple):
|
|
81
|
+
_array(value, out)
|
|
82
|
+
elif isinstance(value, dict):
|
|
83
|
+
_object(value, out)
|
|
84
|
+
else:
|
|
85
|
+
raise TypeError(f"JCS cannot serialize {type(value).__name__}")
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def _array(value, out: list[str]) -> None:
|
|
89
|
+
out.append("[")
|
|
90
|
+
for i, item in enumerate(value):
|
|
91
|
+
out.append("," if i else "")
|
|
92
|
+
_serialize(item, out)
|
|
93
|
+
out.append("]")
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def _object(value: dict, out: list[str]) -> None:
|
|
97
|
+
if not all(isinstance(k, str) for k in value):
|
|
98
|
+
raise TypeError("JCS object keys must be strings")
|
|
99
|
+
out.append("{")
|
|
100
|
+
for i, key in enumerate(sorted(value, key=_utf16_key)):
|
|
101
|
+
out.append(("," if i else "") + _string(key) + ":")
|
|
102
|
+
_serialize(value[key], out)
|
|
103
|
+
out.append("}")
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def canonicalize(value) -> bytes:
|
|
107
|
+
"""`value` as RFC 8785 canonical JSON, UTF-8 encoded."""
|
|
108
|
+
out: list[str] = []
|
|
109
|
+
_serialize(value, out)
|
|
110
|
+
return "".join(out).encode("utf-8")
|