setspec 0.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. setspec-0.2.0/.editorconfig +23 -0
  2. setspec-0.2.0/.github/workflows/ci.yml +150 -0
  3. setspec-0.2.0/.github/workflows/release.yml +56 -0
  4. setspec-0.2.0/.gitignore +100 -0
  5. setspec-0.2.0/.importlinter +27 -0
  6. setspec-0.2.0/.pre-commit-config.yaml +22 -0
  7. setspec-0.2.0/CHANGELOG.md +247 -0
  8. setspec-0.2.0/CONTRIBUTING.md +50 -0
  9. setspec-0.2.0/LICENSE +201 -0
  10. setspec-0.2.0/PKG-INFO +160 -0
  11. setspec-0.2.0/README.md +129 -0
  12. setspec-0.2.0/SECURITY.md +39 -0
  13. setspec-0.2.0/docs/README.md +8 -0
  14. setspec-0.2.0/docs/packages/setspec/development-plan.md +293 -0
  15. setspec-0.2.0/docs/packages/setspec/spec.md +271 -0
  16. setspec-0.2.0/pyproject.toml +99 -0
  17. setspec-0.2.0/requirements/README.md +48 -0
  18. setspec-0.2.0/requirements/ci.lock +781 -0
  19. setspec-0.2.0/requirements/release.in +6 -0
  20. setspec-0.2.0/requirements/release.lock +493 -0
  21. setspec-0.2.0/src/setspec/__about__.py +3 -0
  22. setspec-0.2.0/src/setspec/__init__.py +114 -0
  23. setspec-0.2.0/src/setspec/artifacts.py +4 -0
  24. setspec-0.2.0/src/setspec/base.py +256 -0
  25. setspec-0.2.0/src/setspec/benchmark/v1.py +594 -0
  26. setspec-0.2.0/src/setspec/capability/v1.py +394 -0
  27. setspec-0.2.0/src/setspec/envelope.py +468 -0
  28. setspec-0.2.0/src/setspec/error/v1.py +4 -0
  29. setspec-0.2.0/src/setspec/errors.py +68 -0
  30. setspec-0.2.0/src/setspec/event/v1.py +4 -0
  31. setspec-0.2.0/src/setspec/goal/v1.py +390 -0
  32. setspec-0.2.0/src/setspec/goldens/.gitkeep +0 -0
  33. setspec-0.2.0/src/setspec/machine/v1.py +131 -0
  34. setspec-0.2.0/src/setspec/metrics.py +184 -0
  35. setspec-0.2.0/src/setspec/model/v1.py +169 -0
  36. setspec-0.2.0/src/setspec/provenance.py +59 -0
  37. setspec-0.2.0/src/setspec/py.typed +0 -0
  38. setspec-0.2.0/src/setspec/schemas/.gitkeep +0 -0
  39. setspec-0.2.0/src/setspec/serialization.py +330 -0
  40. setspec-0.2.0/src/setspec/vocabulary.py +205 -0
  41. setspec-0.2.0/tests/conftest.py +75 -0
  42. setspec-0.2.0/tests/contract/test_cross_version.py +4 -0
  43. setspec-0.2.0/tests/contract/test_goldens.py +4 -0
  44. setspec-0.2.0/tests/contract/test_schema_snapshots.py +4 -0
  45. setspec-0.2.0/tests/contract/test_version_negotiation.py +189 -0
  46. setspec-0.2.0/tests/unit/test_base.py +163 -0
  47. setspec-0.2.0/tests/unit/test_envelope.py +254 -0
  48. setspec-0.2.0/tests/unit/test_errors.py +4 -0
  49. setspec-0.2.0/tests/unit/test_events.py +4 -0
  50. setspec-0.2.0/tests/unit/test_metrics.py +175 -0
  51. setspec-0.2.0/tests/unit/test_payloads_benchmark.py +482 -0
  52. setspec-0.2.0/tests/unit/test_payloads_capability.py +411 -0
  53. setspec-0.2.0/tests/unit/test_payloads_goal.py +499 -0
  54. setspec-0.2.0/tests/unit/test_serialization.py +223 -0
  55. setspec-0.2.0/tests/unit/test_vocabulary.py +225 -0
@@ -0,0 +1,23 @@
1
+ root = true
2
+
3
+ [*]
4
+ charset = utf-8
5
+ end_of_line = lf
6
+ insert_final_newline = true
7
+ trim_trailing_whitespace = true
8
+ indent_style = space
9
+ indent_size = 4
10
+
11
+ [*.py]
12
+ indent_size = 4
13
+ max_line_length = 100
14
+
15
+ [*.{toml,yml,yaml,json}]
16
+ indent_size = 2
17
+
18
+ [*.md]
19
+ trim_trailing_whitespace = false
20
+ max_line_length = off
21
+
22
+ [Makefile]
23
+ indent_style = tab
@@ -0,0 +1,150 @@
1
+ name: CI
2
+
3
+ on:
4
+ push:
5
+ branches: [main]
6
+ pull_request:
7
+
8
+ jobs:
9
+ format:
10
+ runs-on: ubuntu-latest
11
+ steps:
12
+ - uses: actions/checkout@v4
13
+ - uses: actions/setup-python@v5
14
+ with: { python-version: "3.12" }
15
+ - run: pip install ruff
16
+ - run: ruff format --check .
17
+
18
+ lint:
19
+ runs-on: ubuntu-latest
20
+ steps:
21
+ - uses: actions/checkout@v4
22
+ - uses: actions/setup-python@v5
23
+ with: { python-version: "3.12" }
24
+ - run: pip install ruff
25
+ - run: ruff check .
26
+
27
+ types:
28
+ runs-on: ubuntu-latest
29
+ steps:
30
+ - uses: actions/checkout@v4
31
+ - uses: actions/setup-python@v5
32
+ with: { python-version: "3.12" }
33
+ - run: pip install --require-hashes -r requirements/ci.lock
34
+ - run: pip install . --no-deps
35
+ - run: mypy src tests
36
+
37
+ boundaries:
38
+ runs-on: ubuntu-latest
39
+ steps:
40
+ - uses: actions/checkout@v4
41
+ - uses: actions/setup-python@v5
42
+ with: { python-version: "3.12" }
43
+ - run: pip install --require-hashes -r requirements/ci.lock
44
+ - run: pip install . --no-deps
45
+ - run: lint-imports
46
+
47
+ tests:
48
+ runs-on: ubuntu-latest
49
+ strategy:
50
+ matrix:
51
+ python-version: ["3.12", "3.13"]
52
+ steps:
53
+ - uses: actions/checkout@v4
54
+ - uses: actions/setup-python@v5
55
+ with: { python-version: "${{ matrix.python-version }}" }
56
+ - run: pip install --require-hashes -r requirements/ci.lock
57
+ - run: pip install . --no-deps
58
+ - run: pytest -m "not live and not performance" --cov --cov-report=xml
59
+
60
+ tests-314-early-warning:
61
+ runs-on: ubuntu-latest
62
+ continue-on-error: true
63
+ steps:
64
+ - uses: actions/checkout@v4
65
+ - uses: actions/setup-python@v5
66
+ with: { python-version: "3.14" }
67
+ # Deliberately unlocked: this job resolves from the ranges in `pyproject.toml`, because
68
+ # pinning versions that have no 3.14 wheels would defeat the purpose of an early warning
69
+ # (requirements/README.md).
70
+ - run: pip install -e ".[dev]"
71
+ - run: pytest -m "not live and not performance"
72
+
73
+ coverage:
74
+ needs: [tests]
75
+ runs-on: ubuntu-latest
76
+ steps:
77
+ - uses: actions/checkout@v4
78
+ - uses: actions/setup-python@v5
79
+ with: { python-version: "3.12" }
80
+ - run: pip install --require-hashes -r requirements/ci.lock
81
+ - run: pip install . --no-deps
82
+ - run: pytest -m "not live and not performance" --cov --cov-report=term-missing --cov-fail-under=95
83
+
84
+ contracts:
85
+ runs-on: ubuntu-latest
86
+ steps:
87
+ - uses: actions/checkout@v4
88
+ - uses: actions/setup-python@v5
89
+ with: { python-version: "3.12" }
90
+ - run: pip install --require-hashes -r requirements/ci.lock
91
+ - run: pip install . --no-deps
92
+ - run: pytest -m contract
93
+
94
+ security:
95
+ runs-on: ubuntu-latest
96
+ steps:
97
+ - uses: actions/checkout@v4
98
+ # gitleaks scans *history*, and `actions/checkout` fetches a single commit by default.
99
+ # For a push it is handed `<first-pushed>^..<last-pushed>`, so the parent of the first
100
+ # pushed commit has to be in the object store; in a depth-1 clone it is not, and git
101
+ # answers "unknown revision", which the action reports as exit code 1. It fails the same
102
+ # way whether or not a secret exists, so a green run would not have meant anything either.
103
+ with: { fetch-depth: 0 }
104
+ - uses: actions/setup-python@v5
105
+ with: { python-version: "3.12" }
106
+ - run: pip install pip-audit
107
+ # Audit the locked set, not the job's own environment: a bare `pip-audit` here inspects an
108
+ # environment containing only pip-audit itself and reports nothing (Security Standards §11).
109
+ - run: pip-audit --require-hashes -r requirements/ci.lock
110
+ - run: pip-audit --require-hashes -r requirements/release.lock
111
+ - uses: gitleaks/gitleaks-action@v2
112
+ env:
113
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
114
+
115
+ build:
116
+ runs-on: ubuntu-latest
117
+ steps:
118
+ - uses: actions/checkout@v4
119
+ - uses: actions/setup-python@v5
120
+ with: { python-version: "3.12" }
121
+ # The same pinned build chain the release uses, so a wheel that builds here is the wheel
122
+ # that ships: `--no-isolation` takes the backend from this lock rather than re-resolving
123
+ # hatchling from the index at build time.
124
+ - run: pip install --require-hashes -r requirements/release.lock
125
+ - run: python -m build --no-isolation
126
+ - run: twine check dist/*
127
+ - uses: actions/upload-artifact@v4
128
+ with: { name: dist, path: dist/ }
129
+
130
+ install-check:
131
+ needs: [build]
132
+ runs-on: ubuntu-latest
133
+ steps:
134
+ - uses: actions/checkout@v4
135
+ - uses: actions/setup-python@v5
136
+ with: { python-version: "3.12" }
137
+ - uses: actions/download-artifact@v4
138
+ with: { name: dist, path: dist/ }
139
+ - run: pip install dist/*.whl
140
+ - run: python -c "import setspec"
141
+ # The marker is easy to add to the source tree and easy to leave out of the built wheel, and
142
+ # the failure is silent: a consumer's `mypy --strict` treats every import from an unmarked
143
+ # package as untyped rather than erroring. Asserted against the *installed* wheel.
144
+ - name: py.typed ships in the wheel
145
+ run: |
146
+ python -c "
147
+ import pathlib, setspec
148
+ marker = pathlib.Path(setspec.__file__).parent / 'py.typed'
149
+ assert marker.is_file(), 'py.typed missing from the installed wheel'
150
+ "
@@ -0,0 +1,56 @@
1
+ name: Release
2
+
3
+ on:
4
+ push:
5
+ tags: ["v*.*.*"]
6
+ workflow_dispatch: # manual TestPyPI dry run; see publish-testpypi below
7
+
8
+ permissions:
9
+ id-token: write # required for PyPI Trusted Publishing
10
+ contents: write # required to create the GitHub release
11
+
12
+ jobs:
13
+ release:
14
+ if: github.event_name == 'push'
15
+ runs-on: ubuntu-latest
16
+ environment: pypi # must match the Environment name set on the PyPI trusted publisher
17
+ steps:
18
+ - uses: actions/checkout@v4
19
+ - uses: actions/setup-python@v5
20
+ with: { python-version: "3.12" }
21
+ - run: pip install --require-hashes -r requirements/release.lock
22
+ - run: python -m build --no-isolation
23
+ - run: twine check dist/*
24
+ - run: pip install "dist/$(ls dist | grep .whl)[dev]"
25
+ - run: pytest -m "not live and not performance"
26
+ - name: Publish to PyPI
27
+ uses: pypa/gh-action-pypi-publish@release/v1
28
+ - name: Create GitHub release
29
+ uses: softprops/action-gh-release@v2
30
+ with:
31
+ generate_release_notes: true
32
+ files: dist/*
33
+
34
+ publish-testpypi:
35
+ # Manual only, via Actions -> Release -> Run workflow. Packaging and Release Standards §6
36
+ # requires a successful TestPyPI publish ahead of a package's first real release; 0.2.0 is
37
+ # this package's first published version, so run this once before tagging v0.2.0. Later
38
+ # releases may skip it, or use it again as a dry run.
39
+ if: github.event_name == 'workflow_dispatch'
40
+ runs-on: ubuntu-latest
41
+ # No `environment:` here, matching the other package repositories. The OIDC claim must match
42
+ # the trusted publisher exactly: declaring an environment the TestPyPI publisher was not
43
+ # configured with fails the exchange. Add one only alongside configuring it there.
44
+ steps:
45
+ - uses: actions/checkout@v4
46
+ - uses: actions/setup-python@v5
47
+ with: { python-version: "3.12" }
48
+ - run: pip install --require-hashes -r requirements/release.lock
49
+ - run: python -m build --no-isolation
50
+ - run: twine check dist/*
51
+ - run: pip install "dist/$(ls dist | grep .whl)[dev]"
52
+ - run: pytest -m "not live and not performance"
53
+ - name: Publish to TestPyPI
54
+ uses: pypa/gh-action-pypi-publish@release/v1
55
+ with:
56
+ repository-url: https://test.pypi.org/legacy/
@@ -0,0 +1,100 @@
1
+ # ---- Python ----
2
+ __pycache__/
3
+ *.py[cod]
4
+ *$py.class
5
+ *.so
6
+ .Python
7
+ build/
8
+ develop-eggs/
9
+ dist/
10
+ downloads/
11
+ eggs/
12
+ .eggs/
13
+ lib/
14
+ lib64/
15
+ parts/
16
+ sdist/
17
+ var/
18
+ wheels/
19
+ *.egg-info/
20
+ .installed.cfg
21
+ *.egg
22
+ MANIFEST
23
+
24
+ # ---- Packaging / build backends ----
25
+ pip-wheel-metadata/
26
+ share/python-wheels/
27
+
28
+ # ---- Testing / coverage ----
29
+ .pytest_cache/
30
+ .cache
31
+ .coverage
32
+ .coverage.*
33
+ coverage.xml
34
+ *.cover
35
+ *.py,cover
36
+ htmlcov/
37
+ nosetests.xml
38
+ .hypothesis/
39
+
40
+ # ---- Type checking / linting ----
41
+ .mypy_cache/
42
+ .dmypy.json
43
+ dmypy.json
44
+ .ruff_cache/
45
+ .pytype/
46
+
47
+ # ---- Virtual environments ----
48
+ .venv/
49
+ venv/
50
+ ENV/
51
+ env/
52
+ env.bak/
53
+ venv.bak/
54
+ .python-version
55
+
56
+ # ---- Distribution / dependency locking ----
57
+ # requirements/*.lock is deliberately NOT ignored. The lock files are committed
58
+ # inputs: CI and release jobs install from them with --require-hashes (Packaging
59
+ # and Release Standards §4, Security Standards §11), so an ignored lock file
60
+ # means a checkout that cannot build.
61
+
62
+ # ---- Editors / OS ----
63
+ .vscode/
64
+ .idea/
65
+ *.swp
66
+ *.swo
67
+ .DS_Store
68
+ Thumbs.db
69
+
70
+ # ---- Suite runtime state (this component's local data) ----
71
+ # The application writes to XDG paths at runtime (~/.config, ~/.local/share,
72
+ # ~/.local/state per Master Architecture §1.2), never inside the repository.
73
+ # These entries only guard against a developer pointing XDG_* at the repo
74
+ # during local testing.
75
+ .local/
76
+ .config/
77
+ *.sqlite3
78
+ *.sqlite3-journal
79
+ *.sqlite3-wal
80
+ *.sqlite3-shm
81
+ /data/
82
+ /logs/
83
+ /backups/
84
+ /artifacts/
85
+ /exports/
86
+
87
+ # ---- Secrets ----
88
+ # Per Security Standards §8: secrets are never committed. Config files may
89
+ # only name where a secret comes from (*_env / *_file), never the value.
90
+ .env
91
+ .env.*
92
+ *.key
93
+ *.pem
94
+ secrets.toml
95
+
96
+ # ---- Node-free JS assets (MirrorWall consumers may still use a local tool) ----
97
+ node_modules/
98
+
99
+ # ---- Build artifacts from docs generation ----
100
+ docs/api/openapi-v1.json.tmp
@@ -0,0 +1,27 @@
1
+ [importlinter]
2
+ root_packages =
3
+ setspec
4
+ # The forbidden modules below are external distributions, not submodules of setspec, so the graph
5
+ # has to include external packages for the contracts to be able to see them at all.
6
+ include_external_packages = True
7
+
8
+ [importlinter:contract:no-application-imports]
9
+ name = SetSpec must not import applications
10
+ type = forbidden
11
+ source_modules =
12
+ setspec
13
+ forbidden_modules =
14
+ freeweight
15
+ loadcoach
16
+ ideapress
17
+
18
+ [importlinter:contract:no-capability-packages]
19
+ name = SetSpec must not import capability packages
20
+ type = forbidden
21
+ source_modules =
22
+ setspec
23
+ forbidden_modules =
24
+ modelrack
25
+ sweatmeter
26
+ weightsdb
27
+ mirrorwall
@@ -0,0 +1,22 @@
1
+ repos:
2
+ - repo: https://github.com/astral-sh/ruff-pre-commit
3
+ rev: v0.6.9
4
+ hooks:
5
+ - id: ruff
6
+ args: [--fix]
7
+ - id: ruff-format
8
+ - repo: https://github.com/pre-commit/pre-commit-hooks
9
+ rev: v4.6.0
10
+ hooks:
11
+ - id: trailing-whitespace
12
+ - id: end-of-file-fixer
13
+ - id: check-toml
14
+ - id: check-json
15
+ - id: check-added-large-files
16
+ - id: check-merge-conflict
17
+ - id: mixed-line-ending
18
+ args: [--fix=lf]
19
+ - repo: https://github.com/gitleaks/gitleaks
20
+ rev: v8.18.4
21
+ hooks:
22
+ - id: gitleaks
@@ -0,0 +1,247 @@
1
+ # Changelog
2
+
3
+ All notable changes to `setspec` are documented here.
4
+ Format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/); versioning follows
5
+ [Semantic Versioning](https://semver.org/), pre-1.0 per
6
+ packaging and release standards §3.
7
+
8
+ ## [Unreleased]
9
+
10
+ ### Added
11
+ - **`metric_key` on `MetricValueFields`.** The model declared value, unit, aggregation, direction,
12
+ sample count and dispersion — and nothing saying *which metric it is*. Both
13
+ `BenchmarkResultFields.metrics` and `BenchmarkRunSummaryFields.aggregate_metrics` carry sequences
14
+ of it, so a consumer receiving a run summary got a list of numbers it could not attribute, chart,
15
+ compare across runs, or check for the metric it was looking for. Required rather than optional:
16
+ an unattributable number is not a measurement. Constrained to lower snake case, because a metric
17
+ key is an identifier consumers match on and `ttft_ms` / `TTFT_ms` from two producers would be two
18
+ metrics to every reader and one to every author.
19
+
20
+ The payload stays at schema **1.0**: the suite is pre-freeze, no consumer outside it reads these
21
+ documents yet, and results produced during development are not being retained. After M3 the same
22
+ change would be a minor.
23
+
24
+ - **The release workflow can publish, and can rehearse first.** `release.yml` gained the
25
+ `publish-testpypi` job every other package repository has — manual only, via *Actions → Release →
26
+ Run workflow* — because packaging and release standards §6 requires a successful TestPyPI publish
27
+ ahead of a package's **first** real release, and `0.2.0` is this one. The `release` job gained
28
+ `environment: pypi`, which must match the Environment name configured on the PyPI trusted
29
+ publisher; without it the OIDC exchange has nothing to match and the publish is rejected.
30
+ - **`requirements/ci.lock`, `release.in` and `release.lock`**, hash-pinned — required by packaging
31
+ standards §4 and present in `baseaicore`, `modelrack` and `sweatmeter`, absent here. Every
32
+ blocking CI job now installs the locked toolchain and then `pip install . --no-deps`; the build
33
+ jobs use the release lock with `--no-isolation`, so the wheel CI checks is produced by the same
34
+ pinned backend as the wheel that ships. Without them every run re-resolved, and a new `ruff` or
35
+ `mypy` release could change the result with no commit to explain it.
36
+
37
+ The 3.14 early-warning job stays unlocked on purpose: pinning versions that have no 3.14 wheels
38
+ would defeat the point of an early warning.
39
+
40
+ Verified by installing both locks under `--require-hashes` into a clean interpreter and running
41
+ every CI job against them — format, lint, mypy, import-linter, tests, coverage, contracts,
42
+ `pip-audit` and the workflow's own build and `twine check`.
43
+ - Capability vocabulary **1.1**: the reserved root `user`, whose specializations (`user.<slug>`)
44
+ carry FreeWeight's user-authored goal evidence (ADR-0032 §1). One root added once closes the
45
+ question permanently — the existing open-ended specialization rule accepts every goal any user
46
+ will ever write, so no future rubric is a vocabulary change. `RESERVED_ROOTS` is exported
47
+ alongside `CAPABILITIES`.
48
+ - `benchmark.goal_pack` and `benchmark.calibration_report` (`setspec.goal.v1`), both registered in
49
+ `SUPPORTED_SCHEMAS` and listed in `DRAFT_SCHEMAS`. The calibration report is a payload in its own
50
+ right rather than a field group on evidence because it is meaningful precisely when **no**
51
+ evidence was emitted: a goal below its calibration gate produces no evidence record at all
52
+ (ADR-0032 §3).
53
+ - The goal-sourced field group on `capability.evidence`: `judge_validity_factor`, `goal_hash`,
54
+ `goal_pack_version`, `score_method_mix`, `judge_set`, `calibration` and `uncalibrated`
55
+ (ADR-0032 §5). Every field is optional and absent on a non-goal record, and
56
+ `judge_validity_factor` defaults to `1.0` — so **no previously valid document changed meaning
57
+ and no existing confidence value changed**, which a regression test asserts directly.
58
+
59
+ ### Changed
60
+
61
+ - A bare reserved root is now refused by `validate_capability` and reported `False` by
62
+ `is_known_capability`. `user` is a namespace, not a capability: a payload claiming
63
+ `capability_id: "user"` has lost the identity that is the entire point of the namespace.
64
+ This affects no term that existed before 1.1.
65
+ - **Forward-compatibility leniency narrows, as it must.** A payload declaring
66
+ `vocabulary_version: "1.1"` with a root this build does not know was previously accepted — `1.1`
67
+ was a newer minor than `1.0`, so the spec §13 exception applied. It is now refused, because
68
+ `1.1` is this build's own vocabulary and there is nothing newer about it. This is what forward
69
+ compatibility *means* rather than a regression: leniency exists to carry terms from a future
70
+ this build has not seen, and it evaporates for a version this build has arrived at. A payload
71
+ declaring `1.2` or later is unaffected. Producers on `1.1` must emit roots that exist at `1.1`.
72
+
73
+ ### Fixed
74
+ - **Coverage measured a directory nothing imports.** `[tool.coverage.run] source` named
75
+ `src/setspec`, the source *path*, while CI installs the built distribution — so the moment the
76
+ jobs stopped using an editable install, coverage reported **0 %** and failed the 95 % floor for a
77
+ reason unrelated to the tests. It now names the importable package, with `[tool.coverage.paths]`
78
+ folding the source tree and site-packages into one name, matching `baseaicore`. Both install
79
+ modes report 100 %.
80
+ - **`pip-audit` in CI was auditing nothing.** A bare `pip-audit` inspects the job's own
81
+ environment, which contained only `pip-audit` itself, so that job had been reporting clean on an
82
+ empty set. It now audits both locks.
83
+ - **`install-check` did not verify that `py.typed` reaches the wheel.** The marker is easy to add
84
+ to the source tree and easy to leave out of the build, and the failure is silent: a consumer's
85
+ `mypy --strict` treats an unmarked package as untyped rather than erroring. Asserted against the
86
+ installed wheel now. (It does ship — this closes the hole, it does not fix a break.)
87
+
88
+ - The package ships a `py.typed` marker, matching every other implemented package in the suite
89
+ (`baseaicore`, `modelrack`, `sweatmeter`). Without it, `mypy --strict` in a consuming repository
90
+ cannot see this package's types at all and treats every import from it as untyped.
91
+
92
+ ## [0.2.0] — 2026-08-23
93
+
94
+ Phase 2 of the [development plan](docs/packages/setspec/development-plan.md): provisional
95
+ benchmark and evidence payloads. FreeWeight now has concrete models to build against —
96
+ `model.identity`, `machine.profile`, `benchmark.result`, `benchmark.run_summary`,
97
+ `capability.evidence` and `benchmark.evidence_bundle` — all registered in `SUPPORTED_SCHEMAS` at
98
+ `1.0` but explicitly **draft**: Phase 4 may still reshape a field once FreeWeight has produced real
99
+ results against it, per this phase's own known risk (guessing the result shape before its producer
100
+ exists).
101
+
102
+ ### Added
103
+ - `setspec.model.v1`: `ModelIdentityOut`/`In`, the exchange form of `baseaicore.ModelIdentity` plus
104
+ a `ModelDescriptor` snapshot. `canonical_id` and `identity_confidence` are carried on the wire
105
+ but recomputed from the identity triple and checked on every construction — a producer that
106
+ materializes them inconsistently is rejected, not silently trusted.
107
+ - `setspec.machine.v1`: `MachineProfileOut`/`In`, `GpuProfileFields`, `StorageDeviceFields` — a
108
+ field-for-field mirror of `baseaicore.MachineProfile`, including which fields are required
109
+ versus optional. `machine_fingerprint` is carried but deliberately **not** re-verified, matching
110
+ `MachineProfile`'s own documented reason: its inclusion policy may change after the profile was
111
+ written, and a historically valid profile must still reconstruct exactly as stored.
112
+ - `setspec.benchmark.v1`: `BenchmarkResultOut`/`In` and `BenchmarkRunSummaryOut`/`In`, built from
113
+ Machine Identity §6's minimum
114
+ provenance set — the one place a result's required shape was already normative before any
115
+ FreeWeight code existed. `runtime_profile_hash` is recomputed from the embedded profile and
116
+ checked, the same reasoning as `canonical_id`. Every other hash-shaped field (`manifest_hash`,
117
+ `prompt_subset_hash`, `reproducibility_fingerprint`) is validated only as non-empty: this package
118
+ cannot yet compute FreeWeight's own hashes, and guessing a format risks rejecting the first real
119
+ result over a formatting nuance instead of the risk Phase 2's tests actually target.
120
+ - `setspec.capability.v1`: `CapabilityEvidenceOut`/`In` and `EvidenceBundleOut`/`In`, reproducing
121
+ ADR-0022 §1's normative field table
122
+ verbatim. `measured_at` later than `computed_at` is rejected as incoherent — the schema-level
123
+ enforcement of ADR-0022 §2's rule that recomputing evidence must not make it look fresher.
124
+ - `setspec.provenance`: `EnvironmentFields`, the provider/driver/OS block shared by
125
+ `benchmark.result`, `benchmark.run_summary` and `capability.evidence`. It gets a neutral home
126
+ for the same reason `setspec.metrics` has one: a sub-model used by two payload types belongs to
127
+ neither, and had it lived in `benchmark/v1.py`, the day `benchmark.result` needed a changed
128
+ environment shape whoever edited that class would silently have changed `capability.evidence`
129
+ v1 too — a frozen contract mutating because someone edited a different payload's module.
130
+ - `setspec.envelope.DRAFT_SCHEMAS`: which registered schemas are still provisional. The
131
+ development plan asks for draft status to be visible *in the API*, not only in prose. It is a
132
+ separate set rather than a `"1.0-draft"` version string because the latter would have to be
133
+ parsed by `SchemaVersion`, whose contract is that a version is exactly two integers and
134
+ non-canonical spellings are refused — loosening that permanently, to describe a condition that
135
+ ends at Phase 4, would weaken the version format every frozen schema depends on. Freezing a
136
+ schema is one deletion from this set.
137
+ - `benchmark.result` gains the provenance Machine Identity §6
138
+ calls "optional but strongly recommended, and required for any benchmark whose numbers depend on
139
+ them": `measurement_class` (cold/warm/cache_reused/n-a), `telemetry_summary` (peak VRAM,
140
+ peak/mean power, max temperature, throttle flag) and `raw_response_ref` — plus `error_code`,
141
+ `error_text`, `completed_cases` and `total_cases` from FreeWeight's own `run_tests` row. These
142
+ had to be declared rather than left out: the writer half of each pair forbids unknown keys, so a
143
+ field this schema does not name is one a producer *cannot emit at all*. A memory or energy
144
+ benchmark — whose numbers §6 says depend on telemetry — could not have exported a conformant
145
+ result, and a `failed` result had no field in which to say why it failed.
146
+ - `RuntimeProfileFields.profile_hash`, a public property delegating to
147
+ `baseaicore.RuntimeProfile`. A consumer needs the hash for the same reason a producer does —
148
+ two results are comparable only if their profiles hash identically — and previously the only
149
+ way to it was a private helper reached across class boundaries behind two `noqa: SLF001`
150
+ waivers.
151
+ - `setspec.vocabulary`: `CAPABILITIES`, `CAPABILITY_VOCABULARY_VERSION` (`"1.0"`),
152
+ `validate_capability`, `is_known_capability`. A specialization validates against its **root**
153
+ rather than needing individual enumeration — `coding.rust` is accepted the moment `coding` is
154
+ known — and an unknown ID is accepted leniently only when the payload declares a newer *minor*
155
+ of the vocabulary's current major, never across a major.
156
+ - `setspec.base`: `WireSequence`, an ordered-collection field type alongside `WireEnum`. Strict
157
+ mode validates the *input Python type*, and a bare `tuple[T, ...]` under `strict=True` accepts
158
+ only an actual `tuple` — never the `list` every JSON array deserializes to — so every
159
+ ordered-collection field in Phase 2 needed this to accept a real document at all.
160
+ - `Aggregation` gains `P50`, `RATIO` and `RAW`, matching aggregation kinds FreeWeight's own data
161
+ model already names that Phase 1's set did not cover.
162
+
163
+ ### Changed
164
+ - `PayloadDefinition` now sets `extra="allow"` itself, rather than leaving pydantic's default
165
+ (`extra="ignore"`). A definition nested inside another payload — `model: ModelIdentityFields`
166
+ inside `CapabilityEvidenceFields` — is typed as the bare definition, not a generated `Out`/`In`,
167
+ so its own default governs unknown-key handling regardless of which half of the *outer* pair is
168
+ in use; `ignore` would have silently dropped an unrecognized field from every nested object in
169
+ both directions, which is precisely the loss ADR-0009 rule 4
170
+ exists to prevent. This is additive for every Phase 1 payload, which had no nested fields to
171
+ expose the gap.
172
+
173
+ ### Fixed
174
+ - `completed_cases` greater than `total_cases` is now rejected, the same arithmetic honesty the
175
+ timestamp ordering checks already enforced: "12 of 10 cases done" is a producer bug, and a
176
+ consumer rendering it as a percentage would show one above 100%.
177
+ - A garbled sentence in `setspec.machine.v1`'s module docstring, left by an edit, made the
178
+ explanation of why `machine_fingerprint` is *not* re-verified unreadable — the one place this
179
+ package carries a hash-shaped field without checking it, so the reason it is deliberate rather
180
+ than an oversight is exactly the thing that had to be legible.
181
+ - A test fixture (`tests/unit/test_payloads_benchmark.py`) embedded one module-level dict by
182
+ reference into every result it built; a test that mutated its own copy was actually mutating the
183
+ shared fixture in place, corrupting every test built after it in file execution order. It passed
184
+ under Python 3.13 by luck of `pytest-randomly`'s seed and failed outright under 3.14, whose
185
+ ordering exposed it — the bug was reference aliasing, not a Python-version incompatibility.
186
+ Fixed by building the dict fresh on every call; verified stable across five randomized runs on
187
+ both interpreters.
188
+
189
+ ## [0.1.0] — 2026-08-23
190
+
191
+ Phase 1 of the [development plan](docs/packages/setspec/development-plan.md): the envelope,
192
+ versioning and serialization core. Any payload can now be wrapped, versioned, validated,
193
+ serialized canonically and rejected correctly when its major version is unsupported. No payload
194
+ type is registered yet — Phase 2 adds the first ones against this machinery, which is why
195
+ `SUPPORTED_SCHEMAS` ships empty rather than pre-declaring shapes that do not exist.
196
+
197
+ ### Added
198
+ - `envelope`: `SchemaEnvelope`, `GeneratorInfo`, `SchemaVersion`, `SUPPORTED_SCHEMAS`,
199
+ `load_envelope`, `dump_envelope`. The reader policy of
200
+ ADR-0009 rule 3 is enforced in both directions: a
201
+ newer minor within a supported major is accepted with its unknown fields intact, an unsupported
202
+ major is refused with `SchemaVersionUnsupported` naming the schema, the received version and
203
+ every supported one — and is never partially parsed.
204
+ - `serialization`: canonical JSON via `baseaicore.canonical_json` (delegated, not reimplemented, so
205
+ a payload hashes identically to every other hashed structure in the suite); `MeasurementField`,
206
+ mapping `UNSUPPORTED ↔ "unsupported"` and refusing `null`, bools and numeric strings; and
207
+ `TimestampField`, RFC 3339 UTC at millisecond precision with naive datetimes rejected.
208
+ - `serialization`: `parse_json` with size and depth guards applied *before* the parser runs, so a
209
+ hostile document is refused by a named limit rather than by a `RecursionError`. Limits are the
210
+ module constants `MAX_PAYLOAD_BYTES` (16 MiB, matching API Standards §10) and
211
+ `MAX_PAYLOAD_DEPTH` (64); callers may pass tighter ones per call.
212
+ - `base`: `PayloadDefinition`, `StrictPayload`, `PreservingPayload` and `payload_models`, which
213
+ generates the `Out`/`In` pair required by
214
+ ADR-0009 rule 4 from one definition — so a writer
215
+ cannot emit a field it does not know and a reader cannot strip one. The round-trip contract is
216
+ asserted per class, never across the pair.
217
+ - `base`: `WireEnum`, for enum fields that must accept their own string values from a parsed
218
+ document while the bases otherwise validate strictly.
219
+ - `metrics`: `MetricValueOut`/`MetricValueIn` and `Aggregation`. The
220
+ ADR-0016 §6 invariants are structural rather than
221
+ advisory: a real value must report at least one supported sample, an unsupported value must
222
+ report none, and dispersion needs at least two — so `value=0.0, sample_count=0` cannot be
223
+ serialized at all.
224
+ - `errors`: `SchemaVersionUnsupported` (code `SCHEMA_VERSION_UNSUPPORTED`), with
225
+ `baseaicore.ValidationError` re-exported so callers import both from one place.
226
+
227
+ ### Changed
228
+ - Coverage floor raised from 85 % to 95 %, the number [spec §18](docs/packages/setspec/spec.md)
229
+ and every phase's acceptance criteria actually state; the scaffold shipped 85 in both
230
+ `pyproject.toml` and the CI job. Current coverage is 100 %.
231
+ - `.importlinter`: the second contract listed no forbidden modules at all, and its inline
232
+ `root_packages`/`source_modules` values were read character by character — `lint-imports` failed
233
+ with `Could not find package 's'` and therefore checked nothing. Rewritten as newline-separated
234
+ lists with `include_external_packages`, and renamed from `no-sibling-packages` to
235
+ `no-capability-packages`: `modelrack`, `sweatmeter`, `weightsdb` and `mirrorwall` sit in the
236
+ layer *above* SetSpec, so they were never siblings.
237
+
238
+ ### Fixed
239
+ - `ruff format --check .` failed on nine vendored documents under `docs/`. ruff 0.16 formats Python
240
+ code blocks inside markdown, and those files are byte-identical copies of the suite's master
241
+ documents shared with nine other repositories — reformatting them here would desync every copy.
242
+ `docs/` is now excluded from ruff instead.
243
+
244
+ ### Security
245
+ - `pytest` moved from `>=8,<9` to `>=9.0.3,<10`, excluding PYSEC-2026-1845 (vulnerable
246
+ `/tmp/pytest-of-{user}` handling, affecting pytest through 9.0.2). Matches the pin BaseAiCore
247
+ already moved to; the suite passes unchanged on pytest 9.