setspec 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- setspec-0.2.0/.editorconfig +23 -0
- setspec-0.2.0/.github/workflows/ci.yml +150 -0
- setspec-0.2.0/.github/workflows/release.yml +56 -0
- setspec-0.2.0/.gitignore +100 -0
- setspec-0.2.0/.importlinter +27 -0
- setspec-0.2.0/.pre-commit-config.yaml +22 -0
- setspec-0.2.0/CHANGELOG.md +247 -0
- setspec-0.2.0/CONTRIBUTING.md +50 -0
- setspec-0.2.0/LICENSE +201 -0
- setspec-0.2.0/PKG-INFO +160 -0
- setspec-0.2.0/README.md +129 -0
- setspec-0.2.0/SECURITY.md +39 -0
- setspec-0.2.0/docs/README.md +8 -0
- setspec-0.2.0/docs/packages/setspec/development-plan.md +293 -0
- setspec-0.2.0/docs/packages/setspec/spec.md +271 -0
- setspec-0.2.0/pyproject.toml +99 -0
- setspec-0.2.0/requirements/README.md +48 -0
- setspec-0.2.0/requirements/ci.lock +781 -0
- setspec-0.2.0/requirements/release.in +6 -0
- setspec-0.2.0/requirements/release.lock +493 -0
- setspec-0.2.0/src/setspec/__about__.py +3 -0
- setspec-0.2.0/src/setspec/__init__.py +114 -0
- setspec-0.2.0/src/setspec/artifacts.py +4 -0
- setspec-0.2.0/src/setspec/base.py +256 -0
- setspec-0.2.0/src/setspec/benchmark/v1.py +594 -0
- setspec-0.2.0/src/setspec/capability/v1.py +394 -0
- setspec-0.2.0/src/setspec/envelope.py +468 -0
- setspec-0.2.0/src/setspec/error/v1.py +4 -0
- setspec-0.2.0/src/setspec/errors.py +68 -0
- setspec-0.2.0/src/setspec/event/v1.py +4 -0
- setspec-0.2.0/src/setspec/goal/v1.py +390 -0
- setspec-0.2.0/src/setspec/goldens/.gitkeep +0 -0
- setspec-0.2.0/src/setspec/machine/v1.py +131 -0
- setspec-0.2.0/src/setspec/metrics.py +184 -0
- setspec-0.2.0/src/setspec/model/v1.py +169 -0
- setspec-0.2.0/src/setspec/provenance.py +59 -0
- setspec-0.2.0/src/setspec/py.typed +0 -0
- setspec-0.2.0/src/setspec/schemas/.gitkeep +0 -0
- setspec-0.2.0/src/setspec/serialization.py +330 -0
- setspec-0.2.0/src/setspec/vocabulary.py +205 -0
- setspec-0.2.0/tests/conftest.py +75 -0
- setspec-0.2.0/tests/contract/test_cross_version.py +4 -0
- setspec-0.2.0/tests/contract/test_goldens.py +4 -0
- setspec-0.2.0/tests/contract/test_schema_snapshots.py +4 -0
- setspec-0.2.0/tests/contract/test_version_negotiation.py +189 -0
- setspec-0.2.0/tests/unit/test_base.py +163 -0
- setspec-0.2.0/tests/unit/test_envelope.py +254 -0
- setspec-0.2.0/tests/unit/test_errors.py +4 -0
- setspec-0.2.0/tests/unit/test_events.py +4 -0
- setspec-0.2.0/tests/unit/test_metrics.py +175 -0
- setspec-0.2.0/tests/unit/test_payloads_benchmark.py +482 -0
- setspec-0.2.0/tests/unit/test_payloads_capability.py +411 -0
- setspec-0.2.0/tests/unit/test_payloads_goal.py +499 -0
- setspec-0.2.0/tests/unit/test_serialization.py +223 -0
- setspec-0.2.0/tests/unit/test_vocabulary.py +225 -0
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
root = true
|
|
2
|
+
|
|
3
|
+
[*]
|
|
4
|
+
charset = utf-8
|
|
5
|
+
end_of_line = lf
|
|
6
|
+
insert_final_newline = true
|
|
7
|
+
trim_trailing_whitespace = true
|
|
8
|
+
indent_style = space
|
|
9
|
+
indent_size = 4
|
|
10
|
+
|
|
11
|
+
[*.py]
|
|
12
|
+
indent_size = 4
|
|
13
|
+
max_line_length = 100
|
|
14
|
+
|
|
15
|
+
[*.{toml,yml,yaml,json}]
|
|
16
|
+
indent_size = 2
|
|
17
|
+
|
|
18
|
+
[*.md]
|
|
19
|
+
trim_trailing_whitespace = false
|
|
20
|
+
max_line_length = off
|
|
21
|
+
|
|
22
|
+
[Makefile]
|
|
23
|
+
indent_style = tab
|
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
name: CI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [main]
|
|
6
|
+
pull_request:
|
|
7
|
+
|
|
8
|
+
jobs:
|
|
9
|
+
format:
|
|
10
|
+
runs-on: ubuntu-latest
|
|
11
|
+
steps:
|
|
12
|
+
- uses: actions/checkout@v4
|
|
13
|
+
- uses: actions/setup-python@v5
|
|
14
|
+
with: { python-version: "3.12" }
|
|
15
|
+
- run: pip install ruff
|
|
16
|
+
- run: ruff format --check .
|
|
17
|
+
|
|
18
|
+
lint:
|
|
19
|
+
runs-on: ubuntu-latest
|
|
20
|
+
steps:
|
|
21
|
+
- uses: actions/checkout@v4
|
|
22
|
+
- uses: actions/setup-python@v5
|
|
23
|
+
with: { python-version: "3.12" }
|
|
24
|
+
- run: pip install ruff
|
|
25
|
+
- run: ruff check .
|
|
26
|
+
|
|
27
|
+
types:
|
|
28
|
+
runs-on: ubuntu-latest
|
|
29
|
+
steps:
|
|
30
|
+
- uses: actions/checkout@v4
|
|
31
|
+
- uses: actions/setup-python@v5
|
|
32
|
+
with: { python-version: "3.12" }
|
|
33
|
+
- run: pip install --require-hashes -r requirements/ci.lock
|
|
34
|
+
- run: pip install . --no-deps
|
|
35
|
+
- run: mypy src tests
|
|
36
|
+
|
|
37
|
+
boundaries:
|
|
38
|
+
runs-on: ubuntu-latest
|
|
39
|
+
steps:
|
|
40
|
+
- uses: actions/checkout@v4
|
|
41
|
+
- uses: actions/setup-python@v5
|
|
42
|
+
with: { python-version: "3.12" }
|
|
43
|
+
- run: pip install --require-hashes -r requirements/ci.lock
|
|
44
|
+
- run: pip install . --no-deps
|
|
45
|
+
- run: lint-imports
|
|
46
|
+
|
|
47
|
+
tests:
|
|
48
|
+
runs-on: ubuntu-latest
|
|
49
|
+
strategy:
|
|
50
|
+
matrix:
|
|
51
|
+
python-version: ["3.12", "3.13"]
|
|
52
|
+
steps:
|
|
53
|
+
- uses: actions/checkout@v4
|
|
54
|
+
- uses: actions/setup-python@v5
|
|
55
|
+
with: { python-version: "${{ matrix.python-version }}" }
|
|
56
|
+
- run: pip install --require-hashes -r requirements/ci.lock
|
|
57
|
+
- run: pip install . --no-deps
|
|
58
|
+
- run: pytest -m "not live and not performance" --cov --cov-report=xml
|
|
59
|
+
|
|
60
|
+
tests-314-early-warning:
|
|
61
|
+
runs-on: ubuntu-latest
|
|
62
|
+
continue-on-error: true
|
|
63
|
+
steps:
|
|
64
|
+
- uses: actions/checkout@v4
|
|
65
|
+
- uses: actions/setup-python@v5
|
|
66
|
+
with: { python-version: "3.14" }
|
|
67
|
+
# Deliberately unlocked: this job resolves from the ranges in `pyproject.toml`, because
|
|
68
|
+
# pinning versions that have no 3.14 wheels would defeat the purpose of an early warning
|
|
69
|
+
# (requirements/README.md).
|
|
70
|
+
- run: pip install -e ".[dev]"
|
|
71
|
+
- run: pytest -m "not live and not performance"
|
|
72
|
+
|
|
73
|
+
coverage:
|
|
74
|
+
needs: [tests]
|
|
75
|
+
runs-on: ubuntu-latest
|
|
76
|
+
steps:
|
|
77
|
+
- uses: actions/checkout@v4
|
|
78
|
+
- uses: actions/setup-python@v5
|
|
79
|
+
with: { python-version: "3.12" }
|
|
80
|
+
- run: pip install --require-hashes -r requirements/ci.lock
|
|
81
|
+
- run: pip install . --no-deps
|
|
82
|
+
- run: pytest -m "not live and not performance" --cov --cov-report=term-missing --cov-fail-under=95
|
|
83
|
+
|
|
84
|
+
contracts:
|
|
85
|
+
runs-on: ubuntu-latest
|
|
86
|
+
steps:
|
|
87
|
+
- uses: actions/checkout@v4
|
|
88
|
+
- uses: actions/setup-python@v5
|
|
89
|
+
with: { python-version: "3.12" }
|
|
90
|
+
- run: pip install --require-hashes -r requirements/ci.lock
|
|
91
|
+
- run: pip install . --no-deps
|
|
92
|
+
- run: pytest -m contract
|
|
93
|
+
|
|
94
|
+
security:
|
|
95
|
+
runs-on: ubuntu-latest
|
|
96
|
+
steps:
|
|
97
|
+
- uses: actions/checkout@v4
|
|
98
|
+
# gitleaks scans *history*, and `actions/checkout` fetches a single commit by default.
|
|
99
|
+
# For a push it is handed `<first-pushed>^..<last-pushed>`, so the parent of the first
|
|
100
|
+
# pushed commit has to be in the object store; in a depth-1 clone it is not, and git
|
|
101
|
+
# answers "unknown revision", which the action reports as exit code 1. It fails the same
|
|
102
|
+
# way whether or not a secret exists, so a green run would not have meant anything either.
|
|
103
|
+
with: { fetch-depth: 0 }
|
|
104
|
+
- uses: actions/setup-python@v5
|
|
105
|
+
with: { python-version: "3.12" }
|
|
106
|
+
- run: pip install pip-audit
|
|
107
|
+
# Audit the locked set, not the job's own environment: a bare `pip-audit` here inspects an
|
|
108
|
+
# environment containing only pip-audit itself and reports nothing (Security Standards §11).
|
|
109
|
+
- run: pip-audit --require-hashes -r requirements/ci.lock
|
|
110
|
+
- run: pip-audit --require-hashes -r requirements/release.lock
|
|
111
|
+
- uses: gitleaks/gitleaks-action@v2
|
|
112
|
+
env:
|
|
113
|
+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
114
|
+
|
|
115
|
+
build:
|
|
116
|
+
runs-on: ubuntu-latest
|
|
117
|
+
steps:
|
|
118
|
+
- uses: actions/checkout@v4
|
|
119
|
+
- uses: actions/setup-python@v5
|
|
120
|
+
with: { python-version: "3.12" }
|
|
121
|
+
# The same pinned build chain the release uses, so a wheel that builds here is the wheel
|
|
122
|
+
# that ships: `--no-isolation` takes the backend from this lock rather than re-resolving
|
|
123
|
+
# hatchling from the index at build time.
|
|
124
|
+
- run: pip install --require-hashes -r requirements/release.lock
|
|
125
|
+
- run: python -m build --no-isolation
|
|
126
|
+
- run: twine check dist/*
|
|
127
|
+
- uses: actions/upload-artifact@v4
|
|
128
|
+
with: { name: dist, path: dist/ }
|
|
129
|
+
|
|
130
|
+
install-check:
|
|
131
|
+
needs: [build]
|
|
132
|
+
runs-on: ubuntu-latest
|
|
133
|
+
steps:
|
|
134
|
+
- uses: actions/checkout@v4
|
|
135
|
+
- uses: actions/setup-python@v5
|
|
136
|
+
with: { python-version: "3.12" }
|
|
137
|
+
- uses: actions/download-artifact@v4
|
|
138
|
+
with: { name: dist, path: dist/ }
|
|
139
|
+
- run: pip install dist/*.whl
|
|
140
|
+
- run: python -c "import setspec"
|
|
141
|
+
# The marker is easy to add to the source tree and easy to leave out of the built wheel, and
|
|
142
|
+
# the failure is silent: a consumer's `mypy --strict` treats every import from an unmarked
|
|
143
|
+
# package as untyped rather than erroring. Asserted against the *installed* wheel.
|
|
144
|
+
- name: py.typed ships in the wheel
|
|
145
|
+
run: |
|
|
146
|
+
python -c "
|
|
147
|
+
import pathlib, setspec
|
|
148
|
+
marker = pathlib.Path(setspec.__file__).parent / 'py.typed'
|
|
149
|
+
assert marker.is_file(), 'py.typed missing from the installed wheel'
|
|
150
|
+
"
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
name: Release
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
tags: ["v*.*.*"]
|
|
6
|
+
workflow_dispatch: # manual TestPyPI dry run; see publish-testpypi below
|
|
7
|
+
|
|
8
|
+
permissions:
|
|
9
|
+
id-token: write # required for PyPI Trusted Publishing
|
|
10
|
+
contents: write # required to create the GitHub release
|
|
11
|
+
|
|
12
|
+
jobs:
|
|
13
|
+
release:
|
|
14
|
+
if: github.event_name == 'push'
|
|
15
|
+
runs-on: ubuntu-latest
|
|
16
|
+
environment: pypi # must match the Environment name set on the PyPI trusted publisher
|
|
17
|
+
steps:
|
|
18
|
+
- uses: actions/checkout@v4
|
|
19
|
+
- uses: actions/setup-python@v5
|
|
20
|
+
with: { python-version: "3.12" }
|
|
21
|
+
- run: pip install --require-hashes -r requirements/release.lock
|
|
22
|
+
- run: python -m build --no-isolation
|
|
23
|
+
- run: twine check dist/*
|
|
24
|
+
- run: pip install "dist/$(ls dist | grep .whl)[dev]"
|
|
25
|
+
- run: pytest -m "not live and not performance"
|
|
26
|
+
- name: Publish to PyPI
|
|
27
|
+
uses: pypa/gh-action-pypi-publish@release/v1
|
|
28
|
+
- name: Create GitHub release
|
|
29
|
+
uses: softprops/action-gh-release@v2
|
|
30
|
+
with:
|
|
31
|
+
generate_release_notes: true
|
|
32
|
+
files: dist/*
|
|
33
|
+
|
|
34
|
+
publish-testpypi:
|
|
35
|
+
# Manual only, via Actions -> Release -> Run workflow. Packaging and Release Standards §6
|
|
36
|
+
# requires a successful TestPyPI publish ahead of a package's first real release; 0.2.0 is
|
|
37
|
+
# this package's first published version, so run this once before tagging v0.2.0. Later
|
|
38
|
+
# releases may skip it, or use it again as a dry run.
|
|
39
|
+
if: github.event_name == 'workflow_dispatch'
|
|
40
|
+
runs-on: ubuntu-latest
|
|
41
|
+
# No `environment:` here, matching the other package repositories. The OIDC claim must match
|
|
42
|
+
# the trusted publisher exactly: declaring an environment the TestPyPI publisher was not
|
|
43
|
+
# configured with fails the exchange. Add one only alongside configuring it there.
|
|
44
|
+
steps:
|
|
45
|
+
- uses: actions/checkout@v4
|
|
46
|
+
- uses: actions/setup-python@v5
|
|
47
|
+
with: { python-version: "3.12" }
|
|
48
|
+
- run: pip install --require-hashes -r requirements/release.lock
|
|
49
|
+
- run: python -m build --no-isolation
|
|
50
|
+
- run: twine check dist/*
|
|
51
|
+
- run: pip install "dist/$(ls dist | grep .whl)[dev]"
|
|
52
|
+
- run: pytest -m "not live and not performance"
|
|
53
|
+
- name: Publish to TestPyPI
|
|
54
|
+
uses: pypa/gh-action-pypi-publish@release/v1
|
|
55
|
+
with:
|
|
56
|
+
repository-url: https://test.pypi.org/legacy/
|
setspec-0.2.0/.gitignore
ADDED
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
# ---- Python ----
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[cod]
|
|
4
|
+
*$py.class
|
|
5
|
+
*.so
|
|
6
|
+
.Python
|
|
7
|
+
build/
|
|
8
|
+
develop-eggs/
|
|
9
|
+
dist/
|
|
10
|
+
downloads/
|
|
11
|
+
eggs/
|
|
12
|
+
.eggs/
|
|
13
|
+
lib/
|
|
14
|
+
lib64/
|
|
15
|
+
parts/
|
|
16
|
+
sdist/
|
|
17
|
+
var/
|
|
18
|
+
wheels/
|
|
19
|
+
*.egg-info/
|
|
20
|
+
.installed.cfg
|
|
21
|
+
*.egg
|
|
22
|
+
MANIFEST
|
|
23
|
+
|
|
24
|
+
# ---- Packaging / build backends ----
|
|
25
|
+
pip-wheel-metadata/
|
|
26
|
+
share/python-wheels/
|
|
27
|
+
|
|
28
|
+
# ---- Testing / coverage ----
|
|
29
|
+
.pytest_cache/
|
|
30
|
+
.cache
|
|
31
|
+
.coverage
|
|
32
|
+
.coverage.*
|
|
33
|
+
coverage.xml
|
|
34
|
+
*.cover
|
|
35
|
+
*.py,cover
|
|
36
|
+
htmlcov/
|
|
37
|
+
nosetests.xml
|
|
38
|
+
.hypothesis/
|
|
39
|
+
|
|
40
|
+
# ---- Type checking / linting ----
|
|
41
|
+
.mypy_cache/
|
|
42
|
+
.dmypy.json
|
|
43
|
+
dmypy.json
|
|
44
|
+
.ruff_cache/
|
|
45
|
+
.pytype/
|
|
46
|
+
|
|
47
|
+
# ---- Virtual environments ----
|
|
48
|
+
.venv/
|
|
49
|
+
venv/
|
|
50
|
+
ENV/
|
|
51
|
+
env/
|
|
52
|
+
env.bak/
|
|
53
|
+
venv.bak/
|
|
54
|
+
.python-version
|
|
55
|
+
|
|
56
|
+
# ---- Distribution / dependency locking ----
|
|
57
|
+
# requirements/*.lock is deliberately NOT ignored. The lock files are committed
|
|
58
|
+
# inputs: CI and release jobs install from them with --require-hashes (Packaging
|
|
59
|
+
# and Release Standards §4, Security Standards §11), so an ignored lock file
|
|
60
|
+
# means a checkout that cannot build.
|
|
61
|
+
|
|
62
|
+
# ---- Editors / OS ----
|
|
63
|
+
.vscode/
|
|
64
|
+
.idea/
|
|
65
|
+
*.swp
|
|
66
|
+
*.swo
|
|
67
|
+
.DS_Store
|
|
68
|
+
Thumbs.db
|
|
69
|
+
|
|
70
|
+
# ---- Suite runtime state (this component's local data) ----
|
|
71
|
+
# The application writes to XDG paths at runtime (~/.config, ~/.local/share,
|
|
72
|
+
# ~/.local/state per Master Architecture §1.2), never inside the repository.
|
|
73
|
+
# These entries only guard against a developer pointing XDG_* at the repo
|
|
74
|
+
# during local testing.
|
|
75
|
+
.local/
|
|
76
|
+
.config/
|
|
77
|
+
*.sqlite3
|
|
78
|
+
*.sqlite3-journal
|
|
79
|
+
*.sqlite3-wal
|
|
80
|
+
*.sqlite3-shm
|
|
81
|
+
/data/
|
|
82
|
+
/logs/
|
|
83
|
+
/backups/
|
|
84
|
+
/artifacts/
|
|
85
|
+
/exports/
|
|
86
|
+
|
|
87
|
+
# ---- Secrets ----
|
|
88
|
+
# Per Security Standards §8: secrets are never committed. Config files may
|
|
89
|
+
# only name where a secret comes from (*_env / *_file), never the value.
|
|
90
|
+
.env
|
|
91
|
+
.env.*
|
|
92
|
+
*.key
|
|
93
|
+
*.pem
|
|
94
|
+
secrets.toml
|
|
95
|
+
|
|
96
|
+
# ---- Node-free JS assets (MirrorWall consumers may still use a local tool) ----
|
|
97
|
+
node_modules/
|
|
98
|
+
|
|
99
|
+
# ---- Build artifacts from docs generation ----
|
|
100
|
+
docs/api/openapi-v1.json.tmp
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
[importlinter]
|
|
2
|
+
root_packages =
|
|
3
|
+
setspec
|
|
4
|
+
# The forbidden modules below are external distributions, not submodules of setspec, so the graph
|
|
5
|
+
# has to include external packages for the contracts to be able to see them at all.
|
|
6
|
+
include_external_packages = True
|
|
7
|
+
|
|
8
|
+
[importlinter:contract:no-application-imports]
|
|
9
|
+
name = SetSpec must not import applications
|
|
10
|
+
type = forbidden
|
|
11
|
+
source_modules =
|
|
12
|
+
setspec
|
|
13
|
+
forbidden_modules =
|
|
14
|
+
freeweight
|
|
15
|
+
loadcoach
|
|
16
|
+
ideapress
|
|
17
|
+
|
|
18
|
+
[importlinter:contract:no-capability-packages]
|
|
19
|
+
name = SetSpec must not import capability packages
|
|
20
|
+
type = forbidden
|
|
21
|
+
source_modules =
|
|
22
|
+
setspec
|
|
23
|
+
forbidden_modules =
|
|
24
|
+
modelrack
|
|
25
|
+
sweatmeter
|
|
26
|
+
weightsdb
|
|
27
|
+
mirrorwall
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
repos:
|
|
2
|
+
- repo: https://github.com/astral-sh/ruff-pre-commit
|
|
3
|
+
rev: v0.6.9
|
|
4
|
+
hooks:
|
|
5
|
+
- id: ruff
|
|
6
|
+
args: [--fix]
|
|
7
|
+
- id: ruff-format
|
|
8
|
+
- repo: https://github.com/pre-commit/pre-commit-hooks
|
|
9
|
+
rev: v4.6.0
|
|
10
|
+
hooks:
|
|
11
|
+
- id: trailing-whitespace
|
|
12
|
+
- id: end-of-file-fixer
|
|
13
|
+
- id: check-toml
|
|
14
|
+
- id: check-json
|
|
15
|
+
- id: check-added-large-files
|
|
16
|
+
- id: check-merge-conflict
|
|
17
|
+
- id: mixed-line-ending
|
|
18
|
+
args: [--fix=lf]
|
|
19
|
+
- repo: https://github.com/gitleaks/gitleaks
|
|
20
|
+
rev: v8.18.4
|
|
21
|
+
hooks:
|
|
22
|
+
- id: gitleaks
|
|
@@ -0,0 +1,247 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to `setspec` are documented here.
|
|
4
|
+
Format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/); versioning follows
|
|
5
|
+
[Semantic Versioning](https://semver.org/), pre-1.0 per
|
|
6
|
+
packaging and release standards §3.
|
|
7
|
+
|
|
8
|
+
## [Unreleased]
|
|
9
|
+
|
|
10
|
+
### Added
|
|
11
|
+
- **`metric_key` on `MetricValueFields`.** The model declared value, unit, aggregation, direction,
|
|
12
|
+
sample count and dispersion — and nothing saying *which metric it is*. Both
|
|
13
|
+
`BenchmarkResultFields.metrics` and `BenchmarkRunSummaryFields.aggregate_metrics` carry sequences
|
|
14
|
+
of it, so a consumer receiving a run summary got a list of numbers it could not attribute, chart,
|
|
15
|
+
compare across runs, or check for the metric it was looking for. Required rather than optional:
|
|
16
|
+
an unattributable number is not a measurement. Constrained to lower snake case, because a metric
|
|
17
|
+
key is an identifier consumers match on and `ttft_ms` / `TTFT_ms` from two producers would be two
|
|
18
|
+
metrics to every reader and one to every author.
|
|
19
|
+
|
|
20
|
+
The payload stays at schema **1.0**: the suite is pre-freeze, no consumer outside it reads these
|
|
21
|
+
documents yet, and results produced during development are not being retained. After M3 the same
|
|
22
|
+
change would be a minor.
|
|
23
|
+
|
|
24
|
+
- **The release workflow can publish, and can rehearse first.** `release.yml` gained the
|
|
25
|
+
`publish-testpypi` job every other package repository has — manual only, via *Actions → Release →
|
|
26
|
+
Run workflow* — because packaging and release standards §6 requires a successful TestPyPI publish
|
|
27
|
+
ahead of a package's **first** real release, and `0.2.0` is this one. The `release` job gained
|
|
28
|
+
`environment: pypi`, which must match the Environment name configured on the PyPI trusted
|
|
29
|
+
publisher; without it the OIDC exchange has nothing to match and the publish is rejected.
|
|
30
|
+
- **`requirements/ci.lock`, `release.in` and `release.lock`**, hash-pinned — required by packaging
|
|
31
|
+
standards §4 and present in `baseaicore`, `modelrack` and `sweatmeter`, absent here. Every
|
|
32
|
+
blocking CI job now installs the locked toolchain and then `pip install . --no-deps`; the build
|
|
33
|
+
jobs use the release lock with `--no-isolation`, so the wheel CI checks is produced by the same
|
|
34
|
+
pinned backend as the wheel that ships. Without them every run re-resolved, and a new `ruff` or
|
|
35
|
+
`mypy` release could change the result with no commit to explain it.
|
|
36
|
+
|
|
37
|
+
The 3.14 early-warning job stays unlocked on purpose: pinning versions that have no 3.14 wheels
|
|
38
|
+
would defeat the point of an early warning.
|
|
39
|
+
|
|
40
|
+
Verified by installing both locks under `--require-hashes` into a clean interpreter and running
|
|
41
|
+
every CI job against them — format, lint, mypy, import-linter, tests, coverage, contracts,
|
|
42
|
+
`pip-audit` and the workflow's own build and `twine check`.
|
|
43
|
+
- Capability vocabulary **1.1**: the reserved root `user`, whose specializations (`user.<slug>`)
|
|
44
|
+
carry FreeWeight's user-authored goal evidence (ADR-0032 §1). One root added once closes the
|
|
45
|
+
question permanently — the existing open-ended specialization rule accepts every goal any user
|
|
46
|
+
will ever write, so no future rubric is a vocabulary change. `RESERVED_ROOTS` is exported
|
|
47
|
+
alongside `CAPABILITIES`.
|
|
48
|
+
- `benchmark.goal_pack` and `benchmark.calibration_report` (`setspec.goal.v1`), both registered in
|
|
49
|
+
`SUPPORTED_SCHEMAS` and listed in `DRAFT_SCHEMAS`. The calibration report is a payload in its own
|
|
50
|
+
right rather than a field group on evidence because it is meaningful precisely when **no**
|
|
51
|
+
evidence was emitted: a goal below its calibration gate produces no evidence record at all
|
|
52
|
+
(ADR-0032 §3).
|
|
53
|
+
- The goal-sourced field group on `capability.evidence`: `judge_validity_factor`, `goal_hash`,
|
|
54
|
+
`goal_pack_version`, `score_method_mix`, `judge_set`, `calibration` and `uncalibrated`
|
|
55
|
+
(ADR-0032 §5). Every field is optional and absent on a non-goal record, and
|
|
56
|
+
`judge_validity_factor` defaults to `1.0` — so **no previously valid document changed meaning
|
|
57
|
+
and no existing confidence value changed**, which a regression test asserts directly.
|
|
58
|
+
|
|
59
|
+
### Changed
|
|
60
|
+
|
|
61
|
+
- A bare reserved root is now refused by `validate_capability` and reported `False` by
|
|
62
|
+
`is_known_capability`. `user` is a namespace, not a capability: a payload claiming
|
|
63
|
+
`capability_id: "user"` has lost the identity that is the entire point of the namespace.
|
|
64
|
+
This affects no term that existed before 1.1.
|
|
65
|
+
- **Forward-compatibility leniency narrows, as it must.** A payload declaring
|
|
66
|
+
`vocabulary_version: "1.1"` with a root this build does not know was previously accepted — `1.1`
|
|
67
|
+
was a newer minor than `1.0`, so the spec §13 exception applied. It is now refused, because
|
|
68
|
+
`1.1` is this build's own vocabulary and there is nothing newer about it. This is what forward
|
|
69
|
+
compatibility *means* rather than a regression: leniency exists to carry terms from a future
|
|
70
|
+
this build has not seen, and it evaporates for a version this build has arrived at. A payload
|
|
71
|
+
declaring `1.2` or later is unaffected. Producers on `1.1` must emit roots that exist at `1.1`.
|
|
72
|
+
|
|
73
|
+
### Fixed
|
|
74
|
+
- **Coverage measured a directory nothing imports.** `[tool.coverage.run] source` named
|
|
75
|
+
`src/setspec`, the source *path*, while CI installs the built distribution — so the moment the
|
|
76
|
+
jobs stopped using an editable install, coverage reported **0 %** and failed the 95 % floor for a
|
|
77
|
+
reason unrelated to the tests. It now names the importable package, with `[tool.coverage.paths]`
|
|
78
|
+
folding the source tree and site-packages into one name, matching `baseaicore`. Both install
|
|
79
|
+
modes report 100 %.
|
|
80
|
+
- **`pip-audit` in CI was auditing nothing.** A bare `pip-audit` inspects the job's own
|
|
81
|
+
environment, which contained only `pip-audit` itself, so that job had been reporting clean on an
|
|
82
|
+
empty set. It now audits both locks.
|
|
83
|
+
- **`install-check` did not verify that `py.typed` reaches the wheel.** The marker is easy to add
|
|
84
|
+
to the source tree and easy to leave out of the build, and the failure is silent: a consumer's
|
|
85
|
+
`mypy --strict` treats an unmarked package as untyped rather than erroring. Asserted against the
|
|
86
|
+
installed wheel now. (It does ship — this closes the hole, it does not fix a break.)
|
|
87
|
+
|
|
88
|
+
- The package ships a `py.typed` marker, matching every other implemented package in the suite
|
|
89
|
+
(`baseaicore`, `modelrack`, `sweatmeter`). Without it, `mypy --strict` in a consuming repository
|
|
90
|
+
cannot see this package's types at all and treats every import from it as untyped.
|
|
91
|
+
|
|
92
|
+
## [0.2.0] — 2026-08-23
|
|
93
|
+
|
|
94
|
+
Phase 2 of the [development plan](docs/packages/setspec/development-plan.md): provisional
|
|
95
|
+
benchmark and evidence payloads. FreeWeight now has concrete models to build against —
|
|
96
|
+
`model.identity`, `machine.profile`, `benchmark.result`, `benchmark.run_summary`,
|
|
97
|
+
`capability.evidence` and `benchmark.evidence_bundle` — all registered in `SUPPORTED_SCHEMAS` at
|
|
98
|
+
`1.0` but explicitly **draft**: Phase 4 may still reshape a field once FreeWeight has produced real
|
|
99
|
+
results against it, per this phase's own known risk (guessing the result shape before its producer
|
|
100
|
+
exists).
|
|
101
|
+
|
|
102
|
+
### Added
|
|
103
|
+
- `setspec.model.v1`: `ModelIdentityOut`/`In`, the exchange form of `baseaicore.ModelIdentity` plus
|
|
104
|
+
a `ModelDescriptor` snapshot. `canonical_id` and `identity_confidence` are carried on the wire
|
|
105
|
+
but recomputed from the identity triple and checked on every construction — a producer that
|
|
106
|
+
materializes them inconsistently is rejected, not silently trusted.
|
|
107
|
+
- `setspec.machine.v1`: `MachineProfileOut`/`In`, `GpuProfileFields`, `StorageDeviceFields` — a
|
|
108
|
+
field-for-field mirror of `baseaicore.MachineProfile`, including which fields are required
|
|
109
|
+
versus optional. `machine_fingerprint` is carried but deliberately **not** re-verified, matching
|
|
110
|
+
`MachineProfile`'s own documented reason: its inclusion policy may change after the profile was
|
|
111
|
+
written, and a historically valid profile must still reconstruct exactly as stored.
|
|
112
|
+
- `setspec.benchmark.v1`: `BenchmarkResultOut`/`In` and `BenchmarkRunSummaryOut`/`In`, built from
|
|
113
|
+
Machine Identity §6's minimum
|
|
114
|
+
provenance set — the one place a result's required shape was already normative before any
|
|
115
|
+
FreeWeight code existed. `runtime_profile_hash` is recomputed from the embedded profile and
|
|
116
|
+
checked, the same reasoning as `canonical_id`. Every other hash-shaped field (`manifest_hash`,
|
|
117
|
+
`prompt_subset_hash`, `reproducibility_fingerprint`) is validated only as non-empty: this package
|
|
118
|
+
cannot yet compute FreeWeight's own hashes, and guessing a format risks rejecting the first real
|
|
119
|
+
result over a formatting nuance instead of the risk Phase 2's tests actually target.
|
|
120
|
+
- `setspec.capability.v1`: `CapabilityEvidenceOut`/`In` and `EvidenceBundleOut`/`In`, reproducing
|
|
121
|
+
ADR-0022 §1's normative field table
|
|
122
|
+
verbatim. `measured_at` later than `computed_at` is rejected as incoherent — the schema-level
|
|
123
|
+
enforcement of ADR-0022 §2's rule that recomputing evidence must not make it look fresher.
|
|
124
|
+
- `setspec.provenance`: `EnvironmentFields`, the provider/driver/OS block shared by
|
|
125
|
+
`benchmark.result`, `benchmark.run_summary` and `capability.evidence`. It gets a neutral home
|
|
126
|
+
for the same reason `setspec.metrics` has one: a sub-model used by two payload types belongs to
|
|
127
|
+
neither, and had it lived in `benchmark/v1.py`, the day `benchmark.result` needed a changed
|
|
128
|
+
environment shape whoever edited that class would silently have changed `capability.evidence`
|
|
129
|
+
v1 too — a frozen contract mutating because someone edited a different payload's module.
|
|
130
|
+
- `setspec.envelope.DRAFT_SCHEMAS`: which registered schemas are still provisional. The
|
|
131
|
+
development plan asks for draft status to be visible *in the API*, not only in prose. It is a
|
|
132
|
+
separate set rather than a `"1.0-draft"` version string because the latter would have to be
|
|
133
|
+
parsed by `SchemaVersion`, whose contract is that a version is exactly two integers and
|
|
134
|
+
non-canonical spellings are refused — loosening that permanently, to describe a condition that
|
|
135
|
+
ends at Phase 4, would weaken the version format every frozen schema depends on. Freezing a
|
|
136
|
+
schema is one deletion from this set.
|
|
137
|
+
- `benchmark.result` gains the provenance Machine Identity §6
|
|
138
|
+
calls "optional but strongly recommended, and required for any benchmark whose numbers depend on
|
|
139
|
+
them": `measurement_class` (cold/warm/cache_reused/n-a), `telemetry_summary` (peak VRAM,
|
|
140
|
+
peak/mean power, max temperature, throttle flag) and `raw_response_ref` — plus `error_code`,
|
|
141
|
+
`error_text`, `completed_cases` and `total_cases` from FreeWeight's own `run_tests` row. These
|
|
142
|
+
had to be declared rather than left out: the writer half of each pair forbids unknown keys, so a
|
|
143
|
+
field this schema does not name is one a producer *cannot emit at all*. A memory or energy
|
|
144
|
+
benchmark — whose numbers §6 says depend on telemetry — could not have exported a conformant
|
|
145
|
+
result, and a `failed` result had no field in which to say why it failed.
|
|
146
|
+
- `RuntimeProfileFields.profile_hash`, a public property delegating to
|
|
147
|
+
`baseaicore.RuntimeProfile`. A consumer needs the hash for the same reason a producer does —
|
|
148
|
+
two results are comparable only if their profiles hash identically — and previously the only
|
|
149
|
+
way to it was a private helper reached across class boundaries behind two `noqa: SLF001`
|
|
150
|
+
waivers.
|
|
151
|
+
- `setspec.vocabulary`: `CAPABILITIES`, `CAPABILITY_VOCABULARY_VERSION` (`"1.0"`),
|
|
152
|
+
`validate_capability`, `is_known_capability`. A specialization validates against its **root**
|
|
153
|
+
rather than needing individual enumeration — `coding.rust` is accepted the moment `coding` is
|
|
154
|
+
known — and an unknown ID is accepted leniently only when the payload declares a newer *minor*
|
|
155
|
+
of the vocabulary's current major, never across a major.
|
|
156
|
+
- `setspec.base`: `WireSequence`, an ordered-collection field type alongside `WireEnum`. Strict
|
|
157
|
+
mode validates the *input Python type*, and a bare `tuple[T, ...]` under `strict=True` accepts
|
|
158
|
+
only an actual `tuple` — never the `list` every JSON array deserializes to — so every
|
|
159
|
+
ordered-collection field in Phase 2 needed this to accept a real document at all.
|
|
160
|
+
- `Aggregation` gains `P50`, `RATIO` and `RAW`, matching aggregation kinds FreeWeight's own data
|
|
161
|
+
model already names that Phase 1's set did not cover.
|
|
162
|
+
|
|
163
|
+
### Changed
|
|
164
|
+
- `PayloadDefinition` now sets `extra="allow"` itself, rather than leaving pydantic's default
|
|
165
|
+
(`extra="ignore"`). A definition nested inside another payload — `model: ModelIdentityFields`
|
|
166
|
+
inside `CapabilityEvidenceFields` — is typed as the bare definition, not a generated `Out`/`In`,
|
|
167
|
+
so its own default governs unknown-key handling regardless of which half of the *outer* pair is
|
|
168
|
+
in use; `ignore` would have silently dropped an unrecognized field from every nested object in
|
|
169
|
+
both directions, which is precisely the loss ADR-0009 rule 4
|
|
170
|
+
exists to prevent. This is additive for every Phase 1 payload, which had no nested fields to
|
|
171
|
+
expose the gap.
|
|
172
|
+
|
|
173
|
+
### Fixed
|
|
174
|
+
- `completed_cases` greater than `total_cases` is now rejected, the same arithmetic honesty the
|
|
175
|
+
timestamp ordering checks already enforced: "12 of 10 cases done" is a producer bug, and a
|
|
176
|
+
consumer rendering it as a percentage would show one above 100%.
|
|
177
|
+
- A garbled sentence in `setspec.machine.v1`'s module docstring, left by an edit, made the
|
|
178
|
+
explanation of why `machine_fingerprint` is *not* re-verified unreadable — the one place this
|
|
179
|
+
package carries a hash-shaped field without checking it, so the reason it is deliberate rather
|
|
180
|
+
than an oversight is exactly the thing that had to be legible.
|
|
181
|
+
- A test fixture (`tests/unit/test_payloads_benchmark.py`) embedded one module-level dict by
|
|
182
|
+
reference into every result it built; a test that mutated its own copy was actually mutating the
|
|
183
|
+
shared fixture in place, corrupting every test built after it in file execution order. It passed
|
|
184
|
+
under Python 3.13 by luck of `pytest-randomly`'s seed and failed outright under 3.14, whose
|
|
185
|
+
ordering exposed it — the bug was reference aliasing, not a Python-version incompatibility.
|
|
186
|
+
Fixed by building the dict fresh on every call; verified stable across five randomized runs on
|
|
187
|
+
both interpreters.
|
|
188
|
+
|
|
189
|
+
## [0.1.0] — 2026-08-23
|
|
190
|
+
|
|
191
|
+
Phase 1 of the [development plan](docs/packages/setspec/development-plan.md): the envelope,
|
|
192
|
+
versioning and serialization core. Any payload can now be wrapped, versioned, validated,
|
|
193
|
+
serialized canonically and rejected correctly when its major version is unsupported. No payload
|
|
194
|
+
type is registered yet — Phase 2 adds the first ones against this machinery, which is why
|
|
195
|
+
`SUPPORTED_SCHEMAS` ships empty rather than pre-declaring shapes that do not exist.
|
|
196
|
+
|
|
197
|
+
### Added
|
|
198
|
+
- `envelope`: `SchemaEnvelope`, `GeneratorInfo`, `SchemaVersion`, `SUPPORTED_SCHEMAS`,
|
|
199
|
+
`load_envelope`, `dump_envelope`. The reader policy of
|
|
200
|
+
ADR-0009 rule 3 is enforced in both directions: a
|
|
201
|
+
newer minor within a supported major is accepted with its unknown fields intact, an unsupported
|
|
202
|
+
major is refused with `SchemaVersionUnsupported` naming the schema, the received version and
|
|
203
|
+
every supported one — and is never partially parsed.
|
|
204
|
+
- `serialization`: canonical JSON via `baseaicore.canonical_json` (delegated, not reimplemented, so
|
|
205
|
+
a payload hashes identically to every other hashed structure in the suite); `MeasurementField`,
|
|
206
|
+
mapping `UNSUPPORTED ↔ "unsupported"` and refusing `null`, bools and numeric strings; and
|
|
207
|
+
`TimestampField`, RFC 3339 UTC at millisecond precision with naive datetimes rejected.
|
|
208
|
+
- `serialization`: `parse_json` with size and depth guards applied *before* the parser runs, so a
|
|
209
|
+
hostile document is refused by a named limit rather than by a `RecursionError`. Limits are the
|
|
210
|
+
module constants `MAX_PAYLOAD_BYTES` (16 MiB, matching API Standards §10) and
|
|
211
|
+
`MAX_PAYLOAD_DEPTH` (64); callers may pass tighter ones per call.
|
|
212
|
+
- `base`: `PayloadDefinition`, `StrictPayload`, `PreservingPayload` and `payload_models`, which
|
|
213
|
+
generates the `Out`/`In` pair required by
|
|
214
|
+
ADR-0009 rule 4 from one definition — so a writer
|
|
215
|
+
cannot emit a field it does not know and a reader cannot strip one. The round-trip contract is
|
|
216
|
+
asserted per class, never across the pair.
|
|
217
|
+
- `base`: `WireEnum`, for enum fields that must accept their own string values from a parsed
|
|
218
|
+
document while the bases otherwise validate strictly.
|
|
219
|
+
- `metrics`: `MetricValueOut`/`MetricValueIn` and `Aggregation`. The
|
|
220
|
+
ADR-0016 §6 invariants are structural rather than
|
|
221
|
+
advisory: a real value must report at least one supported sample, an unsupported value must
|
|
222
|
+
report none, and dispersion needs at least two — so `value=0.0, sample_count=0` cannot be
|
|
223
|
+
serialized at all.
|
|
224
|
+
- `errors`: `SchemaVersionUnsupported` (code `SCHEMA_VERSION_UNSUPPORTED`), with
|
|
225
|
+
`baseaicore.ValidationError` re-exported so callers import both from one place.
|
|
226
|
+
|
|
227
|
+
### Changed
|
|
228
|
+
- Coverage floor raised from 85 % to 95 %, the number [spec §18](docs/packages/setspec/spec.md)
|
|
229
|
+
and every phase's acceptance criteria actually state; the scaffold shipped 85 in both
|
|
230
|
+
`pyproject.toml` and the CI job. Current coverage is 100 %.
|
|
231
|
+
- `.importlinter`: the second contract listed no forbidden modules at all, and its inline
|
|
232
|
+
`root_packages`/`source_modules` values were read character by character — `lint-imports` failed
|
|
233
|
+
with `Could not find package 's'` and therefore checked nothing. Rewritten as newline-separated
|
|
234
|
+
lists with `include_external_packages`, and renamed from `no-sibling-packages` to
|
|
235
|
+
`no-capability-packages`: `modelrack`, `sweatmeter`, `weightsdb` and `mirrorwall` sit in the
|
|
236
|
+
layer *above* SetSpec, so they were never siblings.
|
|
237
|
+
|
|
238
|
+
### Fixed
|
|
239
|
+
- `ruff format --check .` failed on nine vendored documents under `docs/`. ruff 0.16 formats Python
|
|
240
|
+
code blocks inside markdown, and those files are byte-identical copies of the suite's master
|
|
241
|
+
documents shared with nine other repositories — reformatting them here would desync every copy.
|
|
242
|
+
`docs/` is now excluded from ruff instead.
|
|
243
|
+
|
|
244
|
+
### Security
|
|
245
|
+
- `pytest` moved from `>=8,<9` to `>=9.0.3,<10`, excluding PYSEC-2026-1845 (vulnerable
|
|
246
|
+
`/tmp/pytest-of-{user}` handling, affecting pytest through 9.0.2). Matches the pin BaseAiCore
|
|
247
|
+
already moved to; the suite passes unchanged on pytest 9.
|