clio-schemas 0.2.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. clio_schemas-0.2.1/.gitignore +21 -0
  2. clio_schemas-0.2.1/PKG-INFO +203 -0
  3. clio_schemas-0.2.1/README.md +183 -0
  4. clio_schemas-0.2.1/SCAFFOLD_REPORT.md +167 -0
  5. clio_schemas-0.2.1/ci-drafts/clio-agent-schema-check.yml +47 -0
  6. clio_schemas-0.2.1/ci-drafts/clio-schemas-ci.yml +82 -0
  7. clio_schemas-0.2.1/ci-drafts/gact-tui-ts-gen.yml +49 -0
  8. clio_schemas-0.2.1/pyproject.toml +79 -0
  9. clio_schemas-0.2.1/scripts/check_version_bump.py +112 -0
  10. clio_schemas-0.2.1/src/clio_schemas/__init__.py +61 -0
  11. clio_schemas-0.2.1/src/clio_schemas/constants.py +20 -0
  12. clio_schemas-0.2.1/src/clio_schemas/export.py +346 -0
  13. clio_schemas-0.2.1/src/clio_schemas/models.py +466 -0
  14. clio_schemas-0.2.1/src/clio_schemas/py.typed +0 -0
  15. clio_schemas-0.2.1/src/clio_schemas/schemas/HASHES.json +13 -0
  16. clio_schemas-0.2.1/src/clio_schemas/schemas/artifact_record.json +245 -0
  17. clio_schemas-0.2.1/src/clio_schemas/schemas/artifact_version.json +212 -0
  18. clio_schemas-0.2.1/src/clio_schemas/schemas/environment_record.json +88 -0
  19. clio_schemas-0.2.1/src/clio_schemas/schemas/identity_evidence.json +66 -0
  20. clio_schemas-0.2.1/src/clio_schemas/schemas/index.json +667 -0
  21. clio_schemas-0.2.1/src/clio_schemas/schemas/instrument.json +32 -0
  22. clio_schemas-0.2.1/src/clio_schemas/schemas/prov_edge.json +129 -0
  23. clio_schemas-0.2.1/src/clio_schemas/schemas/transform_record.json +393 -0
  24. clio_schemas-0.2.1/tests/golden/_models.ts +256 -0
  25. clio_schemas-0.2.1/tests/golden/artifact_record.ts +16 -0
  26. clio_schemas-0.2.1/tests/golden/artifact_version.ts +15 -0
  27. clio_schemas-0.2.1/tests/golden/environment_record.ts +11 -0
  28. clio_schemas-0.2.1/tests/golden/identity_evidence.ts +11 -0
  29. clio_schemas-0.2.1/tests/golden/index.ts +8 -0
  30. clio_schemas-0.2.1/tests/golden/instrument.ts +10 -0
  31. clio_schemas-0.2.1/tests/golden/prov_edge.ts +12 -0
  32. clio_schemas-0.2.1/tests/golden/transform_record.ts +20 -0
  33. clio_schemas-0.2.1/tests/legacy_golden_json/artifact_record.json +1 -0
  34. clio_schemas-0.2.1/tests/legacy_golden_json/artifact_version.json +1 -0
  35. clio_schemas-0.2.1/tests/legacy_golden_json/identity_evidence.json +1 -0
  36. clio_schemas-0.2.1/tests/legacy_golden_json/instrument.json +1 -0
  37. clio_schemas-0.2.1/tests/legacy_golden_json/prov_edge.json +1 -0
  38. clio_schemas-0.2.1/tests/legacy_golden_json/transform_record.json +1 -0
  39. clio_schemas-0.2.1/tests/test_legacy_parity.py +185 -0
  40. clio_schemas-0.2.1/tests/test_roundtrip.py +164 -0
  41. clio_schemas-0.2.1/tests/test_version_bump.py +51 -0
  42. clio_schemas-0.2.1/tools/ts-gen/package-lock.json +214 -0
  43. clio_schemas-0.2.1/tools/ts-gen/package.json +17 -0
  44. clio_schemas-0.2.1/tools/ts-gen/schemas-to-ts.mjs +177 -0
  45. clio_schemas-0.2.1/tools/ts-gen/tsconfig.check.json +15 -0
@@ -0,0 +1,21 @@
1
+ # Python
2
+ __pycache__/
3
+ *.pyc
4
+ .venv/
5
+ .pytest_cache/
6
+ .ruff_cache/
7
+ dist/
8
+ build/
9
+ *.egg-info/
10
+
11
+ # Node
12
+ tools/ts-gen/node_modules/
13
+
14
+ # Generated TS output (regenerated in CI; committed goldens live in tests/golden)
15
+ generated-ts/
16
+ .ts-fresh/
17
+
18
+ # Consumer copy-out target for the committed schemas (repo-local temp).
19
+ # NOTE: the CANONICAL committed schemas live at src/clio_schemas/schemas/ and
20
+ # are tracked — do NOT ignore that path.
21
+ /schemas/
@@ -0,0 +1,203 @@
1
+ Metadata-Version: 2.4
2
+ Name: clio-schemas
3
+ Version: 0.2.1
4
+ Summary: Canonical shared record schemas (pydantic v2) for clio-agent, clio-relay, and gact-tui.
5
+ Project-URL: Homepage, https://github.com/iowarp/clio-schemas
6
+ Project-URL: Repository, https://github.com/iowarp/clio-schemas
7
+ Author: IOWarp
8
+ License: MIT
9
+ Keywords: clio,json-schema,pydantic,schemas
10
+ Classifier: Programming Language :: Python :: 3.12
11
+ Classifier: Programming Language :: Python :: 3.13
12
+ Classifier: Typing :: Typed
13
+ Requires-Python: >=3.12
14
+ Requires-Dist: pydantic<3,>=2.7
15
+ Provides-Extra: dev
16
+ Requires-Dist: pyright>=1.1.370; extra == 'dev'
17
+ Requires-Dist: pytest>=8; extra == 'dev'
18
+ Requires-Dist: ruff>=0.6; extra == 'dev'
19
+ Description-Content-Type: text/markdown
20
+
21
+ # clio-schemas
22
+
23
+ **The single source of truth for the record shapes shared across the CLIO
24
+ system.** `clio-agent`, `clio-relay`, and `gact-tui` all speak the same wire
25
+ records — artifact versions/chains and transform provenance records. Historically
26
+ each service hand-wrote its own copy of these types and
27
+ they drifted. This package makes the shapes canonical: they are defined once as
28
+ [pydantic](https://docs.pydantic.dev/) v2 models here and exported to JSON
29
+ Schema. Python consumers import the models; TypeScript consumers **generate**
30
+ types from the JSON Schema shipped inside the package. Nobody hand-writes a
31
+ shared shape again.
32
+
33
+ > **Status: P2.1 records landed (version 0.2.0 / issue #1120).** The bootstrap
34
+ > placeholders are gone. `ArtifactVersion`, `ArtifactRecord`, `ProvEdge`,
35
+ > `TransformRecord`, their nested value models, and their enums are canonical.
36
+
37
+ ---
38
+
39
+ ## Why the schemas are immutable package resources (design decision)
40
+
41
+ An exact `clio-schemas` pin, on its own, does **not** guarantee two services see
42
+ the same schema *bytes*: if each consumer *regenerated* JSON Schema from the
43
+ pydantic models under its own resolved pydantic version, a pydantic point release
44
+ could subtly change the output and drift would return through the back door.
45
+
46
+ So the canonical JSON Schemas are **built once, committed, and shipped as
47
+ immutable package resources** inside the wheel (`clio_schemas/schemas/*.json`),
48
+ alongside a canonical hash manifest (`HASHES.json`). Consumers **copy** those
49
+ committed bytes — they never regenerate. An exact pin therefore *does* determine
50
+ the bytes. Regenerating from the models is a repo-local developer command
51
+ (`--regenerate`), gated to the single **locked pydantic version**, so the
52
+ committed artifacts can only change deliberately, in this repo.
53
+
54
+ ## What's in the box
55
+
56
+ ```
57
+ clio-schemas/
58
+ ├── pyproject.toml # uv-compatible, py>=3.12, pydantic v2, hatchling
59
+ ├── src/clio_schemas/
60
+ │ ├── __init__.py # public exports + __version__
61
+ │ ├── constants.py # LOCKED_PYDANTIC_VERSION, file names
62
+ │ ├── models.py # canonical models + ClioSchemaBase + registry
63
+ │ ├── export.py # copy / check / regenerate / verify
64
+ │ ├── py.typed # ships type information
65
+ │ └── schemas/ # COMMITTED immutable artifacts (in the wheel)
66
+ │ ├── artifact_version.json # per-model, self-contained
67
+ │ ├── artifact_record.json # per-model, self-contained
68
+ │ ├── transform_record.json # plus provenance/value model schemas
69
+ │ ├── index.json # aggregate: shared $defs emitted once
70
+ │ └── HASHES.json # canonical sha256 manifest
71
+ ├── tools/ts-gen/
72
+ │ ├── schemas-to-ts.mjs # JSON Schema dir -> TS module graph (deterministic)
73
+ │ ├── tsconfig.check.json # isolated strict typecheck of generated TS
74
+ │ └── package.json / package-lock.json
75
+ ├── scripts/check_version_bump.py # CI: schema change ⇒ version bump
76
+ ├── tests/
77
+ │ ├── test_roundtrip.py # JSON/TS golden anti-drift tests
78
+ │ ├── test_legacy_parity.py # original clio-agent byte fixtures
79
+ │ ├── test_version_bump.py # unit tests for the bump-enforcement core
80
+ │ └── golden/ # committed golden TS directory
81
+ │ ├── _models.ts index.ts artifact_version.ts transform_record.ts ...
82
+ └── ci-drafts/ # workflow drafts for this repo + both consumers
83
+ ```
84
+
85
+ ## The pipeline
86
+
87
+ ```
88
+ pydantic models ──render (locked pydantic)──▶ committed *.json + HASHES.json ──ship in wheel──▶
89
+ consumer copies bytes ──schemas-to-ts.mjs──▶ TypeScript module graph
90
+ ```
91
+
92
+ Every hop is **deterministic**: JSON is emitted with sorted keys and a stable
93
+ `$defs` order; the TypeScript generator uses a fixed banner (no timestamp),
94
+ sorted file order, and pinned formatting. The aggregate `index.json` carries all
95
+ models under a single shared `$defs`, so the generator emits each shared
96
+ definition (e.g. the `ArtifactKind` enum and nested `IdentityEvidence`) **exactly
97
+ once** — the generated module graph has no duplicate declarations.
98
+
99
+ ## Export command modes
100
+
101
+ ```bash
102
+ # Consumer: copy the immutable committed schemas into ./schemas
103
+ uv run python -m clio_schemas.export --out schemas
104
+
105
+ # Consumer CI: verify a directory matches the committed bytes exactly
106
+ # (rejects stale, missing, AND unexpected/orphaned files)
107
+ uv run python -m clio_schemas.export --out schemas --check
108
+
109
+ # DEV: re-render the committed package resources from the models
110
+ # (refuses unless the installed pydantic == LOCKED_PYDANTIC_VERSION)
111
+ uv run python -m clio_schemas.export --regenerate
112
+
113
+ # REPO CI: assert committed artifacts are canonical (match models + hashes)
114
+ uv run python -m clio_schemas.export --verify
115
+ ```
116
+
117
+ ## Quick start
118
+
119
+ ```bash
120
+ uv sync --extra dev # install (locked pydantic)
121
+ uv run python -m clio_schemas.export --verify # artifacts are canonical
122
+ uv run pytest # golden round-trip + conformance
123
+
124
+ cd tools/ts-gen && npm ci && npm run check # generate TS + strict typecheck
125
+ ```
126
+
127
+ ## Legacy extraction compatibility
128
+
129
+ The P2.1 artifact/provenance records use `LegacyToleranceBase` so their
130
+ clio-agent behavior remains byte- and validation-compatible: unknown keys are
131
+ ignored, ordinary Pydantic coercion remains enabled, immutable records stay
132
+ frozen, and `ArtifactRecord` stays mutable. Moving existing records to the
133
+ strict `ClioSchemaBase` contract would be a wire change; that coordinated
134
+ convergence is tracked in iowarp/clio-agent#1121.
135
+
136
+ ## How to add a model
137
+
138
+ 1. Define the pydantic v2 model in `src/clio_schemas/models.py`, inheriting
139
+ `ClioSchemaBase` (strict wire semantics: `extra="forbid"`, `strict=True`,
140
+ `frozen=True`). Give every field a `Field(description=...)` — descriptions flow
141
+ into the JSON Schema and the generated TS doc comments.
142
+ 2. Append the class to the `EXPORTED_MODELS` tuple in the same file.
143
+ 3. Regenerate the committed artifacts and the golden TS, then bump the version:
144
+ ```bash
145
+ uv run python -m clio_schemas.export --regenerate # updates schemas/ + HASHES.json
146
+ cd tools/ts-gen && node schemas-to-ts.mjs \
147
+ --in ../../src/clio_schemas/schemas --out ../../tests/golden
148
+ ```
149
+ 4. Bump `__version__` in `src/clio_schemas/__init__.py` **and** `version` in
150
+ `pyproject.toml` in lockstep (CI enforces a bump whenever `HASHES.json`
151
+ changes), then `uv run pytest`.
152
+
153
+ ## Versioning policy — exact-pin lockstep
154
+
155
+ Consumers pin an **exact** version (`clio-schemas==X.Y.Z`, not `>=`). A schema is
156
+ a contract between multiple services; a range would let two services resolve
157
+ different shapes and reintroduce drift. Therefore:
158
+
159
+ - **Any** change to a committed schema (detected via `HASHES.json`) requires a
160
+ `clio-schemas` version bump — enforced mechanically in CI by
161
+ `scripts/check_version_bump.py`, which compares the hash manifest against the
162
+ merge base and fails if the version was not incremented.
163
+ - All consumers (`clio-agent`, `clio-relay`, `gact-tui`) update their pin to the
164
+ new exact version **in lockstep**, in the same coordinated change.
165
+ - The version lives in exactly two places that must agree: `pyproject.toml`
166
+ `version` and `clio_schemas.__version__`; CI verifies they match.
167
+
168
+ ### Wire evolution
169
+
170
+ Exact-pin lockstep is the bootstrap rule. As the real records land and the
171
+ system runs mixed versions during rollouts, the intended evolution discipline
172
+ is:
173
+
174
+ - **Additive, reader-first staging.** Additive changes (new optional field, new
175
+ enum member) ship to *readers* before *writers*: deploy the version that can
176
+ *accept* the new shape everywhere first, then deploy the writers that *emit*
177
+ it. Readers must ignore-or-tolerate unknown-but-optional additions during the
178
+ transition. Extracted P2.1 records preserve their legacy reader-tolerant
179
+ behavior through `LegacyToleranceBase`; new records use `ClioSchemaBase`.
180
+ - **N / N-1 compatibility.** Once real schemas exist, adjacent versions are
181
+ expected to interoperate: a service on version *N* and one on *N-1* must be
182
+ able to exchange the records they share for the duration of a rollout.
183
+ Breaking (non-additive) changes require a two-step migration across at least
184
+ one intermediate version, never a flag-day.
185
+ - **Rollback ordering.** Roll back in the reverse of deploy order: retire the
186
+ *writers* of a new shape before the *readers* that understand it, so a rolled
187
+ -back writer never emits a shape an already-rolled-back reader would reject.
188
+
189
+ Convergence of the extracted records on stricter validation is tracked by
190
+ iowarp/clio-agent#1121 and requires a coordinated consumer migration.
191
+
192
+ ## How consumers regenerate
193
+
194
+ - **clio-agent (Python):** pin `clio-schemas==X.Y.Z`, `import clio_schemas`. Its
195
+ CI copies the committed schemas out of the installed package and `--check`s any
196
+ vendored copy for exact-byte + file-set equality (see
197
+ `ci-drafts/clio-agent-schema-check.yml`).
198
+ - **clio-relay (Python):** same as clio-agent — import the models, pin exact.
199
+ - **gact-tui (TypeScript):** pin `clio-schemas==X.Y.Z`, copy the shipped JSON
200
+ Schema out of the package, and run the in-repo generator (`tools/ts-gen`,
201
+ pinned via its lockfile) to produce `.ts`. Its CI regenerates into a clean dir
202
+ and compares exact file *sets* + bytes so untracked/orphaned files are caught
203
+ (see `ci-drafts/gact-tui-ts-gen.yml`).
@@ -0,0 +1,183 @@
1
+ # clio-schemas
2
+
3
+ **The single source of truth for the record shapes shared across the CLIO
4
+ system.** `clio-agent`, `clio-relay`, and `gact-tui` all speak the same wire
5
+ records — artifact versions/chains and transform provenance records. Historically
6
+ each service hand-wrote its own copy of these types and
7
+ they drifted. This package makes the shapes canonical: they are defined once as
8
+ [pydantic](https://docs.pydantic.dev/) v2 models here and exported to JSON
9
+ Schema. Python consumers import the models; TypeScript consumers **generate**
10
+ types from the JSON Schema shipped inside the package. Nobody hand-writes a
11
+ shared shape again.
12
+
13
+ > **Status: P2.1 records landed (version 0.2.0 / issue #1120).** The bootstrap
14
+ > placeholders are gone. `ArtifactVersion`, `ArtifactRecord`, `ProvEdge`,
15
+ > `TransformRecord`, their nested value models, and their enums are canonical.
16
+
17
+ ---
18
+
19
+ ## Why the schemas are immutable package resources (design decision)
20
+
21
+ An exact `clio-schemas` pin, on its own, does **not** guarantee two services see
22
+ the same schema *bytes*: if each consumer *regenerated* JSON Schema from the
23
+ pydantic models under its own resolved pydantic version, a pydantic point release
24
+ could subtly change the output and drift would return through the back door.
25
+
26
+ So the canonical JSON Schemas are **built once, committed, and shipped as
27
+ immutable package resources** inside the wheel (`clio_schemas/schemas/*.json`),
28
+ alongside a canonical hash manifest (`HASHES.json`). Consumers **copy** those
29
+ committed bytes — they never regenerate. An exact pin therefore *does* determine
30
+ the bytes. Regenerating from the models is a repo-local developer command
31
+ (`--regenerate`), gated to the single **locked pydantic version**, so the
32
+ committed artifacts can only change deliberately, in this repo.
33
+
34
+ ## What's in the box
35
+
36
+ ```
37
+ clio-schemas/
38
+ ├── pyproject.toml # uv-compatible, py>=3.12, pydantic v2, hatchling
39
+ ├── src/clio_schemas/
40
+ │ ├── __init__.py # public exports + __version__
41
+ │ ├── constants.py # LOCKED_PYDANTIC_VERSION, file names
42
+ │ ├── models.py # canonical models + ClioSchemaBase + registry
43
+ │ ├── export.py # copy / check / regenerate / verify
44
+ │ ├── py.typed # ships type information
45
+ │ └── schemas/ # COMMITTED immutable artifacts (in the wheel)
46
+ │ ├── artifact_version.json # per-model, self-contained
47
+ │ ├── artifact_record.json # per-model, self-contained
48
+ │ ├── transform_record.json # plus provenance/value model schemas
49
+ │ ├── index.json # aggregate: shared $defs emitted once
50
+ │ └── HASHES.json # canonical sha256 manifest
51
+ ├── tools/ts-gen/
52
+ │ ├── schemas-to-ts.mjs # JSON Schema dir -> TS module graph (deterministic)
53
+ │ ├── tsconfig.check.json # isolated strict typecheck of generated TS
54
+ │ └── package.json / package-lock.json
55
+ ├── scripts/check_version_bump.py # CI: schema change ⇒ version bump
56
+ ├── tests/
57
+ │ ├── test_roundtrip.py # JSON/TS golden anti-drift tests
58
+ │ ├── test_legacy_parity.py # original clio-agent byte fixtures
59
+ │ ├── test_version_bump.py # unit tests for the bump-enforcement core
60
+ │ └── golden/ # committed golden TS directory
61
+ │ ├── _models.ts index.ts artifact_version.ts transform_record.ts ...
62
+ └── ci-drafts/ # workflow drafts for this repo + both consumers
63
+ ```
64
+
65
+ ## The pipeline
66
+
67
+ ```
68
+ pydantic models ──render (locked pydantic)──▶ committed *.json + HASHES.json ──ship in wheel──▶
69
+ consumer copies bytes ──schemas-to-ts.mjs──▶ TypeScript module graph
70
+ ```
71
+
72
+ Every hop is **deterministic**: JSON is emitted with sorted keys and a stable
73
+ `$defs` order; the TypeScript generator uses a fixed banner (no timestamp),
74
+ sorted file order, and pinned formatting. The aggregate `index.json` carries all
75
+ models under a single shared `$defs`, so the generator emits each shared
76
+ definition (e.g. the `ArtifactKind` enum and nested `IdentityEvidence`) **exactly
77
+ once** — the generated module graph has no duplicate declarations.
78
+
79
+ ## Export command modes
80
+
81
+ ```bash
82
+ # Consumer: copy the immutable committed schemas into ./schemas
83
+ uv run python -m clio_schemas.export --out schemas
84
+
85
+ # Consumer CI: verify a directory matches the committed bytes exactly
86
+ # (rejects stale, missing, AND unexpected/orphaned files)
87
+ uv run python -m clio_schemas.export --out schemas --check
88
+
89
+ # DEV: re-render the committed package resources from the models
90
+ # (refuses unless the installed pydantic == LOCKED_PYDANTIC_VERSION)
91
+ uv run python -m clio_schemas.export --regenerate
92
+
93
+ # REPO CI: assert committed artifacts are canonical (match models + hashes)
94
+ uv run python -m clio_schemas.export --verify
95
+ ```
96
+
97
+ ## Quick start
98
+
99
+ ```bash
100
+ uv sync --extra dev # install (locked pydantic)
101
+ uv run python -m clio_schemas.export --verify # artifacts are canonical
102
+ uv run pytest # golden round-trip + conformance
103
+
104
+ cd tools/ts-gen && npm ci && npm run check # generate TS + strict typecheck
105
+ ```
106
+
107
+ ## Legacy extraction compatibility
108
+
109
+ The P2.1 artifact/provenance records use `LegacyToleranceBase` so their
110
+ clio-agent behavior remains byte- and validation-compatible: unknown keys are
111
+ ignored, ordinary Pydantic coercion remains enabled, immutable records stay
112
+ frozen, and `ArtifactRecord` stays mutable. Moving existing records to the
113
+ strict `ClioSchemaBase` contract would be a wire change; that coordinated
114
+ convergence is tracked in iowarp/clio-agent#1121.
115
+
116
+ ## How to add a model
117
+
118
+ 1. Define the pydantic v2 model in `src/clio_schemas/models.py`, inheriting
119
+ `ClioSchemaBase` (strict wire semantics: `extra="forbid"`, `strict=True`,
120
+ `frozen=True`). Give every field a `Field(description=...)` — descriptions flow
121
+ into the JSON Schema and the generated TS doc comments.
122
+ 2. Append the class to the `EXPORTED_MODELS` tuple in the same file.
123
+ 3. Regenerate the committed artifacts and the golden TS, then bump the version:
124
+ ```bash
125
+ uv run python -m clio_schemas.export --regenerate # updates schemas/ + HASHES.json
126
+ cd tools/ts-gen && node schemas-to-ts.mjs \
127
+ --in ../../src/clio_schemas/schemas --out ../../tests/golden
128
+ ```
129
+ 4. Bump `__version__` in `src/clio_schemas/__init__.py` **and** `version` in
130
+ `pyproject.toml` in lockstep (CI enforces a bump whenever `HASHES.json`
131
+ changes), then `uv run pytest`.
132
+
133
+ ## Versioning policy — exact-pin lockstep
134
+
135
+ Consumers pin an **exact** version (`clio-schemas==X.Y.Z`, not `>=`). A schema is
136
+ a contract between multiple services; a range would let two services resolve
137
+ different shapes and reintroduce drift. Therefore:
138
+
139
+ - **Any** change to a committed schema (detected via `HASHES.json`) requires a
140
+ `clio-schemas` version bump — enforced mechanically in CI by
141
+ `scripts/check_version_bump.py`, which compares the hash manifest against the
142
+ merge base and fails if the version was not incremented.
143
+ - All consumers (`clio-agent`, `clio-relay`, `gact-tui`) update their pin to the
144
+ new exact version **in lockstep**, in the same coordinated change.
145
+ - The version lives in exactly two places that must agree: `pyproject.toml`
146
+ `version` and `clio_schemas.__version__`; CI verifies they match.
147
+
148
+ ### Wire evolution
149
+
150
+ Exact-pin lockstep is the bootstrap rule. As the real records land and the
151
+ system runs mixed versions during rollouts, the intended evolution discipline
152
+ is:
153
+
154
+ - **Additive, reader-first staging.** Additive changes (new optional field, new
155
+ enum member) ship to *readers* before *writers*: deploy the version that can
156
+ *accept* the new shape everywhere first, then deploy the writers that *emit*
157
+ it. Readers must ignore-or-tolerate unknown-but-optional additions during the
158
+ transition. Extracted P2.1 records preserve their legacy reader-tolerant
159
+ behavior through `LegacyToleranceBase`; new records use `ClioSchemaBase`.
160
+ - **N / N-1 compatibility.** Once real schemas exist, adjacent versions are
161
+ expected to interoperate: a service on version *N* and one on *N-1* must be
162
+ able to exchange the records they share for the duration of a rollout.
163
+ Breaking (non-additive) changes require a two-step migration across at least
164
+ one intermediate version, never a flag-day.
165
+ - **Rollback ordering.** Roll back in the reverse of deploy order: retire the
166
+ *writers* of a new shape before the *readers* that understand it, so a rolled
167
+ -back writer never emits a shape an already-rolled-back reader would reject.
168
+
169
+ Convergence of the extracted records on stricter validation is tracked by
170
+ iowarp/clio-agent#1121 and requires a coordinated consumer migration.
171
+
172
+ ## How consumers regenerate
173
+
174
+ - **clio-agent (Python):** pin `clio-schemas==X.Y.Z`, `import clio_schemas`. Its
175
+ CI copies the committed schemas out of the installed package and `--check`s any
176
+ vendored copy for exact-byte + file-set equality (see
177
+ `ci-drafts/clio-agent-schema-check.yml`).
178
+ - **clio-relay (Python):** same as clio-agent — import the models, pin exact.
179
+ - **gact-tui (TypeScript):** pin `clio-schemas==X.Y.Z`, copy the shipped JSON
180
+ Schema out of the package, and run the in-repo generator (`tools/ts-gen`,
181
+ pinned via its lockfile) to produce `.ts`. Its CI regenerates into a clean dir
182
+ and compares exact file *sets* + bytes so untracked/orphaned files are caught
183
+ (see `ci-drafts/gact-tui-ts-gen.yml`).
@@ -0,0 +1,167 @@
1
+ # clio-schemas scaffold — build report (issue #1110, P0.6)
2
+
3
+ Pipeline-only bootstrap of the `iowarp/clio-schemas` package: a pydantic v2 ->
4
+ JSON Schema -> TypeScript loop, with the canonical schemas shipped as immutable
5
+ package resources, a golden anti-drift test, strict wire conformance, and CI
6
+ drafts for this repo and both consumers. This directory is intended to become
7
+ the real repo verbatim after review.
8
+
9
+ ## CHANGES AFTER REVIEW (Codex verdict: no-ship — findings accepted)
10
+
11
+ Each accepted finding, the fix, and the proof it works.
12
+
13
+ 1. **Design correction — immutable package resources, not consumer-side
14
+ regeneration.** An exact pin does not fix schema *bytes* if consumers
15
+ regenerate under their own pydantic.
16
+ - *Fix:* canonical JSON Schemas are rendered once (under the locked pydantic)
17
+ and committed as package resources in `src/clio_schemas/schemas/*.json`,
18
+ shipped inside the wheel; a canonical `HASHES.json` (sha256) is committed
19
+ alongside. `export.py` now has consumer modes that **copy** those bytes
20
+ (`--out DIR`) or **verify** a copy (`--out DIR --check`), and dev/CI modes
21
+ that render from the models (`--regenerate`, `--verify`) gated to
22
+ `LOCKED_PYDANTIC_VERSION = 2.13.4`.
23
+ - *Proof:* `uv run python -m clio_schemas.export --verify` → "OK: 4 committed
24
+ schema file(s) are canonical"; wheel listing shows `clio_schemas/schemas/{schema_probe,probe_batch,index,HASHES}.json`
25
+ + `py.typed`; `--regenerate`/`--verify` raise a clear error under any other
26
+ pydantic (guard `_require_locked_pydantic`).
27
+
28
+ 2. **Golden covers EVERY model + full-directory set equality.**
29
+ - *Fix:* `test_golden_directory_set_and_bytes` compares the complete generated
30
+ dir vs the complete golden dir (filename-set equality + byte content);
31
+ `test_every_model_has_a_committed_schema` and
32
+ `test_every_model_has_a_golden_ts_module` are parameterised over
33
+ `EXPORTED_MODELS`.
34
+ - *Proof:* 22 tests pass; `diff -rN tests/golden generated-ts` → "GOLDEN
35
+ MATCHES FRESH GENERATION".
36
+
37
+ 3. **Stale-artifact detection via clean temp dir + exact file SETS.**
38
+ - *Fix:* the TS generator wipes its `--out` dir first (orphans cannot
39
+ survive); the round-trip test generates into a fresh `tmp_path`; `export
40
+ --check` rejects unexpected `.json` (`unexpected:` / `missing:` / `stale:`).
41
+ The gact-tui CI draft now regenerates into a clean dir and uses `diff -rN`
42
+ (not `git diff`, which misses untracked/orphans).
43
+ - *Proof:* injecting `rogue.json` → `--check` exits 1 with "unexpected:
44
+ rogue.json"; corrupting a file → exit 1 "stale: schema_probe.json".
45
+
46
+ 4. **CI enforces version bump on schema change.**
47
+ - *Fix:* `scripts/check_version_bump.py` compares `HASHES.json` against the
48
+ merge base and fails if it changed without an `__version__` increment; pure
49
+ core (`requires_bump`, `is_increment`) is unit-tested in
50
+ `tests/test_version_bump.py`. Wired into `ci-drafts/clio-schemas-ci.yml`.
51
+ - *Proof:* `check_version_bump.py` runs clean; 6 unit tests cover
52
+ unchanged/changed/bumped/downgrade cases.
53
+
54
+ 5. **Strict wire validation + negative conformance tests.**
55
+ - *Fix:* `ClioSchemaBase(ConfigDict(extra="forbid", strict=True,
56
+ frozen=True))`; both models inherit it. Tests assert `sequence="7"` (str
57
+ for int) is rejected, unknown keys are rejected, and records are frozen.
58
+ - *Proof:* `test_string_for_int_is_rejected`, `test_unknown_key_is_rejected`,
59
+ `test_records_are_frozen` all pass.
60
+
61
+ 6. **Multi-root readiness — shared definitions emitted once.**
62
+ - *Fix:* second model `ProbeBatch` references `SchemaProbe` (nested) and reuses
63
+ `ProbeKind` (shared enum). The exporter builds an aggregate `index.json` via
64
+ pydantic `models_json_schema` (single shared `$defs`); the TS generator
65
+ compiles that once into `_models.ts` (declarations), emits thin per-model
66
+ re-export modules, and an `index.ts` barrel. A `$ref`-sibling flattening
67
+ pass in the generator prevents json-schema-to-typescript from duplicating a
68
+ ref'd type (it had produced `ProbeKind` + `ProbeKind1`).
69
+ - *Proof:* declaration scan across `generated-ts` shows every type exactly
70
+ once (`ProbeKind` count 1); `test_no_duplicate_declarations_in_barrel`
71
+ passes.
72
+
73
+ 7. **Packaging ships the schemas; documented consumption path.**
74
+ - *Fix:* `[tool.hatch.build.targets.wheel].artifacts` force-includes
75
+ `schemas/*.json` + `py.typed`; sdist includes `scripts`/`ci-drafts`. README
76
+ documents that gact-tui copies the committed schemas from the installed
77
+ package (pin ⇒ bytes) and runs the in-repo generator pinned by its lockfile.
78
+ - *Proof:* wheel zip listing (see item 1) contains all four JSON files +
79
+ `py.typed`; `uv build` produces wheel + sdist.
80
+
81
+ 8. **README wire-evolution paragraph.**
82
+ - *Fix:* added "Wire evolution" subsection: additive-reader-first staging,
83
+ N/N-1 compatibility once real schemas land (P2.1 designs the full protocol),
84
+ and rollback-in-reverse-of-deploy ordering. Exact-pin lockstep retained for
85
+ the bootstrap.
86
+
87
+ ## File inventory
88
+
89
+ | File | Purpose |
90
+ | --- | --- |
91
+ | `pyproject.toml` | hatchling build, `requires-python>=3.12`, `pydantic>=2.7,<3`; wheel force-includes `schemas/*.json` + `py.typed`; ruff/pytest/pyright config; `dev` extra. |
92
+ | `src/clio_schemas/__init__.py` | Public exports (models, `ClioSchemaBase`, `EXPORTED_MODELS`, `LOCKED_PYDANTIC_VERSION`) + `__version__`. |
93
+ | `src/clio_schemas/constants.py` | `LOCKED_PYDANTIC_VERSION`, aggregate/hash file names (import-cheap, avoids a `runpy` warning). |
94
+ | `src/clio_schemas/models.py` | `ClioSchemaBase` (strict/forbid/frozen), `ProbeKind` (shared enum), `SchemaProbe`, `ProbeBatch` (references SchemaProbe), `EXPORTED_MODELS`. |
95
+ | `src/clio_schemas/export.py` | Deterministic render + `copy` / `check` / `regenerate` / `verify` modes; aggregate schema; hash manifest; pydantic-lock guard; dir-diff (missing/unexpected/stale). |
96
+ | `src/clio_schemas/py.typed` | PEP 561 marker. |
97
+ | `src/clio_schemas/schemas/*.json` | **Committed immutable artifacts:** `schema_probe.json`, `probe_batch.json`, `index.json` (aggregate), `HASHES.json`. |
98
+ | `scripts/check_version_bump.py` | CI: schema-hash change ⇒ version-bump enforcement (pure, unit-tested core). |
99
+ | `tools/ts-gen/schemas-to-ts.mjs` | Aggregate -> `_models.ts` (once) + per-model re-export modules + `index.ts` barrel; `$ref`-sibling flattening; deterministic (fixed banner, sorted, LF). |
100
+ | `tools/ts-gen/package.json` + `package-lock.json` | Pins `json-schema-to-typescript@15.0.4` + `typescript@5.6.3`; `generate`/`check` scripts. |
101
+ | `tools/ts-gen/tsconfig.check.json` | Isolated strict typecheck (`types: []`, `lib: es2020`). |
102
+ | `tests/test_roundtrip.py` | Committed-canonical, per-model coverage, golden dir set+bytes, no-dup-declarations, strict conformance (negative) tests. |
103
+ | `tests/test_version_bump.py` | Unit tests for the version-bump core. |
104
+ | `tests/golden/*.ts` | Committed golden TS module graph (`_models.ts`, `index.ts`, `probe_batch.ts`, `schema_probe.ts`). |
105
+ | `ci-drafts/clio-schemas-ci.yml` | Own repo: ruff/pyright, `--verify`, version-pin agreement, version-bump script, golden round-trip, TS typecheck. |
106
+ | `ci-drafts/clio-agent-schema-check.yml` | Consumer: exact pin + `--out schemas --check` byte/set equality. |
107
+ | `ci-drafts/gact-tui-ts-gen.yml` | Consumer: copy shipped schemas, generate TS into clean dir, typecheck, `diff -rN` set+byte equality. |
108
+ | `README.md` | Purpose, immutable-resource design rationale, pipeline, export modes, add-a-model, exact-pin lockstep + wire-evolution, consumer regeneration. |
109
+ | `.gitignore` | Ignores venv/caches/dist/node_modules/`generated-ts`/`.ts-fresh`/top-level `schemas/`; keeps `src/clio_schemas/schemas/` tracked. |
110
+ | `uv.lock` | Resolved lock (pins pydantic 2.13.4 = LOCKED_PYDANTIC_VERSION). |
111
+
112
+ ## Verify locally
113
+
114
+ ```bash
115
+ cd clio-schemas-scaffold
116
+
117
+ uv sync --extra dev
118
+ uv run python -m clio_schemas.export --verify # committed artifacts canonical
119
+ uv run python -m clio_schemas.export --out /tmp/s # consumer copy
120
+ uv run python -m clio_schemas.export --out /tmp/s --check # exit 0 = matches
121
+ uv run --extra dev ruff check src tests scripts
122
+ uv run --extra dev ruff format --check src tests scripts
123
+ uv run --extra dev pyright src tests scripts
124
+ uv run --extra dev pytest # 22 passed
125
+ uv run python scripts/check_version_bump.py --base origin/main
126
+ uv build # wheel + sdist (schemas inside)
127
+
128
+ cd tools/ts-gen && npm ci && npm run check # generate TS + strict typecheck
129
+ ```
130
+
131
+ ## Verified in this session (all green)
132
+
133
+ - `--verify`: 4 committed schema files canonical (match models + HASHES under
134
+ locked pydantic 2.13.4).
135
+ - `pytest`: **22 passed** — committed-canonical, per-model coverage (×2), golden
136
+ dir set+bytes, no-duplicate-declarations, negative conformance (str-for-int,
137
+ unknown-key, frozen), version-bump unit tests.
138
+ - Consumer copy + `--check` pass; injected unexpected file and corrupted file
139
+ each fail `--check` with a typed reason.
140
+ - `ruff check` + `ruff format --check` + `pyright` (src/tests/scripts) clean.
141
+ - TS generation: `_models.ts` declares every type exactly once (`ProbeKind`
142
+ shared enum once); per-model re-export modules + barrel; strict `tsc` passes;
143
+ `diff -rN tests/golden generated-ts` identical.
144
+ - `uv build`: wheel + sdist; wheel contains `schemas/*.json` + `py.typed`.
145
+ - `check_version_bump.py` runs clean.
146
+
147
+ ## Not verified / caveats
148
+
149
+ - **CI YAML files are drafts, not executed** in a real Actions runner. Step
150
+ logic mirrors commands verified locally, but action versions
151
+ (`astral-sh/setup-uv@v5`, `actions/setup-node@v4`) and consumer-repo paths
152
+ (e.g. gact-tui `src/generated/schemas`) must be reconciled on adoption.
153
+ - **Consumer snippets assume a published/installable `clio-schemas==0.1.0`.**
154
+ Until the repo is published (PyPI or git pin), `uv pip install
155
+ "clio-schemas==0.1.0"` / dependency-pin steps will not resolve. Locally it
156
+ installs from source (`uv sync`, `uv build`).
157
+ - **`LOCKED_PYDANTIC_VERSION` is hardcoded to 2.13.4** (matching `uv.lock`).
158
+ Bumping pydantic is a deliberate action: update the constant, `--regenerate`,
159
+ and version-bump. `--verify`/`--regenerate` correctly refuse under any other
160
+ installed pydantic — this is by design, not a bug.
161
+ - **`json-schema-to-typescript` determinism is pinned by version** (15.0.4), not
162
+ proven across versions; the golden guards it and a bump requires regenerating
163
+ the golden.
164
+ - **`check_version_bump.py` merge-base logic** is exercised only via its pure
165
+ unit-tested core here (the scaffold dir is not a git repo with an `origin/main`
166
+ history); in a real repo it diffs against the base ref.
167
+ - Windows note: the generator normalises CRLF->LF; verified on win32.
@@ -0,0 +1,47 @@
1
+ # DRAFT — job snippet to add to clio-agent's CI (e.g. a new job in
2
+ # `.github/workflows/ci.yml`, or a standalone workflow).
3
+ #
4
+ # Purpose (acceptance criterion #2 of #1110): clio-agent resolves the
5
+ # clio-schemas dependency (exact pin) and verifies that any schemas it vendors
6
+ # match the IMMUTABLE bytes shipped inside the installed clio-schemas package.
7
+ # Because consumers *copy* the committed artifacts rather than regenerate them,
8
+ # the exact pin fully determines the bytes — this job proves that holds.
9
+ #
10
+ # Prerequisite in clio-agent: add an exact pin to pyproject.toml, e.g.
11
+ # dependencies = [ ..., "clio-schemas==0.1.0" ]
12
+ # If clio-agent vendors the schemas for offline use, commit them under a known
13
+ # dir (e.g. `schemas/`); otherwise it can consume them straight from the package.
14
+
15
+ jobs:
16
+ clio-schemas-export-check:
17
+ runs-on: ubuntu-latest
18
+ steps:
19
+ - uses: actions/checkout@v4
20
+
21
+ - name: Install uv
22
+ uses: astral-sh/setup-uv@v5
23
+ with:
24
+ python-version: "3.12"
25
+
26
+ - name: Sync deps (resolves the exact clio-schemas pin)
27
+ run: uv sync --extra dev
28
+
29
+ - name: Confirm exact-pin lockstep is installed
30
+ run: |
31
+ uv run python - <<'PY'
32
+ import clio_schemas
33
+ print("clio-schemas", clio_schemas.__version__)
34
+ PY
35
+
36
+ # Option A — clio-agent vendors the schemas under ./schemas:
37
+ # verify the vendored copy is byte-identical to the installed package
38
+ # (rejects stale, missing, AND unexpected/orphaned files).
39
+ - name: Vendored schemas match the installed package (exact bytes + set)
40
+ run: uv run python -m clio_schemas.export --out schemas --check
41
+
42
+ # Option B — CI refreshes the vendored copy into a CLEAN dir and fails on
43
+ # any drift, including untracked/orphaned files that `git diff` alone
44
+ # would miss:
45
+ # rm -rf schemas
46
+ # uv run python -m clio_schemas.export --out schemas
47
+ # git add -A schemas && git diff --cached --exit-code -- schemas