overture-schema-system 0.1.1.dev0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- overture_schema_system-0.1.1.dev0/PKG-INFO +328 -0
- overture_schema_system-0.1.1.dev0/README.md +313 -0
- overture_schema_system-0.1.1.dev0/pyproject.toml +60 -0
- overture_schema_system-0.1.1.dev0/pyproject.toml.orig +58 -0
- overture_schema_system-0.1.1.dev0/src/overture/schema/system/__init__.py +180 -0
- overture_schema_system-0.1.1.dev0/src/overture/schema/system/_json_schema.py +404 -0
- overture_schema_system-0.1.1.dev0/src/overture/schema/system/case.py +26 -0
- overture_schema_system-0.1.1.dev0/src/overture/schema/system/create_model.py +60 -0
- overture_schema_system-0.1.1.dev0/src/overture/schema/system/discovery/__init__.py +29 -0
- overture_schema_system-0.1.1.dev0/src/overture/schema/system/discovery/discovery.py +294 -0
- overture_schema_system-0.1.1.dev0/src/overture/schema/system/discovery/entry_point.py +119 -0
- overture_schema_system-0.1.1.dev0/src/overture/schema/system/discovery/keys.py +41 -0
- overture_schema_system-0.1.1.dev0/src/overture/schema/system/discovery/tag.py +102 -0
- overture_schema_system-0.1.1.dev0/src/overture/schema/system/discovery/tag_providers.py +35 -0
- overture_schema_system-0.1.1.dev0/src/overture/schema/system/discovery/types.py +18 -0
- overture_schema_system-0.1.1.dev0/src/overture/schema/system/doc.py +88 -0
- overture_schema_system-0.1.1.dev0/src/overture/schema/system/feature.py +810 -0
- overture_schema_system-0.1.1.dev0/src/overture/schema/system/field_constraint/__init__.py +44 -0
- overture_schema_system-0.1.1.dev0/src/overture/schema/system/field_constraint/collection.py +91 -0
- overture_schema_system-0.1.1.dev0/src/overture/schema/system/field_constraint/field_constraint.py +87 -0
- overture_schema_system-0.1.1.dev0/src/overture/schema/system/field_constraint/string.py +267 -0
- overture_schema_system-0.1.1.dev0/src/overture/schema/system/field_path.py +485 -0
- overture_schema_system-0.1.1.dev0/src/overture/schema/system/geometric/__init__.py +25 -0
- overture_schema_system-0.1.1.dev0/src/overture/schema/system/geometric/bbox.py +285 -0
- overture_schema_system-0.1.1.dev0/src/overture/schema/system/geometric/geom.py +555 -0
- overture_schema_system-0.1.1.dev0/src/overture/schema/system/json_schema.py +204 -0
- overture_schema_system-0.1.1.dev0/src/overture/schema/system/metadata.py +325 -0
- overture_schema_system-0.1.1.dev0/src/overture/schema/system/model_constraint/__init__.py +40 -0
- overture_schema_system-0.1.1.dev0/src/overture/schema/system/model_constraint/forbid_if.py +150 -0
- overture_schema_system-0.1.1.dev0/src/overture/schema/system/model_constraint/min_fields_set.py +134 -0
- overture_schema_system-0.1.1.dev0/src/overture/schema/system/model_constraint/model_constraint.py +595 -0
- overture_schema_system-0.1.1.dev0/src/overture/schema/system/model_constraint/no_extra_fields.py +81 -0
- overture_schema_system-0.1.1.dev0/src/overture/schema/system/model_constraint/radio_group.py +159 -0
- overture_schema_system-0.1.1.dev0/src/overture/schema/system/model_constraint/require_any_of.py +122 -0
- overture_schema_system-0.1.1.dev0/src/overture/schema/system/model_constraint/require_any_true.py +166 -0
- overture_schema_system-0.1.1.dev0/src/overture/schema/system/model_constraint/require_if.py +159 -0
- overture_schema_system-0.1.1.dev0/src/overture/schema/system/numeric.py +103 -0
- overture_schema_system-0.1.1.dev0/src/overture/schema/system/optionality.py +87 -0
- overture_schema_system-0.1.1.dev0/src/overture/schema/system/py.typed +0 -0
- overture_schema_system-0.1.1.dev0/src/overture/schema/system/ref/__init__.py +4 -0
- overture_schema_system-0.1.1.dev0/src/overture/schema/system/ref/id.py +62 -0
- overture_schema_system-0.1.1.dev0/src/overture/schema/system/ref/ref.py +112 -0
- overture_schema_system-0.1.1.dev0/src/overture/schema/system/string.py +176 -0
- overture_schema_system-0.1.1.dev0/src/overture/schema/system/testing/__init__.py +13 -0
- overture_schema_system-0.1.1.dev0/src/overture/schema/system/testing/golden.py +61 -0
- overture_schema_system-0.1.1.dev0/src/overture/schema/system/testing/plugin.py +23 -0
- overture_schema_system-0.1.1.dev0/src/overture/schema/system/typing_util.py +48 -0
|
@@ -0,0 +1,328 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: overture-schema-system
|
|
3
|
+
Version: 0.1.1.dev0
|
|
4
|
+
Summary: Foundational types at the base of the Overture Maps schema system
|
|
5
|
+
License-Expression: MIT
|
|
6
|
+
Requires-Dist: pydantic>=2.13.0
|
|
7
|
+
Requires-Dist: shapely>=2.0.0
|
|
8
|
+
Requires-Dist: typing-extensions>=4.16
|
|
9
|
+
Maintainer: Overture Maps Schema Working Group
|
|
10
|
+
Requires-Python: >=3.10
|
|
11
|
+
Project-URL: Homepage, https://overturemaps.org
|
|
12
|
+
Project-URL: Source, https://github.com/OvertureMaps/schema
|
|
13
|
+
Project-URL: Issues, https://github.com/OvertureMaps/schema/issues
|
|
14
|
+
Description-Content-Type: text/markdown
|
|
15
|
+
|
|
16
|
+
# Overture Schema System
|
|
17
|
+
|
|
18
|
+
Write Pydantic models once, get validated data that serializes correctly to JSON, Parquet, and Spark. This package provides the numeric and geometric types, constraint decorators, and GeoJSON-aware base class that make Pydantic models portable across serialization targets.
|
|
19
|
+
|
|
20
|
+
## Installation
|
|
21
|
+
|
|
22
|
+
```bash
|
|
23
|
+
pip install overture-schema-system
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
## Feature
|
|
27
|
+
|
|
28
|
+
GeoJSON-compatible Pydantic base model. Subclasses serialize to the GeoJSON format automatically -- `geometry` and `id` at the top level, everything else under `properties` -- and validate from it:
|
|
29
|
+
|
|
30
|
+
```python
|
|
31
|
+
from overture.schema.system.feature import Feature
|
|
32
|
+
from overture.schema.system.geometric import Geometry
|
|
33
|
+
from overture.schema.system.numeric import float32
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
class Mountain(Feature):
|
|
37
|
+
name: str
|
|
38
|
+
max_elevation: float32
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
m = Mountain(
|
|
42
|
+
geometry=Geometry.from_wkt("POINT(86.9252 27.9888)"),
|
|
43
|
+
name="Mount Everest",
|
|
44
|
+
max_elevation=8848.86,
|
|
45
|
+
)
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
## Numeric Types
|
|
49
|
+
|
|
50
|
+
Using `int` and `float` in a Pydantic model produces valid Python but loses information downstream -- an `int` field becomes a 64-bit integer in Parquet, Arrow, and Spark StructTypes, even when a single byte would hold every value. How wide a field needs to be is the schema author's choice, but `int` and `float` give Pydantic no way to record it. The numeric types (`uint8`, `int32`, `float32`, etc.) do: each declares a width that maps to the correct wire type in every serialization target, so a single portable declaration round-trips cleanly between Python, Parquet, Spark, and JSON Schema:
|
|
51
|
+
|
|
52
|
+
```python
|
|
53
|
+
from pydantic import BaseModel
|
|
54
|
+
from overture.schema.system.numeric import uint8, float32
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
class Building(BaseModel):
|
|
58
|
+
height: float32 | None = None
|
|
59
|
+
num_floors: uint8 | None = None
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
Integer types: `uint8`, `uint16`, `uint32`, `int8`, `int16`, `int32`, `int64`. Float types: `float32`, `float64`.
|
|
63
|
+
|
|
64
|
+
## Geometric Types
|
|
65
|
+
|
|
66
|
+
`Geometry` and `BBox` wrap Shapely and GeoJSON-compatible geometry and bounding box values so they can participate in a Pydantic model as fields, with `GeometryType` and `GeometryTypeConstraint` available to restrict a `Geometry` field to specific geometry types:
|
|
67
|
+
|
|
68
|
+
```python
|
|
69
|
+
from overture.schema.system.geometric import (
|
|
70
|
+
Geometry,
|
|
71
|
+
GeometryType,
|
|
72
|
+
GeometryTypeConstraint,
|
|
73
|
+
)
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
Types: `Geometry`, `BBox`, `GeometryType`, `GeometryTypeConstraint`.
|
|
77
|
+
|
|
78
|
+
## String Types
|
|
79
|
+
|
|
80
|
+
Validated string types that carry their constraints into generated JSON Schemas and downstream code generation. Using `CountryCodeAlpha2` instead of `str` means Pydantic rejects `"USA"` at validation time, JSON Schema gets the right pattern, and codegen tools produce typed output:
|
|
81
|
+
|
|
82
|
+
```python
|
|
83
|
+
from overture.schema.system.string import CountryCodeAlpha2, LanguageTag
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
Available types: `CountryCodeAlpha2`, `RegionCode`, `LanguageTag`, `HexColor`, `JsonPointer`, `PhoneNumber`, `StrippedString`, `SnakeCaseString`, `NoWhitespaceString`, `WikidataId`.
|
|
87
|
+
|
|
88
|
+
## Field Constraints
|
|
89
|
+
|
|
90
|
+
Annotations for Pydantic fields that enforce domain rules beyond what the type alone expresses. Each constraint produces the corresponding JSON Schema keywords (e.g., `pattern`, `uniqueItems`) and is introspectable by code generation tools -- unlike Pydantic's `@field_validator`, which runs in Python only. Apply via `Annotated`:
|
|
91
|
+
|
|
92
|
+
```python
|
|
93
|
+
from typing import Annotated
|
|
94
|
+
from pydantic import BaseModel, Field
|
|
95
|
+
from overture.schema.system.field_constraint import (
|
|
96
|
+
UniqueItemsConstraint,
|
|
97
|
+
PatternConstraint,
|
|
98
|
+
)
|
|
99
|
+
|
|
100
|
+
OsmIdConstraint = PatternConstraint(
|
|
101
|
+
pattern=r"^[nwr]\d+$",
|
|
102
|
+
error_message="invalid OSM ID format: {value}. Must be n123, w123, or r123.",
|
|
103
|
+
)
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
class MyModel(BaseModel):
|
|
107
|
+
osm_id: Annotated[str, OsmIdConstraint]
|
|
108
|
+
tags: Annotated[list[str], UniqueItemsConstraint()] = Field(min_length=1)
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
Built-in constraints include `PatternConstraint`, `StrippedConstraint`, `UniqueItemsConstraint`, and all the string-type constraints (`CountryCodeAlpha2Constraint`, `HexColorConstraint`, etc.). All produce error messages with domain context.
|
|
112
|
+
|
|
113
|
+
## Model Constraints
|
|
114
|
+
|
|
115
|
+
Class-level decorators for cross-field validation -- relationships between fields that no single field annotation can express. Each decorator produces corresponding JSON Schema constructs (`if`/`then`, `anyOf`, etc.) and is introspectable for code generation:
|
|
116
|
+
|
|
117
|
+
```python
|
|
118
|
+
from pydantic import BaseModel
|
|
119
|
+
from overture.schema.system.model_constraint import require_any_of
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
@require_any_of("email", "phone")
|
|
123
|
+
class Contact(BaseModel):
|
|
124
|
+
email: str | None = None
|
|
125
|
+
phone: str | None = None
|
|
126
|
+
```
|
|
127
|
+
|
|
128
|
+
- `@require_any_of("a", "b", ...)` -- at least one field must be non-None
|
|
129
|
+
- `@require_any_true(cond1, cond2, ...)` -- at least one condition must evaluate to true
|
|
130
|
+
- `@radio_group("a", "b", ...)` -- at most one field may be truthy
|
|
131
|
+
- `@require_if("target", condition)` -- field required when condition holds
|
|
132
|
+
- `@forbid_if("target", condition)` -- field forbidden when condition holds
|
|
133
|
+
- `@min_fields_set(n, "a", "b", ...)` -- at least *n* fields must be set
|
|
134
|
+
- `@no_extra_fields` -- reject unrecognized fields (equivalent to `model_config = ConfigDict(extra="forbid")`)
|
|
135
|
+
|
|
136
|
+
## References
|
|
137
|
+
|
|
138
|
+
Foreign-key-style annotations that describe relationships between models. These carry no runtime enforcement but provide metadata for code generation and documentation tools:
|
|
139
|
+
|
|
140
|
+
```python
|
|
141
|
+
from typing import Annotated
|
|
142
|
+
from overture.schema.system.ref import Id, Identified, Reference, Relationship
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
class Park(Identified):
|
|
146
|
+
pass
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
class ParkBench(Identified):
|
|
150
|
+
park_id: Annotated[Id, Reference(Relationship.COMPOSITION, Park, role="part_of")]
|
|
151
|
+
```
|
|
152
|
+
|
|
153
|
+
## Discovery
|
|
154
|
+
|
|
155
|
+
Packages register models on the `overture.models` Python entry point group. Each entry maps a name to a class import path:
|
|
156
|
+
|
|
157
|
+
```toml
|
|
158
|
+
[project.entry-points."overture.models"]
|
|
159
|
+
building = "overture.schema.buildings:Building"
|
|
160
|
+
building_part = "overture.schema.buildings:BuildingPart"
|
|
161
|
+
```
|
|
162
|
+
|
|
163
|
+
`discover_models()` walks the group, loads each entry point, and returns a dict keyed by `ModelKey`. Consumers iterate over the result without knowing which package owns any given model -- the CLI and codegen tools both run discovery to assemble their working set.
|
|
164
|
+
|
|
165
|
+
A `ModelKey` carries the entry point `name`, its `entry_point` value (`"module:Class"`), and a `frozenset[str]` of tags. [Tagging](#tagging) is how those tags get attached.
|
|
166
|
+
|
|
167
|
+
## Tagging
|
|
168
|
+
|
|
169
|
+
Tags classify discovered models. A package registers [tag providers](#providers) on `overture.tag_providers`; when `discover_models` runs, it asks every provider which tags apply to each model and attaches the resulting set to its `ModelKey`. Downstream tools read those tags -- the CLI's `--tag` filter, codegen's grouping logic, anything that reasons about a model without importing it.
|
|
170
|
+
|
|
171
|
+
```python
|
|
172
|
+
from overture.schema.system.discovery import (
|
|
173
|
+
TagSelector,
|
|
174
|
+
discover_models,
|
|
175
|
+
filter_models,
|
|
176
|
+
)
|
|
177
|
+
|
|
178
|
+
models = discover_models()
|
|
179
|
+
|
|
180
|
+
selected = filter_models(
|
|
181
|
+
models,
|
|
182
|
+
TagSelector(include_any=("feature",), exclude_any=("draft",)),
|
|
183
|
+
)
|
|
184
|
+
```
|
|
185
|
+
|
|
186
|
+
### Format
|
|
187
|
+
|
|
188
|
+
Tags follow `[namespace:]predicate[=value]`:
|
|
189
|
+
|
|
190
|
+
- **Plain** -- `feature`, `overture`
|
|
191
|
+
- **Namespaced** -- `system:extension`
|
|
192
|
+
- **Key/value** -- `overture:theme=buildings`
|
|
193
|
+
|
|
194
|
+
`:` separates namespace from predicate -- one level only, no nested colons. `=` introduces a discrete value, one per tag. Predicate and namespace parts are lowercase alphanumeric (with `_`, `.`, `-`); values also accept uppercase. Matching is case-sensitive throughout.
|
|
195
|
+
|
|
196
|
+
Helpers in `overture.schema.system.discovery.tag` parse structured tags:
|
|
197
|
+
|
|
198
|
+
- `is_valid_tag(tag)` -- check whether a string matches the format
|
|
199
|
+
- `get_namespace(tag)` -- extract the namespace prefix, or `""` for a plain tag
|
|
200
|
+
- `get_values_for_key(tags, "overture:theme")` -- extract values from k/v tags with the given key
|
|
201
|
+
|
|
202
|
+
### Providers
|
|
203
|
+
|
|
204
|
+
A tag provider is a callable registered on the `overture.tag_providers` entry point group. Discovery passes it the concrete `BaseModel` subclasses for the entry point and a copy of the tags accumulated so far; tags it adds are merged into the running set after passing the reservation checks below.
|
|
205
|
+
|
|
206
|
+
```python
|
|
207
|
+
from collections.abc import Iterable
|
|
208
|
+
from pydantic import BaseModel
|
|
209
|
+
from overture.schema.system.discovery import ModelKey
|
|
210
|
+
from overture.schema.system.feature import Feature
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
def feature_provider(
|
|
214
|
+
types: Iterable[type[BaseModel]],
|
|
215
|
+
key: ModelKey,
|
|
216
|
+
tags: set[str],
|
|
217
|
+
) -> set[str]:
|
|
218
|
+
if any(issubclass(tp, Feature) for tp in types):
|
|
219
|
+
tags.add("feature")
|
|
220
|
+
return tags
|
|
221
|
+
```
|
|
222
|
+
|
|
223
|
+
```toml
|
|
224
|
+
[project.entry-points."overture.tag_providers"]
|
|
225
|
+
feature = "overture.schema.system.discovery.tag_providers:feature_provider"
|
|
226
|
+
```
|
|
227
|
+
|
|
228
|
+
Tags from one provider are visible to providers that run later, but execution order is unspecified -- a provider must not depend on tags added by another. Provider exceptions are caught, logged, and discarded; they do not abort discovery.
|
|
229
|
+
|
|
230
|
+
Discovery resolves the entry-point value to concrete classes before invoking providers. For class entries that yields a one-element iterable; for discriminated-union features (e.g. `Segment`, which loads as `Annotated[Union[...], Field(...)]`) it yields every arm. Providers therefore work uniformly with `issubclass` and never need to walk type expressions themselves.
|
|
231
|
+
|
|
232
|
+
### Reservation
|
|
233
|
+
|
|
234
|
+
Specific plain tags and namespaces are reserved for designated packages. For example:
|
|
235
|
+
|
|
236
|
+
| Tag or namespace | Owning package |
|
|
237
|
+
|---|---|
|
|
238
|
+
| `feature` (tag) | `overture-schema-system` |
|
|
239
|
+
| `system:` (namespace) | `overture-schema-system` |
|
|
240
|
+
| `overture` (tag) | `overture-schema-common` |
|
|
241
|
+
| `overture:` (namespace) | `overture-schema-common` |
|
|
242
|
+
|
|
243
|
+
When a provider attempts to set a reserved tag from an unauthorized package, discovery logs a warning and discards the tag.
|
|
244
|
+
|
|
245
|
+
### Built-in Providers
|
|
246
|
+
|
|
247
|
+
- **`feature`** (in `system`) -- adds `feature` if any concrete arm is a `Feature` subclass.
|
|
248
|
+
- **`overture`** (in `common`) -- adds `overture` if any concrete arm is an `OvertureFeature` subclass: the model is built on Overture's feature model, as distinct from `feature`, which says only that it is a `Feature`. Consumers that need to ask that question read the tag rather than importing `OvertureFeature`. The tag does not assert that the type belongs to the Overture schema -- a third-party `OvertureFeature` subclass receives it too, and the reservation governs who may emit the tag, not which models get it.
|
|
249
|
+
- **`theme`** (in `common`) -- adds `overture:theme={theme}` for each `OvertureFeature` referenced. A discriminated-union feature whose arms span multiple themes contributes one tag per distinct theme.
|
|
250
|
+
|
|
251
|
+
### Selecting Models by Tag
|
|
252
|
+
|
|
253
|
+
`filter_models(models, selector)` applies `TagSelector` predicates against each `ModelKey.tags`:
|
|
254
|
+
|
|
255
|
+
- `include_any` -- OR scope; at least one tag must match (empty: no scope filter)
|
|
256
|
+
- `require_all` -- AND narrowing; every tag must be present (empty: no narrowing)
|
|
257
|
+
- `exclude_any` -- OR-NOT subtraction; any match drops the model
|
|
258
|
+
|
|
259
|
+
An empty selector returns the input unchanged.
|
|
260
|
+
|
|
261
|
+
## Also Included
|
|
262
|
+
|
|
263
|
+
- **Optionality** -- `Omitable[T]` models JSON Schema's "may be absent but not null" semantics, which Pydantic's `T | None` conflates with nullable.
|
|
264
|
+
- **DocumentedEnum** -- base class for enumerations whose members carry their own docstrings, enabling code generation tools to produce documented output.
|
|
265
|
+
- **Metadata** -- internal key-value store used by model constraints to attach data to classes.
|
|
266
|
+
- **JSON Schema** -- schema generator that treats `T | None = None` as "omit when unset" rather than Pydantic's default "nullable with null default." Also handles unions of models.
|
|
267
|
+
|
|
268
|
+
## Baseline Testing
|
|
269
|
+
|
|
270
|
+
`overture.schema.system.testing` provides a pytest plugin for golden-file baseline tests of generated JSON Schemas. Implementers building feature packages use it to detect unintended schema drift.
|
|
271
|
+
|
|
272
|
+
### Helpers
|
|
273
|
+
|
|
274
|
+
- `assert_golden(actual, golden_path, *, update)` -- compare a string against a golden file. On mismatch, raises `AssertionError` with a unified diff. When `update=True`, writes `actual` to `golden_path` instead of comparing.
|
|
275
|
+
- `assert_json_schema_golden(model_or_union, golden_path, *, update)` -- generate the JSON Schema for a Pydantic model (or discriminated union type alias) via `overture.schema.system.json_schema` and delegate to `assert_golden`.
|
|
276
|
+
|
|
277
|
+
### Opting In
|
|
278
|
+
|
|
279
|
+
Activation is opt-in so the `--update-baselines` flag does not pollute pytest runs of packages that do not declare it. Add the plugin to your package's `pyproject.toml`:
|
|
280
|
+
|
|
281
|
+
```toml
|
|
282
|
+
[project.entry-points.pytest11]
|
|
283
|
+
overture_baselines = "overture.schema.system.testing.plugin"
|
|
284
|
+
```
|
|
285
|
+
|
|
286
|
+
The plugin registers:
|
|
287
|
+
|
|
288
|
+
- `--update-baselines` -- pytest CLI flag.
|
|
289
|
+
- `update_baselines` -- bool fixture, true when the flag is passed.
|
|
290
|
+
|
|
291
|
+
### Writing a Baseline Test
|
|
292
|
+
|
|
293
|
+
```python
|
|
294
|
+
from pathlib import Path
|
|
295
|
+
|
|
296
|
+
import pytest
|
|
297
|
+
from overture.schema.system.testing import assert_json_schema_golden
|
|
298
|
+
|
|
299
|
+
from yourpackage import Mountain # the model you're locking down
|
|
300
|
+
|
|
301
|
+
GOLDEN = Path(__file__).parent / "mountain_baseline_schema.json"
|
|
302
|
+
|
|
303
|
+
|
|
304
|
+
@pytest.mark.baseline
|
|
305
|
+
def test_mountain_json_schema(update_baselines: bool) -> None:
|
|
306
|
+
assert_json_schema_golden(Mountain, GOLDEN, update=update_baselines)
|
|
307
|
+
```
|
|
308
|
+
|
|
309
|
+
Convention: the golden file lives next to the test, named `<feature>_baseline_schema.json`.
|
|
310
|
+
|
|
311
|
+
Mark each baseline test with `@pytest.mark.baseline` so it can be selected or skipped via `pytest -m baseline` / `pytest -m "not baseline"`. Register the marker in your pyproject:
|
|
312
|
+
|
|
313
|
+
```toml
|
|
314
|
+
[tool.pytest.ini_options]
|
|
315
|
+
markers = [
|
|
316
|
+
"baseline: golden file baseline tests",
|
|
317
|
+
]
|
|
318
|
+
```
|
|
319
|
+
|
|
320
|
+
### Updating Baselines
|
|
321
|
+
|
|
322
|
+
After an intentional schema change:
|
|
323
|
+
|
|
324
|
+
```bash
|
|
325
|
+
pytest -m baseline --update-baselines
|
|
326
|
+
```
|
|
327
|
+
|
|
328
|
+
Inspect `git diff` on the regenerated golden files to confirm the changes are intended before committing.
|
|
@@ -0,0 +1,313 @@
|
|
|
1
|
+
# Overture Schema System
|
|
2
|
+
|
|
3
|
+
Write Pydantic models once, get validated data that serializes correctly to JSON, Parquet, and Spark. This package provides the numeric and geometric types, constraint decorators, and GeoJSON-aware base class that make Pydantic models portable across serialization targets.
|
|
4
|
+
|
|
5
|
+
## Installation
|
|
6
|
+
|
|
7
|
+
```bash
|
|
8
|
+
pip install overture-schema-system
|
|
9
|
+
```
|
|
10
|
+
|
|
11
|
+
## Feature
|
|
12
|
+
|
|
13
|
+
GeoJSON-compatible Pydantic base model. Subclasses serialize to the GeoJSON format automatically -- `geometry` and `id` at the top level, everything else under `properties` -- and validate from it:
|
|
14
|
+
|
|
15
|
+
```python
|
|
16
|
+
from overture.schema.system.feature import Feature
|
|
17
|
+
from overture.schema.system.geometric import Geometry
|
|
18
|
+
from overture.schema.system.numeric import float32
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class Mountain(Feature):
|
|
22
|
+
name: str
|
|
23
|
+
max_elevation: float32
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
m = Mountain(
|
|
27
|
+
geometry=Geometry.from_wkt("POINT(86.9252 27.9888)"),
|
|
28
|
+
name="Mount Everest",
|
|
29
|
+
max_elevation=8848.86,
|
|
30
|
+
)
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
## Numeric Types
|
|
34
|
+
|
|
35
|
+
Using `int` and `float` in a Pydantic model produces valid Python but loses information downstream -- an `int` field becomes a 64-bit integer in Parquet, Arrow, and Spark StructTypes, even when a single byte would hold every value. How wide a field needs to be is the schema author's choice, but `int` and `float` give Pydantic no way to record it. The numeric types (`uint8`, `int32`, `float32`, etc.) do: each declares a width that maps to the correct wire type in every serialization target, so a single portable declaration round-trips cleanly between Python, Parquet, Spark, and JSON Schema:
|
|
36
|
+
|
|
37
|
+
```python
|
|
38
|
+
from pydantic import BaseModel
|
|
39
|
+
from overture.schema.system.numeric import uint8, float32
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
class Building(BaseModel):
|
|
43
|
+
height: float32 | None = None
|
|
44
|
+
num_floors: uint8 | None = None
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
Integer types: `uint8`, `uint16`, `uint32`, `int8`, `int16`, `int32`, `int64`. Float types: `float32`, `float64`.
|
|
48
|
+
|
|
49
|
+
## Geometric Types
|
|
50
|
+
|
|
51
|
+
`Geometry` and `BBox` wrap Shapely and GeoJSON-compatible geometry and bounding box values so they can participate in a Pydantic model as fields, with `GeometryType` and `GeometryTypeConstraint` available to restrict a `Geometry` field to specific geometry types:
|
|
52
|
+
|
|
53
|
+
```python
|
|
54
|
+
from overture.schema.system.geometric import (
|
|
55
|
+
Geometry,
|
|
56
|
+
GeometryType,
|
|
57
|
+
GeometryTypeConstraint,
|
|
58
|
+
)
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
Types: `Geometry`, `BBox`, `GeometryType`, `GeometryTypeConstraint`.
|
|
62
|
+
|
|
63
|
+
## String Types
|
|
64
|
+
|
|
65
|
+
Validated string types that carry their constraints into generated JSON Schemas and downstream code generation. Using `CountryCodeAlpha2` instead of `str` means Pydantic rejects `"USA"` at validation time, JSON Schema gets the right pattern, and codegen tools produce typed output:
|
|
66
|
+
|
|
67
|
+
```python
|
|
68
|
+
from overture.schema.system.string import CountryCodeAlpha2, LanguageTag
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
Available types: `CountryCodeAlpha2`, `RegionCode`, `LanguageTag`, `HexColor`, `JsonPointer`, `PhoneNumber`, `StrippedString`, `SnakeCaseString`, `NoWhitespaceString`, `WikidataId`.
|
|
72
|
+
|
|
73
|
+
## Field Constraints
|
|
74
|
+
|
|
75
|
+
Annotations for Pydantic fields that enforce domain rules beyond what the type alone expresses. Each constraint produces the corresponding JSON Schema keywords (e.g., `pattern`, `uniqueItems`) and is introspectable by code generation tools -- unlike Pydantic's `@field_validator`, which runs in Python only. Apply via `Annotated`:
|
|
76
|
+
|
|
77
|
+
```python
|
|
78
|
+
from typing import Annotated
|
|
79
|
+
from pydantic import BaseModel, Field
|
|
80
|
+
from overture.schema.system.field_constraint import (
|
|
81
|
+
UniqueItemsConstraint,
|
|
82
|
+
PatternConstraint,
|
|
83
|
+
)
|
|
84
|
+
|
|
85
|
+
OsmIdConstraint = PatternConstraint(
|
|
86
|
+
pattern=r"^[nwr]\d+$",
|
|
87
|
+
error_message="invalid OSM ID format: {value}. Must be n123, w123, or r123.",
|
|
88
|
+
)
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
class MyModel(BaseModel):
|
|
92
|
+
osm_id: Annotated[str, OsmIdConstraint]
|
|
93
|
+
tags: Annotated[list[str], UniqueItemsConstraint()] = Field(min_length=1)
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
Built-in constraints include `PatternConstraint`, `StrippedConstraint`, `UniqueItemsConstraint`, and all the string-type constraints (`CountryCodeAlpha2Constraint`, `HexColorConstraint`, etc.). All produce error messages with domain context.
|
|
97
|
+
|
|
98
|
+
## Model Constraints
|
|
99
|
+
|
|
100
|
+
Class-level decorators for cross-field validation -- relationships between fields that no single field annotation can express. Each decorator produces corresponding JSON Schema constructs (`if`/`then`, `anyOf`, etc.) and is introspectable for code generation:
|
|
101
|
+
|
|
102
|
+
```python
|
|
103
|
+
from pydantic import BaseModel
|
|
104
|
+
from overture.schema.system.model_constraint import require_any_of
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
@require_any_of("email", "phone")
|
|
108
|
+
class Contact(BaseModel):
|
|
109
|
+
email: str | None = None
|
|
110
|
+
phone: str | None = None
|
|
111
|
+
```
|
|
112
|
+
|
|
113
|
+
- `@require_any_of("a", "b", ...)` -- at least one field must be non-None
|
|
114
|
+
- `@require_any_true(cond1, cond2, ...)` -- at least one condition must evaluate to true
|
|
115
|
+
- `@radio_group("a", "b", ...)` -- at most one field may be truthy
|
|
116
|
+
- `@require_if("target", condition)` -- field required when condition holds
|
|
117
|
+
- `@forbid_if("target", condition)` -- field forbidden when condition holds
|
|
118
|
+
- `@min_fields_set(n, "a", "b", ...)` -- at least *n* fields must be set
|
|
119
|
+
- `@no_extra_fields` -- reject unrecognized fields (equivalent to `model_config = ConfigDict(extra="forbid")`)
|
|
120
|
+
|
|
121
|
+
## References
|
|
122
|
+
|
|
123
|
+
Foreign-key-style annotations that describe relationships between models. These carry no runtime enforcement but provide metadata for code generation and documentation tools:
|
|
124
|
+
|
|
125
|
+
```python
|
|
126
|
+
from typing import Annotated
|
|
127
|
+
from overture.schema.system.ref import Id, Identified, Reference, Relationship
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
class Park(Identified):
|
|
131
|
+
pass
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
class ParkBench(Identified):
|
|
135
|
+
park_id: Annotated[Id, Reference(Relationship.COMPOSITION, Park, role="part_of")]
|
|
136
|
+
```
|
|
137
|
+
|
|
138
|
+
## Discovery
|
|
139
|
+
|
|
140
|
+
Packages register models on the `overture.models` Python entry point group. Each entry maps a name to a class import path:
|
|
141
|
+
|
|
142
|
+
```toml
|
|
143
|
+
[project.entry-points."overture.models"]
|
|
144
|
+
building = "overture.schema.buildings:Building"
|
|
145
|
+
building_part = "overture.schema.buildings:BuildingPart"
|
|
146
|
+
```
|
|
147
|
+
|
|
148
|
+
`discover_models()` walks the group, loads each entry point, and returns a dict keyed by `ModelKey`. Consumers iterate over the result without knowing which package owns any given model -- the CLI and codegen tools both run discovery to assemble their working set.
|
|
149
|
+
|
|
150
|
+
A `ModelKey` carries the entry point `name`, its `entry_point` value (`"module:Class"`), and a `frozenset[str]` of tags. [Tagging](#tagging) is how those tags get attached.
|
|
151
|
+
|
|
152
|
+
## Tagging
|
|
153
|
+
|
|
154
|
+
Tags classify discovered models. A package registers [tag providers](#providers) on `overture.tag_providers`; when `discover_models` runs, it asks every provider which tags apply to each model and attaches the resulting set to its `ModelKey`. Downstream tools read those tags -- the CLI's `--tag` filter, codegen's grouping logic, anything that reasons about a model without importing it.
|
|
155
|
+
|
|
156
|
+
```python
|
|
157
|
+
from overture.schema.system.discovery import (
|
|
158
|
+
TagSelector,
|
|
159
|
+
discover_models,
|
|
160
|
+
filter_models,
|
|
161
|
+
)
|
|
162
|
+
|
|
163
|
+
models = discover_models()
|
|
164
|
+
|
|
165
|
+
selected = filter_models(
|
|
166
|
+
models,
|
|
167
|
+
TagSelector(include_any=("feature",), exclude_any=("draft",)),
|
|
168
|
+
)
|
|
169
|
+
```
|
|
170
|
+
|
|
171
|
+
### Format
|
|
172
|
+
|
|
173
|
+
Tags follow `[namespace:]predicate[=value]`:
|
|
174
|
+
|
|
175
|
+
- **Plain** -- `feature`, `overture`
|
|
176
|
+
- **Namespaced** -- `system:extension`
|
|
177
|
+
- **Key/value** -- `overture:theme=buildings`
|
|
178
|
+
|
|
179
|
+
`:` separates namespace from predicate -- one level only, no nested colons. `=` introduces a discrete value, one per tag. Predicate and namespace parts are lowercase alphanumeric (with `_`, `.`, `-`); values also accept uppercase. Matching is case-sensitive throughout.
|
|
180
|
+
|
|
181
|
+
Helpers in `overture.schema.system.discovery.tag` parse structured tags:
|
|
182
|
+
|
|
183
|
+
- `is_valid_tag(tag)` -- check whether a string matches the format
|
|
184
|
+
- `get_namespace(tag)` -- extract the namespace prefix, or `""` for a plain tag
|
|
185
|
+
- `get_values_for_key(tags, "overture:theme")` -- extract values from k/v tags with the given key
|
|
186
|
+
|
|
187
|
+
### Providers
|
|
188
|
+
|
|
189
|
+
A tag provider is a callable registered on the `overture.tag_providers` entry point group. Discovery passes it the concrete `BaseModel` subclasses for the entry point and a copy of the tags accumulated so far; tags it adds are merged into the running set after passing the reservation checks below.
|
|
190
|
+
|
|
191
|
+
```python
|
|
192
|
+
from collections.abc import Iterable
|
|
193
|
+
from pydantic import BaseModel
|
|
194
|
+
from overture.schema.system.discovery import ModelKey
|
|
195
|
+
from overture.schema.system.feature import Feature
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def feature_provider(
|
|
199
|
+
types: Iterable[type[BaseModel]],
|
|
200
|
+
key: ModelKey,
|
|
201
|
+
tags: set[str],
|
|
202
|
+
) -> set[str]:
|
|
203
|
+
if any(issubclass(tp, Feature) for tp in types):
|
|
204
|
+
tags.add("feature")
|
|
205
|
+
return tags
|
|
206
|
+
```
|
|
207
|
+
|
|
208
|
+
```toml
|
|
209
|
+
[project.entry-points."overture.tag_providers"]
|
|
210
|
+
feature = "overture.schema.system.discovery.tag_providers:feature_provider"
|
|
211
|
+
```
|
|
212
|
+
|
|
213
|
+
Tags from one provider are visible to providers that run later, but execution order is unspecified -- a provider must not depend on tags added by another. Provider exceptions are caught, logged, and discarded; they do not abort discovery.
|
|
214
|
+
|
|
215
|
+
Discovery resolves the entry-point value to concrete classes before invoking providers. For class entries that yields a one-element iterable; for discriminated-union features (e.g. `Segment`, which loads as `Annotated[Union[...], Field(...)]`) it yields every arm. Providers therefore work uniformly with `issubclass` and never need to walk type expressions themselves.
|
|
216
|
+
|
|
217
|
+
### Reservation
|
|
218
|
+
|
|
219
|
+
Specific plain tags and namespaces are reserved for designated packages. For example:
|
|
220
|
+
|
|
221
|
+
| Tag or namespace | Owning package |
|
|
222
|
+
|---|---|
|
|
223
|
+
| `feature` (tag) | `overture-schema-system` |
|
|
224
|
+
| `system:` (namespace) | `overture-schema-system` |
|
|
225
|
+
| `overture` (tag) | `overture-schema-common` |
|
|
226
|
+
| `overture:` (namespace) | `overture-schema-common` |
|
|
227
|
+
|
|
228
|
+
When a provider attempts to set a reserved tag from an unauthorized package, discovery logs a warning and discards the tag.
|
|
229
|
+
|
|
230
|
+
### Built-in Providers
|
|
231
|
+
|
|
232
|
+
- **`feature`** (in `system`) -- adds `feature` if any concrete arm is a `Feature` subclass.
|
|
233
|
+
- **`overture`** (in `common`) -- adds `overture` if any concrete arm is an `OvertureFeature` subclass: the model is built on Overture's feature model, as distinct from `feature`, which says only that it is a `Feature`. Consumers that need to ask that question read the tag rather than importing `OvertureFeature`. The tag does not assert that the type belongs to the Overture schema -- a third-party `OvertureFeature` subclass receives it too, and the reservation governs who may emit the tag, not which models get it.
|
|
234
|
+
- **`theme`** (in `common`) -- adds `overture:theme={theme}` for each `OvertureFeature` referenced. A discriminated-union feature whose arms span multiple themes contributes one tag per distinct theme.
|
|
235
|
+
|
|
236
|
+
### Selecting Models by Tag
|
|
237
|
+
|
|
238
|
+
`filter_models(models, selector)` applies `TagSelector` predicates against each `ModelKey.tags`:
|
|
239
|
+
|
|
240
|
+
- `include_any` -- OR scope; at least one tag must match (empty: no scope filter)
|
|
241
|
+
- `require_all` -- AND narrowing; every tag must be present (empty: no narrowing)
|
|
242
|
+
- `exclude_any` -- OR-NOT subtraction; any match drops the model
|
|
243
|
+
|
|
244
|
+
An empty selector returns the input unchanged.
|
|
245
|
+
|
|
246
|
+
## Also Included
|
|
247
|
+
|
|
248
|
+
- **Optionality** -- `Omitable[T]` models JSON Schema's "may be absent but not null" semantics, which Pydantic's `T | None` conflates with nullable.
|
|
249
|
+
- **DocumentedEnum** -- base class for enumerations whose members carry their own docstrings, enabling code generation tools to produce documented output.
|
|
250
|
+
- **Metadata** -- internal key-value store used by model constraints to attach data to classes.
|
|
251
|
+
- **JSON Schema** -- schema generator that treats `T | None = None` as "omit when unset" rather than Pydantic's default "nullable with null default." Also handles unions of models.
|
|
252
|
+
|
|
253
|
+
## Baseline Testing
|
|
254
|
+
|
|
255
|
+
`overture.schema.system.testing` provides a pytest plugin for golden-file baseline tests of generated JSON Schemas. Implementers building feature packages use it to detect unintended schema drift.
|
|
256
|
+
|
|
257
|
+
### Helpers
|
|
258
|
+
|
|
259
|
+
- `assert_golden(actual, golden_path, *, update)` -- compare a string against a golden file. On mismatch, raises `AssertionError` with a unified diff. When `update=True`, writes `actual` to `golden_path` instead of comparing.
|
|
260
|
+
- `assert_json_schema_golden(model_or_union, golden_path, *, update)` -- generate the JSON Schema for a Pydantic model (or discriminated union type alias) via `overture.schema.system.json_schema` and delegate to `assert_golden`.
|
|
261
|
+
|
|
262
|
+
### Opting In
|
|
263
|
+
|
|
264
|
+
Activation is opt-in so the `--update-baselines` flag does not pollute pytest runs of packages that do not declare it. Add the plugin to your package's `pyproject.toml`:
|
|
265
|
+
|
|
266
|
+
```toml
|
|
267
|
+
[project.entry-points.pytest11]
|
|
268
|
+
overture_baselines = "overture.schema.system.testing.plugin"
|
|
269
|
+
```
|
|
270
|
+
|
|
271
|
+
The plugin registers:
|
|
272
|
+
|
|
273
|
+
- `--update-baselines` -- pytest CLI flag.
|
|
274
|
+
- `update_baselines` -- bool fixture, true when the flag is passed.
|
|
275
|
+
|
|
276
|
+
### Writing a Baseline Test
|
|
277
|
+
|
|
278
|
+
```python
|
|
279
|
+
from pathlib import Path
|
|
280
|
+
|
|
281
|
+
import pytest
|
|
282
|
+
from overture.schema.system.testing import assert_json_schema_golden
|
|
283
|
+
|
|
284
|
+
from yourpackage import Mountain # the model you're locking down
|
|
285
|
+
|
|
286
|
+
GOLDEN = Path(__file__).parent / "mountain_baseline_schema.json"
|
|
287
|
+
|
|
288
|
+
|
|
289
|
+
@pytest.mark.baseline
|
|
290
|
+
def test_mountain_json_schema(update_baselines: bool) -> None:
|
|
291
|
+
assert_json_schema_golden(Mountain, GOLDEN, update=update_baselines)
|
|
292
|
+
```
|
|
293
|
+
|
|
294
|
+
Convention: the golden file lives next to the test, named `<feature>_baseline_schema.json`.
|
|
295
|
+
|
|
296
|
+
Mark each baseline test with `@pytest.mark.baseline` so it can be selected or skipped via `pytest -m baseline` / `pytest -m "not baseline"`. Register the marker in your pyproject:
|
|
297
|
+
|
|
298
|
+
```toml
|
|
299
|
+
[tool.pytest.ini_options]
|
|
300
|
+
markers = [
|
|
301
|
+
"baseline: golden file baseline tests",
|
|
302
|
+
]
|
|
303
|
+
```
|
|
304
|
+
|
|
305
|
+
### Updating Baselines
|
|
306
|
+
|
|
307
|
+
After an intentional schema change:
|
|
308
|
+
|
|
309
|
+
```bash
|
|
310
|
+
pytest -m baseline --update-baselines
|
|
311
|
+
```
|
|
312
|
+
|
|
313
|
+
Inspect `git diff` on the regenerated golden files to confirm the changes are intended before committing.
|