overture-schema-cli 0.1.1.dev0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- overture/schema/cli/__init__.py +29 -0
- overture/schema/cli/__main__.py +6 -0
- overture/schema/cli/commands.py +930 -0
- overture/schema/cli/data_display.py +716 -0
- overture/schema/cli/docstrings.py +21 -0
- overture/schema/cli/error_formatting.py +587 -0
- overture/schema/cli/output.py +29 -0
- overture/schema/cli/py.typed +0 -0
- overture/schema/cli/tag_options.py +69 -0
- overture/schema/cli/type_analysis.py +448 -0
- overture/schema/cli/types.py +17 -0
- overture_schema_cli-0.1.1.dev0.dist-info/METADATA +38 -0
- overture_schema_cli-0.1.1.dev0.dist-info/RECORD +15 -0
- overture_schema_cli-0.1.1.dev0.dist-info/WHEEL +4 -0
- overture_schema_cli-0.1.1.dev0.dist-info/entry_points.txt +3 -0
|
@@ -0,0 +1,930 @@
|
|
|
1
|
+
"""Click-based CLI for overture-schema package."""
|
|
2
|
+
|
|
3
|
+
import builtins
|
|
4
|
+
import io
|
|
5
|
+
import json
|
|
6
|
+
import sys
|
|
7
|
+
from collections import Counter, defaultdict
|
|
8
|
+
from functools import reduce
|
|
9
|
+
from operator import or_
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
from typing import Annotated, Any, Literal, cast, get_args, get_origin
|
|
12
|
+
|
|
13
|
+
import click
|
|
14
|
+
import yaml
|
|
15
|
+
from pydantic import BaseModel, Field, Tag, TypeAdapter, ValidationError
|
|
16
|
+
from rich.console import Console
|
|
17
|
+
from rich.text import Text
|
|
18
|
+
from yamlcore import CoreLoader # type: ignore
|
|
19
|
+
|
|
20
|
+
from overture.schema.common import OvertureFeature
|
|
21
|
+
from overture.schema.system.discovery import (
|
|
22
|
+
ModelDict,
|
|
23
|
+
ModelKey,
|
|
24
|
+
TagSelector,
|
|
25
|
+
discover_models,
|
|
26
|
+
filter_models,
|
|
27
|
+
)
|
|
28
|
+
from overture.schema.system.discovery.tag import get_values_for_key
|
|
29
|
+
from overture.schema.system.feature import Feature
|
|
30
|
+
from overture.schema.system.json_schema import json_schema
|
|
31
|
+
|
|
32
|
+
from .error_formatting import (
|
|
33
|
+
format_validation_error,
|
|
34
|
+
format_validation_errors_verbose,
|
|
35
|
+
group_errors_by_discriminator,
|
|
36
|
+
select_most_likely_errors,
|
|
37
|
+
)
|
|
38
|
+
from .tag_options import build_selector, tag_selection_options
|
|
39
|
+
from .type_analysis import StructuralTuple, get_item_index, introspect_union
|
|
40
|
+
from .types import ErrorLocation, UnionType, ValidationErrorDict
|
|
41
|
+
|
|
42
|
+
# Console instances for rich output
|
|
43
|
+
stdout = Console(highlight=False)
|
|
44
|
+
stderr = Console(highlight=False, file=sys.stderr)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _is_geojson_feature(data: dict) -> bool:
|
|
48
|
+
"""Check if data is in GeoJSON Feature format."""
|
|
49
|
+
return data.get("type") == "Feature" and "properties" in data
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def _can_discriminate(model_class: object) -> bool:
|
|
53
|
+
"""Check if a model can participate in a discriminated union.
|
|
54
|
+
|
|
55
|
+
Returns True if the model is an OvertureFeature with a single literal 'type' value.
|
|
56
|
+
"""
|
|
57
|
+
if not (isinstance(model_class, type) and issubclass(model_class, OvertureFeature)):
|
|
58
|
+
return False
|
|
59
|
+
|
|
60
|
+
return _type_literal(cast(type[OvertureFeature], model_class)) is not None
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def _type_literal(feature_class: type[OvertureFeature]) -> str | None:
|
|
64
|
+
"""Extract the literal value from an OvertureFeature's 'type' field.
|
|
65
|
+
|
|
66
|
+
Returns the literal type value, or None if not a single literal.
|
|
67
|
+
"""
|
|
68
|
+
if "type" not in feature_class.model_fields:
|
|
69
|
+
return None
|
|
70
|
+
|
|
71
|
+
type_annotation = feature_class.model_fields["type"].annotation
|
|
72
|
+
|
|
73
|
+
# Unwrap Annotated if present
|
|
74
|
+
while get_origin(type_annotation) is Annotated:
|
|
75
|
+
type_annotation = get_args(type_annotation)[0]
|
|
76
|
+
|
|
77
|
+
# Check if it's a Literal with a single value
|
|
78
|
+
if get_origin(type_annotation) is Literal:
|
|
79
|
+
args = get_args(type_annotation)
|
|
80
|
+
if len(args) == 1 and isinstance(args[0], str):
|
|
81
|
+
return args[0]
|
|
82
|
+
|
|
83
|
+
return None
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def _discriminated_union(feature_classes: tuple[type[OvertureFeature], ...]) -> Any: # noqa: ANN401
|
|
87
|
+
"""Create a discriminated union of Overture features on the 'type' field."""
|
|
88
|
+
if not feature_classes:
|
|
89
|
+
return None
|
|
90
|
+
elif len(feature_classes) == 1:
|
|
91
|
+
# Single model doesn't need a discriminated union
|
|
92
|
+
return feature_classes[0]
|
|
93
|
+
|
|
94
|
+
return Annotated[
|
|
95
|
+
reduce(
|
|
96
|
+
or_,
|
|
97
|
+
(Annotated[f, Tag(cast(str, _type_literal(f)))] for f in feature_classes),
|
|
98
|
+
),
|
|
99
|
+
Field(discriminator=Feature.field_discriminator("type", *feature_classes)),
|
|
100
|
+
]
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def create_union_type_from_models(
|
|
104
|
+
models: ModelDict,
|
|
105
|
+
) -> UnionType:
|
|
106
|
+
"""Create a union type from a dict of models.
|
|
107
|
+
|
|
108
|
+
Uses discriminated unions for OvertureFeatures when possible for better performance.
|
|
109
|
+
|
|
110
|
+
Args
|
|
111
|
+
----
|
|
112
|
+
models: Dict mapping ModelKey to Pydantic model classes
|
|
113
|
+
|
|
114
|
+
Returns
|
|
115
|
+
-------
|
|
116
|
+
Union type suitable for TypeAdapter
|
|
117
|
+
"""
|
|
118
|
+
if not models:
|
|
119
|
+
raise ValueError("No models provided")
|
|
120
|
+
|
|
121
|
+
model_list = list(models.values())
|
|
122
|
+
|
|
123
|
+
# Separate models that can be discriminated from those that cannot
|
|
124
|
+
discriminated_models = tuple(
|
|
125
|
+
cast(type[OvertureFeature], m) for m in model_list if _can_discriminate(m)
|
|
126
|
+
)
|
|
127
|
+
discriminated_union = _discriminated_union(discriminated_models)
|
|
128
|
+
|
|
129
|
+
non_discriminated_models = [m for m in model_list if not _can_discriminate(m)]
|
|
130
|
+
# Use None only if list is empty, otherwise build union
|
|
131
|
+
non_discriminated_union = (
|
|
132
|
+
reduce(or_, non_discriminated_models) if non_discriminated_models else None
|
|
133
|
+
)
|
|
134
|
+
|
|
135
|
+
# Combine discriminated and non-discriminated unions
|
|
136
|
+
if discriminated_union and non_discriminated_union:
|
|
137
|
+
return discriminated_union | non_discriminated_union
|
|
138
|
+
elif discriminated_union:
|
|
139
|
+
return discriminated_union
|
|
140
|
+
elif non_discriminated_union:
|
|
141
|
+
return non_discriminated_union
|
|
142
|
+
else:
|
|
143
|
+
raise RuntimeError("No valid models found")
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def validate_feature(data: dict, model_type: UnionType) -> BaseModel:
|
|
147
|
+
"""Validate a single feature against the model type.
|
|
148
|
+
|
|
149
|
+
Args
|
|
150
|
+
----
|
|
151
|
+
data: Feature data to validate (GeoJSON or flat format)
|
|
152
|
+
model_type: Union type for validation
|
|
153
|
+
|
|
154
|
+
Returns
|
|
155
|
+
-------
|
|
156
|
+
Validated model instance
|
|
157
|
+
|
|
158
|
+
Raises
|
|
159
|
+
------
|
|
160
|
+
ValidationError: If validation fails
|
|
161
|
+
"""
|
|
162
|
+
adapter = TypeAdapter(model_type)
|
|
163
|
+
if isinstance(data, dict) and _is_geojson_feature(data):
|
|
164
|
+
# Use validate_json to trigger the model's GeoJSON handling
|
|
165
|
+
return cast(BaseModel, adapter.validate_json(json.dumps(data)))
|
|
166
|
+
return cast(BaseModel, adapter.validate_python(data))
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
def validate_features(data: list, model_type: UnionType) -> list[BaseModel]:
|
|
170
|
+
"""Validate a list of features against the model type.
|
|
171
|
+
|
|
172
|
+
Args
|
|
173
|
+
----
|
|
174
|
+
data: List of feature data to validate (GeoJSON or flat format)
|
|
175
|
+
model_type: Union type for validation
|
|
176
|
+
|
|
177
|
+
Returns
|
|
178
|
+
-------
|
|
179
|
+
List of validated model instances
|
|
180
|
+
|
|
181
|
+
Raises
|
|
182
|
+
------
|
|
183
|
+
ValidationError: If validation fails
|
|
184
|
+
"""
|
|
185
|
+
# Check if any items are GeoJSON features
|
|
186
|
+
has_geojson = any(
|
|
187
|
+
isinstance(item, dict) and _is_geojson_feature(item) for item in data
|
|
188
|
+
)
|
|
189
|
+
|
|
190
|
+
list_type = list[model_type] # type: ignore[misc,valid-type]
|
|
191
|
+
adapter = TypeAdapter(list_type)
|
|
192
|
+
|
|
193
|
+
if has_geojson:
|
|
194
|
+
# Use validate_json to trigger the model's GeoJSON handling
|
|
195
|
+
return cast(list[BaseModel], adapter.validate_json(json.dumps(data)))
|
|
196
|
+
return cast(list[BaseModel], adapter.validate_python(data))
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
def resolve_types(
|
|
200
|
+
selector: TagSelector = TagSelector(),
|
|
201
|
+
*,
|
|
202
|
+
type_names: tuple[str, ...] = (),
|
|
203
|
+
) -> UnionType:
|
|
204
|
+
"""Resolve a TagSelector + type-names into a Pydantic union type."""
|
|
205
|
+
models = discover_models()
|
|
206
|
+
models = filter_models(models, selector, type_names=type_names)
|
|
207
|
+
|
|
208
|
+
if not models:
|
|
209
|
+
raise ValueError("No models found matching the specified criteria")
|
|
210
|
+
|
|
211
|
+
return create_union_type_from_models(models)
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
def get_source_name(filename: Path) -> str:
|
|
215
|
+
"""Get display name for input source.
|
|
216
|
+
|
|
217
|
+
Args
|
|
218
|
+
----
|
|
219
|
+
filename: Path to input file or "-" for stdin
|
|
220
|
+
|
|
221
|
+
Returns
|
|
222
|
+
-------
|
|
223
|
+
Display name: "<stdin>" for stdin input, otherwise the filename
|
|
224
|
+
"""
|
|
225
|
+
return "<stdin>" if str(filename) == "-" else str(filename)
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
# Every `# noqa: D301` below is the same waiver, against `pydocstyle` (see
|
|
229
|
+
# the docformat-only target). D301 wants a raw string wherever a docstring
|
|
230
|
+
# contains a backslash, but `\b` here is Click's no-rewrap marker: a raw
|
|
231
|
+
# string hands Click two literal characters and every example block collapses
|
|
232
|
+
# into one paragraph. Any new command with an Examples block needs the waiver
|
|
233
|
+
# too. Note the placement is pydocstyle's -- ruff reports D301 at the
|
|
234
|
+
# docstring line instead, so selecting ruff's `D` rules would need its own.
|
|
235
|
+
@click.group()
|
|
236
|
+
@click.version_option(package_name="overture-schema")
|
|
237
|
+
def cli() -> None: # noqa: D301
|
|
238
|
+
"""Overture Schema command-line interface.
|
|
239
|
+
|
|
240
|
+
Provides validation, schema generation, and type discovery for Overture Maps data.
|
|
241
|
+
|
|
242
|
+
\b
|
|
243
|
+
Examples:
|
|
244
|
+
# Validate a file
|
|
245
|
+
$ overture-schema validate data.json
|
|
246
|
+
\b
|
|
247
|
+
# Validate from stdin
|
|
248
|
+
$ overture-schema validate - < data.json
|
|
249
|
+
\b
|
|
250
|
+
# List available types
|
|
251
|
+
$ overture-schema list-types
|
|
252
|
+
\b
|
|
253
|
+
# Generate JSON schema
|
|
254
|
+
$ overture-schema json-schema --tag overture:theme=buildings
|
|
255
|
+
\b
|
|
256
|
+
# Validate specific types
|
|
257
|
+
$ overture-schema validate --tag overture:theme=buildings data.json
|
|
258
|
+
"""
|
|
259
|
+
pass
|
|
260
|
+
|
|
261
|
+
|
|
262
|
+
def load_input(filename: Path) -> tuple[dict | list, str]:
|
|
263
|
+
"""Load and parse input from file or stdin.
|
|
264
|
+
|
|
265
|
+
Args
|
|
266
|
+
----
|
|
267
|
+
filename: Path to input file, or "-" for stdin
|
|
268
|
+
|
|
269
|
+
Returns
|
|
270
|
+
-------
|
|
271
|
+
Tuple of (parsed_data, source_name)
|
|
272
|
+
|
|
273
|
+
Raises
|
|
274
|
+
------
|
|
275
|
+
yaml.YAMLError: If input is invalid YAML/JSON
|
|
276
|
+
SystemExit: If filename doesn't exist or isn't a file
|
|
277
|
+
"""
|
|
278
|
+
if str(filename) == "-":
|
|
279
|
+
# Read all stdin content
|
|
280
|
+
content = sys.stdin.read()
|
|
281
|
+
|
|
282
|
+
# Try to detect JSONL format (newline-delimited JSON)
|
|
283
|
+
# JSONL has multiple non-empty lines, each containing a complete JSON object
|
|
284
|
+
lines = [line.strip() for line in content.strip().split("\n") if line.strip()]
|
|
285
|
+
|
|
286
|
+
if len(lines) > 1:
|
|
287
|
+
# Attempt to parse as JSONL
|
|
288
|
+
try:
|
|
289
|
+
parsed_lines = [json.loads(line) for line in lines]
|
|
290
|
+
return parsed_lines, "<stdin>"
|
|
291
|
+
except json.JSONDecodeError:
|
|
292
|
+
# Not valid JSONL, fall through to YAML parser
|
|
293
|
+
pass
|
|
294
|
+
|
|
295
|
+
# Parse as single YAML/JSON document
|
|
296
|
+
data = yaml.load(io.StringIO(content), Loader=CoreLoader)
|
|
297
|
+
return data, "<stdin>"
|
|
298
|
+
|
|
299
|
+
if not filename.is_file():
|
|
300
|
+
raise click.UsageError(f"'{filename}' is not a file.")
|
|
301
|
+
|
|
302
|
+
# Warn about unexpected file extensions
|
|
303
|
+
if filename.suffix not in {".json", ".yaml", ".yml", ".geojson"}:
|
|
304
|
+
click.echo(
|
|
305
|
+
f"Warning: File '{filename}' has unexpected extension. "
|
|
306
|
+
f"Expecting .json, .yaml, .yml, or .geojson",
|
|
307
|
+
err=True,
|
|
308
|
+
)
|
|
309
|
+
|
|
310
|
+
# Use YAML-1.2-compliant loader (YAML-1.2 dropped support for yes/no boolean values)
|
|
311
|
+
with filename.open("r", encoding="utf-8") as f:
|
|
312
|
+
data = yaml.load(f, Loader=CoreLoader)
|
|
313
|
+
|
|
314
|
+
return data, str(filename)
|
|
315
|
+
|
|
316
|
+
|
|
317
|
+
def perform_validation(data: dict | list, model_type: UnionType) -> None:
|
|
318
|
+
"""Validate data based on its structure.
|
|
319
|
+
|
|
320
|
+
Automatically detects and handles three input formats:
|
|
321
|
+
- Single feature (dict)
|
|
322
|
+
- List of features (list)
|
|
323
|
+
- GeoJSON FeatureCollection (dict with type="FeatureCollection")
|
|
324
|
+
|
|
325
|
+
Args
|
|
326
|
+
----
|
|
327
|
+
data : dict | list
|
|
328
|
+
Parsed data to validate
|
|
329
|
+
model_type : UnionType
|
|
330
|
+
Union type for validation
|
|
331
|
+
|
|
332
|
+
Raises
|
|
333
|
+
------
|
|
334
|
+
ValidationError
|
|
335
|
+
If validation fails
|
|
336
|
+
"""
|
|
337
|
+
if isinstance(data, list):
|
|
338
|
+
# List of features
|
|
339
|
+
validate_features(data, model_type)
|
|
340
|
+
elif isinstance(data, dict) and data.get("type") == "FeatureCollection":
|
|
341
|
+
# GeoJSON FeatureCollection
|
|
342
|
+
validate_features(data["features"], model_type)
|
|
343
|
+
else:
|
|
344
|
+
# Single feature
|
|
345
|
+
validate_feature(data, model_type)
|
|
346
|
+
|
|
347
|
+
|
|
348
|
+
def compute_collection_statistics(
|
|
349
|
+
item_types: dict[int, builtins.type[BaseModel] | None],
|
|
350
|
+
filtered_errors: list,
|
|
351
|
+
) -> tuple[
|
|
352
|
+
int,
|
|
353
|
+
Counter[builtins.type[BaseModel] | None],
|
|
354
|
+
dict[builtins.type[BaseModel], set[int]],
|
|
355
|
+
]:
|
|
356
|
+
"""Compute validation statistics for heterogeneous collections.
|
|
357
|
+
|
|
358
|
+
Args
|
|
359
|
+
----
|
|
360
|
+
item_types : dict[int, type[BaseModel] | None]
|
|
361
|
+
Mapping from item index to detected model type
|
|
362
|
+
filtered_errors : list
|
|
363
|
+
List of filtered validation errors
|
|
364
|
+
|
|
365
|
+
Returns
|
|
366
|
+
-------
|
|
367
|
+
tuple
|
|
368
|
+
Tuple of (items_without_errors, type_counts, items_with_errors_by_type)
|
|
369
|
+
"""
|
|
370
|
+
# Compute statistics: group items by type
|
|
371
|
+
type_counts: Counter[builtins.type[BaseModel] | None] = Counter(item_types.values())
|
|
372
|
+
|
|
373
|
+
# Determine total number of items (max index + 1, or count from data)
|
|
374
|
+
max_index = max(item_types.keys()) if item_types else -1
|
|
375
|
+
total_items = max_index + 1
|
|
376
|
+
|
|
377
|
+
# Count items with errors per type
|
|
378
|
+
items_with_errors_by_type: dict[builtins.type[BaseModel], set[int]] = {}
|
|
379
|
+
for err in filtered_errors:
|
|
380
|
+
idx = get_item_index(err["loc"])
|
|
381
|
+
if idx is not None and idx in item_types:
|
|
382
|
+
model_type_cls = item_types[idx]
|
|
383
|
+
if model_type_cls is not None:
|
|
384
|
+
if model_type_cls not in items_with_errors_by_type:
|
|
385
|
+
items_with_errors_by_type[model_type_cls] = set()
|
|
386
|
+
items_with_errors_by_type[model_type_cls].add(idx)
|
|
387
|
+
|
|
388
|
+
# Count items without any errors
|
|
389
|
+
items_without_errors = total_items - len(
|
|
390
|
+
{
|
|
391
|
+
idx
|
|
392
|
+
for idx in item_types.keys()
|
|
393
|
+
if any(get_item_index(err["loc"]) == idx for err in filtered_errors)
|
|
394
|
+
}
|
|
395
|
+
)
|
|
396
|
+
|
|
397
|
+
return items_without_errors, type_counts, items_with_errors_by_type
|
|
398
|
+
|
|
399
|
+
|
|
400
|
+
def print_collection_statistics(
|
|
401
|
+
items_without_errors: int,
|
|
402
|
+
type_counts: Counter[builtins.type[BaseModel] | None],
|
|
403
|
+
items_with_errors_by_type: dict[builtins.type[BaseModel], set[int]],
|
|
404
|
+
stderr: Console,
|
|
405
|
+
) -> None:
|
|
406
|
+
"""Print validation statistics for heterogeneous collections.
|
|
407
|
+
|
|
408
|
+
Args
|
|
409
|
+
----
|
|
410
|
+
items_without_errors : int
|
|
411
|
+
Count of items with no validation errors
|
|
412
|
+
type_counts : Counter[type[BaseModel] | None]
|
|
413
|
+
Counter of items by model type
|
|
414
|
+
items_with_errors_by_type : dict[type[BaseModel], set[int]]
|
|
415
|
+
Mapping from model type to set of item indices with errors
|
|
416
|
+
stderr : Console
|
|
417
|
+
Console for stderr output
|
|
418
|
+
"""
|
|
419
|
+
stderr.print(" [dim]Collection statistics:[/dim]")
|
|
420
|
+
|
|
421
|
+
# Show items without errors first
|
|
422
|
+
# TODO: Once we switch to parse_features (instead of validate_features),
|
|
423
|
+
# we can include type information for items without errors by parsing
|
|
424
|
+
# the input and tracking which items validated successfully and their types.
|
|
425
|
+
# This would allow output like: "Building: 2 confirmed (no errors)"
|
|
426
|
+
if items_without_errors > 0:
|
|
427
|
+
stderr.print(
|
|
428
|
+
f" • {items_without_errors} item{'s' if items_without_errors != 1 else ''} with no errors",
|
|
429
|
+
style="dim",
|
|
430
|
+
)
|
|
431
|
+
|
|
432
|
+
# Show per-type statistics
|
|
433
|
+
for model_type_cls, count in type_counts.most_common():
|
|
434
|
+
if model_type_cls is not None:
|
|
435
|
+
items_with_errors = len(
|
|
436
|
+
items_with_errors_by_type.get(model_type_cls, set())
|
|
437
|
+
)
|
|
438
|
+
valid_count = count - items_with_errors
|
|
439
|
+
|
|
440
|
+
if valid_count > 0:
|
|
441
|
+
stderr.print(
|
|
442
|
+
f" • {model_type_cls.__name__}: {valid_count} confirmed, {items_with_errors} with errors",
|
|
443
|
+
style="dim",
|
|
444
|
+
)
|
|
445
|
+
else:
|
|
446
|
+
stderr.print(
|
|
447
|
+
f" • {model_type_cls.__name__} (probable): {items_with_errors} item{'s' if items_with_errors != 1 else ''} with errors",
|
|
448
|
+
style="dim",
|
|
449
|
+
)
|
|
450
|
+
stderr.print()
|
|
451
|
+
|
|
452
|
+
|
|
453
|
+
def _best_fit_model(
|
|
454
|
+
item_data: object,
|
|
455
|
+
candidate_models: tuple[type[BaseModel], ...],
|
|
456
|
+
) -> tuple[type[BaseModel], list[ValidationErrorDict]] | None:
|
|
457
|
+
"""Find the candidate model that a single item best fits.
|
|
458
|
+
|
|
459
|
+
Validates `item_data` against each candidate and returns the model with
|
|
460
|
+
the fewest validation errors (the fewest changes needed to make the data
|
|
461
|
+
valid), together with those errors. Ties are broken deterministically by
|
|
462
|
+
candidate order. Returns `None` if no candidate produced errors (nothing
|
|
463
|
+
to re-home).
|
|
464
|
+
|
|
465
|
+
Parameters
|
|
466
|
+
----------
|
|
467
|
+
item_data : object
|
|
468
|
+
The single feature's data (GeoJSON or flat dict).
|
|
469
|
+
candidate_models : tuple[type[BaseModel], ...]
|
|
470
|
+
Candidate model classes to try, in a stable order.
|
|
471
|
+
|
|
472
|
+
Returns
|
|
473
|
+
-------
|
|
474
|
+
tuple[type[BaseModel], list[ValidationErrorDict]] | None
|
|
475
|
+
A (model, errors) tuple for the best fit, or None if no candidate
|
|
476
|
+
produced errors.
|
|
477
|
+
"""
|
|
478
|
+
best: tuple[int, type[BaseModel], list[ValidationErrorDict]] | None = None
|
|
479
|
+
for model in candidate_models:
|
|
480
|
+
try:
|
|
481
|
+
validate_feature(cast(dict, item_data), model)
|
|
482
|
+
except ValidationError as exc:
|
|
483
|
+
errs = exc.errors()
|
|
484
|
+
else:
|
|
485
|
+
# Item validated cleanly against this candidate; it is not the
|
|
486
|
+
# source of a failure, so it is not a useful "best fit" to report.
|
|
487
|
+
continue
|
|
488
|
+
if best is None or len(errs) < best[0]:
|
|
489
|
+
best = (len(errs), model, errs)
|
|
490
|
+
if best is None:
|
|
491
|
+
return None
|
|
492
|
+
return best[1], best[2]
|
|
493
|
+
|
|
494
|
+
|
|
495
|
+
def _revalidate_undiscriminatable_items(
|
|
496
|
+
errors: list[ValidationErrorDict],
|
|
497
|
+
original_data: dict | list | None,
|
|
498
|
+
candidate_models: tuple[type[BaseModel], ...],
|
|
499
|
+
) -> tuple[list[ValidationErrorDict], dict[int, type[BaseModel]]]:
|
|
500
|
+
"""Replace undiscriminatable-item noise with best-fit field errors.
|
|
501
|
+
|
|
502
|
+
When a list item cannot be discriminated (its errors are *entirely*
|
|
503
|
+
`union_tag_not_found`), pydantic cannot select a union branch and reports
|
|
504
|
+
only the opaque "unable to extract tag" message with no field-level detail,
|
|
505
|
+
resulting in useless validation output.
|
|
506
|
+
|
|
507
|
+
This function re-validates those non-discriminatable items against each
|
|
508
|
+
candidate model, picks the best fitting candidate model (fewest errors, see
|
|
509
|
+
`_best_fit_model`), and replaces the item's errors with that model's field
|
|
510
|
+
errors.
|
|
511
|
+
|
|
512
|
+
Items that already have at least one concrete (non-`union_tag_not_found`)
|
|
513
|
+
error are left untouched because for them, pydantic has already produced
|
|
514
|
+
useful field-level detail (as happens when the union contains a plain,
|
|
515
|
+
non-discriminated member).
|
|
516
|
+
|
|
517
|
+
Parameters
|
|
518
|
+
----------
|
|
519
|
+
errors : list[ValidationErrorDict]
|
|
520
|
+
The raw validation errors from `ValidationError.errors()`.
|
|
521
|
+
original_data : dict | list | None
|
|
522
|
+
The original parsed input (only lists are handled here).
|
|
523
|
+
candidate_models : tuple[type[BaseModel], ...]
|
|
524
|
+
Candidate model classes to try, in a stable order.
|
|
525
|
+
|
|
526
|
+
Returns
|
|
527
|
+
-------
|
|
528
|
+
tuple[list[ValidationErrorDict], dict[int, type[BaseModel]]]
|
|
529
|
+
A tuple of (possibly-augmented errors, {item_index: best_fit_model}).
|
|
530
|
+
The mapping is used to label each re-homed item's type in the display.
|
|
531
|
+
"""
|
|
532
|
+
if not isinstance(original_data, list):
|
|
533
|
+
return errors, {}
|
|
534
|
+
|
|
535
|
+
errors_by_item: dict[int | None, list[ValidationErrorDict]] = defaultdict(list)
|
|
536
|
+
for error in errors:
|
|
537
|
+
errors_by_item[get_item_index(error["loc"])].append(error)
|
|
538
|
+
|
|
539
|
+
augmented: list[ValidationErrorDict] = []
|
|
540
|
+
best_fit_types: dict[int, type[BaseModel]] = {}
|
|
541
|
+
|
|
542
|
+
for item_idx, item_errors in errors_by_item.items():
|
|
543
|
+
if (
|
|
544
|
+
item_idx is None
|
|
545
|
+
or not (0 <= item_idx < len(original_data))
|
|
546
|
+
or not all(e.get("type") == "union_tag_not_found" for e in item_errors)
|
|
547
|
+
):
|
|
548
|
+
augmented.extend(item_errors)
|
|
549
|
+
continue
|
|
550
|
+
|
|
551
|
+
best = _best_fit_model(original_data[item_idx], candidate_models)
|
|
552
|
+
if best is None:
|
|
553
|
+
augmented.extend(item_errors)
|
|
554
|
+
continue
|
|
555
|
+
|
|
556
|
+
best_model, best_errors = best
|
|
557
|
+
best_fit_types[item_idx] = best_model
|
|
558
|
+
for err in best_errors:
|
|
559
|
+
rehomed = dict(err)
|
|
560
|
+
rehomed["loc"] = (item_idx, *err["loc"])
|
|
561
|
+
augmented.append(cast(ValidationErrorDict, rehomed))
|
|
562
|
+
|
|
563
|
+
return augmented, best_fit_types
|
|
564
|
+
|
|
565
|
+
|
|
566
|
+
def handle_validation_error(
|
|
567
|
+
e: ValidationError,
|
|
568
|
+
model_type: UnionType,
|
|
569
|
+
stderr: Console,
|
|
570
|
+
original_data: dict | list | None = None,
|
|
571
|
+
show_fields: list[str] | None = None,
|
|
572
|
+
) -> None:
|
|
573
|
+
"""Handle and format validation errors with rich contextual information.
|
|
574
|
+
|
|
575
|
+
Groups errors by discriminator, selects most likely error groups, and provides
|
|
576
|
+
helpful diagnostics for heterogeneous collections and ambiguous types.
|
|
577
|
+
|
|
578
|
+
Args
|
|
579
|
+
----
|
|
580
|
+
e : ValidationError
|
|
581
|
+
ValidationError from pydantic
|
|
582
|
+
model_type : UnionType
|
|
583
|
+
Union type used for validation
|
|
584
|
+
stderr : Console
|
|
585
|
+
Console for stderr output
|
|
586
|
+
original_data : dict | list | None
|
|
587
|
+
Original input data for error display
|
|
588
|
+
show_fields : list[str] | None
|
|
589
|
+
List of field names to display alongside errors
|
|
590
|
+
"""
|
|
591
|
+
# Compute metadata once upfront
|
|
592
|
+
metadata = introspect_union(model_type)
|
|
593
|
+
|
|
594
|
+
# Create cache for structural tuple computation (optimizes systematic errors)
|
|
595
|
+
structural_cache: dict[ErrorLocation, StructuralTuple] = {}
|
|
596
|
+
|
|
597
|
+
# For list items that cannot be discriminated at all (errors are entirely
|
|
598
|
+
# union_tag_not_found), replace the opaque "unable to extract tag" noise
|
|
599
|
+
# with field-level errors from the best-fit candidate model, re-homed under
|
|
600
|
+
# the item index so they render as normal field errors.
|
|
601
|
+
errors, best_fit_item_types = _revalidate_undiscriminatable_items(
|
|
602
|
+
e.errors(),
|
|
603
|
+
original_data,
|
|
604
|
+
tuple(dict.fromkeys(metadata.discriminator_to_model.values())),
|
|
605
|
+
)
|
|
606
|
+
|
|
607
|
+
# Group errors by discriminator path and select most likely group(s)
|
|
608
|
+
error_groups = group_errors_by_discriminator(errors, metadata, structural_cache)
|
|
609
|
+
filtered_errors, is_tied, is_heterogeneous, item_types = select_most_likely_errors(
|
|
610
|
+
error_groups,
|
|
611
|
+
metadata=metadata,
|
|
612
|
+
all_errors=errors,
|
|
613
|
+
structural_cache=structural_cache,
|
|
614
|
+
)
|
|
615
|
+
|
|
616
|
+
# Label re-homed best-fit items with their inferred type.
|
|
617
|
+
item_types.update(best_fit_item_types)
|
|
618
|
+
|
|
619
|
+
# Show heterogeneity warning if collection has mixed types
|
|
620
|
+
if is_heterogeneous:
|
|
621
|
+
stderr.print(
|
|
622
|
+
" ⚠ Heterogeneous collection: Data contains multiple feature types. Consider:",
|
|
623
|
+
style="yellow",
|
|
624
|
+
)
|
|
625
|
+
stderr.print(
|
|
626
|
+
" • Validating each type separately with --tag, --filter, "
|
|
627
|
+
"--exclude, or --type",
|
|
628
|
+
style="dim",
|
|
629
|
+
)
|
|
630
|
+
stderr.print()
|
|
631
|
+
|
|
632
|
+
# Compute and display statistics if there are errors to report
|
|
633
|
+
if filtered_errors:
|
|
634
|
+
items_without_errors, type_counts, items_with_errors_by_type = (
|
|
635
|
+
compute_collection_statistics(item_types, filtered_errors)
|
|
636
|
+
)
|
|
637
|
+
print_collection_statistics(
|
|
638
|
+
items_without_errors, type_counts, items_with_errors_by_type, stderr
|
|
639
|
+
)
|
|
640
|
+
|
|
641
|
+
# Show tie indicator if multiple groups had same error count
|
|
642
|
+
elif is_tied:
|
|
643
|
+
stderr.print(
|
|
644
|
+
" ⚠ Ambiguous: Data matches multiple types equally. Consider:",
|
|
645
|
+
style="yellow",
|
|
646
|
+
)
|
|
647
|
+
stderr.print(
|
|
648
|
+
" • Specifying --tag or --type to narrow validation", style="dim"
|
|
649
|
+
)
|
|
650
|
+
stderr.print(" • Adding discriminator fields to clarify intent", style="dim")
|
|
651
|
+
stderr.print()
|
|
652
|
+
|
|
653
|
+
# Group errors by item
|
|
654
|
+
|
|
655
|
+
errors_by_item: dict[int | None, list] = defaultdict(list)
|
|
656
|
+
for error in filtered_errors:
|
|
657
|
+
item_idx = get_item_index(error["loc"])
|
|
658
|
+
errors_by_item[item_idx].append(error)
|
|
659
|
+
|
|
660
|
+
# Display errors grouped by item
|
|
661
|
+
|
|
662
|
+
for item_idx, item_errors in errors_by_item.items():
|
|
663
|
+
# Determine item type
|
|
664
|
+
error_item_type = None
|
|
665
|
+
if item_idx is not None and item_idx in item_types:
|
|
666
|
+
error_item_type = item_types.get(item_idx)
|
|
667
|
+
|
|
668
|
+
# Try verbose display first
|
|
669
|
+
displayed = format_validation_errors_verbose(
|
|
670
|
+
item_errors,
|
|
671
|
+
stderr,
|
|
672
|
+
metadata=metadata,
|
|
673
|
+
item_type=error_item_type,
|
|
674
|
+
structural_cache=structural_cache,
|
|
675
|
+
original_data=original_data,
|
|
676
|
+
item_index=item_idx,
|
|
677
|
+
show_fields=show_fields,
|
|
678
|
+
)
|
|
679
|
+
|
|
680
|
+
# Fall back to non-verbose format if verbose couldn't display
|
|
681
|
+
if not displayed:
|
|
682
|
+
for i, error in enumerate(item_errors):
|
|
683
|
+
format_validation_error(
|
|
684
|
+
error,
|
|
685
|
+
stderr,
|
|
686
|
+
metadata=metadata,
|
|
687
|
+
show_model_hint=(i == 0),
|
|
688
|
+
item_type=error_item_type,
|
|
689
|
+
show_item_type=is_heterogeneous,
|
|
690
|
+
structural_cache=structural_cache,
|
|
691
|
+
original_data=original_data,
|
|
692
|
+
show_feature_data=False,
|
|
693
|
+
)
|
|
694
|
+
|
|
695
|
+
|
|
696
|
+
def handle_generic_error(e: Exception, filename: Path, error_type: str) -> None:
|
|
697
|
+
"""Handle generic errors during validation.
|
|
698
|
+
|
|
699
|
+
Args
|
|
700
|
+
----
|
|
701
|
+
e : Exception
|
|
702
|
+
Exception that occurred
|
|
703
|
+
filename : Path
|
|
704
|
+
Input filename or "-" for stdin
|
|
705
|
+
error_type : str
|
|
706
|
+
Type of error for user-friendly message
|
|
707
|
+
|
|
708
|
+
Raises
|
|
709
|
+
------
|
|
710
|
+
click.UsageError
|
|
711
|
+
Always, with formatted error message
|
|
712
|
+
"""
|
|
713
|
+
source_name = get_source_name(filename)
|
|
714
|
+
|
|
715
|
+
if error_type == "yaml":
|
|
716
|
+
raise click.UsageError(f"'{source_name}' contains invalid input: {e}")
|
|
717
|
+
elif error_type == "value":
|
|
718
|
+
raise click.UsageError(str(e))
|
|
719
|
+
elif error_type == "key":
|
|
720
|
+
raise click.UsageError(f"Invalid data structure - missing key: {e}")
|
|
721
|
+
else:
|
|
722
|
+
raise click.UsageError(f"Error processing {source_name}: {e}")
|
|
723
|
+
|
|
724
|
+
|
|
725
|
+
@cli.command()
|
|
726
|
+
@click.argument("filename", type=click.Path(path_type=Path), required=True)
|
|
727
|
+
@tag_selection_options
|
|
728
|
+
@click.option(
|
|
729
|
+
"--type",
|
|
730
|
+
"types",
|
|
731
|
+
multiple=True,
|
|
732
|
+
help="Specific type to validate against (e.g., building, segment)",
|
|
733
|
+
)
|
|
734
|
+
@click.option(
|
|
735
|
+
"--show-field",
|
|
736
|
+
"show_fields",
|
|
737
|
+
multiple=True,
|
|
738
|
+
help="Field to display alongside errors (e.g., id, version). Can be repeated.",
|
|
739
|
+
)
|
|
740
|
+
def validate(
|
|
741
|
+
filename: Path,
|
|
742
|
+
tags: tuple[str, ...],
|
|
743
|
+
filters: tuple[str, ...],
|
|
744
|
+
excludes: tuple[str, ...],
|
|
745
|
+
types: tuple[str, ...],
|
|
746
|
+
show_fields: tuple[str, ...],
|
|
747
|
+
) -> None: # noqa: D301
|
|
748
|
+
"""Validate Overture Maps data against schemas.
|
|
749
|
+
|
|
750
|
+
Read from FILENAME or stdin if FILENAME is '-'.
|
|
751
|
+
Supports JSON, YAML, and GeoJSON formats.
|
|
752
|
+
|
|
753
|
+
\b
|
|
754
|
+
Examples:
|
|
755
|
+
# Validate a file
|
|
756
|
+
$ overture-schema validate data.json
|
|
757
|
+
\b
|
|
758
|
+
# Validate from stdin
|
|
759
|
+
$ overture-schema validate - < data.json
|
|
760
|
+
\b
|
|
761
|
+
# Validate only buildings
|
|
762
|
+
$ overture-schema validate --tag overture:theme=buildings data.json
|
|
763
|
+
\b
|
|
764
|
+
# Validate specific type
|
|
765
|
+
$ overture-schema validate --type building data.json
|
|
766
|
+
\b
|
|
767
|
+
# Two themes at once (repeatable; scope is their union)
|
|
768
|
+
$ overture-schema validate --tag overture:theme=buildings \\
|
|
769
|
+
--tag overture:theme=places data.json
|
|
770
|
+
\b
|
|
771
|
+
# Only types built on the Overture feature model
|
|
772
|
+
$ overture-schema validate --tag overture data.json
|
|
773
|
+
"""
|
|
774
|
+
# Resolve model type first (errors here are ValueErrors, not ValidationErrors)
|
|
775
|
+
try:
|
|
776
|
+
model_type = resolve_types(
|
|
777
|
+
build_selector(tags, filters, excludes), type_names=types
|
|
778
|
+
)
|
|
779
|
+
except ValueError as e:
|
|
780
|
+
handle_generic_error(e, filename, "value")
|
|
781
|
+
return
|
|
782
|
+
|
|
783
|
+
# Load input (errors here are YAMLErrors or ValueErrors, not ValidationErrors)
|
|
784
|
+
try:
|
|
785
|
+
data, source_name = load_input(filename)
|
|
786
|
+
except yaml.YAMLError as e:
|
|
787
|
+
handle_generic_error(e, filename, "yaml")
|
|
788
|
+
return
|
|
789
|
+
except KeyError as e:
|
|
790
|
+
handle_generic_error(e, filename, "key")
|
|
791
|
+
return
|
|
792
|
+
|
|
793
|
+
# Perform validation (now model_type and data are guaranteed to be defined)
|
|
794
|
+
try:
|
|
795
|
+
perform_validation(data, model_type)
|
|
796
|
+
stdout.print(f"✓ Successfully validated {source_name}")
|
|
797
|
+
except ValidationError as e:
|
|
798
|
+
handle_validation_error(
|
|
799
|
+
e, model_type, stderr, original_data=data, show_fields=list(show_fields)
|
|
800
|
+
)
|
|
801
|
+
sys.exit(1)
|
|
802
|
+
|
|
803
|
+
|
|
804
|
+
@cli.command("json-schema")
|
|
805
|
+
@tag_selection_options
|
|
806
|
+
@click.option(
|
|
807
|
+
"--type",
|
|
808
|
+
"types",
|
|
809
|
+
multiple=True,
|
|
810
|
+
help="Specific type to generate schema for (e.g., building, segment)",
|
|
811
|
+
)
|
|
812
|
+
def json_schema_command(
|
|
813
|
+
tags: tuple[str, ...],
|
|
814
|
+
filters: tuple[str, ...],
|
|
815
|
+
excludes: tuple[str, ...],
|
|
816
|
+
types: tuple[str, ...],
|
|
817
|
+
) -> None: # noqa: D301
|
|
818
|
+
"""Generate JSON schema for Overture Maps types.
|
|
819
|
+
|
|
820
|
+
Outputs a JSON Schema document to stdout that can be used for validation
|
|
821
|
+
or documentation purposes.
|
|
822
|
+
|
|
823
|
+
\b
|
|
824
|
+
Examples:
|
|
825
|
+
# All types
|
|
826
|
+
$ overture-schema json-schema > schema.json
|
|
827
|
+
\b
|
|
828
|
+
# Buildings theme by tag
|
|
829
|
+
$ overture-schema json-schema --tag overture:theme=buildings
|
|
830
|
+
\b
|
|
831
|
+
# Specific types
|
|
832
|
+
$ overture-schema json-schema --type building
|
|
833
|
+
\b
|
|
834
|
+
# Two themes at once (repeatable; scope is their union)
|
|
835
|
+
$ overture-schema json-schema --tag overture:theme=buildings \\
|
|
836
|
+
--tag overture:theme=places
|
|
837
|
+
\b
|
|
838
|
+
# Only types built on the Overture feature model
|
|
839
|
+
$ overture-schema json-schema --tag overture
|
|
840
|
+
"""
|
|
841
|
+
try:
|
|
842
|
+
model_type = resolve_types(
|
|
843
|
+
build_selector(tags, filters, excludes), type_names=types
|
|
844
|
+
)
|
|
845
|
+
schema = json_schema(model_type)
|
|
846
|
+
# Use plain print for JSON output to avoid Rich formatting
|
|
847
|
+
print(json.dumps(schema, indent=2, sort_keys=True))
|
|
848
|
+
except ValueError as e:
|
|
849
|
+
raise click.UsageError(str(e)) from e
|
|
850
|
+
|
|
851
|
+
|
|
852
|
+
@cli.command("list-types")
|
|
853
|
+
@tag_selection_options
|
|
854
|
+
@click.option(
|
|
855
|
+
"--group-by",
|
|
856
|
+
help="Group types by a key/value tag's key, as in "
|
|
857
|
+
"--group-by overture:theme. "
|
|
858
|
+
"Plain and namespaced tags have no value to group by and are "
|
|
859
|
+
"ignored here.",
|
|
860
|
+
)
|
|
861
|
+
def list_types(
|
|
862
|
+
tags: tuple[str, ...],
|
|
863
|
+
filters: tuple[str, ...],
|
|
864
|
+
excludes: tuple[str, ...],
|
|
865
|
+
group_by: str | None,
|
|
866
|
+
) -> None: # noqa: D301
|
|
867
|
+
"""List all available types.
|
|
868
|
+
|
|
869
|
+
Displays all registered models and can be organized by grouping.
|
|
870
|
+
|
|
871
|
+
\b
|
|
872
|
+
Examples:
|
|
873
|
+
# List all types
|
|
874
|
+
$ overture-schema list-types
|
|
875
|
+
\b
|
|
876
|
+
# One theme
|
|
877
|
+
$ overture-schema list-types --tag overture:theme=buildings
|
|
878
|
+
\b
|
|
879
|
+
# Group the listing by theme
|
|
880
|
+
$ overture-schema list-types --group-by overture:theme
|
|
881
|
+
"""
|
|
882
|
+
try:
|
|
883
|
+
models = discover_models()
|
|
884
|
+
models = filter_models(models, build_selector(tags, filters, excludes))
|
|
885
|
+
|
|
886
|
+
if group_by:
|
|
887
|
+
grouped_models: dict[str, set[ModelKey]] = {}
|
|
888
|
+
|
|
889
|
+
for key in models.keys():
|
|
890
|
+
if groups := get_values_for_key(key.tags, group_by):
|
|
891
|
+
for group in groups:
|
|
892
|
+
grouped_models.setdefault(group, set()).add(key)
|
|
893
|
+
|
|
894
|
+
padding = (
|
|
895
|
+
max(
|
|
896
|
+
(len(key.name) for keys in grouped_models.values() for key in keys),
|
|
897
|
+
default=0,
|
|
898
|
+
)
|
|
899
|
+
+ 2
|
|
900
|
+
)
|
|
901
|
+
|
|
902
|
+
for group, keys in sorted(grouped_models.items()):
|
|
903
|
+
stdout.print(
|
|
904
|
+
f"[green bold]{group_by}={group} ({len(keys)})[/green bold]"
|
|
905
|
+
)
|
|
906
|
+
for key in sorted(keys, key=lambda k: k.name):
|
|
907
|
+
model = Text()
|
|
908
|
+
model.append("→ ", style="bright_black")
|
|
909
|
+
model.append(key.name, style="bold cyan")
|
|
910
|
+
model.pad_right(max(1, padding - len(key.name)))
|
|
911
|
+
model.append(" ".join(sorted(key.tags)))
|
|
912
|
+
stdout.print(model)
|
|
913
|
+
stdout.print()
|
|
914
|
+
|
|
915
|
+
else:
|
|
916
|
+
padding = max((len(key.name) for key in models.keys()), default=0) + 2
|
|
917
|
+
|
|
918
|
+
for key in sorted(models.keys(), key=lambda k: k.name):
|
|
919
|
+
model = Text()
|
|
920
|
+
model.append(key.name, style="bold cyan")
|
|
921
|
+
model.pad_right(max(1, padding - len(key.name)))
|
|
922
|
+
model.append(" ".join(sorted(key.tags)))
|
|
923
|
+
stdout.print(model)
|
|
924
|
+
|
|
925
|
+
except Exception as e:
|
|
926
|
+
click.echo(f"Error listing types: {e}", err=True)
|
|
927
|
+
|
|
928
|
+
|
|
929
|
+
if __name__ == "__main__":
|
|
930
|
+
cli()
|