overture-schema-codegen 0.1.1.dev0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- overture/schema/codegen/__init__.py +1 -0
- overture/schema/codegen/cli.py +228 -0
- overture/schema/codegen/extraction/__init__.py +0 -0
- overture/schema/codegen/extraction/docstring.py +46 -0
- overture/schema/codegen/extraction/enum_extraction.py +40 -0
- overture/schema/codegen/extraction/examples.py +367 -0
- overture/schema/codegen/extraction/field.py +172 -0
- overture/schema/codegen/extraction/field_constraints.py +185 -0
- overture/schema/codegen/extraction/field_walk.py +275 -0
- overture/schema/codegen/extraction/length_constraints.py +49 -0
- overture/schema/codegen/extraction/literal_alternatives.py +26 -0
- overture/schema/codegen/extraction/model_constraints.py +252 -0
- overture/schema/codegen/extraction/model_extraction.py +240 -0
- overture/schema/codegen/extraction/newtype_extraction.py +73 -0
- overture/schema/codegen/extraction/numeric_extraction.py +74 -0
- overture/schema/codegen/extraction/pydantic_extraction.py +33 -0
- overture/schema/codegen/extraction/specs.py +295 -0
- overture/schema/codegen/extraction/type_analyzer.py +693 -0
- overture/schema/codegen/extraction/type_registry.py +137 -0
- overture/schema/codegen/extraction/union_extraction.py +270 -0
- overture/schema/codegen/layout/__init__.py +0 -0
- overture/schema/codegen/layout/module_layout.py +139 -0
- overture/schema/codegen/layout/type_collection.py +122 -0
- overture/schema/codegen/markdown/__init__.py +0 -0
- overture/schema/codegen/markdown/link_computation.py +70 -0
- overture/schema/codegen/markdown/path_assignment.py +114 -0
- overture/schema/codegen/markdown/pipeline.py +198 -0
- overture/schema/codegen/markdown/renderer.py +641 -0
- overture/schema/codegen/markdown/reverse_references.py +169 -0
- overture/schema/codegen/markdown/templates/_used_by.md.jinja2 +10 -0
- overture/schema/codegen/markdown/templates/enum.md.jinja2 +13 -0
- overture/schema/codegen/markdown/templates/feature.md.jinja2 +45 -0
- overture/schema/codegen/markdown/templates/geometric.md.jinja2 +11 -0
- overture/schema/codegen/markdown/templates/newtype.md.jinja2 +17 -0
- overture/schema/codegen/markdown/templates/numeric.md.jinja2 +27 -0
- overture/schema/codegen/markdown/templates/pydantic_type.md.jinja2 +8 -0
- overture/schema/codegen/markdown/type_format.py +383 -0
- overture/schema/codegen/py.typed +0 -0
- overture/schema/codegen/pyspark/__init__.py +1 -0
- overture/schema/codegen/pyspark/_primitive_fill.py +23 -0
- overture/schema/codegen/pyspark/_render_common.py +477 -0
- overture/schema/codegen/pyspark/check_builder.py +961 -0
- overture/schema/codegen/pyspark/check_ir.py +223 -0
- overture/schema/codegen/pyspark/constraint_dispatch.py +753 -0
- overture/schema/codegen/pyspark/pipeline.py +220 -0
- overture/schema/codegen/pyspark/renderer.py +816 -0
- overture/schema/codegen/pyspark/schema_builder.py +187 -0
- overture/schema/codegen/pyspark/templates/_check_function.py.jinja2 +10 -0
- overture/schema/codegen/pyspark/templates/model_module.py.jinja2 +83 -0
- overture/schema/codegen/pyspark/templates/test_module.py.jinja2 +129 -0
- overture/schema/codegen/pyspark/test_data/__init__.py +9 -0
- overture/schema/codegen/pyspark/test_data/base_row.py +835 -0
- overture/schema/codegen/pyspark/test_data/constraint_values.py +203 -0
- overture/schema/codegen/pyspark/test_data/invalid_value.py +105 -0
- overture/schema/codegen/pyspark/test_data/scaffold.py +390 -0
- overture/schema/codegen/pyspark/test_renderer.py +708 -0
- overture/schema/codegen/spec_discovery.py +66 -0
- overture_schema_codegen-0.1.1.dev0.dist-info/METADATA +13 -0
- overture_schema_codegen-0.1.1.dev0.dist-info/RECORD +61 -0
- overture_schema_codegen-0.1.1.dev0.dist-info/WHEEL +4 -0
- overture_schema_codegen-0.1.1.dev0.dist-info/entry_points.txt +3 -0
|
@@ -0,0 +1,753 @@
|
|
|
1
|
+
"""Constraint type to PySpark expression descriptor dispatch.
|
|
2
|
+
|
|
3
|
+
Pure mapping from constraint objects to expression descriptors.
|
|
4
|
+
No awareness of field paths, list depth, or struct nesting --
|
|
5
|
+
those are composition concerns handled by check_builder.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import re
|
|
11
|
+
from collections.abc import Callable, Mapping
|
|
12
|
+
from dataclasses import dataclass
|
|
13
|
+
from typing import Any, NamedTuple, TypeAlias
|
|
14
|
+
|
|
15
|
+
from annotated_types import Ge, Gt, Interval, Le, Lt, MultipleOf
|
|
16
|
+
from pydantic import Strict
|
|
17
|
+
from pydantic._internal._fields import PydanticMetadata
|
|
18
|
+
|
|
19
|
+
from overture.schema.system.case import to_snake_case
|
|
20
|
+
from overture.schema.system.field_constraint.collection import UniqueItemsConstraint
|
|
21
|
+
from overture.schema.system.field_constraint.string import (
|
|
22
|
+
JsonPointerConstraint,
|
|
23
|
+
PatternConstraint,
|
|
24
|
+
StrippedConstraint,
|
|
25
|
+
)
|
|
26
|
+
from overture.schema.system.field_path import FieldPath
|
|
27
|
+
from overture.schema.system.geometric import GeometryTypeConstraint
|
|
28
|
+
from overture.schema.system.model_constraint import (
|
|
29
|
+
Condition,
|
|
30
|
+
FieldEqCondition,
|
|
31
|
+
ForbidIfConstraint,
|
|
32
|
+
MinFieldsSetConstraint,
|
|
33
|
+
NoExtraFieldsConstraint,
|
|
34
|
+
Not,
|
|
35
|
+
RadioGroupConstraint,
|
|
36
|
+
RequireAnyOfConstraint,
|
|
37
|
+
RequireAnyTrueConstraint,
|
|
38
|
+
RequireIfConstraint,
|
|
39
|
+
)
|
|
40
|
+
from overture.schema.system.ref import Reference
|
|
41
|
+
|
|
42
|
+
from ..extraction.docstring import first_docstring_line
|
|
43
|
+
from ..extraction.field import FieldShape, ModelRef, Primitive
|
|
44
|
+
from ..extraction.field_walk import has_array_layer, terminal_of
|
|
45
|
+
from ..extraction.length_constraints import (
|
|
46
|
+
ArrayMaxLen,
|
|
47
|
+
ArrayMinLen,
|
|
48
|
+
ScalarMaxLen,
|
|
49
|
+
ScalarMinLen,
|
|
50
|
+
)
|
|
51
|
+
from ..extraction.literal_alternatives import LiteralAlternatives
|
|
52
|
+
from ..extraction.specs import FieldSpec
|
|
53
|
+
from ..extraction.type_registry import primitive_spark_category
|
|
54
|
+
from ._primitive_fill import PRIMITIVE_FILL_TABLE
|
|
55
|
+
|
|
56
|
+
__all__ = [
|
|
57
|
+
"ExpressionDescriptor",
|
|
58
|
+
"FieldEq",
|
|
59
|
+
"ForbidIf",
|
|
60
|
+
"MinFieldsSet",
|
|
61
|
+
"ModelConstraintDescriptor",
|
|
62
|
+
"RadioGroup",
|
|
63
|
+
"RequireAnyOf",
|
|
64
|
+
"RequireAnyTrue",
|
|
65
|
+
"RequireIf",
|
|
66
|
+
"dispatch_base_type",
|
|
67
|
+
"dispatch_constraint",
|
|
68
|
+
"dispatch_model_constraint",
|
|
69
|
+
"dispatch_newtype",
|
|
70
|
+
"forbid_if_field_shapes",
|
|
71
|
+
"model_constraint_function",
|
|
72
|
+
"model_mutation_function",
|
|
73
|
+
"parse_field_eq",
|
|
74
|
+
"require_bool_field_eq",
|
|
75
|
+
"require_field_eq",
|
|
76
|
+
]
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
@dataclass(frozen=True, slots=True)
|
|
80
|
+
class ExpressionDescriptor:
|
|
81
|
+
"""Describes a constraint_expressions function call.
|
|
82
|
+
|
|
83
|
+
`function` names the function (e.g., `"check_bounds"`).
|
|
84
|
+
`args` are positional arguments after `col` and `field`.
|
|
85
|
+
`kwargs` are keyword arguments, stored as a tuple of `(name, value)`
|
|
86
|
+
pairs so the descriptor is hashable -- consumers convert with `dict()`
|
|
87
|
+
when they need mapping access.
|
|
88
|
+
`constraint_type` is the Python class of the constraint that
|
|
89
|
+
produced this descriptor (e.g., `NoWhitespaceConstraint`),
|
|
90
|
+
used by test generators to pick pattern-appropriate mutation values.
|
|
91
|
+
`gate` is the structural path to a nullable ancestor struct; when set,
|
|
92
|
+
the renderer wraps the expression in `F.when(gate.isNotNull(), ...)`.
|
|
93
|
+
`label` is a human-readable description used in error messages
|
|
94
|
+
(e.g., `"ISO 3166-1 alpha-2 country code"`).
|
|
95
|
+
`check_name` overrides the Check.name derivation in error_key;
|
|
96
|
+
when None, the renderer strips the `check_` prefix from `function`.
|
|
97
|
+
"""
|
|
98
|
+
|
|
99
|
+
function: str
|
|
100
|
+
args: tuple[object, ...] = ()
|
|
101
|
+
kwargs: tuple[tuple[str, object], ...] = ()
|
|
102
|
+
constraint_type: type | None = None
|
|
103
|
+
gate: FieldPath | None = None
|
|
104
|
+
label: str | None = None
|
|
105
|
+
check_name: str | None = None
|
|
106
|
+
check_nan: bool | None = None
|
|
107
|
+
allow_literals: tuple[object, ...] = ()
|
|
108
|
+
"""Literal values that bypass this check.
|
|
109
|
+
|
|
110
|
+
When non-empty, the renderer wraps the generated call in
|
|
111
|
+
`except_literals(col, call, list(allow_literals))` so that a column
|
|
112
|
+
value matching one of these literals is treated as valid regardless of
|
|
113
|
+
what the check would otherwise report. Populated by `check_builder`
|
|
114
|
+
from `LiteralAlternatives` constraints on terminal scalars (e.g.
|
|
115
|
+
`HttpUrl | Literal[""]`). Never set on `check_required` descriptors.
|
|
116
|
+
"""
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
_BASE_TYPE_DISPATCH: dict[str, tuple[ExpressionDescriptor, ...]] = {
|
|
120
|
+
"HttpUrl": (
|
|
121
|
+
ExpressionDescriptor(function="check_url_format"),
|
|
122
|
+
ExpressionDescriptor(function="check_url_length"),
|
|
123
|
+
),
|
|
124
|
+
"EmailStr": (ExpressionDescriptor(function="check_email"),),
|
|
125
|
+
"BBox": (ExpressionDescriptor(function="check_bbox_completeness"),),
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
_NEWTYPE_DISPATCH: dict[str, tuple[ExpressionDescriptor, ...]] = {
|
|
129
|
+
"LinearlyReferencedRange": (
|
|
130
|
+
ExpressionDescriptor(function="check_linear_range_length"),
|
|
131
|
+
ExpressionDescriptor(function="check_linear_range_bounds"),
|
|
132
|
+
ExpressionDescriptor(function="check_linear_range_order"),
|
|
133
|
+
),
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
# re.UNICODE is Python's implicit default on compiled `str` patterns and needs
|
|
138
|
+
# no translation -- Java's regex engine is Unicode-aware without a flag.
|
|
139
|
+
# re.IGNORECASE maps to the inline `(?i)` flag Spark's rlike honors. A new
|
|
140
|
+
# supported flag with a visible matching effect also belongs in
|
|
141
|
+
# `field_constraints._DISPLAY_FLAG_LETTERS`, or docs will hide its behavior.
|
|
142
|
+
_SUPPORTED_PATTERN_FLAGS = re.IGNORECASE | re.UNICODE
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def compiled_pattern_source(pattern: re.Pattern[str]) -> str:
|
|
146
|
+
"""Return the Spark-regex source string for a compiled `re.Pattern`.
|
|
147
|
+
|
|
148
|
+
A compiled `re.Pattern` is the only Pydantic carrier for a flagged pattern
|
|
149
|
+
(a bare `Field(pattern=str)` cannot express `re.I`). Translates the flags
|
|
150
|
+
Spark's `rlike` can honor into inline prefixes -- `re.IGNORECASE` becomes
|
|
151
|
+
`(?i)`, the idiom `constraint_expressions.check_url_format` already uses.
|
|
152
|
+
The ASCII/Unicode case-folding divergence between Java and Python is the
|
|
153
|
+
same accepted divergence documented at `check_pattern`.
|
|
154
|
+
|
|
155
|
+
Raises
|
|
156
|
+
------
|
|
157
|
+
NotImplementedError
|
|
158
|
+
For any flag without a faithful `rlike` translation (e.g.
|
|
159
|
+
`re.MULTILINE`), naming the flag rather than silently dropping it.
|
|
160
|
+
"""
|
|
161
|
+
unsupported = re.RegexFlag(pattern.flags & ~_SUPPORTED_PATTERN_FLAGS)
|
|
162
|
+
if unsupported:
|
|
163
|
+
raise NotImplementedError(
|
|
164
|
+
f"check_pattern cannot translate regex flag {unsupported!r} to Spark rlike"
|
|
165
|
+
)
|
|
166
|
+
source = pattern.pattern
|
|
167
|
+
# Only IGNORECASE emits a prefix; UNICODE passes the gate but is a no-op
|
|
168
|
+
# (Java is Unicode-aware unflagged). A new supported flag needs its own
|
|
169
|
+
# translation clause here, or it will pass the gate and be silently dropped.
|
|
170
|
+
if pattern.flags & re.IGNORECASE:
|
|
171
|
+
source = f"(?i){source}"
|
|
172
|
+
return source
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
def normalize_anchor(pattern: str) -> str:
|
|
176
|
+
r"""Replace trailing `$` with `\z` for Java/Spark regex compatibility.
|
|
177
|
+
|
|
178
|
+
Uses backslash-parity to distinguish a real anchor from an escaped
|
|
179
|
+
literal `$`. Counts the run of backslashes immediately before the
|
|
180
|
+
final `$`: an even count means `$` is unescaped (convert to `\z`);
|
|
181
|
+
an odd count means it is an escaped literal `$` (leave unchanged).
|
|
182
|
+
"""
|
|
183
|
+
if not pattern.endswith("$"):
|
|
184
|
+
return pattern
|
|
185
|
+
prefix = pattern[:-1] # strip the trailing $
|
|
186
|
+
backslashes = len(prefix) - len(prefix.rstrip("\\"))
|
|
187
|
+
if backslashes % 2 == 0:
|
|
188
|
+
return prefix + r"\z"
|
|
189
|
+
return pattern
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
def _pattern_check_name(constraint: PatternConstraint) -> str:
|
|
193
|
+
"""Derive a snake_case check name from the constraint class name."""
|
|
194
|
+
if type(constraint) is PatternConstraint:
|
|
195
|
+
return "pattern"
|
|
196
|
+
return to_snake_case(type(constraint).__name__.removesuffix("Constraint"))
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
def _pattern_label(constraint: PatternConstraint) -> str:
|
|
200
|
+
"""Extract a human-readable label from a PatternConstraint."""
|
|
201
|
+
if constraint.description:
|
|
202
|
+
return constraint.description
|
|
203
|
+
if (summary := first_docstring_line(type(constraint).__doc__)) is not None:
|
|
204
|
+
return summary.rstrip(".")
|
|
205
|
+
name = type(constraint).__name__.removesuffix("Constraint")
|
|
206
|
+
return to_snake_case(name).replace("_", " ")
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
_ConstraintHandler = Callable[[Any, str | None], ExpressionDescriptor | None]
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
_BOUND_ATTRS = ("ge", "gt", "le", "lt")
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
def _dispatch_bounds(
|
|
216
|
+
constraint: Ge | Gt | Le | Lt | Interval,
|
|
217
|
+
base_type: str | None,
|
|
218
|
+
) -> ExpressionDescriptor:
|
|
219
|
+
"""Extract bound kwargs from an annotated_types constraint.
|
|
220
|
+
|
|
221
|
+
Coerces integer bound values to float on float-typed columns so
|
|
222
|
+
that generated test mutations match the Spark DoubleType column.
|
|
223
|
+
"""
|
|
224
|
+
is_float = base_type is not None and primitive_spark_category(base_type) == "float"
|
|
225
|
+
kwargs: list[tuple[str, object]] = []
|
|
226
|
+
for attr in _BOUND_ATTRS:
|
|
227
|
+
value = getattr(constraint, attr, None)
|
|
228
|
+
if value is not None:
|
|
229
|
+
if is_float and isinstance(value, int) and not isinstance(value, bool):
|
|
230
|
+
value = float(value)
|
|
231
|
+
kwargs.append((attr, value))
|
|
232
|
+
check_nan: bool | None = False if base_type is not None and not is_float else None
|
|
233
|
+
return ExpressionDescriptor(
|
|
234
|
+
function="check_bounds", kwargs=tuple(kwargs), check_nan=check_nan
|
|
235
|
+
)
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
def _dispatch_multiple_of(
|
|
239
|
+
constraint: MultipleOf,
|
|
240
|
+
_base_type: str | None,
|
|
241
|
+
) -> ExpressionDescriptor:
|
|
242
|
+
"""Map `Field(multiple_of=n)` to a check_multiple_of descriptor.
|
|
243
|
+
|
|
244
|
+
`check_multiple_of(col, n)` tests `col % n == 0`; `multiple_of=1` is the
|
|
245
|
+
integral (whole-number) case. The divisor rides in `args`, so any positive
|
|
246
|
+
`n` dispatches without special-casing.
|
|
247
|
+
"""
|
|
248
|
+
return ExpressionDescriptor(
|
|
249
|
+
function="check_multiple_of", args=(constraint.multiple_of,)
|
|
250
|
+
)
|
|
251
|
+
|
|
252
|
+
|
|
253
|
+
def _dispatch_pattern(
|
|
254
|
+
constraint: PatternConstraint,
|
|
255
|
+
_base_type: str | None,
|
|
256
|
+
) -> ExpressionDescriptor:
|
|
257
|
+
"""Map a PatternConstraint (or subclass) to a check_pattern descriptor.
|
|
258
|
+
|
|
259
|
+
The Python `re` pattern source is embedded verbatim (anchor and inline
|
|
260
|
+
flags aside) into a Java `rlike`. The two engines diverge on Unicode
|
|
261
|
+
shorthand classes and `.` line-terminator handling; that is an accepted
|
|
262
|
+
divergence, documented at `constraint_expressions.check_pattern`.
|
|
263
|
+
"""
|
|
264
|
+
return ExpressionDescriptor(
|
|
265
|
+
function="check_pattern",
|
|
266
|
+
args=(normalize_anchor(compiled_pattern_source(constraint.pattern)),),
|
|
267
|
+
constraint_type=type(constraint),
|
|
268
|
+
label=_pattern_label(constraint),
|
|
269
|
+
check_name=_pattern_check_name(constraint),
|
|
270
|
+
)
|
|
271
|
+
|
|
272
|
+
|
|
273
|
+
def _raw_pattern(constraint: object) -> str | None:
|
|
274
|
+
"""Return the Spark-regex source of raw pydantic `Field(pattern=)`, or None.
|
|
275
|
+
|
|
276
|
+
Pydantic represents `Field(pattern=...)` as a `PydanticMetadata` marker
|
|
277
|
+
(the private `_PydanticGeneralMetadata`) carrying the pattern as either a
|
|
278
|
+
`str` (`Field(pattern="...")`) or a compiled `re.Pattern`
|
|
279
|
+
(`Field(pattern=re.compile(...))` -- the only carrier for a flagged,
|
|
280
|
+
e.g. case-insensitive, pattern). The schema's own `PatternConstraint` is
|
|
281
|
+
handled earlier; raw metadata reaches here from `dict[K, V]` keys/values
|
|
282
|
+
that used `Field(pattern=)` rather than a schema constraint class
|
|
283
|
+
(e.g. `Sources.license_priority`).
|
|
284
|
+
|
|
285
|
+
The `PydanticMetadata` check -- not merely a `.pattern` attribute --
|
|
286
|
+
keeps `dispatch_constraint`'s fallback contract intact: an unrelated future
|
|
287
|
+
constraint that happens to expose a `.pattern` still raises `TypeError`
|
|
288
|
+
rather than being silently dispatched as a `check_pattern`. A compiled
|
|
289
|
+
pattern carrying an untranslatable flag raises `NotImplementedError` via
|
|
290
|
+
`compiled_pattern_source`.
|
|
291
|
+
"""
|
|
292
|
+
if not isinstance(constraint, PydanticMetadata):
|
|
293
|
+
return None
|
|
294
|
+
pattern = getattr(constraint, "pattern", None)
|
|
295
|
+
if isinstance(pattern, str):
|
|
296
|
+
return pattern
|
|
297
|
+
if isinstance(pattern, re.Pattern):
|
|
298
|
+
return compiled_pattern_source(pattern)
|
|
299
|
+
return None
|
|
300
|
+
|
|
301
|
+
|
|
302
|
+
# Ordered: the first matching entry wins, so any subclass relationship
|
|
303
|
+
# between keys must place the more-specific class first. StrippedConstraint
|
|
304
|
+
# subclasses PatternConstraint, so it must appear before the PatternConstraint
|
|
305
|
+
# fallback entry.
|
|
306
|
+
_CONSTRAINT_DISPATCH: list[tuple[type | tuple[type, ...], _ConstraintHandler]] = [
|
|
307
|
+
# LiteralAlternatives is a modifier threaded onto the field's other
|
|
308
|
+
# descriptors as allow_literals (by check_builder), not a standalone check.
|
|
309
|
+
((Reference, Strict, LiteralAlternatives), lambda _c, _bt: None),
|
|
310
|
+
((Ge, Gt, Le, Lt, Interval), _dispatch_bounds),
|
|
311
|
+
(
|
|
312
|
+
ArrayMinLen,
|
|
313
|
+
lambda c, _bt: ExpressionDescriptor(
|
|
314
|
+
function="check_array_min_length", args=(c.min_length,)
|
|
315
|
+
),
|
|
316
|
+
),
|
|
317
|
+
(
|
|
318
|
+
ArrayMaxLen,
|
|
319
|
+
lambda c, _bt: ExpressionDescriptor(
|
|
320
|
+
function="check_array_max_length", args=(c.max_length,)
|
|
321
|
+
),
|
|
322
|
+
),
|
|
323
|
+
(
|
|
324
|
+
ScalarMinLen,
|
|
325
|
+
lambda c, _bt: ExpressionDescriptor(
|
|
326
|
+
function="check_string_min_length", args=(c.min_length,)
|
|
327
|
+
),
|
|
328
|
+
),
|
|
329
|
+
(
|
|
330
|
+
ScalarMaxLen,
|
|
331
|
+
lambda c, _bt: ExpressionDescriptor(
|
|
332
|
+
function="check_string_max_length", args=(c.max_length,)
|
|
333
|
+
),
|
|
334
|
+
),
|
|
335
|
+
(
|
|
336
|
+
StrippedConstraint,
|
|
337
|
+
lambda c, _bt: ExpressionDescriptor(
|
|
338
|
+
function="check_stripped", constraint_type=type(c)
|
|
339
|
+
),
|
|
340
|
+
),
|
|
341
|
+
(
|
|
342
|
+
JsonPointerConstraint,
|
|
343
|
+
lambda c, _bt: ExpressionDescriptor(
|
|
344
|
+
function="check_json_pointer", constraint_type=type(c)
|
|
345
|
+
),
|
|
346
|
+
),
|
|
347
|
+
(PatternConstraint, _dispatch_pattern),
|
|
348
|
+
# check_struct_unique uses Spark's array_distinct: structural equality on
|
|
349
|
+
# whole elements, against the raw stored values. Pydantic's
|
|
350
|
+
# UniqueItemsConstraint on list[HttpUrl] compares *normalized* URLs
|
|
351
|
+
# (trailing-slash, lowercase host/scheme), so it catches duplicates that
|
|
352
|
+
# differ only in normalization. We accept that difference -- the PySpark
|
|
353
|
+
# check catches exact duplicates only.
|
|
354
|
+
(
|
|
355
|
+
UniqueItemsConstraint,
|
|
356
|
+
lambda _c, _bt: ExpressionDescriptor(function="check_struct_unique"),
|
|
357
|
+
),
|
|
358
|
+
(
|
|
359
|
+
GeometryTypeConstraint,
|
|
360
|
+
lambda c, _bt: ExpressionDescriptor(
|
|
361
|
+
function="check_geometry_type", args=tuple(c.allowed_types)
|
|
362
|
+
),
|
|
363
|
+
),
|
|
364
|
+
(MultipleOf, _dispatch_multiple_of),
|
|
365
|
+
]
|
|
366
|
+
|
|
367
|
+
|
|
368
|
+
def dispatch_constraint(
|
|
369
|
+
constraint: object,
|
|
370
|
+
*,
|
|
371
|
+
base_type: str | None = None,
|
|
372
|
+
) -> ExpressionDescriptor | None:
|
|
373
|
+
"""Map a constraint object to an expression descriptor.
|
|
374
|
+
|
|
375
|
+
Parameters
|
|
376
|
+
----------
|
|
377
|
+
constraint
|
|
378
|
+
The constraint object from `ConstraintSource.constraint`. Length
|
|
379
|
+
constraints arrive as `ArrayMinLen` / `ArrayMaxLen` /
|
|
380
|
+
`ScalarMinLen` / `ScalarMaxLen` -- the typed variants emitted
|
|
381
|
+
by `extraction.type_analyzer.attach_constraints`.
|
|
382
|
+
base_type
|
|
383
|
+
The field's terminal-scalar base type, used to detect float
|
|
384
|
+
bounds.
|
|
385
|
+
|
|
386
|
+
Returns
|
|
387
|
+
-------
|
|
388
|
+
ExpressionDescriptor or None
|
|
389
|
+
`None` for explicitly skipped constraints (Reference, Strict).
|
|
390
|
+
|
|
391
|
+
Raises
|
|
392
|
+
------
|
|
393
|
+
TypeError
|
|
394
|
+
For unrecognized constraint types.
|
|
395
|
+
"""
|
|
396
|
+
for key_types, handler in _CONSTRAINT_DISPATCH:
|
|
397
|
+
if isinstance(constraint, key_types):
|
|
398
|
+
return handler(constraint, base_type)
|
|
399
|
+
raw_pattern = _raw_pattern(constraint)
|
|
400
|
+
if raw_pattern is not None:
|
|
401
|
+
# Raw pydantic `Field(pattern=)` metadata. `constraint_type` stays
|
|
402
|
+
# None (the pydantic class is a private closure type, not a stable
|
|
403
|
+
# key); the curated valid/invalid pair lives in `PATTERN_VALUES`,
|
|
404
|
+
# keyed by the normalized pattern in `args`.
|
|
405
|
+
return ExpressionDescriptor(
|
|
406
|
+
function="check_pattern",
|
|
407
|
+
args=(normalize_anchor(raw_pattern),),
|
|
408
|
+
label="pattern",
|
|
409
|
+
)
|
|
410
|
+
raise TypeError(f"Unhandled constraint type: {type(constraint).__name__}")
|
|
411
|
+
|
|
412
|
+
|
|
413
|
+
def dispatch_newtype(newtype_name: str) -> tuple[ExpressionDescriptor, ...] | None:
|
|
414
|
+
"""Look up a NewType-level expression override.
|
|
415
|
+
|
|
416
|
+
Returns None when the NewType decomposes normally into
|
|
417
|
+
individual constraint dispatches.
|
|
418
|
+
"""
|
|
419
|
+
return _NEWTYPE_DISPATCH.get(newtype_name)
|
|
420
|
+
|
|
421
|
+
|
|
422
|
+
def dispatch_base_type(base_type: str) -> tuple[ExpressionDescriptor, ...] | None:
|
|
423
|
+
"""Look up a base-type-level expression override.
|
|
424
|
+
|
|
425
|
+
Handles primitive types like HttpUrl and EmailStr that carry no
|
|
426
|
+
Annotated constraints but need semantic validation functions.
|
|
427
|
+
"""
|
|
428
|
+
return _BASE_TYPE_DISPATCH.get(base_type)
|
|
429
|
+
|
|
430
|
+
|
|
431
|
+
class FieldEq(NamedTuple):
|
|
432
|
+
"""An unwrapped `FieldEqCondition`, with `negated` set when wrapped in `Not`."""
|
|
433
|
+
|
|
434
|
+
field_name: str
|
|
435
|
+
value: object
|
|
436
|
+
negated: bool
|
|
437
|
+
|
|
438
|
+
|
|
439
|
+
def parse_field_eq(condition: Condition) -> FieldEq | None:
|
|
440
|
+
"""Unwrap a `FieldEqCondition` or `Not(FieldEqCondition)`.
|
|
441
|
+
|
|
442
|
+
Returns a `FieldEq` triple for either shape, or `None` for any
|
|
443
|
+
other condition. `negated` is True iff the condition was wrapped
|
|
444
|
+
in `Not`.
|
|
445
|
+
"""
|
|
446
|
+
match condition:
|
|
447
|
+
case Not(inner=FieldEqCondition(field_name=fn, value=v)):
|
|
448
|
+
return FieldEq(fn, v, True)
|
|
449
|
+
case FieldEqCondition(field_name=fn, value=v):
|
|
450
|
+
return FieldEq(fn, v, False)
|
|
451
|
+
case _:
|
|
452
|
+
return None
|
|
453
|
+
|
|
454
|
+
|
|
455
|
+
def require_field_eq(condition: Condition) -> FieldEq:
|
|
456
|
+
"""Unwrap a field-equality condition, raising on any other shape.
|
|
457
|
+
|
|
458
|
+
The strict companion to `parse_field_eq`, for callers that only
|
|
459
|
+
handle `FieldEqCondition` / `Not(FieldEqCondition)`: a new condition
|
|
460
|
+
subtype fails loudly here, in one place, rather than slipping through
|
|
461
|
+
several independent `None` checks with drifting error messages.
|
|
462
|
+
"""
|
|
463
|
+
parsed = parse_field_eq(condition)
|
|
464
|
+
if parsed is None:
|
|
465
|
+
raise TypeError(f"Unhandled condition type: {type(condition).__name__}")
|
|
466
|
+
return parsed
|
|
467
|
+
|
|
468
|
+
|
|
469
|
+
def require_bool_field_eq(condition: Condition) -> FieldEq:
|
|
470
|
+
"""Unwrap a positive boolean `FieldEqCondition`, raising otherwise.
|
|
471
|
+
|
|
472
|
+
`require_any_true` PySpark support is limited to positive boolean-flag
|
|
473
|
+
equalities: the runtime coalesces a null condition to False (sound only
|
|
474
|
+
for a positive equality), and the test-data disabling value is the
|
|
475
|
+
boolean's negation. Negation or a non-boolean value raises here, so
|
|
476
|
+
every consumer -- dispatch, base-row synthesis, test rendering -- accepts
|
|
477
|
+
exactly the same set rather than each enforcing a different subset.
|
|
478
|
+
"""
|
|
479
|
+
field_eq = require_field_eq(condition)
|
|
480
|
+
if field_eq.negated or not isinstance(field_eq.value, bool):
|
|
481
|
+
raise TypeError(
|
|
482
|
+
"require_any_true PySpark generation supports only positive "
|
|
483
|
+
f"boolean FieldEqConditions; got {condition!r}"
|
|
484
|
+
)
|
|
485
|
+
return field_eq
|
|
486
|
+
|
|
487
|
+
|
|
488
|
+
@dataclass(frozen=True, slots=True)
|
|
489
|
+
class RequireAnyOf:
|
|
490
|
+
"""Descriptor for `check_require_any_of`: at least one field must be set."""
|
|
491
|
+
|
|
492
|
+
field_names: tuple[str, ...]
|
|
493
|
+
|
|
494
|
+
|
|
495
|
+
@dataclass(frozen=True, slots=True)
|
|
496
|
+
class RequireAnyTrue:
|
|
497
|
+
"""Descriptor for `check_require_any_true`: at least one condition holds.
|
|
498
|
+
|
|
499
|
+
Carries the raw `Condition`s so the renderer can lower each to a
|
|
500
|
+
Spark boolean expression (reusing the `require_if` condition path).
|
|
501
|
+
Unlike `RequireAnyOf`, which tests field presence, each condition
|
|
502
|
+
tests a field value -- the divisions case is all
|
|
503
|
+
`FieldEqCondition(field, True)`.
|
|
504
|
+
|
|
505
|
+
Construction enforces the positive-boolean invariant (via
|
|
506
|
+
`require_bool_field_eq`), so every instance is valid by construction and
|
|
507
|
+
the `field_names` and renderer paths need not re-check.
|
|
508
|
+
"""
|
|
509
|
+
|
|
510
|
+
conditions: tuple[Condition, ...]
|
|
511
|
+
|
|
512
|
+
def __post_init__(self) -> None:
|
|
513
|
+
for c in self.conditions:
|
|
514
|
+
require_bool_field_eq(c)
|
|
515
|
+
|
|
516
|
+
@property
|
|
517
|
+
def field_names(self) -> tuple[str, ...]:
|
|
518
|
+
"""Fields the conditions reference -- the columns this constraint governs.
|
|
519
|
+
|
|
520
|
+
Derived from `conditions` so `RequireAnyTrue` presents the same
|
|
521
|
+
`field_names` interface as the presence-based descriptors, letting
|
|
522
|
+
generic consumers (read-column tracking, node filtering) treat every
|
|
523
|
+
model-constraint descriptor uniformly. The value-vs-presence
|
|
524
|
+
distinction is carried by the descriptor type, not this attribute.
|
|
525
|
+
"""
|
|
526
|
+
return tuple(require_field_eq(c).field_name for c in self.conditions)
|
|
527
|
+
|
|
528
|
+
|
|
529
|
+
@dataclass(frozen=True, slots=True)
|
|
530
|
+
class RadioGroup:
|
|
531
|
+
"""Descriptor for `check_radio_group`: exactly one boolean field must be True."""
|
|
532
|
+
|
|
533
|
+
field_names: tuple[str, ...]
|
|
534
|
+
|
|
535
|
+
|
|
536
|
+
@dataclass(frozen=True, slots=True)
|
|
537
|
+
class RequireIf:
|
|
538
|
+
"""Descriptor for `check_require_if`: field required when condition holds."""
|
|
539
|
+
|
|
540
|
+
field_names: tuple[str, ...]
|
|
541
|
+
condition: Condition
|
|
542
|
+
|
|
543
|
+
|
|
544
|
+
@dataclass(frozen=True, slots=True)
|
|
545
|
+
class ForbidIf:
|
|
546
|
+
"""Descriptor for `check_forbid_if`: field must be absent when condition holds.
|
|
547
|
+
|
|
548
|
+
`field_shapes` pairs non-string-default field names with their `FieldShape`
|
|
549
|
+
so the test renderer can emit type-appropriate `fill_values` literals.
|
|
550
|
+
Stored as a tuple of `(name, shape)` pairs so the descriptor is
|
|
551
|
+
hashable; consumers convert with `dict()` when they need mapping
|
|
552
|
+
access. String fields are omitted because the renderer defaults to
|
|
553
|
+
`""` for them, which is correct. Arrays, model references, and
|
|
554
|
+
non-string scalars (int/uint/float/bool) require an explicit entry
|
|
555
|
+
so the renderer emits a typed literal (`[{}]`, `{}`, `0`, `False`, etc.).
|
|
556
|
+
"""
|
|
557
|
+
|
|
558
|
+
field_names: tuple[str, ...]
|
|
559
|
+
condition: Condition
|
|
560
|
+
field_shapes: tuple[tuple[str, FieldShape], ...]
|
|
561
|
+
|
|
562
|
+
|
|
563
|
+
@dataclass(frozen=True, slots=True)
|
|
564
|
+
class MinFieldsSet:
|
|
565
|
+
"""Descriptor for `check_min_fields_set`: at least `count` fields set.
|
|
566
|
+
|
|
567
|
+
Matches Pydantic's `model_fields_set` semantics: required fields are
|
|
568
|
+
always set (the constructor requires them) and contribute to the count
|
|
569
|
+
alongside any explicitly-set optional fields. Both kinds are passed to
|
|
570
|
+
the runtime check.
|
|
571
|
+
"""
|
|
572
|
+
|
|
573
|
+
field_names: tuple[str, ...]
|
|
574
|
+
count: int
|
|
575
|
+
|
|
576
|
+
|
|
577
|
+
ModelConstraintDescriptor: TypeAlias = (
|
|
578
|
+
RequireAnyOf | RequireAnyTrue | RadioGroup | RequireIf | ForbidIf | MinFieldsSet
|
|
579
|
+
)
|
|
580
|
+
"""One variant per model-constraint kind.
|
|
581
|
+
|
|
582
|
+
Each variant carries only the fields meaningful for that constraint;
|
|
583
|
+
`ForbidIf` adds `field_shapes` for non-string targets so the test
|
|
584
|
+
renderer can emit type-appropriate `fill_values` literals.
|
|
585
|
+
"""
|
|
586
|
+
|
|
587
|
+
|
|
588
|
+
def _first_required_leaf(field_spec: FieldSpec) -> str | None:
|
|
589
|
+
"""Return the name of the first required field in a MODEL-kind `FieldSpec`.
|
|
590
|
+
|
|
591
|
+
Returns `None` for fields whose terminal is anything but a
|
|
592
|
+
`ModelRef` (scalars, arrays, `UnionRef`s, etc.). The
|
|
593
|
+
`RequireAnyOf` unwrapping uses this to drill into a struct's
|
|
594
|
+
required leaf when one exists; non-struct terminals leave the
|
|
595
|
+
field name unwrapped, which is the correct behavior for scalars
|
|
596
|
+
and arrays. `UnionRef` returns `None` because picking one arm's
|
|
597
|
+
required leaf would silently bias the constraint to that arm.
|
|
598
|
+
"""
|
|
599
|
+
if has_array_layer(field_spec.shape):
|
|
600
|
+
return None
|
|
601
|
+
terminal = terminal_of(field_spec.shape)
|
|
602
|
+
if not isinstance(terminal, ModelRef):
|
|
603
|
+
return None
|
|
604
|
+
for sub in terminal.model.fields:
|
|
605
|
+
if sub.is_required:
|
|
606
|
+
return sub.name
|
|
607
|
+
return None
|
|
608
|
+
|
|
609
|
+
|
|
610
|
+
def _unwrap_require_any_of_names(
|
|
611
|
+
field_names: tuple[str, ...],
|
|
612
|
+
by_name: dict[str, FieldSpec],
|
|
613
|
+
) -> tuple[str, ...]:
|
|
614
|
+
"""Replace struct field names with their first required leaf path."""
|
|
615
|
+
result = []
|
|
616
|
+
for name in field_names:
|
|
617
|
+
field_spec = by_name.get(name)
|
|
618
|
+
leaf = _first_required_leaf(field_spec) if field_spec is not None else None
|
|
619
|
+
result.append(f"{name}.{leaf}" if leaf is not None else name)
|
|
620
|
+
return tuple(result)
|
|
621
|
+
|
|
622
|
+
|
|
623
|
+
def _needs_explicit_fill(shape: FieldShape) -> bool:
|
|
624
|
+
"""Whether `shape` needs an explicit (non-default-string) fill value.
|
|
625
|
+
|
|
626
|
+
Arrays and model references need `[{}]` / `{}` fill. Non-string
|
|
627
|
+
scalars (int/uint/float/bool families) need a typed fill (0, False,
|
|
628
|
+
etc.). Plain string scalars are omitted -- the `""` default is correct.
|
|
629
|
+
"""
|
|
630
|
+
if has_array_layer(shape):
|
|
631
|
+
return True
|
|
632
|
+
terminal = terminal_of(shape)
|
|
633
|
+
if isinstance(terminal, ModelRef):
|
|
634
|
+
return True
|
|
635
|
+
if not isinstance(terminal, Primitive):
|
|
636
|
+
return False
|
|
637
|
+
return primitive_spark_category(terminal.base_type) in PRIMITIVE_FILL_TABLE
|
|
638
|
+
|
|
639
|
+
|
|
640
|
+
def forbid_if_field_shapes(
|
|
641
|
+
field_names: tuple[str, ...],
|
|
642
|
+
shape_by_name: Mapping[str, FieldShape],
|
|
643
|
+
) -> tuple[tuple[str, FieldShape], ...]:
|
|
644
|
+
"""Build the `field_shapes` pairs for non-string ForbidIf targets.
|
|
645
|
+
|
|
646
|
+
Keeps fields whose shape is an array, a model reference, or a
|
|
647
|
+
non-string scalar (int/uint/float/bool families). String fields are
|
|
648
|
+
omitted because the test renderer defaults their fill value to `""`
|
|
649
|
+
without needing the shape.
|
|
650
|
+
"""
|
|
651
|
+
return tuple(
|
|
652
|
+
(name, shape)
|
|
653
|
+
for name in field_names
|
|
654
|
+
if (shape := shape_by_name.get(name)) is not None
|
|
655
|
+
and _needs_explicit_fill(shape)
|
|
656
|
+
)
|
|
657
|
+
|
|
658
|
+
|
|
659
|
+
def dispatch_model_constraint(
|
|
660
|
+
constraint: object,
|
|
661
|
+
fields: list[FieldSpec],
|
|
662
|
+
) -> tuple[ModelConstraintDescriptor, ...]:
|
|
663
|
+
"""Map a model-level constraint to fully constructed typed descriptors.
|
|
664
|
+
|
|
665
|
+
Parameters
|
|
666
|
+
----------
|
|
667
|
+
constraint
|
|
668
|
+
The model constraint object.
|
|
669
|
+
fields
|
|
670
|
+
All fields of the model. Branches consult them as needed --
|
|
671
|
+
`RequireAnyOf` and `ForbidIf` index by name, `MinFieldsSet`
|
|
672
|
+
enumerates every field (required and optional).
|
|
673
|
+
|
|
674
|
+
Returns
|
|
675
|
+
-------
|
|
676
|
+
tuple of ModelConstraintDescriptor
|
|
677
|
+
Empty tuple for explicitly skipped constraints (NoExtraFields).
|
|
678
|
+
Most kinds return a single-element tuple. Multi-field
|
|
679
|
+
`@require_if` / `@forbid_if` split into one descriptor per
|
|
680
|
+
target field because the runtime check functions take a single
|
|
681
|
+
target column each.
|
|
682
|
+
|
|
683
|
+
Raises
|
|
684
|
+
------
|
|
685
|
+
TypeError
|
|
686
|
+
For unrecognized constraint types.
|
|
687
|
+
"""
|
|
688
|
+
match constraint:
|
|
689
|
+
case NoExtraFieldsConstraint():
|
|
690
|
+
return ()
|
|
691
|
+
case RequireAnyOfConstraint():
|
|
692
|
+
unwrapped = _unwrap_require_any_of_names(
|
|
693
|
+
constraint.field_names, {f.name: f for f in fields}
|
|
694
|
+
)
|
|
695
|
+
return (RequireAnyOf(field_names=unwrapped),)
|
|
696
|
+
case RequireAnyTrueConstraint():
|
|
697
|
+
# `RequireAnyTrue.__post_init__` enforces the positive-boolean set
|
|
698
|
+
# the whole pipeline (runtime, renderer, base row, test renderer)
|
|
699
|
+
# depends on, so a bad condition raises here at construction.
|
|
700
|
+
return (RequireAnyTrue(conditions=constraint.conditions),)
|
|
701
|
+
case RadioGroupConstraint():
|
|
702
|
+
return (RadioGroup(field_names=constraint.field_names),)
|
|
703
|
+
case RequireIfConstraint():
|
|
704
|
+
# `@require_if(["a", "b"], cond)` means "all of a, b required when
|
|
705
|
+
# cond" -- one runtime check per field, since check_require_if
|
|
706
|
+
# takes a single target column.
|
|
707
|
+
return tuple(
|
|
708
|
+
RequireIf(field_names=(name,), condition=constraint.condition)
|
|
709
|
+
for name in constraint.field_names
|
|
710
|
+
)
|
|
711
|
+
case ForbidIfConstraint():
|
|
712
|
+
shapes_by_field = forbid_if_field_shapes(
|
|
713
|
+
constraint.field_names,
|
|
714
|
+
{f.name: f.shape for f in fields},
|
|
715
|
+
)
|
|
716
|
+
per_field_shapes = dict(shapes_by_field)
|
|
717
|
+
return tuple(
|
|
718
|
+
ForbidIf(
|
|
719
|
+
field_names=(name,),
|
|
720
|
+
condition=constraint.condition,
|
|
721
|
+
field_shapes=(
|
|
722
|
+
((name, per_field_shapes[name]),)
|
|
723
|
+
if name in per_field_shapes
|
|
724
|
+
else ()
|
|
725
|
+
),
|
|
726
|
+
)
|
|
727
|
+
for name in constraint.field_names
|
|
728
|
+
)
|
|
729
|
+
case MinFieldsSetConstraint():
|
|
730
|
+
all_names = tuple(f.name for f in fields)
|
|
731
|
+
return (MinFieldsSet(field_names=all_names, count=constraint.count),)
|
|
732
|
+
case _:
|
|
733
|
+
raise TypeError(f"Unhandled model constraint: {type(constraint).__name__}")
|
|
734
|
+
|
|
735
|
+
|
|
736
|
+
_MODEL_CONSTRAINT_DISPATCH: dict[type[ModelConstraintDescriptor], tuple[str, str]] = {
|
|
737
|
+
RequireAnyOf: ("check_require_any_of", "mutate_require_any_of"),
|
|
738
|
+
RequireAnyTrue: ("check_require_any_true", "mutate_require_any_true"),
|
|
739
|
+
RadioGroup: ("check_radio_group", "mutate_radio_group"),
|
|
740
|
+
RequireIf: ("check_require_if", "mutate_require_if"),
|
|
741
|
+
ForbidIf: ("check_forbid_if", "mutate_forbid_if"),
|
|
742
|
+
MinFieldsSet: ("check_min_fields_set", "mutate_min_fields_set"),
|
|
743
|
+
}
|
|
744
|
+
|
|
745
|
+
|
|
746
|
+
def model_constraint_function(d: ModelConstraintDescriptor) -> str:
|
|
747
|
+
"""Map a `ModelConstraintDescriptor` variant to its runtime function name."""
|
|
748
|
+
return _MODEL_CONSTRAINT_DISPATCH[type(d)][0]
|
|
749
|
+
|
|
750
|
+
|
|
751
|
+
def model_mutation_function(d: ModelConstraintDescriptor) -> str:
|
|
752
|
+
"""Map a `ModelConstraintDescriptor` variant to its test mutation helper."""
|
|
753
|
+
return _MODEL_CONSTRAINT_DISPATCH[type(d)][1]
|