calc-flow-python 4.0.0__cp313-abi3-win_amd64.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- calc_flow/__init__.py +169 -0
- calc_flow/_native.pyd +0 -0
- calc_flow/_native.pyi +366 -0
- calc_flow/array.py +1324 -0
- calc_flow/capabilities.py +775 -0
- calc_flow/config.py +219 -0
- calc_flow/errors.py +25 -0
- calc_flow/join_spec.py +156 -0
- calc_flow/pipeline.py +1123 -0
- calc_flow/py.typed +0 -0
- calc_flow/runtime.py +979 -0
- calc_flow/store.py +138 -0
- calc_flow/symbolic/__init__.py +65 -0
- calc_flow/symbolic/_generated_rolling_kernels.py +23 -0
- calc_flow/symbolic/analyzer.py +2280 -0
- calc_flow/symbolic/domains.py +77 -0
- calc_flow/symbolic/errors.py +58 -0
- calc_flow/symbolic/expr.py +662 -0
- calc_flow/symbolic/lower/__init__.py +31 -0
- calc_flow/symbolic/lower/planners.py +1270 -0
- calc_flow/symbolic/lower/program.py +840 -0
- calc_flow/symbolic/lower/segments.py +836 -0
- calc_flow/symbolic/lower/strategies.py +1472 -0
- calc_flow/symbolic/nodes.py +603 -0
- calc_flow/symbolic/ops.py +1155 -0
- calc_flow/symbolic/optimizer.py +600 -0
- calc_flow/symbolic/program.py +377 -0
- calc_flow/symbolic/types.py +110 -0
- calc_flow/symbolic/windows.py +153 -0
- calc_flow/udf.py +19 -0
- calc_flow_python-4.0.0.dist-info/METADATA +376 -0
- calc_flow_python-4.0.0.dist-info/RECORD +35 -0
- calc_flow_python-4.0.0.dist-info/WHEEL +4 -0
- calc_flow_python-4.0.0.dist-info/licenses/LICENSE +202 -0
- calc_flow_python-4.0.0.dist-info/sboms/calc-flow-python.cyclonedx.json +10081 -0
|
@@ -0,0 +1,2280 @@
|
|
|
1
|
+
"""Static analysis of symbolic programs.
|
|
2
|
+
|
|
3
|
+
The analyzer infers value types, domains, lineages, state requirements, and
|
|
4
|
+
stream safety from the declaration graph alone — no data object, source, sink,
|
|
5
|
+
or runner is accepted — and reports every finding as an immutable
|
|
6
|
+
``AnalysisIssue`` with a stable path rooted at a named program output or input.
|
|
7
|
+
|
|
8
|
+
Path grammar (frozen by this stage): a program output roots ``outputs.<name>``
|
|
9
|
+
and a declared input roots ``inputs.<name>``/``static_inputs.<name>``. A
|
|
10
|
+
derived field of a table node at path ``P`` is defined by the expression at
|
|
11
|
+
``P.<field_name>``; an argument of a node with primitive ``prim`` at path ``P``
|
|
12
|
+
sits at ``P.<prim>.<role>``; the failing aspect of an operand appends
|
|
13
|
+
``.dtype``, ``.lineage``, ``.shape[<i>]``, and so on.
|
|
14
|
+
|
|
15
|
+
Value-type inference proves only what the capability snapshot proves: identical
|
|
16
|
+
operand types, the frozen rolling/cross-section output-type table, and exact
|
|
17
|
+
field resolution. Cross-type arithmetic needs an explicit ``row.cast``; no
|
|
18
|
+
competing Python promotion table exists here.
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
from __future__ import annotations
|
|
22
|
+
|
|
23
|
+
from collections.abc import Callable
|
|
24
|
+
from dataclasses import dataclass
|
|
25
|
+
from typing import Final
|
|
26
|
+
|
|
27
|
+
from calc_flow.capabilities import RuntimeCapabilities
|
|
28
|
+
from calc_flow.pipeline import Runtime
|
|
29
|
+
from calc_flow.symbolic.domains import type_name
|
|
30
|
+
from calc_flow.symbolic.nodes import (
|
|
31
|
+
CBool,
|
|
32
|
+
CDType,
|
|
33
|
+
CEnum,
|
|
34
|
+
CFloat,
|
|
35
|
+
CInt,
|
|
36
|
+
CMap,
|
|
37
|
+
CNull,
|
|
38
|
+
CSeq,
|
|
39
|
+
CStr,
|
|
40
|
+
CValue,
|
|
41
|
+
Node,
|
|
42
|
+
)
|
|
43
|
+
from calc_flow.symbolic.types import CompileMode, Field
|
|
44
|
+
|
|
45
|
+
_MODES: Final[tuple[str, ...]] = ("batch", "stream")
|
|
46
|
+
_EVENT_TIME_TYPE: Final = "timestamp[us, UTC]"
|
|
47
|
+
_FLOATING_TYPES: Final = ("float32", "float64")
|
|
48
|
+
_SIGNED_INT_TYPES: Final = ("int8", "int16", "int32", "int64")
|
|
49
|
+
_UNSIGNED_INT_TYPES: Final = ("uint8", "uint16", "uint32", "uint64")
|
|
50
|
+
_NUMERIC_TYPES: Final = frozenset(
|
|
51
|
+
(*_SIGNED_INT_TYPES, *_UNSIGNED_INT_TYPES, *_FLOATING_TYPES)
|
|
52
|
+
)
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def _is_array_parameter(node: Node, /) -> bool:
|
|
56
|
+
kind = node.attr("kind") if node.op.name == "parameter" else None
|
|
57
|
+
return isinstance(kind, CEnum) and kind.variant == "array"
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def _sum_output_type(input_type: str | None, /) -> str:
|
|
61
|
+
if input_type in _SIGNED_INT_TYPES:
|
|
62
|
+
return "int64"
|
|
63
|
+
if input_type in _UNSIGNED_INT_TYPES:
|
|
64
|
+
return "uint64"
|
|
65
|
+
return "float64"
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def _rolling_output_type(primitive: str, input_type: str | None, /) -> str | None:
|
|
69
|
+
if primitive == "count":
|
|
70
|
+
return "uint64"
|
|
71
|
+
if primitive == "sum":
|
|
72
|
+
return _sum_output_type(input_type)
|
|
73
|
+
if primitive in ("min", "max"):
|
|
74
|
+
return input_type
|
|
75
|
+
return "float64"
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def _entity_field_is_valid(field: Field | None, /) -> bool:
|
|
79
|
+
return field is not None
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def _entity_field_message(field_name: str, /) -> str:
|
|
83
|
+
return f"entity field {field_name!r} is not in the input schema"
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def _sequence_field_is_valid(field: Field | None, /) -> bool:
|
|
87
|
+
return (
|
|
88
|
+
field is not None
|
|
89
|
+
and not field.nullable
|
|
90
|
+
and field.data_type not in _FLOATING_TYPES
|
|
91
|
+
)
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def _sequence_field_message(_field_name: str, /) -> str:
|
|
95
|
+
return (
|
|
96
|
+
"sequence fields must be non-null with a portable total order;"
|
|
97
|
+
" floating sequence fields are forbidden"
|
|
98
|
+
)
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
_FRAME_ATTRIBUTES: Final[dict[str, str]] = {"rows": "size", "duration": "micros"}
|
|
102
|
+
_ARITHMETIC: Final = frozenset({"add", "sub", "mul", "truediv"})
|
|
103
|
+
_COMPARISONS: Final = frozenset({"eq", "ne", "lt", "le", "gt", "ge"})
|
|
104
|
+
_BOOLEANS: Final = frozenset({"and", "or"})
|
|
105
|
+
_UNARY_NUMERIC: Final = frozenset({"log", "exp", "sqrt"})
|
|
106
|
+
_ROW_LOCAL_PRIMITIVES: Final = frozenset(
|
|
107
|
+
{
|
|
108
|
+
"column_ref",
|
|
109
|
+
"literal",
|
|
110
|
+
"add",
|
|
111
|
+
"sub",
|
|
112
|
+
"mul",
|
|
113
|
+
"truediv",
|
|
114
|
+
"neg",
|
|
115
|
+
"eq",
|
|
116
|
+
"ne",
|
|
117
|
+
"lt",
|
|
118
|
+
"le",
|
|
119
|
+
"gt",
|
|
120
|
+
"ge",
|
|
121
|
+
"and",
|
|
122
|
+
"or",
|
|
123
|
+
"not",
|
|
124
|
+
"where",
|
|
125
|
+
"coalesce",
|
|
126
|
+
"log",
|
|
127
|
+
"exp",
|
|
128
|
+
"sqrt",
|
|
129
|
+
"abs",
|
|
130
|
+
"clip",
|
|
131
|
+
"cast",
|
|
132
|
+
}
|
|
133
|
+
)
|
|
134
|
+
_ROLLING_AGGREGATES: Final = frozenset(
|
|
135
|
+
{"count", "sum", "mean", "min", "max", "variance", "stddev"}
|
|
136
|
+
)
|
|
137
|
+
_ROLLING_DDOF: Final = frozenset({"variance", "stddev"})
|
|
138
|
+
_ROLLING_PRIMITIVES: Final = _ROLLING_AGGREGATES | frozenset(
|
|
139
|
+
{"lag", "delta", "ewma", "covariance", "correlation"}
|
|
140
|
+
)
|
|
141
|
+
_CROSS_SECTION: Final = frozenset(
|
|
142
|
+
{
|
|
143
|
+
"rank",
|
|
144
|
+
"percentile",
|
|
145
|
+
"demean",
|
|
146
|
+
"zscore",
|
|
147
|
+
"winsorize",
|
|
148
|
+
"top",
|
|
149
|
+
"bottom",
|
|
150
|
+
"mean_fill",
|
|
151
|
+
}
|
|
152
|
+
)
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
@dataclass(frozen=True, slots=True)
|
|
156
|
+
class AnalysisIssue:
|
|
157
|
+
"""One analysis finding with its stable path, code, and message."""
|
|
158
|
+
|
|
159
|
+
path: str
|
|
160
|
+
code: str
|
|
161
|
+
message: str
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
@dataclass(frozen=True, slots=True)
|
|
165
|
+
class AnalysisResult:
|
|
166
|
+
"""The immutable outcome of analyzing one program in one mode."""
|
|
167
|
+
|
|
168
|
+
mode: CompileMode
|
|
169
|
+
program_fingerprint: str
|
|
170
|
+
capability_session_id: str
|
|
171
|
+
capability_revision: int
|
|
172
|
+
issues: tuple[AnalysisIssue, ...]
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
@dataclass(frozen=True, slots=True)
|
|
176
|
+
class TableFacts:
|
|
177
|
+
"""The inferred schema, lineage, ordering, and state of one table value."""
|
|
178
|
+
|
|
179
|
+
schema: tuple[Field, ...]
|
|
180
|
+
lineage: str | None
|
|
181
|
+
state: frozenset[str]
|
|
182
|
+
event_time: str | None
|
|
183
|
+
entity_by: tuple[str, ...]
|
|
184
|
+
sequence_by: tuple[str, ...]
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
@dataclass(frozen=True, slots=True)
|
|
188
|
+
class ColumnFacts:
|
|
189
|
+
"""The inferred value type and lineage of one column expression."""
|
|
190
|
+
|
|
191
|
+
data_type: str | None
|
|
192
|
+
nullable: bool
|
|
193
|
+
lineage: str | None
|
|
194
|
+
state: frozenset[str]
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
@dataclass(frozen=True, slots=True)
|
|
198
|
+
class ArrayFacts:
|
|
199
|
+
"""The inferred backend, dtype, shape, and row lineage of one array."""
|
|
200
|
+
|
|
201
|
+
backend: str | None
|
|
202
|
+
dtype: str | None
|
|
203
|
+
shape: tuple[int | str, ...]
|
|
204
|
+
lineage: str | None
|
|
205
|
+
state: frozenset[str]
|
|
206
|
+
|
|
207
|
+
|
|
208
|
+
def _cstr(value: CValue, /) -> str | None:
|
|
209
|
+
return value.value if isinstance(value, CStr) else None
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
def _ctype_str(value: CValue, /) -> str | None:
|
|
213
|
+
if isinstance(value, CStr):
|
|
214
|
+
return value.value
|
|
215
|
+
if isinstance(value, CDType):
|
|
216
|
+
return value.name
|
|
217
|
+
return None
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
def _cint(value: CValue, /) -> int | None:
|
|
221
|
+
return value.value if isinstance(value, CInt) else None
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
def _cstr_seq(value: CValue, /) -> tuple[str, ...]:
|
|
225
|
+
if isinstance(value, CSeq):
|
|
226
|
+
return tuple(item.value for item in value.items if isinstance(item, CStr))
|
|
227
|
+
return ()
|
|
228
|
+
|
|
229
|
+
|
|
230
|
+
def _declared_type_name(data_type: CValue, /) -> str | None:
|
|
231
|
+
if isinstance(data_type, CStr):
|
|
232
|
+
return data_type.value
|
|
233
|
+
if isinstance(data_type, CDType):
|
|
234
|
+
return data_type.name
|
|
235
|
+
return None
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
def _declared_field(item: CValue, /) -> Field | None:
|
|
239
|
+
if not isinstance(item, CMap):
|
|
240
|
+
return None
|
|
241
|
+
name = item.get("name")
|
|
242
|
+
nullable = item.get("nullable")
|
|
243
|
+
if not isinstance(name, CStr) or not isinstance(nullable, CBool):
|
|
244
|
+
return None
|
|
245
|
+
type_name_value = _declared_type_name(item.get("data_type"))
|
|
246
|
+
if type_name_value is None:
|
|
247
|
+
return None
|
|
248
|
+
return Field(name.value, type_name_value, nullable.value)
|
|
249
|
+
|
|
250
|
+
|
|
251
|
+
def _schema_fields(value: CValue, /) -> tuple[Field, ...]:
|
|
252
|
+
if not isinstance(value, CSeq):
|
|
253
|
+
return ()
|
|
254
|
+
fields = [
|
|
255
|
+
field for item in value.items if (field := _declared_field(item)) is not None
|
|
256
|
+
]
|
|
257
|
+
return tuple(fields)
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
def _resolves_to_input_column(node: Node, /) -> bool:
|
|
261
|
+
"""Whether the row-local lowerer resolves ``node`` to one source column."""
|
|
262
|
+
|
|
263
|
+
if node.op.name != "column_ref" or not node.args:
|
|
264
|
+
return False
|
|
265
|
+
return _table_field_resolves_to_input(
|
|
266
|
+
node.args[0],
|
|
267
|
+
_cstr(node.attr("name")) or "",
|
|
268
|
+
)
|
|
269
|
+
|
|
270
|
+
|
|
271
|
+
def _table_field_resolves_to_input(table: Node, field_name: str, /) -> bool:
|
|
272
|
+
operation = table.op.name
|
|
273
|
+
if operation in ("table_input", "stream_join"):
|
|
274
|
+
return True
|
|
275
|
+
if operation in ("project", "filter"):
|
|
276
|
+
return _table_field_resolves_to_input(table.args[0], field_name)
|
|
277
|
+
if operation != "with_columns":
|
|
278
|
+
return False
|
|
279
|
+
names = _cstr_seq(table.attr("names"))
|
|
280
|
+
if field_name not in names:
|
|
281
|
+
return _table_field_resolves_to_input(table.args[0], field_name)
|
|
282
|
+
index = names.index(field_name)
|
|
283
|
+
return _resolves_to_input_column(table.args[index + 1])
|
|
284
|
+
|
|
285
|
+
|
|
286
|
+
def _stateful_operand_is_stageable(node: Node, /) -> bool:
|
|
287
|
+
"""Whether an operand can be scheduled before its stateful consumer."""
|
|
288
|
+
|
|
289
|
+
operation = node.op.name
|
|
290
|
+
if operation == "column_ref":
|
|
291
|
+
if not node.args:
|
|
292
|
+
return True
|
|
293
|
+
return _table_field_is_stageable(
|
|
294
|
+
node.args[0],
|
|
295
|
+
_cstr(node.attr("name")) or "",
|
|
296
|
+
)
|
|
297
|
+
if operation == "literal":
|
|
298
|
+
return True
|
|
299
|
+
if operation not in _ROW_LOCAL_PRIMITIVES and operation not in _ROLLING_PRIMITIVES:
|
|
300
|
+
return False
|
|
301
|
+
return all(_stateful_operand_is_stageable(argument) for argument in node.args)
|
|
302
|
+
|
|
303
|
+
|
|
304
|
+
def _table_field_is_stageable(table: Node, field_name: str, /) -> bool:
|
|
305
|
+
operation = table.op.name
|
|
306
|
+
if operation in ("table_input", "stream_join"):
|
|
307
|
+
return True
|
|
308
|
+
if operation in ("project", "filter"):
|
|
309
|
+
return _table_field_is_stageable(table.args[0], field_name)
|
|
310
|
+
if operation != "with_columns":
|
|
311
|
+
return False
|
|
312
|
+
names = _cstr_seq(table.attr("names"))
|
|
313
|
+
if field_name not in names:
|
|
314
|
+
return _table_field_is_stageable(table.args[0], field_name)
|
|
315
|
+
index = names.index(field_name)
|
|
316
|
+
return _stateful_operand_is_stageable(table.args[index + 1])
|
|
317
|
+
|
|
318
|
+
|
|
319
|
+
def _contains_stateful_primitive(node: Node, /) -> bool:
|
|
320
|
+
if node.op.name in _ROLLING_PRIMITIVES or node.op.name in _CROSS_SECTION:
|
|
321
|
+
return True
|
|
322
|
+
return any(_contains_stateful_primitive(argument) for argument in node.args)
|
|
323
|
+
|
|
324
|
+
|
|
325
|
+
class _Analyzer:
|
|
326
|
+
"""One analysis pass over one program, mode, and capability snapshot."""
|
|
327
|
+
|
|
328
|
+
def __init__(
|
|
329
|
+
self,
|
|
330
|
+
mode: CompileMode,
|
|
331
|
+
declared: frozenset[str],
|
|
332
|
+
portable_types: frozenset[str],
|
|
333
|
+
supports_array_kind: bool,
|
|
334
|
+
) -> None:
|
|
335
|
+
self._mode = mode
|
|
336
|
+
self._declared = declared
|
|
337
|
+
self._portable_types = portable_types
|
|
338
|
+
self._supports_array_kind = supports_array_kind
|
|
339
|
+
self._issues: list[AnalysisIssue] = []
|
|
340
|
+
self._table_cache: dict[str, TableFacts] = {}
|
|
341
|
+
self._column_cache: dict[str, ColumnFacts] = {}
|
|
342
|
+
self._array_cache: dict[str, ArrayFacts] = {}
|
|
343
|
+
self._undeclared_reported: set[str] = set()
|
|
344
|
+
self._temporal_lineages: set[str] = set()
|
|
345
|
+
|
|
346
|
+
def issue(self, path: str, code: str, message: str, /) -> None:
|
|
347
|
+
self._issues.append(AnalysisIssue(path, code, message))
|
|
348
|
+
|
|
349
|
+
# -- declaration-level checks ------------------------------------------
|
|
350
|
+
|
|
351
|
+
def check_input_declaration(self, node: Node, root: str, name: str, /) -> None:
|
|
352
|
+
if _is_array_parameter(node) and not self._supports_array_kind:
|
|
353
|
+
self.issue(
|
|
354
|
+
f"{root}.{name}",
|
|
355
|
+
"capability_mismatch",
|
|
356
|
+
"the capability snapshot does not support the array batch kind",
|
|
357
|
+
)
|
|
358
|
+
for index, field in enumerate(_schema_fields(node.attr("schema"))):
|
|
359
|
+
if field.data_type not in self._portable_types:
|
|
360
|
+
self.issue(
|
|
361
|
+
f"{root}.{name}.schema[{index}].data_type",
|
|
362
|
+
"capability_mismatch",
|
|
363
|
+
f"type {field.data_type!r} is not portable in the selected"
|
|
364
|
+
" runtime capability snapshot",
|
|
365
|
+
)
|
|
366
|
+
|
|
367
|
+
def check_array_output_kind(self, output_name: str, /) -> None:
|
|
368
|
+
if not self._supports_array_kind:
|
|
369
|
+
self.issue(
|
|
370
|
+
f"outputs.{output_name}",
|
|
371
|
+
"capability_mismatch",
|
|
372
|
+
"the capability snapshot does not support the array batch kind",
|
|
373
|
+
)
|
|
374
|
+
|
|
375
|
+
def check_stream_ordering(self, node: Node, root: str, name: str, /) -> None:
|
|
376
|
+
schema = {field.name: field for field in _schema_fields(node.attr("schema"))}
|
|
377
|
+
base = f"{root}.{name}"
|
|
378
|
+
self._ordering_event_time(node, schema, base)
|
|
379
|
+
self._ordering_key_fields(
|
|
380
|
+
node.attr("entity_by"),
|
|
381
|
+
schema,
|
|
382
|
+
f"{base}.entity_by",
|
|
383
|
+
"temporal work in stream mode requires a non-empty entity key",
|
|
384
|
+
_entity_field_is_valid,
|
|
385
|
+
_entity_field_message,
|
|
386
|
+
)
|
|
387
|
+
self._ordering_key_fields(
|
|
388
|
+
node.attr("sequence_by"),
|
|
389
|
+
schema,
|
|
390
|
+
f"{base}.sequence_by",
|
|
391
|
+
"temporal work in stream mode requires a non-empty sequence key",
|
|
392
|
+
_sequence_field_is_valid,
|
|
393
|
+
_sequence_field_message,
|
|
394
|
+
)
|
|
395
|
+
|
|
396
|
+
def _ordering_event_time(
|
|
397
|
+
self, node: Node, schema: dict[str, Field], base: str, /
|
|
398
|
+
) -> None:
|
|
399
|
+
event_time = _cstr(node.attr("event_time"))
|
|
400
|
+
field = None if event_time is None else schema.get(event_time)
|
|
401
|
+
if field is None or field.data_type != _EVENT_TIME_TYPE or field.nullable:
|
|
402
|
+
message = (
|
|
403
|
+
"temporal work in stream mode requires a declared event-time column"
|
|
404
|
+
if event_time is None
|
|
405
|
+
else "the event-time column must be a non-null timestamp[us, UTC] field"
|
|
406
|
+
)
|
|
407
|
+
self.issue(f"{base}.event_time", "ordering_required", message)
|
|
408
|
+
|
|
409
|
+
def _ordering_key_fields(
|
|
410
|
+
self,
|
|
411
|
+
declared: CValue,
|
|
412
|
+
schema: dict[str, Field],
|
|
413
|
+
base: str,
|
|
414
|
+
empty_message: str,
|
|
415
|
+
field_is_valid,
|
|
416
|
+
field_message,
|
|
417
|
+
/,
|
|
418
|
+
) -> None:
|
|
419
|
+
names = _cstr_seq(declared)
|
|
420
|
+
if not names:
|
|
421
|
+
self.issue(base, "ordering_required", empty_message)
|
|
422
|
+
return
|
|
423
|
+
for index, field_name in enumerate(names):
|
|
424
|
+
if not field_is_valid(schema.get(field_name)):
|
|
425
|
+
self.issue(
|
|
426
|
+
f"{base}[{index}]",
|
|
427
|
+
"ordering_required",
|
|
428
|
+
field_message(field_name),
|
|
429
|
+
)
|
|
430
|
+
|
|
431
|
+
def check_unbounded_output(self, output_name: str, facts: ArrayFacts, /) -> None:
|
|
432
|
+
if self._mode == "stream" and facts.lineage is not None:
|
|
433
|
+
self.issue(
|
|
434
|
+
f"outputs.{output_name}",
|
|
435
|
+
"unbounded_state",
|
|
436
|
+
"an array output with row-axis lineage over a stream table has"
|
|
437
|
+
" unbounded state; attach it to its table or express it through"
|
|
438
|
+
" an explicit window",
|
|
439
|
+
)
|
|
440
|
+
|
|
441
|
+
@property
|
|
442
|
+
def issues(self) -> tuple[AnalysisIssue, ...]:
|
|
443
|
+
return tuple(
|
|
444
|
+
sorted(
|
|
445
|
+
self._issues,
|
|
446
|
+
key=lambda issue: (issue.path, issue.code, issue.message),
|
|
447
|
+
)
|
|
448
|
+
)
|
|
449
|
+
|
|
450
|
+
@property
|
|
451
|
+
def temporal_lineages(self) -> frozenset[str]:
|
|
452
|
+
return frozenset(self._temporal_lineages)
|
|
453
|
+
|
|
454
|
+
# -- table analysis ------------------------------------------------------
|
|
455
|
+
|
|
456
|
+
def table(self, node: Node, path: str, /) -> TableFacts:
|
|
457
|
+
cached = self._table_cache.get(node.digest)
|
|
458
|
+
if cached is not None:
|
|
459
|
+
return cached
|
|
460
|
+
facts = self._analyze_table(node, path)
|
|
461
|
+
self._table_cache[node.digest] = facts
|
|
462
|
+
return facts
|
|
463
|
+
|
|
464
|
+
def _analyze_table(self, node: Node, path: str, /) -> TableFacts:
|
|
465
|
+
name = node.op.name
|
|
466
|
+
if name in ("table_input", "parameter"):
|
|
467
|
+
return self._table_declaration(node, path)
|
|
468
|
+
if name == "project":
|
|
469
|
+
return self._project_table(node, path)
|
|
470
|
+
if name == "filter":
|
|
471
|
+
return self._filter_table(node, path)
|
|
472
|
+
if name == "with_columns":
|
|
473
|
+
return self._with_columns_table(node, path)
|
|
474
|
+
if name == "attach_columns":
|
|
475
|
+
return self._attach_columns_table(node, path)
|
|
476
|
+
if name == "stream_join":
|
|
477
|
+
return self._stream_join_table(node, path)
|
|
478
|
+
if name in ("window_tumbling", "window_hopping"):
|
|
479
|
+
return self._window_table(node, path)
|
|
480
|
+
self.issue(
|
|
481
|
+
path,
|
|
482
|
+
"unknown_primitive_version",
|
|
483
|
+
f"primitive {name!r} does not produce a table value",
|
|
484
|
+
)
|
|
485
|
+
return TableFacts((), None, frozenset(), None, (), ())
|
|
486
|
+
|
|
487
|
+
def _table_declaration(self, node: Node, path: str, /) -> TableFacts:
|
|
488
|
+
name = _cstr(node.attr("name"))
|
|
489
|
+
is_parameter = node.op.name == "parameter"
|
|
490
|
+
root = "static_inputs" if is_parameter else "inputs"
|
|
491
|
+
if (
|
|
492
|
+
name is not None
|
|
493
|
+
and node.digest not in self._declared
|
|
494
|
+
and name not in self._undeclared_reported
|
|
495
|
+
):
|
|
496
|
+
self._undeclared_reported.add(name)
|
|
497
|
+
self.issue(
|
|
498
|
+
f"{root}.{name}",
|
|
499
|
+
"unresolved_type",
|
|
500
|
+
f"input {name!r} is referenced by program outputs but not"
|
|
501
|
+
" declared in Program inputs",
|
|
502
|
+
)
|
|
503
|
+
return TableFacts(
|
|
504
|
+
schema=_schema_fields(node.attr("schema")),
|
|
505
|
+
lineage=name,
|
|
506
|
+
state=frozenset({"static"}) if is_parameter else frozenset(),
|
|
507
|
+
event_time=_cstr(node.attr("event_time")),
|
|
508
|
+
entity_by=_cstr_seq(node.attr("entity_by")),
|
|
509
|
+
sequence_by=_cstr_seq(node.attr("sequence_by")),
|
|
510
|
+
)
|
|
511
|
+
|
|
512
|
+
def _project_table(self, node: Node, path: str, /) -> TableFacts:
|
|
513
|
+
child = self.table(node.args[0], f"{path}.project.value")
|
|
514
|
+
columns = _cstr_seq(node.attr("columns"))
|
|
515
|
+
by_name = {field.name: field for field in child.schema}
|
|
516
|
+
projected: list[Field] = []
|
|
517
|
+
for index, column in enumerate(columns):
|
|
518
|
+
field = by_name.get(column)
|
|
519
|
+
if field is None:
|
|
520
|
+
self.issue(
|
|
521
|
+
f"{path}.project.columns[{index}]",
|
|
522
|
+
"unresolved_type",
|
|
523
|
+
f"unknown field {column!r} in the projected table schema",
|
|
524
|
+
)
|
|
525
|
+
else:
|
|
526
|
+
projected.append(field)
|
|
527
|
+
retained = frozenset(columns)
|
|
528
|
+
return TableFacts(
|
|
529
|
+
tuple(projected),
|
|
530
|
+
child.lineage,
|
|
531
|
+
child.state,
|
|
532
|
+
child.event_time if child.event_time in retained else None,
|
|
533
|
+
child.entity_by if set(child.entity_by) <= retained else (),
|
|
534
|
+
child.sequence_by if set(child.sequence_by) <= retained else (),
|
|
535
|
+
)
|
|
536
|
+
|
|
537
|
+
def _filter_table(self, node: Node, path: str, /) -> TableFacts:
|
|
538
|
+
child = self.table(node.args[0], f"{path}.filter.value")
|
|
539
|
+
predicate = self.column(node.args[1], f"{path}.filter.predicate")
|
|
540
|
+
if "stream_join" in child.state and _contains_stateful_primitive(node.args[1]):
|
|
541
|
+
self._require_post_join_ordering(
|
|
542
|
+
child,
|
|
543
|
+
f"{path}.filter.value",
|
|
544
|
+
"stateful work after a symbolic stream join",
|
|
545
|
+
)
|
|
546
|
+
if predicate.lineage is not None and predicate.lineage != child.lineage:
|
|
547
|
+
self.issue(
|
|
548
|
+
f"{path}.filter.predicate.lineage",
|
|
549
|
+
"schema_mismatch",
|
|
550
|
+
f"the filter predicate mixes table lineage {child.lineage!r}"
|
|
551
|
+
f" with columns from {predicate.lineage!r}",
|
|
552
|
+
)
|
|
553
|
+
if predicate.data_type is not None and predicate.data_type != "bool":
|
|
554
|
+
self.issue(
|
|
555
|
+
f"{path}.filter.predicate.dtype",
|
|
556
|
+
"unsupported_type",
|
|
557
|
+
"a filter predicate must be a boolean column expression",
|
|
558
|
+
)
|
|
559
|
+
return child
|
|
560
|
+
|
|
561
|
+
def _with_columns_table(self, node: Node, path: str, /) -> TableFacts:
|
|
562
|
+
child = self.table(node.args[0], f"{path}.with_columns.value")
|
|
563
|
+
names = _cstr_seq(node.attr("names"))
|
|
564
|
+
fields = list(child.schema)
|
|
565
|
+
existing = {field.name for field in child.schema}
|
|
566
|
+
states: set[str] = set(child.state)
|
|
567
|
+
for index, name in enumerate(names):
|
|
568
|
+
expression = node.args[index + 1]
|
|
569
|
+
facts = self.column(expression, f"{path}.{name}")
|
|
570
|
+
states |= facts.state
|
|
571
|
+
if facts.lineage is not None and facts.lineage != child.lineage:
|
|
572
|
+
self.issue(
|
|
573
|
+
f"{path}.{name}.lineage",
|
|
574
|
+
"schema_mismatch",
|
|
575
|
+
f"feature {name!r} mixes table lineage {child.lineage!r}"
|
|
576
|
+
f" with columns from {facts.lineage!r}",
|
|
577
|
+
)
|
|
578
|
+
if name in existing:
|
|
579
|
+
self.issue(
|
|
580
|
+
f"{path}.{name}",
|
|
581
|
+
"duplicate_name",
|
|
582
|
+
f"derived field {name!r} collides with an existing schema field",
|
|
583
|
+
)
|
|
584
|
+
elif facts.data_type is not None:
|
|
585
|
+
fields.append(Field(name, facts.data_type, facts.nullable))
|
|
586
|
+
existing.add(name)
|
|
587
|
+
if "stream_join" in child.state and any(
|
|
588
|
+
_contains_stateful_primitive(expression) for expression in node.args[1:]
|
|
589
|
+
):
|
|
590
|
+
self._require_post_join_ordering(
|
|
591
|
+
child,
|
|
592
|
+
f"{path}.with_columns.value",
|
|
593
|
+
"stateful work after a symbolic stream join",
|
|
594
|
+
)
|
|
595
|
+
return TableFacts(
|
|
596
|
+
tuple(fields),
|
|
597
|
+
child.lineage,
|
|
598
|
+
frozenset(states),
|
|
599
|
+
child.event_time,
|
|
600
|
+
child.entity_by,
|
|
601
|
+
child.sequence_by,
|
|
602
|
+
)
|
|
603
|
+
|
|
604
|
+
def _attach_columns_table(self, node: Node, path: str, /) -> TableFacts:
|
|
605
|
+
child = self.table(node.args[0], f"{path}.attach_columns.value")
|
|
606
|
+
array = self.array(node.args[1], f"{path}.attach_columns.array")
|
|
607
|
+
names = _cstr_seq(node.attr("names"))
|
|
608
|
+
self._attach_lineage_check(array, child, path)
|
|
609
|
+
self._attach_width_check(array, names, path)
|
|
610
|
+
fields = self._attached_fields(child, array, names, path)
|
|
611
|
+
return TableFacts(
|
|
612
|
+
tuple(fields),
|
|
613
|
+
child.lineage,
|
|
614
|
+
child.state | array.state,
|
|
615
|
+
child.event_time,
|
|
616
|
+
child.entity_by,
|
|
617
|
+
child.sequence_by,
|
|
618
|
+
)
|
|
619
|
+
|
|
620
|
+
def _stream_join_table(self, node: Node, path: str, /) -> TableFacts:
|
|
621
|
+
role = f"{path}.stream_join"
|
|
622
|
+
left = self.table(node.args[0], f"{role}.left")
|
|
623
|
+
right = self.table(node.args[1], f"{role}.right")
|
|
624
|
+
if self._mode != "stream":
|
|
625
|
+
self.issue(
|
|
626
|
+
role,
|
|
627
|
+
"unsupported_mode",
|
|
628
|
+
"symbolic stream_join is available only in stream mode",
|
|
629
|
+
)
|
|
630
|
+
self._stream_join_side(
|
|
631
|
+
left,
|
|
632
|
+
"left",
|
|
633
|
+
role,
|
|
634
|
+
_cstr(node.attr("left_event_time")),
|
|
635
|
+
)
|
|
636
|
+
self._stream_join_side(
|
|
637
|
+
right,
|
|
638
|
+
"right",
|
|
639
|
+
role,
|
|
640
|
+
_cstr(node.attr("right_event_time")),
|
|
641
|
+
)
|
|
642
|
+
self._stream_join_event_time(
|
|
643
|
+
left,
|
|
644
|
+
_cstr(node.attr("left_event_time")),
|
|
645
|
+
f"{role}.left_event_time",
|
|
646
|
+
)
|
|
647
|
+
self._stream_join_event_time(
|
|
648
|
+
right,
|
|
649
|
+
_cstr(node.attr("right_event_time")),
|
|
650
|
+
f"{role}.right_event_time",
|
|
651
|
+
)
|
|
652
|
+
self._stream_join_keys(node, left, right, role)
|
|
653
|
+
left_prefix = _cstr(node.attr("left_prefix")) or "left"
|
|
654
|
+
right_prefix = _cstr(node.attr("right_prefix")) or "right"
|
|
655
|
+
output_schema = self._stream_join_schema(
|
|
656
|
+
left,
|
|
657
|
+
right,
|
|
658
|
+
left_prefix,
|
|
659
|
+
right_prefix,
|
|
660
|
+
)
|
|
661
|
+
output_event_time = _cstr(node.attr("output_event_time"))
|
|
662
|
+
output_entity_by = _cstr_seq(node.attr("output_entity_by"))
|
|
663
|
+
output_sequence_by = _cstr_seq(node.attr("output_sequence_by"))
|
|
664
|
+
if node.op.version >= 2 or any(
|
|
665
|
+
(output_event_time, output_entity_by, output_sequence_by)
|
|
666
|
+
):
|
|
667
|
+
self._check_join_output_ordering(
|
|
668
|
+
node,
|
|
669
|
+
left,
|
|
670
|
+
right,
|
|
671
|
+
output_schema,
|
|
672
|
+
output_event_time,
|
|
673
|
+
output_entity_by,
|
|
674
|
+
output_sequence_by,
|
|
675
|
+
role,
|
|
676
|
+
)
|
|
677
|
+
return TableFacts(
|
|
678
|
+
output_schema,
|
|
679
|
+
node.digest,
|
|
680
|
+
left.state | right.state | frozenset({"stream_join"}),
|
|
681
|
+
output_event_time,
|
|
682
|
+
output_entity_by,
|
|
683
|
+
output_sequence_by,
|
|
684
|
+
)
|
|
685
|
+
|
|
686
|
+
def _stream_join_side(
|
|
687
|
+
self,
|
|
688
|
+
facts: TableFacts,
|
|
689
|
+
side_name: str,
|
|
690
|
+
role: str,
|
|
691
|
+
selected_event_time: str | None,
|
|
692
|
+
/,
|
|
693
|
+
) -> None:
|
|
694
|
+
if "stream_join" in facts.state:
|
|
695
|
+
self._require_post_join_ordering(
|
|
696
|
+
facts,
|
|
697
|
+
f"{role}.{side_name}",
|
|
698
|
+
"a joined result used by another symbolic stream join",
|
|
699
|
+
)
|
|
700
|
+
if facts.event_time is not None and selected_event_time != facts.event_time:
|
|
701
|
+
self.issue(
|
|
702
|
+
f"{role}.{side_name}_event_time",
|
|
703
|
+
"ordering_required",
|
|
704
|
+
"nested join event time must match the declared post-join"
|
|
705
|
+
f" event time {facts.event_time!r}",
|
|
706
|
+
)
|
|
707
|
+
if facts.lineage is not None:
|
|
708
|
+
self._temporal_lineages.add(facts.lineage)
|
|
709
|
+
|
|
710
|
+
def _require_post_join_ordering(
|
|
711
|
+
self,
|
|
712
|
+
facts: TableFacts,
|
|
713
|
+
path: str,
|
|
714
|
+
purpose: str,
|
|
715
|
+
/,
|
|
716
|
+
) -> None:
|
|
717
|
+
if facts.event_time and facts.entity_by and facts.sequence_by:
|
|
718
|
+
return
|
|
719
|
+
self.issue(
|
|
720
|
+
path,
|
|
721
|
+
"ordering_required",
|
|
722
|
+
f"{purpose} requires explicit output_entity_by,"
|
|
723
|
+
" output_event_time, and output_sequence_by metadata",
|
|
724
|
+
)
|
|
725
|
+
|
|
726
|
+
def _check_join_output_ordering(
|
|
727
|
+
self,
|
|
728
|
+
node: Node,
|
|
729
|
+
left: TableFacts,
|
|
730
|
+
right: TableFacts,
|
|
731
|
+
schema: tuple[Field, ...],
|
|
732
|
+
event_time: str | None,
|
|
733
|
+
entity_by: tuple[str, ...],
|
|
734
|
+
sequence_by: tuple[str, ...],
|
|
735
|
+
role: str,
|
|
736
|
+
/,
|
|
737
|
+
) -> None:
|
|
738
|
+
fields = {field.name: field for field in schema}
|
|
739
|
+
left_prefix = _cstr(node.attr("left_prefix")) or "left"
|
|
740
|
+
right_prefix = _cstr(node.attr("right_prefix")) or "right"
|
|
741
|
+
self._check_join_output_event_time(
|
|
742
|
+
node,
|
|
743
|
+
fields,
|
|
744
|
+
event_time,
|
|
745
|
+
left_prefix,
|
|
746
|
+
right_prefix,
|
|
747
|
+
role,
|
|
748
|
+
)
|
|
749
|
+
self._check_join_output_entities(
|
|
750
|
+
node,
|
|
751
|
+
fields,
|
|
752
|
+
entity_by,
|
|
753
|
+
left_prefix,
|
|
754
|
+
role,
|
|
755
|
+
)
|
|
756
|
+
self._check_join_output_sequences(
|
|
757
|
+
left,
|
|
758
|
+
right,
|
|
759
|
+
fields,
|
|
760
|
+
sequence_by,
|
|
761
|
+
left_prefix,
|
|
762
|
+
right_prefix,
|
|
763
|
+
role,
|
|
764
|
+
)
|
|
765
|
+
|
|
766
|
+
def _check_join_output_event_time(
|
|
767
|
+
self,
|
|
768
|
+
node: Node,
|
|
769
|
+
fields: dict[str, Field],
|
|
770
|
+
event_time: str | None,
|
|
771
|
+
left_prefix: str,
|
|
772
|
+
right_prefix: str,
|
|
773
|
+
role: str,
|
|
774
|
+
/,
|
|
775
|
+
) -> None:
|
|
776
|
+
allowed_event_times = {
|
|
777
|
+
f"{left_prefix}__{_cstr(node.attr('left_event_time'))}",
|
|
778
|
+
f"{right_prefix}__{_cstr(node.attr('right_event_time'))}",
|
|
779
|
+
}
|
|
780
|
+
event_field = None if event_time is None else fields.get(event_time)
|
|
781
|
+
if (
|
|
782
|
+
event_field is None
|
|
783
|
+
or event_field.data_type != _EVENT_TIME_TYPE
|
|
784
|
+
or event_field.nullable
|
|
785
|
+
or event_time not in allowed_event_times
|
|
786
|
+
):
|
|
787
|
+
self.issue(
|
|
788
|
+
f"{role}.output_event_time",
|
|
789
|
+
"ordering_required",
|
|
790
|
+
"post-join event time must be a non-null timestamp[us, UTC] field",
|
|
791
|
+
)
|
|
792
|
+
|
|
793
|
+
def _check_join_output_entities(
|
|
794
|
+
self,
|
|
795
|
+
node: Node,
|
|
796
|
+
fields: dict[str, Field],
|
|
797
|
+
entity_by: tuple[str, ...],
|
|
798
|
+
left_prefix: str,
|
|
799
|
+
role: str,
|
|
800
|
+
/,
|
|
801
|
+
) -> None:
|
|
802
|
+
self._ordering_key_fields(
|
|
803
|
+
CSeq(tuple(CStr(name) for name in entity_by)),
|
|
804
|
+
fields,
|
|
805
|
+
f"{role}.output_entity_by",
|
|
806
|
+
"post-join ordering requires a non-empty entity key",
|
|
807
|
+
_entity_field_is_valid,
|
|
808
|
+
_entity_field_message,
|
|
809
|
+
)
|
|
810
|
+
expected_entities = tuple(
|
|
811
|
+
f"{left_prefix}__{name}" for name in _cstr_seq(node.attr("left_keys"))
|
|
812
|
+
)
|
|
813
|
+
if entity_by and entity_by != expected_entities:
|
|
814
|
+
self.issue(
|
|
815
|
+
f"{role}.output_entity_by",
|
|
816
|
+
"ordering_required",
|
|
817
|
+
"post-join entity keys must be the prefixed left join keys"
|
|
818
|
+
f" {expected_entities!r}",
|
|
819
|
+
)
|
|
820
|
+
|
|
821
|
+
def _check_join_output_sequences(
|
|
822
|
+
self,
|
|
823
|
+
left: TableFacts,
|
|
824
|
+
right: TableFacts,
|
|
825
|
+
fields: dict[str, Field],
|
|
826
|
+
sequence_by: tuple[str, ...],
|
|
827
|
+
left_prefix: str,
|
|
828
|
+
right_prefix: str,
|
|
829
|
+
role: str,
|
|
830
|
+
/,
|
|
831
|
+
) -> None:
|
|
832
|
+
self._ordering_key_fields(
|
|
833
|
+
CSeq(tuple(CStr(name) for name in sequence_by)),
|
|
834
|
+
fields,
|
|
835
|
+
f"{role}.output_sequence_by",
|
|
836
|
+
"post-join ordering requires a non-empty sequence key",
|
|
837
|
+
_sequence_field_is_valid,
|
|
838
|
+
_sequence_field_message,
|
|
839
|
+
)
|
|
840
|
+
expected_sequences = (
|
|
841
|
+
*(f"{left_prefix}__{name}" for name in left.sequence_by),
|
|
842
|
+
*(f"{right_prefix}__{name}" for name in right.sequence_by),
|
|
843
|
+
)
|
|
844
|
+
if sequence_by and sequence_by != expected_sequences:
|
|
845
|
+
self.issue(
|
|
846
|
+
f"{role}.output_sequence_by",
|
|
847
|
+
"ordering_required",
|
|
848
|
+
"post-join sequence keys must concatenate the prefixed left"
|
|
849
|
+
f" and right input sequence keys {expected_sequences!r}",
|
|
850
|
+
)
|
|
851
|
+
|
|
852
|
+
@staticmethod
|
|
853
|
+
def _stream_join_schema(
|
|
854
|
+
left: TableFacts,
|
|
855
|
+
right: TableFacts,
|
|
856
|
+
left_prefix: str,
|
|
857
|
+
right_prefix: str,
|
|
858
|
+
/,
|
|
859
|
+
) -> tuple[Field, ...]:
|
|
860
|
+
left_fields = tuple(
|
|
861
|
+
Field(f"{left_prefix}__{field.name}", field.data_type, field.nullable)
|
|
862
|
+
for field in left.schema
|
|
863
|
+
)
|
|
864
|
+
right_fields = tuple(
|
|
865
|
+
Field(f"{right_prefix}__{field.name}", field.data_type, field.nullable)
|
|
866
|
+
for field in right.schema
|
|
867
|
+
)
|
|
868
|
+
return left_fields + right_fields
|
|
869
|
+
|
|
870
|
+
def _stream_join_event_time(
|
|
871
|
+
self,
|
|
872
|
+
table: TableFacts,
|
|
873
|
+
field_name: str | None,
|
|
874
|
+
path: str,
|
|
875
|
+
/,
|
|
876
|
+
) -> None:
|
|
877
|
+
field = next(
|
|
878
|
+
(field for field in table.schema if field.name == field_name),
|
|
879
|
+
None,
|
|
880
|
+
)
|
|
881
|
+
if field is None:
|
|
882
|
+
self.issue(
|
|
883
|
+
path,
|
|
884
|
+
"unresolved_type",
|
|
885
|
+
f"unknown event-time field {field_name!r} in the joined input schema",
|
|
886
|
+
)
|
|
887
|
+
elif field.data_type != _EVENT_TIME_TYPE or field.nullable:
|
|
888
|
+
self.issue(
|
|
889
|
+
path,
|
|
890
|
+
"ordering_required",
|
|
891
|
+
"join event time must be a non-null timestamp[us, UTC] field",
|
|
892
|
+
)
|
|
893
|
+
|
|
894
|
+
def _stream_join_keys(
|
|
895
|
+
self,
|
|
896
|
+
node: Node,
|
|
897
|
+
left: TableFacts,
|
|
898
|
+
right: TableFacts,
|
|
899
|
+
role: str,
|
|
900
|
+
/,
|
|
901
|
+
) -> None:
|
|
902
|
+
left_by_name = {field.name: field for field in left.schema}
|
|
903
|
+
right_by_name = {field.name: field for field in right.schema}
|
|
904
|
+
left_keys = _cstr_seq(node.attr("left_keys"))
|
|
905
|
+
right_keys = _cstr_seq(node.attr("right_keys"))
|
|
906
|
+
for index, (left_name, right_name) in enumerate(
|
|
907
|
+
zip(left_keys, right_keys, strict=True)
|
|
908
|
+
):
|
|
909
|
+
self._stream_join_key_pair(
|
|
910
|
+
index,
|
|
911
|
+
left_name,
|
|
912
|
+
right_name,
|
|
913
|
+
left_by_name.get(left_name),
|
|
914
|
+
right_by_name.get(right_name),
|
|
915
|
+
role,
|
|
916
|
+
)
|
|
917
|
+
|
|
918
|
+
def _stream_join_key_pair(
|
|
919
|
+
self,
|
|
920
|
+
index: int,
|
|
921
|
+
left_name: str,
|
|
922
|
+
right_name: str,
|
|
923
|
+
left_field: Field | None,
|
|
924
|
+
right_field: Field | None,
|
|
925
|
+
role: str,
|
|
926
|
+
/,
|
|
927
|
+
) -> None:
|
|
928
|
+
if left_field is None:
|
|
929
|
+
self.issue(
|
|
930
|
+
f"{role}.left_keys[{index}]",
|
|
931
|
+
"unresolved_type",
|
|
932
|
+
f"unknown join key {left_name!r} in the left schema",
|
|
933
|
+
)
|
|
934
|
+
if right_field is None:
|
|
935
|
+
self.issue(
|
|
936
|
+
f"{role}.right_keys[{index}]",
|
|
937
|
+
"unresolved_type",
|
|
938
|
+
f"unknown join key {right_name!r} in the right schema",
|
|
939
|
+
)
|
|
940
|
+
if any((left_field is None, right_field is None)):
|
|
941
|
+
return
|
|
942
|
+
if left_field.data_type == right_field.data_type:
|
|
943
|
+
return
|
|
944
|
+
self.issue(
|
|
945
|
+
f"{role}.right_keys[{index}]",
|
|
946
|
+
"schema_mismatch",
|
|
947
|
+
f"right join key type {right_field.data_type!r} does not match"
|
|
948
|
+
f" left type {left_field.data_type!r}",
|
|
949
|
+
)
|
|
950
|
+
|
|
951
|
+
def _attach_lineage_check(
|
|
952
|
+
self, array: ArrayFacts, child: TableFacts, path: str, /
|
|
953
|
+
) -> None:
|
|
954
|
+
if array.lineage is None or array.lineage != child.lineage:
|
|
955
|
+
self.issue(
|
|
956
|
+
f"{path}.attach_columns.array.lineage",
|
|
957
|
+
"schema_mismatch",
|
|
958
|
+
"an attached array must carry the row-axis lineage of the"
|
|
959
|
+
f" target table {child.lineage!r}",
|
|
960
|
+
)
|
|
961
|
+
|
|
962
|
+
def _attach_width_check(
|
|
963
|
+
self, array: ArrayFacts, names: tuple[str, ...], path: str, /
|
|
964
|
+
) -> None:
|
|
965
|
+
if not array.shape:
|
|
966
|
+
return
|
|
967
|
+
width_path = f"{path}.attach_columns.array.shape[{len(array.shape) - 1}]"
|
|
968
|
+
width = array.shape[-1]
|
|
969
|
+
if isinstance(width, str):
|
|
970
|
+
self.issue(
|
|
971
|
+
width_path,
|
|
972
|
+
"unresolved_type",
|
|
973
|
+
"the attached array width is symbolic and cannot be proved"
|
|
974
|
+
" to match the declared names",
|
|
975
|
+
)
|
|
976
|
+
elif width != len(names):
|
|
977
|
+
self.issue(
|
|
978
|
+
width_path,
|
|
979
|
+
"schema_mismatch",
|
|
980
|
+
f"array width {width} does not match {len(names)} declared names",
|
|
981
|
+
)
|
|
982
|
+
|
|
983
|
+
def _attached_fields(
|
|
984
|
+
self,
|
|
985
|
+
child: TableFacts,
|
|
986
|
+
array: ArrayFacts,
|
|
987
|
+
names: tuple[str, ...],
|
|
988
|
+
path: str,
|
|
989
|
+
/,
|
|
990
|
+
) -> list[Field]:
|
|
991
|
+
fields = list(child.schema)
|
|
992
|
+
existing = {field.name for field in child.schema}
|
|
993
|
+
for name in names:
|
|
994
|
+
if name in existing:
|
|
995
|
+
self.issue(
|
|
996
|
+
f"{path}.{name}",
|
|
997
|
+
"duplicate_name",
|
|
998
|
+
f"attached field {name!r} collides with an existing schema field",
|
|
999
|
+
)
|
|
1000
|
+
elif array.dtype is not None:
|
|
1001
|
+
fields.append(Field(name, array.dtype, nullable=True))
|
|
1002
|
+
existing.add(name)
|
|
1003
|
+
return fields
|
|
1004
|
+
|
|
1005
|
+
def _window_table(self, node: Node, path: str, /) -> TableFacts:
|
|
1006
|
+
child = self.table(node.args[0], f"{path}.{node.op.name}.value")
|
|
1007
|
+
if "stream_join" in child.state:
|
|
1008
|
+
self._require_post_join_ordering(
|
|
1009
|
+
child,
|
|
1010
|
+
f"{path}.{node.op.name}.value",
|
|
1011
|
+
"event windows after a symbolic stream join",
|
|
1012
|
+
)
|
|
1013
|
+
if child.lineage is not None:
|
|
1014
|
+
self._temporal_lineages.add(child.lineage)
|
|
1015
|
+
by_name = {field.name: field for field in child.schema}
|
|
1016
|
+
event_time = _cstr(node.attr("event_time"))
|
|
1017
|
+
if event_time is not None and event_time not in by_name:
|
|
1018
|
+
self.issue(
|
|
1019
|
+
f"{path}.{node.op.name}.event_time",
|
|
1020
|
+
"unresolved_type",
|
|
1021
|
+
f"unknown event-time field {event_time!r} in the windowed table schema",
|
|
1022
|
+
)
|
|
1023
|
+
fields = [
|
|
1024
|
+
Field("window_start", _EVENT_TIME_TYPE, nullable=False),
|
|
1025
|
+
Field("window_end", _EVENT_TIME_TYPE, nullable=False),
|
|
1026
|
+
]
|
|
1027
|
+
for index, name in enumerate(_cstr_seq(node.attr("group_by"))):
|
|
1028
|
+
field = by_name.get(name)
|
|
1029
|
+
if field is None:
|
|
1030
|
+
self.issue(
|
|
1031
|
+
f"{path}.{node.op.name}.group_by[{index}]",
|
|
1032
|
+
"unresolved_type",
|
|
1033
|
+
f"unknown group field {name!r} in the windowed table schema",
|
|
1034
|
+
)
|
|
1035
|
+
else:
|
|
1036
|
+
fields.append(field)
|
|
1037
|
+
return TableFacts(
|
|
1038
|
+
tuple(fields),
|
|
1039
|
+
None,
|
|
1040
|
+
child.state | frozenset({"window"}),
|
|
1041
|
+
child.event_time,
|
|
1042
|
+
child.entity_by,
|
|
1043
|
+
child.sequence_by,
|
|
1044
|
+
)
|
|
1045
|
+
|
|
1046
|
+
# -- column analysis -----------------------------------------------------
|
|
1047
|
+
|
|
1048
|
+
def column(self, node: Node, path: str, /) -> ColumnFacts:
|
|
1049
|
+
cached = self._column_cache.get(node.digest)
|
|
1050
|
+
if cached is not None:
|
|
1051
|
+
return cached
|
|
1052
|
+
facts = self._analyze_column(node, path)
|
|
1053
|
+
self._column_cache[node.digest] = facts
|
|
1054
|
+
return facts
|
|
1055
|
+
|
|
1056
|
+
def _analyze_column(self, node: Node, path: str, /) -> ColumnFacts:
|
|
1057
|
+
handler = _COLUMN_HANDLERS.get(node.op.name)
|
|
1058
|
+
if handler is None:
|
|
1059
|
+
self.issue(
|
|
1060
|
+
path,
|
|
1061
|
+
"unknown_primitive_version",
|
|
1062
|
+
f"primitive {node.op.name!r} does not produce a column value",
|
|
1063
|
+
)
|
|
1064
|
+
return ColumnFacts(None, True, None, frozenset())
|
|
1065
|
+
return handler(self, node, path)
|
|
1066
|
+
|
|
1067
|
+
def _column_ref(self, node: Node, path: str, /) -> ColumnFacts:
|
|
1068
|
+
table = self.table(node.args[0], f"{path}.column_ref.value")
|
|
1069
|
+
field_name = _cstr(node.attr("name"))
|
|
1070
|
+
for field in table.schema:
|
|
1071
|
+
if field.name == field_name:
|
|
1072
|
+
return ColumnFacts(
|
|
1073
|
+
field.data_type,
|
|
1074
|
+
field.nullable,
|
|
1075
|
+
table.lineage,
|
|
1076
|
+
table.state,
|
|
1077
|
+
)
|
|
1078
|
+
self.issue(
|
|
1079
|
+
path,
|
|
1080
|
+
"unresolved_type",
|
|
1081
|
+
f"unknown field {field_name!r} in the schema of input {table.lineage!r}",
|
|
1082
|
+
)
|
|
1083
|
+
return ColumnFacts(None, True, table.lineage, table.state)
|
|
1084
|
+
|
|
1085
|
+
def _anchored_operands(
|
|
1086
|
+
self,
|
|
1087
|
+
nodes: tuple[Node, ...],
|
|
1088
|
+
paths: tuple[str, ...],
|
|
1089
|
+
/,
|
|
1090
|
+
initial_anchor: str | None = None,
|
|
1091
|
+
) -> tuple[ColumnFacts, ...]:
|
|
1092
|
+
"""Analyze operands left to right against the first resolved lineage."""
|
|
1093
|
+
|
|
1094
|
+
operands: list[ColumnFacts] = []
|
|
1095
|
+
anchor = initial_anchor
|
|
1096
|
+
for child, operand_path in zip(nodes, paths, strict=True):
|
|
1097
|
+
facts = self._operand(child, operand_path, anchor)
|
|
1098
|
+
operands.append(facts)
|
|
1099
|
+
if anchor is None and facts.lineage is not None:
|
|
1100
|
+
anchor = facts.lineage
|
|
1101
|
+
return tuple(operands)
|
|
1102
|
+
|
|
1103
|
+
@staticmethod
|
|
1104
|
+
def _running_anchor(operands: tuple[ColumnFacts, ...], /) -> str | None:
|
|
1105
|
+
return next(
|
|
1106
|
+
(facts.lineage for facts in operands if facts.lineage is not None),
|
|
1107
|
+
None,
|
|
1108
|
+
)
|
|
1109
|
+
|
|
1110
|
+
def _operand(self, node: Node, path: str, anchor: str | None, /) -> ColumnFacts:
|
|
1111
|
+
if node.op.name == "literal":
|
|
1112
|
+
facts = _literal_facts(node)
|
|
1113
|
+
else:
|
|
1114
|
+
facts = self.column(node, path)
|
|
1115
|
+
if anchor is not None and facts.lineage is not None and facts.lineage != anchor:
|
|
1116
|
+
self.issue(
|
|
1117
|
+
f"{path}.lineage",
|
|
1118
|
+
"schema_mismatch",
|
|
1119
|
+
f"column operands span inputs {anchor!r} and {facts.lineage!r}",
|
|
1120
|
+
)
|
|
1121
|
+
return facts
|
|
1122
|
+
|
|
1123
|
+
def _pair(self, node: Node, path: str, /) -> tuple[ColumnFacts, ColumnFacts] | None:
|
|
1124
|
+
left = self._operand(node.args[0], f"{path}.{node.op.name}.left", None)
|
|
1125
|
+
anchor = left.lineage
|
|
1126
|
+
right = self._operand(node.args[1], f"{path}.{node.op.name}.right", anchor)
|
|
1127
|
+
if (
|
|
1128
|
+
left.data_type is not None
|
|
1129
|
+
and right.data_type is not None
|
|
1130
|
+
and left.data_type != right.data_type
|
|
1131
|
+
):
|
|
1132
|
+
self.issue(
|
|
1133
|
+
f"{path}.{node.op.name}.right.dtype",
|
|
1134
|
+
"unsupported_type",
|
|
1135
|
+
f"no provable common type for {left.data_type!r} and"
|
|
1136
|
+
f" {right.data_type!r}; use row.cast for an explicit conversion",
|
|
1137
|
+
)
|
|
1138
|
+
return None
|
|
1139
|
+
return left, right
|
|
1140
|
+
|
|
1141
|
+
def _arithmetic(self, node: Node, path: str, /) -> ColumnFacts:
|
|
1142
|
+
pair = self._pair(node, path)
|
|
1143
|
+
if pair is None:
|
|
1144
|
+
return ColumnFacts(None, True, None, frozenset())
|
|
1145
|
+
left, right = pair
|
|
1146
|
+
data_type = left.data_type
|
|
1147
|
+
if (
|
|
1148
|
+
node.op.name == "truediv"
|
|
1149
|
+
and data_type is not None
|
|
1150
|
+
and data_type not in _FLOATING_TYPES
|
|
1151
|
+
):
|
|
1152
|
+
self.issue(
|
|
1153
|
+
f"{path}.{node.op.name}.right.dtype",
|
|
1154
|
+
"unsupported_type",
|
|
1155
|
+
"division is only provable for floating columns; cast operands"
|
|
1156
|
+
" explicitly",
|
|
1157
|
+
)
|
|
1158
|
+
data_type = None
|
|
1159
|
+
return ColumnFacts(
|
|
1160
|
+
data_type,
|
|
1161
|
+
left.nullable or right.nullable,
|
|
1162
|
+
left.lineage,
|
|
1163
|
+
left.state | right.state,
|
|
1164
|
+
)
|
|
1165
|
+
|
|
1166
|
+
def _comparison(self, node: Node, path: str, /) -> ColumnFacts:
|
|
1167
|
+
pair = self._pair(node, path)
|
|
1168
|
+
if pair is None:
|
|
1169
|
+
return ColumnFacts(None, True, None, frozenset())
|
|
1170
|
+
left, right = pair
|
|
1171
|
+
return ColumnFacts(
|
|
1172
|
+
"bool",
|
|
1173
|
+
left.nullable or right.nullable,
|
|
1174
|
+
left.lineage,
|
|
1175
|
+
left.state | right.state,
|
|
1176
|
+
)
|
|
1177
|
+
|
|
1178
|
+
def _boolean_pair(self, node: Node, path: str, /) -> ColumnFacts:
|
|
1179
|
+
pair = self._pair(node, path)
|
|
1180
|
+
if pair is None:
|
|
1181
|
+
return ColumnFacts(None, True, None, frozenset())
|
|
1182
|
+
left, right = pair
|
|
1183
|
+
if left.data_type is not None and left.data_type != "bool":
|
|
1184
|
+
self.issue(
|
|
1185
|
+
f"{path}.{node.op.name}.left.dtype",
|
|
1186
|
+
"unsupported_type",
|
|
1187
|
+
"boolean composition requires boolean column operands",
|
|
1188
|
+
)
|
|
1189
|
+
return ColumnFacts(None, True, left.lineage, left.state | right.state)
|
|
1190
|
+
return ColumnFacts(
|
|
1191
|
+
"bool",
|
|
1192
|
+
left.nullable or right.nullable,
|
|
1193
|
+
left.lineage,
|
|
1194
|
+
left.state | right.state,
|
|
1195
|
+
)
|
|
1196
|
+
|
|
1197
|
+
def _unary(self, node: Node, path: str, /) -> ColumnFacts:
|
|
1198
|
+
operand = self._operand(node.args[0], f"{path}.{node.op.name}.value", None)
|
|
1199
|
+
if node.op.name == "not":
|
|
1200
|
+
if operand.data_type is not None and operand.data_type != "bool":
|
|
1201
|
+
self.issue(
|
|
1202
|
+
f"{path}.not.value.dtype",
|
|
1203
|
+
"unsupported_type",
|
|
1204
|
+
"boolean negation requires a boolean column operand",
|
|
1205
|
+
)
|
|
1206
|
+
return ColumnFacts(None, True, operand.lineage, operand.state)
|
|
1207
|
+
return ColumnFacts("bool", operand.nullable, operand.lineage, operand.state)
|
|
1208
|
+
if operand.data_type is not None and operand.data_type not in (
|
|
1209
|
+
*_SIGNED_INT_TYPES,
|
|
1210
|
+
*_UNSIGNED_INT_TYPES,
|
|
1211
|
+
*_FLOATING_TYPES,
|
|
1212
|
+
):
|
|
1213
|
+
self.issue(
|
|
1214
|
+
f"{path}.neg.value.dtype",
|
|
1215
|
+
"unsupported_type",
|
|
1216
|
+
"negation requires a numeric column operand",
|
|
1217
|
+
)
|
|
1218
|
+
return ColumnFacts(None, True, operand.lineage, operand.state)
|
|
1219
|
+
return ColumnFacts(
|
|
1220
|
+
operand.data_type, operand.nullable, operand.lineage, operand.state
|
|
1221
|
+
)
|
|
1222
|
+
|
|
1223
|
+
def _where(self, node: Node, path: str, /) -> ColumnFacts:
|
|
1224
|
+
role = f"{path}.where"
|
|
1225
|
+
operand_paths = (f"{role}.condition", f"{role}.when_true", f"{role}.when_false")
|
|
1226
|
+
operands = self._anchored_operands(node.args, operand_paths)
|
|
1227
|
+
condition, left, right = operands
|
|
1228
|
+
if condition.data_type is not None and condition.data_type != "bool":
|
|
1229
|
+
self.issue(
|
|
1230
|
+
f"{role}.condition.dtype",
|
|
1231
|
+
"unsupported_type",
|
|
1232
|
+
"a conditional requires a boolean condition",
|
|
1233
|
+
)
|
|
1234
|
+
anchor = self._running_anchor(operands)
|
|
1235
|
+
states = _column_state_union(operands)
|
|
1236
|
+
if (
|
|
1237
|
+
left.data_type is not None
|
|
1238
|
+
and right.data_type is not None
|
|
1239
|
+
and left.data_type != right.data_type
|
|
1240
|
+
):
|
|
1241
|
+
self.issue(
|
|
1242
|
+
f"{role}.when_false.dtype",
|
|
1243
|
+
"unsupported_type",
|
|
1244
|
+
f"no provable common type for {left.data_type!r} and"
|
|
1245
|
+
f" {right.data_type!r}; use row.cast for an explicit conversion",
|
|
1246
|
+
)
|
|
1247
|
+
return ColumnFacts(None, True, anchor, states)
|
|
1248
|
+
return ColumnFacts(
|
|
1249
|
+
left.data_type,
|
|
1250
|
+
left.nullable or right.nullable,
|
|
1251
|
+
anchor,
|
|
1252
|
+
states,
|
|
1253
|
+
)
|
|
1254
|
+
|
|
1255
|
+
def _coalesce(self, node: Node, path: str, /) -> ColumnFacts:
|
|
1256
|
+
role = f"{path}.coalesce"
|
|
1257
|
+
operands = self._anchored_operands(
|
|
1258
|
+
node.args,
|
|
1259
|
+
tuple(f"{role}.values[{index}]" for index in range(len(node.args))),
|
|
1260
|
+
)
|
|
1261
|
+
first = operands[0]
|
|
1262
|
+
for index, facts in enumerate(operands[1:], start=1):
|
|
1263
|
+
if (
|
|
1264
|
+
first.data_type is not None
|
|
1265
|
+
and facts.data_type is not None
|
|
1266
|
+
and facts.data_type != first.data_type
|
|
1267
|
+
):
|
|
1268
|
+
self.issue(
|
|
1269
|
+
f"{role}.values[{index}].dtype",
|
|
1270
|
+
"unsupported_type",
|
|
1271
|
+
"coalesce requires operands of one provable type",
|
|
1272
|
+
)
|
|
1273
|
+
return ColumnFacts(
|
|
1274
|
+
None,
|
|
1275
|
+
True,
|
|
1276
|
+
self._running_anchor(operands),
|
|
1277
|
+
_column_state_union(operands),
|
|
1278
|
+
)
|
|
1279
|
+
return ColumnFacts(
|
|
1280
|
+
first.data_type,
|
|
1281
|
+
all(facts.nullable for facts in operands),
|
|
1282
|
+
self._running_anchor(operands),
|
|
1283
|
+
_column_state_union(operands),
|
|
1284
|
+
)
|
|
1285
|
+
|
|
1286
|
+
def _scalar_function(self, node: Node, path: str, /) -> ColumnFacts:
|
|
1287
|
+
operand = self._operand(node.args[0], f"{path}.{node.op.name}.value", None)
|
|
1288
|
+
if operand.data_type is not None and operand.data_type not in (
|
|
1289
|
+
"float32",
|
|
1290
|
+
"float64",
|
|
1291
|
+
):
|
|
1292
|
+
self.issue(
|
|
1293
|
+
f"{path}.{node.op.name}.value.dtype",
|
|
1294
|
+
"unsupported_type",
|
|
1295
|
+
f"{node.op.name} is only provable for floating columns; use"
|
|
1296
|
+
" row.cast for an explicit conversion",
|
|
1297
|
+
)
|
|
1298
|
+
return ColumnFacts(None, True, operand.lineage, operand.state)
|
|
1299
|
+
return ColumnFacts("float64", operand.nullable, operand.lineage, operand.state)
|
|
1300
|
+
|
|
1301
|
+
def _arithmetic_like_unary(self, node: Node, path: str, /) -> ColumnFacts:
|
|
1302
|
+
operand = self._operand(node.args[0], f"{path}.{node.op.name}.value", None)
|
|
1303
|
+
if operand.data_type is not None and operand.data_type not in (
|
|
1304
|
+
*_SIGNED_INT_TYPES,
|
|
1305
|
+
*_UNSIGNED_INT_TYPES,
|
|
1306
|
+
*_FLOATING_TYPES,
|
|
1307
|
+
):
|
|
1308
|
+
self.issue(
|
|
1309
|
+
f"{path}.{node.op.name}.value.dtype",
|
|
1310
|
+
"unsupported_type",
|
|
1311
|
+
f"{node.op.name} requires a numeric column operand",
|
|
1312
|
+
)
|
|
1313
|
+
return ColumnFacts(None, True, operand.lineage, operand.state)
|
|
1314
|
+
return ColumnFacts(
|
|
1315
|
+
operand.data_type, operand.nullable, operand.lineage, operand.state
|
|
1316
|
+
)
|
|
1317
|
+
|
|
1318
|
+
def _clip(self, node: Node, path: str, /) -> ColumnFacts:
|
|
1319
|
+
operand = self._operand(node.args[0], f"{path}.clip.value", None)
|
|
1320
|
+
if operand.data_type is not None and operand.data_type not in (
|
|
1321
|
+
"float32",
|
|
1322
|
+
"float64",
|
|
1323
|
+
):
|
|
1324
|
+
self.issue(
|
|
1325
|
+
f"{path}.clip.value.dtype",
|
|
1326
|
+
"unsupported_type",
|
|
1327
|
+
"clipping is only provable for floating columns",
|
|
1328
|
+
)
|
|
1329
|
+
return ColumnFacts(None, True, operand.lineage, operand.state)
|
|
1330
|
+
return ColumnFacts(
|
|
1331
|
+
operand.data_type, operand.nullable, operand.lineage, operand.state
|
|
1332
|
+
)
|
|
1333
|
+
|
|
1334
|
+
def _cast(self, node: Node, path: str, /) -> ColumnFacts:
|
|
1335
|
+
operand = self._operand(node.args[0], f"{path}.cast.value", None)
|
|
1336
|
+
target = _ctype_str(node.attr("data_type"))
|
|
1337
|
+
return ColumnFacts(target, operand.nullable, operand.lineage, operand.state)
|
|
1338
|
+
|
|
1339
|
+
def _lag_like(self, node: Node, path: str, /) -> ColumnFacts:
|
|
1340
|
+
role = f"{path}.{node.op.name}"
|
|
1341
|
+
operand = self._operand(node.args[0], f"{role}.value", None)
|
|
1342
|
+
self._require_rolling_operand(
|
|
1343
|
+
node.args[0],
|
|
1344
|
+
f"{role}.value",
|
|
1345
|
+
f"rolling {node.op.name} argument must be an input column or"
|
|
1346
|
+
" row-local expression in this release",
|
|
1347
|
+
)
|
|
1348
|
+
if operand.lineage is not None:
|
|
1349
|
+
self._temporal_lineages.add(operand.lineage)
|
|
1350
|
+
periods = _cint(node.attr("periods")) or 1
|
|
1351
|
+
states = operand.state | frozenset({f"rows({periods})"})
|
|
1352
|
+
if node.op.name == "delta" and not self._numeric_or_issue(
|
|
1353
|
+
operand.data_type, f"{role}.value", node.op.name
|
|
1354
|
+
):
|
|
1355
|
+
return ColumnFacts(None, True, operand.lineage, states)
|
|
1356
|
+
return ColumnFacts(operand.data_type, True, operand.lineage, states)
|
|
1357
|
+
|
|
1358
|
+
def _numeric_or_issue(
|
|
1359
|
+
self, data_type: str | None, operand_path: str, primitive: str, /
|
|
1360
|
+
) -> bool:
|
|
1361
|
+
"""Require a provably numeric input; unknown types stay unresolved."""
|
|
1362
|
+
|
|
1363
|
+
if data_type is None or data_type in _NUMERIC_TYPES:
|
|
1364
|
+
return True
|
|
1365
|
+
self.issue(
|
|
1366
|
+
f"{operand_path}.dtype",
|
|
1367
|
+
"unsupported_type",
|
|
1368
|
+
f"{primitive} is only defined for numeric columns; use row.cast"
|
|
1369
|
+
" for an explicit conversion",
|
|
1370
|
+
)
|
|
1371
|
+
return False
|
|
1372
|
+
|
|
1373
|
+
def _frame_state(self, node: Node, /) -> str | None:
|
|
1374
|
+
frame = node.attr("frame")
|
|
1375
|
+
if not isinstance(frame, CMap):
|
|
1376
|
+
return None
|
|
1377
|
+
kind = frame.get("frame")
|
|
1378
|
+
attribute = (
|
|
1379
|
+
_FRAME_ATTRIBUTES.get(kind.variant) if isinstance(kind, CEnum) else None
|
|
1380
|
+
)
|
|
1381
|
+
if attribute is None:
|
|
1382
|
+
return None
|
|
1383
|
+
value = _cint(frame.get(attribute))
|
|
1384
|
+
return None if value is None else f"{kind.variant}({value})"
|
|
1385
|
+
|
|
1386
|
+
def _rolling_aggregate(self, node: Node, path: str, /) -> ColumnFacts:
|
|
1387
|
+
operand = self._operand(node.args[0], f"{path}.{node.op.name}.value", None)
|
|
1388
|
+
self._require_rolling_operand(
|
|
1389
|
+
node.args[0],
|
|
1390
|
+
f"{path}.{node.op.name}.value",
|
|
1391
|
+
f"rolling {node.op.name} argument must be an input column or"
|
|
1392
|
+
" row-local expression in this release",
|
|
1393
|
+
)
|
|
1394
|
+
if operand.lineage is not None:
|
|
1395
|
+
self._temporal_lineages.add(operand.lineage)
|
|
1396
|
+
state = self._frame_state(node)
|
|
1397
|
+
states = operand.state | (
|
|
1398
|
+
frozenset({state}) if state is not None else frozenset()
|
|
1399
|
+
)
|
|
1400
|
+
primitive = node.op.name
|
|
1401
|
+
input_type = operand.data_type
|
|
1402
|
+
if primitive not in ("count", "min", "max") and not self._numeric_or_issue(
|
|
1403
|
+
input_type, f"{path}.{primitive}.value", primitive
|
|
1404
|
+
):
|
|
1405
|
+
return ColumnFacts(None, True, operand.lineage, states)
|
|
1406
|
+
return ColumnFacts(
|
|
1407
|
+
_rolling_output_type(primitive, input_type),
|
|
1408
|
+
True,
|
|
1409
|
+
operand.lineage,
|
|
1410
|
+
states,
|
|
1411
|
+
)
|
|
1412
|
+
|
|
1413
|
+
def _ewma(self, node: Node, path: str, /) -> ColumnFacts:
|
|
1414
|
+
role = f"{path}.ewma"
|
|
1415
|
+
operand = self._operand(node.args[0], f"{role}.value", None)
|
|
1416
|
+
self._require_rolling_operand(
|
|
1417
|
+
node.args[0],
|
|
1418
|
+
f"{role}.value",
|
|
1419
|
+
"rolling ewma argument must be an input column or row-local"
|
|
1420
|
+
" expression in this release",
|
|
1421
|
+
)
|
|
1422
|
+
if operand.lineage is not None:
|
|
1423
|
+
self._temporal_lineages.add(operand.lineage)
|
|
1424
|
+
span = _cint(node.attr("span"))
|
|
1425
|
+
states = operand.state | (
|
|
1426
|
+
frozenset({f"constant(span={span})"}) if span is not None else frozenset()
|
|
1427
|
+
)
|
|
1428
|
+
if not self._numeric_or_issue(operand.data_type, f"{role}.value", "ewma"):
|
|
1429
|
+
return ColumnFacts(None, True, operand.lineage, states)
|
|
1430
|
+
return ColumnFacts("float64", True, operand.lineage, states)
|
|
1431
|
+
|
|
1432
|
+
def _rolling_pair(self, node: Node, path: str, /) -> ColumnFacts:
|
|
1433
|
+
role = f"{path}.{node.op.name}"
|
|
1434
|
+
left = self._operand(node.args[0], f"{role}.left", None)
|
|
1435
|
+
right = self._operand(node.args[1], f"{role}.right", left.lineage)
|
|
1436
|
+
message = (
|
|
1437
|
+
f"rolling {node.op.name} argument must be an input column or"
|
|
1438
|
+
" row-local expression in this release"
|
|
1439
|
+
)
|
|
1440
|
+
self._require_rolling_operand(node.args[0], f"{role}.left", message)
|
|
1441
|
+
self._require_rolling_operand(node.args[1], f"{role}.right", message)
|
|
1442
|
+
if left.lineage is not None:
|
|
1443
|
+
self._temporal_lineages.add(left.lineage)
|
|
1444
|
+
state = self._frame_state(node)
|
|
1445
|
+
states = (
|
|
1446
|
+
left.state
|
|
1447
|
+
| right.state
|
|
1448
|
+
| (frozenset({state}) if state is not None else frozenset())
|
|
1449
|
+
)
|
|
1450
|
+
numeric = self._numeric_or_issue(
|
|
1451
|
+
left.data_type, f"{role}.left", node.op.name
|
|
1452
|
+
) & self._numeric_or_issue(right.data_type, f"{role}.right", node.op.name)
|
|
1453
|
+
if not numeric:
|
|
1454
|
+
return ColumnFacts(None, True, left.lineage, states)
|
|
1455
|
+
return ColumnFacts("float64", True, left.lineage, states)
|
|
1456
|
+
|
|
1457
|
+
def _cross_section(self, node: Node, path: str, /) -> ColumnFacts:
|
|
1458
|
+
role = f"{path}.{node.op.name}"
|
|
1459
|
+
operand = self._operand(node.args[0], f"{role}.value", None)
|
|
1460
|
+
self._require_stageable_stateful_operand(
|
|
1461
|
+
node.args[0],
|
|
1462
|
+
f"{role}.value",
|
|
1463
|
+
f"cross-section {node.op.name} argument must be an input column,"
|
|
1464
|
+
" row-local expression, or rolling result in this release",
|
|
1465
|
+
)
|
|
1466
|
+
if operand.lineage is not None:
|
|
1467
|
+
self._temporal_lineages.add(operand.lineage)
|
|
1468
|
+
group_paths = (
|
|
1469
|
+
f"{role}.event_time",
|
|
1470
|
+
*(f"{role}.partition_by[{index}]" for index in range(len(node.args) - 2)),
|
|
1471
|
+
)
|
|
1472
|
+
group = self._anchored_operands(node.args[1:], group_paths, operand.lineage)
|
|
1473
|
+
grouping_messages = (
|
|
1474
|
+
"cross-section grouping event time must be an input column in this release",
|
|
1475
|
+
*(
|
|
1476
|
+
"cross-section group columns must be input columns in this release"
|
|
1477
|
+
for _ in node.args[2:]
|
|
1478
|
+
),
|
|
1479
|
+
)
|
|
1480
|
+
for group_node, group_path, message in zip(
|
|
1481
|
+
node.args[1:], group_paths, grouping_messages, strict=True
|
|
1482
|
+
):
|
|
1483
|
+
self._require_stateful_input_column(
|
|
1484
|
+
group_node,
|
|
1485
|
+
group_path,
|
|
1486
|
+
message,
|
|
1487
|
+
)
|
|
1488
|
+
states = (
|
|
1489
|
+
operand.state | _column_state_union(group) | frozenset({"cross_section"})
|
|
1490
|
+
)
|
|
1491
|
+
return self._cross_section_output(node.op.name, operand, role, states)
|
|
1492
|
+
|
|
1493
|
+
def _require_stateful_input_column(
|
|
1494
|
+
self,
|
|
1495
|
+
node: Node,
|
|
1496
|
+
path: str,
|
|
1497
|
+
message: str,
|
|
1498
|
+
/,
|
|
1499
|
+
) -> None:
|
|
1500
|
+
if not _resolves_to_input_column(node):
|
|
1501
|
+
self.issue(
|
|
1502
|
+
path,
|
|
1503
|
+
"unsupported_type",
|
|
1504
|
+
message,
|
|
1505
|
+
)
|
|
1506
|
+
|
|
1507
|
+
def _require_rolling_operand(
|
|
1508
|
+
self,
|
|
1509
|
+
node: Node,
|
|
1510
|
+
path: str,
|
|
1511
|
+
message: str,
|
|
1512
|
+
/,
|
|
1513
|
+
) -> None:
|
|
1514
|
+
self._require_stageable_stateful_operand(node, path, message)
|
|
1515
|
+
|
|
1516
|
+
def _require_stageable_stateful_operand(
|
|
1517
|
+
self,
|
|
1518
|
+
node: Node,
|
|
1519
|
+
path: str,
|
|
1520
|
+
message: str,
|
|
1521
|
+
/,
|
|
1522
|
+
) -> None:
|
|
1523
|
+
if not _stateful_operand_is_stageable(node):
|
|
1524
|
+
self.issue(path, "unsupported_type", message)
|
|
1525
|
+
|
|
1526
|
+
def _cross_section_output(
|
|
1527
|
+
self,
|
|
1528
|
+
primitive: str,
|
|
1529
|
+
operand: ColumnFacts,
|
|
1530
|
+
role: str,
|
|
1531
|
+
states: frozenset[str],
|
|
1532
|
+
/,
|
|
1533
|
+
) -> ColumnFacts:
|
|
1534
|
+
unresolved = ColumnFacts(None, True, operand.lineage, states)
|
|
1535
|
+
if primitive in ("winsorize", "mean_fill"):
|
|
1536
|
+
if operand.data_type is not None and operand.data_type not in (
|
|
1537
|
+
"float32",
|
|
1538
|
+
"float64",
|
|
1539
|
+
):
|
|
1540
|
+
self.issue(
|
|
1541
|
+
f"{role}.value.dtype",
|
|
1542
|
+
"unsupported_type",
|
|
1543
|
+
f"{primitive} is only supported for floating columns",
|
|
1544
|
+
)
|
|
1545
|
+
return unresolved
|
|
1546
|
+
return ColumnFacts(operand.data_type, True, operand.lineage, states)
|
|
1547
|
+
if primitive in ("top", "bottom"):
|
|
1548
|
+
if not self._numeric_or_issue(
|
|
1549
|
+
operand.data_type, f"{role}.value", primitive
|
|
1550
|
+
):
|
|
1551
|
+
return unresolved
|
|
1552
|
+
return ColumnFacts("bool", True, operand.lineage, states)
|
|
1553
|
+
if primitive in ("zscore", "demean") and not self._numeric_or_issue(
|
|
1554
|
+
operand.data_type, f"{role}.value", primitive
|
|
1555
|
+
):
|
|
1556
|
+
return unresolved
|
|
1557
|
+
return ColumnFacts("float64", True, operand.lineage, states)
|
|
1558
|
+
|
|
1559
|
+
# -- array analysis ------------------------------------------------------
|
|
1560
|
+
|
|
1561
|
+
def array(self, node: Node, path: str, /) -> ArrayFacts:
|
|
1562
|
+
cached = self._array_cache.get(node.digest)
|
|
1563
|
+
if cached is not None:
|
|
1564
|
+
return cached
|
|
1565
|
+
facts = self._analyze_array(node, path)
|
|
1566
|
+
self._array_cache[node.digest] = facts
|
|
1567
|
+
return facts
|
|
1568
|
+
|
|
1569
|
+
def _analyze_array(self, node: Node, path: str, /) -> ArrayFacts:
|
|
1570
|
+
handler = _ARRAY_HANDLERS.get(node.op.name)
|
|
1571
|
+
if handler is None:
|
|
1572
|
+
self.issue(
|
|
1573
|
+
path,
|
|
1574
|
+
"unknown_primitive_version",
|
|
1575
|
+
f"primitive {node.op.name!r} does not produce an array value",
|
|
1576
|
+
)
|
|
1577
|
+
return ArrayFacts(None, None, (), None, frozenset())
|
|
1578
|
+
return handler(self, node, path)
|
|
1579
|
+
|
|
1580
|
+
def _array_parameter(self, node: Node, path: str, /) -> ArrayFacts:
|
|
1581
|
+
name = _cstr(node.attr("name"))
|
|
1582
|
+
if (
|
|
1583
|
+
name is not None
|
|
1584
|
+
and node.digest not in self._declared
|
|
1585
|
+
and name not in self._undeclared_reported
|
|
1586
|
+
):
|
|
1587
|
+
self._undeclared_reported.add(name)
|
|
1588
|
+
self.issue(
|
|
1589
|
+
f"static_inputs.{name}",
|
|
1590
|
+
"unresolved_type",
|
|
1591
|
+
f"input {name!r} is referenced by program outputs but not"
|
|
1592
|
+
" declared in Program inputs",
|
|
1593
|
+
)
|
|
1594
|
+
shape = tuple(
|
|
1595
|
+
dimension.value
|
|
1596
|
+
for dimension in (
|
|
1597
|
+
node.attr("shape").items if isinstance(node.attr("shape"), CSeq) else ()
|
|
1598
|
+
)
|
|
1599
|
+
if isinstance(dimension, CInt)
|
|
1600
|
+
)
|
|
1601
|
+
return ArrayFacts(
|
|
1602
|
+
_cstr(node.attr("backend")),
|
|
1603
|
+
_ctype_str(node.attr("dtype")),
|
|
1604
|
+
shape,
|
|
1605
|
+
None,
|
|
1606
|
+
frozenset({"static"}),
|
|
1607
|
+
)
|
|
1608
|
+
|
|
1609
|
+
def _from_columns(self, node: Node, path: str, /) -> ArrayFacts:
|
|
1610
|
+
role = f"{path}.from_columns"
|
|
1611
|
+
table = self.table(node.args[0], f"{role}.value")
|
|
1612
|
+
columns = _cstr_seq(node.attr("columns"))
|
|
1613
|
+
data_type = self._from_columns_dtype(columns, table.schema, role)
|
|
1614
|
+
rows: int | str = table.lineage if table.lineage is not None else "rows"
|
|
1615
|
+
return ArrayFacts(
|
|
1616
|
+
_cstr(node.attr("backend")),
|
|
1617
|
+
data_type,
|
|
1618
|
+
(rows, len(columns)),
|
|
1619
|
+
table.lineage,
|
|
1620
|
+
table.state,
|
|
1621
|
+
)
|
|
1622
|
+
|
|
1623
|
+
def _from_columns_dtype(
|
|
1624
|
+
self,
|
|
1625
|
+
columns: tuple[str, ...],
|
|
1626
|
+
schema: tuple[Field, ...],
|
|
1627
|
+
role: str,
|
|
1628
|
+
/,
|
|
1629
|
+
) -> str | None:
|
|
1630
|
+
data_type: str | None = None
|
|
1631
|
+
for index, column in enumerate(columns):
|
|
1632
|
+
field = next((item for item in schema if item.name == column), None)
|
|
1633
|
+
if field is None:
|
|
1634
|
+
self.issue(
|
|
1635
|
+
f"{role}.columns[{index}]",
|
|
1636
|
+
"unresolved_type",
|
|
1637
|
+
f"unknown field {column!r} in the source table schema",
|
|
1638
|
+
)
|
|
1639
|
+
continue
|
|
1640
|
+
if data_type is None:
|
|
1641
|
+
data_type = field.data_type
|
|
1642
|
+
elif field.data_type != data_type:
|
|
1643
|
+
self.issue(
|
|
1644
|
+
f"{role}.columns[{index}].dtype",
|
|
1645
|
+
"unsupported_type",
|
|
1646
|
+
f"from_columns requires one dtype; found {data_type!r} and"
|
|
1647
|
+
f" {field.data_type!r}",
|
|
1648
|
+
)
|
|
1649
|
+
return None
|
|
1650
|
+
return data_type
|
|
1651
|
+
|
|
1652
|
+
def _array_pair_compat(
|
|
1653
|
+
self,
|
|
1654
|
+
left: ArrayFacts,
|
|
1655
|
+
right: ArrayFacts,
|
|
1656
|
+
role: str,
|
|
1657
|
+
lineage_message: str,
|
|
1658
|
+
/,
|
|
1659
|
+
) -> str | None:
|
|
1660
|
+
"""Report dtype, backend, and row-lineage incompatibilities."""
|
|
1661
|
+
|
|
1662
|
+
dtype = self._safe_array_dtype(
|
|
1663
|
+
left.backend or right.backend,
|
|
1664
|
+
left.dtype,
|
|
1665
|
+
right.dtype,
|
|
1666
|
+
role,
|
|
1667
|
+
)
|
|
1668
|
+
self._array_pair_aspects(left, right, role, lineage_message)
|
|
1669
|
+
return dtype
|
|
1670
|
+
|
|
1671
|
+
def _array_pair_aspects(
|
|
1672
|
+
self,
|
|
1673
|
+
left: ArrayFacts,
|
|
1674
|
+
right: ArrayFacts,
|
|
1675
|
+
role: str,
|
|
1676
|
+
lineage_message: str,
|
|
1677
|
+
/,
|
|
1678
|
+
) -> None:
|
|
1679
|
+
self._aspect_mismatch(
|
|
1680
|
+
left.backend,
|
|
1681
|
+
right.backend,
|
|
1682
|
+
f"{role}.right.backend",
|
|
1683
|
+
"implicit cross-backend conversion between {!r} and {!r} is rejected",
|
|
1684
|
+
)
|
|
1685
|
+
if (
|
|
1686
|
+
left.lineage is not None
|
|
1687
|
+
and right.lineage is not None
|
|
1688
|
+
and left.lineage != right.lineage
|
|
1689
|
+
):
|
|
1690
|
+
self.issue(f"{role}.right.lineage", "schema_mismatch", lineage_message)
|
|
1691
|
+
|
|
1692
|
+
def _safe_array_dtype(
|
|
1693
|
+
self,
|
|
1694
|
+
backend: str | None,
|
|
1695
|
+
left: str | None,
|
|
1696
|
+
right: str | None,
|
|
1697
|
+
role: str,
|
|
1698
|
+
/,
|
|
1699
|
+
) -> str | None:
|
|
1700
|
+
if left is None:
|
|
1701
|
+
return right
|
|
1702
|
+
if right is None:
|
|
1703
|
+
return left
|
|
1704
|
+
if backend is not None:
|
|
1705
|
+
from calc_flow.array import _symbolic_result_dtype
|
|
1706
|
+
|
|
1707
|
+
try:
|
|
1708
|
+
return _symbolic_result_dtype(backend, "matmul", left, right)
|
|
1709
|
+
except (KeyError, TypeError, ValueError):
|
|
1710
|
+
pass
|
|
1711
|
+
elif left == right:
|
|
1712
|
+
return left
|
|
1713
|
+
self.issue(
|
|
1714
|
+
f"{role}.right.dtype",
|
|
1715
|
+
"unsupported_type",
|
|
1716
|
+
f"no provable result dtype for {left!r} and {right!r}",
|
|
1717
|
+
)
|
|
1718
|
+
return None
|
|
1719
|
+
|
|
1720
|
+
def _aspect_mismatch(
|
|
1721
|
+
self, left: object, right: object, path: str, template: str, /
|
|
1722
|
+
) -> None:
|
|
1723
|
+
if left is not None and right is not None and left != right:
|
|
1724
|
+
self.issue(path, "unsupported_type", template.format(left, right))
|
|
1725
|
+
|
|
1726
|
+
def _matmul_inner_dims(
|
|
1727
|
+
self, left: ArrayFacts, right: ArrayFacts, role: str, /
|
|
1728
|
+
) -> tuple[int | str, int | str] | None:
|
|
1729
|
+
"""Check the inner-dimension contract; None marks a known mismatch."""
|
|
1730
|
+
|
|
1731
|
+
inner_left = left.shape[1]
|
|
1732
|
+
inner_right = right.shape[0]
|
|
1733
|
+
if inner_left == inner_right:
|
|
1734
|
+
return left.shape[0], right.shape[1]
|
|
1735
|
+
if isinstance(inner_left, int) and isinstance(inner_right, int):
|
|
1736
|
+
self.issue(
|
|
1737
|
+
f"{role}.right.shape[0]",
|
|
1738
|
+
"schema_mismatch",
|
|
1739
|
+
f"matmul inner dimensions {inner_left} and {inner_right} do not match",
|
|
1740
|
+
)
|
|
1741
|
+
return None
|
|
1742
|
+
self.issue(
|
|
1743
|
+
f"{role}.right.shape[0]",
|
|
1744
|
+
"unresolved_type",
|
|
1745
|
+
f"matmul inner dimensions {inner_left!r} and {inner_right!r}"
|
|
1746
|
+
" cannot be proved equal",
|
|
1747
|
+
)
|
|
1748
|
+
return left.shape[0], right.shape[1]
|
|
1749
|
+
|
|
1750
|
+
def _matmul(self, node: Node, path: str, /) -> ArrayFacts:
|
|
1751
|
+
role = f"{path}.matmul"
|
|
1752
|
+
left = self.array(node.args[0], f"{role}.left")
|
|
1753
|
+
right = self.array(node.args[1], f"{role}.right")
|
|
1754
|
+
states = left.state | right.state
|
|
1755
|
+
if len(left.shape) != 2 or len(right.shape) != 2:
|
|
1756
|
+
side = "left" if len(left.shape) != 2 else "right"
|
|
1757
|
+
rank = len(left.shape) if side == "left" else len(right.shape)
|
|
1758
|
+
self.issue(
|
|
1759
|
+
f"{role}.{side}.shape",
|
|
1760
|
+
"schema_mismatch",
|
|
1761
|
+
f"matmul requires rank-2 operands; got rank {rank}",
|
|
1762
|
+
)
|
|
1763
|
+
return ArrayFacts(None, None, (), None, states)
|
|
1764
|
+
domains_valid = self._array_numeric_domain(
|
|
1765
|
+
left.dtype,
|
|
1766
|
+
f"{role}.left",
|
|
1767
|
+
"matmul",
|
|
1768
|
+
) and self._array_numeric_domain(
|
|
1769
|
+
right.dtype,
|
|
1770
|
+
f"{role}.right",
|
|
1771
|
+
"matmul",
|
|
1772
|
+
)
|
|
1773
|
+
dtype = self._array_pair_compat(
|
|
1774
|
+
left, right, role, "matmul operands carry different row-axis lineages"
|
|
1775
|
+
)
|
|
1776
|
+
if not domains_valid:
|
|
1777
|
+
dtype = None
|
|
1778
|
+
output_dims = self._matmul_inner_dims(left, right, role)
|
|
1779
|
+
shape = () if output_dims is None else output_dims
|
|
1780
|
+
return ArrayFacts(
|
|
1781
|
+
left.backend,
|
|
1782
|
+
dtype,
|
|
1783
|
+
shape,
|
|
1784
|
+
left.lineage,
|
|
1785
|
+
states,
|
|
1786
|
+
)
|
|
1787
|
+
|
|
1788
|
+
def _array_operand(self, node: Node, operand_path: str, /) -> ArrayFacts:
|
|
1789
|
+
if node.op.name == "literal":
|
|
1790
|
+
return _array_literal_facts(node)
|
|
1791
|
+
return self.array(node, operand_path)
|
|
1792
|
+
|
|
1793
|
+
def _array_numeric_domain(
|
|
1794
|
+
self,
|
|
1795
|
+
data_type: str | None,
|
|
1796
|
+
operand_path: str,
|
|
1797
|
+
primitive: str,
|
|
1798
|
+
/,
|
|
1799
|
+
) -> bool:
|
|
1800
|
+
if data_type is None or data_type in _NUMERIC_TYPES:
|
|
1801
|
+
return True
|
|
1802
|
+
self.issue(
|
|
1803
|
+
f"{operand_path}.dtype",
|
|
1804
|
+
"unsupported_type",
|
|
1805
|
+
f"{primitive} is only defined for numeric arrays",
|
|
1806
|
+
)
|
|
1807
|
+
return False
|
|
1808
|
+
|
|
1809
|
+
def _array_boolean_domain(
|
|
1810
|
+
self,
|
|
1811
|
+
data_type: str | None,
|
|
1812
|
+
operand_path: str,
|
|
1813
|
+
primitive: str,
|
|
1814
|
+
/,
|
|
1815
|
+
) -> bool:
|
|
1816
|
+
if data_type is None or data_type == "bool":
|
|
1817
|
+
return True
|
|
1818
|
+
self.issue(
|
|
1819
|
+
f"{operand_path}.dtype",
|
|
1820
|
+
"unsupported_type",
|
|
1821
|
+
f"{primitive} is only defined for boolean arrays",
|
|
1822
|
+
)
|
|
1823
|
+
return False
|
|
1824
|
+
|
|
1825
|
+
@staticmethod
|
|
1826
|
+
def _array_dtype_operand(
|
|
1827
|
+
node: Node,
|
|
1828
|
+
facts: ArrayFacts,
|
|
1829
|
+
/,
|
|
1830
|
+
) -> str | bool | int | float | None:
|
|
1831
|
+
if node.op.name != "literal":
|
|
1832
|
+
return facts.dtype
|
|
1833
|
+
value = node.attr("value")
|
|
1834
|
+
if isinstance(value, (CBool, CInt, CFloat)):
|
|
1835
|
+
return value.value
|
|
1836
|
+
return None
|
|
1837
|
+
|
|
1838
|
+
def _array_binary_domain(
|
|
1839
|
+
self,
|
|
1840
|
+
primitive: str,
|
|
1841
|
+
left: ArrayFacts,
|
|
1842
|
+
right: ArrayFacts,
|
|
1843
|
+
role: str,
|
|
1844
|
+
/,
|
|
1845
|
+
) -> bool:
|
|
1846
|
+
if primitive in _BOOLEANS:
|
|
1847
|
+
left_valid = self._array_boolean_domain(
|
|
1848
|
+
left.dtype,
|
|
1849
|
+
f"{role}.left",
|
|
1850
|
+
primitive,
|
|
1851
|
+
)
|
|
1852
|
+
right_valid = self._array_boolean_domain(
|
|
1853
|
+
right.dtype,
|
|
1854
|
+
f"{role}.right",
|
|
1855
|
+
primitive,
|
|
1856
|
+
)
|
|
1857
|
+
return left_valid and right_valid
|
|
1858
|
+
if primitive in _ARITHMETIC or primitive in {"lt", "le", "gt", "ge"}:
|
|
1859
|
+
left_valid = self._array_numeric_domain(
|
|
1860
|
+
left.dtype,
|
|
1861
|
+
f"{role}.left",
|
|
1862
|
+
primitive,
|
|
1863
|
+
)
|
|
1864
|
+
right_valid = self._array_numeric_domain(
|
|
1865
|
+
right.dtype,
|
|
1866
|
+
f"{role}.right",
|
|
1867
|
+
primitive,
|
|
1868
|
+
)
|
|
1869
|
+
return left_valid and right_valid
|
|
1870
|
+
return True
|
|
1871
|
+
|
|
1872
|
+
def _elementwise(self, node: Node, path: str, /) -> ArrayFacts:
|
|
1873
|
+
role = f"{path}.{node.op.name}"
|
|
1874
|
+
primitive = node.op.name
|
|
1875
|
+
left = self._array_operand(node.args[0], f"{role}.left")
|
|
1876
|
+
if len(node.args) == 1:
|
|
1877
|
+
if primitive == "not":
|
|
1878
|
+
domain_valid = self._array_boolean_domain(
|
|
1879
|
+
left.dtype,
|
|
1880
|
+
f"{role}.value",
|
|
1881
|
+
primitive,
|
|
1882
|
+
)
|
|
1883
|
+
else:
|
|
1884
|
+
domain_valid = self._array_numeric_domain(
|
|
1885
|
+
left.dtype,
|
|
1886
|
+
f"{role}.value",
|
|
1887
|
+
primitive,
|
|
1888
|
+
)
|
|
1889
|
+
dtype: str | None = None
|
|
1890
|
+
operand = self._array_dtype_operand(node.args[0], left)
|
|
1891
|
+
if primitive == "not" and domain_valid and left.dtype == "bool":
|
|
1892
|
+
dtype = "bool"
|
|
1893
|
+
elif domain_valid and left.backend is not None and operand is not None:
|
|
1894
|
+
from calc_flow.array import _symbolic_result_dtype
|
|
1895
|
+
|
|
1896
|
+
try:
|
|
1897
|
+
dtype = _symbolic_result_dtype(left.backend, primitive, operand)
|
|
1898
|
+
except (KeyError, TypeError, ValueError):
|
|
1899
|
+
self.issue(
|
|
1900
|
+
f"{role}.value.dtype",
|
|
1901
|
+
"unsupported_type",
|
|
1902
|
+
f"the {left.backend!r} provider cannot prove {primitive} dtype",
|
|
1903
|
+
)
|
|
1904
|
+
return ArrayFacts(left.backend, dtype, left.shape, left.lineage, left.state)
|
|
1905
|
+
right = self._array_operand(node.args[1], f"{role}.right")
|
|
1906
|
+
self._array_pair_aspects(
|
|
1907
|
+
left,
|
|
1908
|
+
right,
|
|
1909
|
+
role,
|
|
1910
|
+
"array operands carry different row-axis lineages",
|
|
1911
|
+
)
|
|
1912
|
+
domain_valid = self._array_binary_domain(primitive, left, right, role)
|
|
1913
|
+
shape = _broadcast_shapes(left.shape, right.shape, role, self)
|
|
1914
|
+
backend = left.backend or right.backend
|
|
1915
|
+
dtype: str | None = None
|
|
1916
|
+
left_operand = self._array_dtype_operand(node.args[0], left)
|
|
1917
|
+
right_operand = self._array_dtype_operand(node.args[1], right)
|
|
1918
|
+
if (
|
|
1919
|
+
domain_valid
|
|
1920
|
+
and backend is not None
|
|
1921
|
+
and left_operand is not None
|
|
1922
|
+
and right_operand is not None
|
|
1923
|
+
):
|
|
1924
|
+
from calc_flow.array import _symbolic_result_dtype
|
|
1925
|
+
|
|
1926
|
+
try:
|
|
1927
|
+
dtype = _symbolic_result_dtype(
|
|
1928
|
+
backend,
|
|
1929
|
+
primitive,
|
|
1930
|
+
left_operand,
|
|
1931
|
+
right_operand,
|
|
1932
|
+
)
|
|
1933
|
+
except (KeyError, TypeError, ValueError):
|
|
1934
|
+
self.issue(
|
|
1935
|
+
f"{role}.right.dtype",
|
|
1936
|
+
"unsupported_type",
|
|
1937
|
+
"the selected provider cannot prove a safe result dtype for"
|
|
1938
|
+
f" {left.dtype!r} and {right.dtype!r}",
|
|
1939
|
+
)
|
|
1940
|
+
return ArrayFacts(
|
|
1941
|
+
backend,
|
|
1942
|
+
dtype,
|
|
1943
|
+
shape,
|
|
1944
|
+
left.lineage if left.lineage is not None else right.lineage,
|
|
1945
|
+
left.state | right.state,
|
|
1946
|
+
)
|
|
1947
|
+
|
|
1948
|
+
|
|
1949
|
+
def _literal_column(node: Node, _path: str, /) -> ColumnFacts:
|
|
1950
|
+
return _literal_facts(node)
|
|
1951
|
+
|
|
1952
|
+
|
|
1953
|
+
_ELEMENTWISE_PRIMITIVES: Final[frozenset[str]] = (
|
|
1954
|
+
_ARITHMETIC | _COMPARISONS | _BOOLEANS | frozenset({"neg", "not"})
|
|
1955
|
+
)
|
|
1956
|
+
|
|
1957
|
+
_ARRAY_HANDLERS: Final[dict[str, Callable[[_Analyzer, Node, str], ArrayFacts]]] = {
|
|
1958
|
+
"parameter": _Analyzer._array_parameter,
|
|
1959
|
+
"from_columns": _Analyzer._from_columns,
|
|
1960
|
+
"matmul": _Analyzer._matmul,
|
|
1961
|
+
**dict.fromkeys(_ELEMENTWISE_PRIMITIVES, _Analyzer._elementwise),
|
|
1962
|
+
}
|
|
1963
|
+
|
|
1964
|
+
_COLUMN_HANDLERS: Final[dict[str, Callable[[_Analyzer, Node, str], ColumnFacts]]] = {
|
|
1965
|
+
"column_ref": _Analyzer._column_ref,
|
|
1966
|
+
"literal": _literal_column,
|
|
1967
|
+
**dict.fromkeys(("add", "sub", "mul", "truediv"), _Analyzer._arithmetic),
|
|
1968
|
+
**dict.fromkeys(_COMPARISONS, _Analyzer._comparison),
|
|
1969
|
+
**dict.fromkeys(_BOOLEANS, _Analyzer._boolean_pair),
|
|
1970
|
+
**dict.fromkeys(("neg", "not"), _Analyzer._unary),
|
|
1971
|
+
"where": _Analyzer._where,
|
|
1972
|
+
"coalesce": _Analyzer._coalesce,
|
|
1973
|
+
**dict.fromkeys(_UNARY_NUMERIC, _Analyzer._scalar_function),
|
|
1974
|
+
"abs": _Analyzer._arithmetic_like_unary,
|
|
1975
|
+
"clip": _Analyzer._clip,
|
|
1976
|
+
"cast": _Analyzer._cast,
|
|
1977
|
+
**dict.fromkeys(("lag", "delta"), _Analyzer._lag_like),
|
|
1978
|
+
"ewma": _Analyzer._ewma,
|
|
1979
|
+
**dict.fromkeys(_ROLLING_AGGREGATES, _Analyzer._rolling_aggregate),
|
|
1980
|
+
**dict.fromkeys(("covariance", "correlation"), _Analyzer._rolling_pair),
|
|
1981
|
+
**dict.fromkeys(_CROSS_SECTION, _Analyzer._cross_section),
|
|
1982
|
+
}
|
|
1983
|
+
|
|
1984
|
+
|
|
1985
|
+
def _column_state_union(
|
|
1986
|
+
operands: tuple[ColumnFacts, ...] | list[ColumnFacts], /
|
|
1987
|
+
) -> frozenset[str]:
|
|
1988
|
+
combined: set[str] = set()
|
|
1989
|
+
for facts in operands:
|
|
1990
|
+
combined |= facts.state
|
|
1991
|
+
return frozenset(combined)
|
|
1992
|
+
|
|
1993
|
+
|
|
1994
|
+
def _array_literal_facts(node: Node, /) -> ArrayFacts:
|
|
1995
|
+
value = node.attr("value")
|
|
1996
|
+
dtype = _literal_dtype(value)
|
|
1997
|
+
return ArrayFacts(None, dtype, (), None, frozenset())
|
|
1998
|
+
|
|
1999
|
+
|
|
2000
|
+
def _literal_dtype(value: CValue, /) -> str | None:
|
|
2001
|
+
if isinstance(value, CNull):
|
|
2002
|
+
return None
|
|
2003
|
+
if isinstance(value, CBool):
|
|
2004
|
+
return "bool"
|
|
2005
|
+
if isinstance(value, CInt):
|
|
2006
|
+
if -(2**63) <= value.value <= 2**63 - 1:
|
|
2007
|
+
return "int64"
|
|
2008
|
+
return "uint64"
|
|
2009
|
+
if isinstance(value, CFloat):
|
|
2010
|
+
return "float64"
|
|
2011
|
+
if isinstance(value, CStr):
|
|
2012
|
+
return "string"
|
|
2013
|
+
return None
|
|
2014
|
+
|
|
2015
|
+
|
|
2016
|
+
def _literal_facts(node: Node, /) -> ColumnFacts:
|
|
2017
|
+
value = node.attr("value")
|
|
2018
|
+
return ColumnFacts(
|
|
2019
|
+
_literal_dtype(value),
|
|
2020
|
+
isinstance(value, CNull),
|
|
2021
|
+
None,
|
|
2022
|
+
frozenset(),
|
|
2023
|
+
)
|
|
2024
|
+
|
|
2025
|
+
|
|
2026
|
+
def _broadcast_shapes(
|
|
2027
|
+
left: tuple[int | str, ...],
|
|
2028
|
+
right: tuple[int | str, ...],
|
|
2029
|
+
role: str,
|
|
2030
|
+
analyzer: _Analyzer,
|
|
2031
|
+
/,
|
|
2032
|
+
) -> tuple[int | str, ...]:
|
|
2033
|
+
rank = max(len(left), len(right))
|
|
2034
|
+
result: list[int | str] = []
|
|
2035
|
+
for index in range(rank):
|
|
2036
|
+
first = left[len(left) - 1 - index] if index < len(left) else 1
|
|
2037
|
+
second = right[len(right) - 1 - index] if index < len(right) else 1
|
|
2038
|
+
result.append(
|
|
2039
|
+
_broadcast_dimension(first, second, role, rank - 1 - index, analyzer)
|
|
2040
|
+
)
|
|
2041
|
+
result.reverse()
|
|
2042
|
+
return tuple(result)
|
|
2043
|
+
|
|
2044
|
+
|
|
2045
|
+
def _broadcast_dimension(
|
|
2046
|
+
first: int | str,
|
|
2047
|
+
second: int | str,
|
|
2048
|
+
role: str,
|
|
2049
|
+
index: int,
|
|
2050
|
+
analyzer: _Analyzer,
|
|
2051
|
+
/,
|
|
2052
|
+
) -> int | str:
|
|
2053
|
+
if first == second:
|
|
2054
|
+
return first
|
|
2055
|
+
if first == 1:
|
|
2056
|
+
return second
|
|
2057
|
+
if second == 1:
|
|
2058
|
+
return first
|
|
2059
|
+
if isinstance(first, int) and isinstance(second, int):
|
|
2060
|
+
analyzer.issue(
|
|
2061
|
+
f"{role}.right.shape[{index}]",
|
|
2062
|
+
"schema_mismatch",
|
|
2063
|
+
f"array shapes {first} and {second} are not broadcast-compatible",
|
|
2064
|
+
)
|
|
2065
|
+
return first
|
|
2066
|
+
analyzer.issue(
|
|
2067
|
+
f"{role}.right.shape[{index}]",
|
|
2068
|
+
"unresolved_type",
|
|
2069
|
+
f"array dimensions {first!r} and {second!r} cannot be proved"
|
|
2070
|
+
" broadcast-compatible",
|
|
2071
|
+
)
|
|
2072
|
+
return first
|
|
2073
|
+
|
|
2074
|
+
|
|
2075
|
+
def _require_runtime(runtime: object, /) -> Runtime:
|
|
2076
|
+
if not isinstance(runtime, Runtime):
|
|
2077
|
+
raise TypeError(
|
|
2078
|
+
f"analyze requires an explicit calc_flow Runtime; got {type_name(runtime)}"
|
|
2079
|
+
)
|
|
2080
|
+
return runtime
|
|
2081
|
+
|
|
2082
|
+
|
|
2083
|
+
def _require_mode(mode: object, /) -> CompileMode:
|
|
2084
|
+
if type(mode) is not str or mode not in _MODES:
|
|
2085
|
+
raise TypeError(
|
|
2086
|
+
f"mode must be 'batch' or 'stream'; got {type_name(mode)}"
|
|
2087
|
+
if type(mode) is not str
|
|
2088
|
+
else f"mode must be 'batch' or 'stream'; got {mode!r}"
|
|
2089
|
+
)
|
|
2090
|
+
return mode # type: ignore[return-value]
|
|
2091
|
+
|
|
2092
|
+
|
|
2093
|
+
def _run(
|
|
2094
|
+
program: object, runtime: Runtime, mode: CompileMode, /
|
|
2095
|
+
) -> tuple[_Analyzer, RuntimeCapabilities]:
|
|
2096
|
+
capabilities = runtime.capabilities()
|
|
2097
|
+
declared = frozenset(value._node.digest for value in program.inputs)
|
|
2098
|
+
portable = frozenset((*capabilities.portable_arrow_types, _EVENT_TIME_TYPE))
|
|
2099
|
+
analyzer = _Analyzer(
|
|
2100
|
+
mode,
|
|
2101
|
+
declared,
|
|
2102
|
+
portable,
|
|
2103
|
+
"array" in capabilities.batch_kinds,
|
|
2104
|
+
)
|
|
2105
|
+
for value in program.inputs:
|
|
2106
|
+
node = value._node
|
|
2107
|
+
root = _declaration_root(node)
|
|
2108
|
+
name = _cstr(node.attr("name")) or ""
|
|
2109
|
+
analyzer.check_input_declaration(node, root, name)
|
|
2110
|
+
_analyze_outputs(program, analyzer)
|
|
2111
|
+
if mode == "stream":
|
|
2112
|
+
_check_stream_ordering_for_inputs(program, analyzer)
|
|
2113
|
+
return analyzer, capabilities
|
|
2114
|
+
|
|
2115
|
+
|
|
2116
|
+
def _analyze_outputs(program: object, analyzer: _Analyzer, /) -> None:
|
|
2117
|
+
from calc_flow.symbolic.expr import ArrayExpr, TableExpr
|
|
2118
|
+
|
|
2119
|
+
for output_name, value in program.outputs:
|
|
2120
|
+
path = f"outputs.{output_name}"
|
|
2121
|
+
if isinstance(value, TableExpr):
|
|
2122
|
+
analyzer.table(value._node, path)
|
|
2123
|
+
elif isinstance(value, ArrayExpr):
|
|
2124
|
+
facts = analyzer.array(value._node, path)
|
|
2125
|
+
analyzer.check_array_output_kind(output_name)
|
|
2126
|
+
analyzer.check_unbounded_output(output_name, facts)
|
|
2127
|
+
|
|
2128
|
+
|
|
2129
|
+
def _declaration_root(node: Node, /) -> str:
|
|
2130
|
+
return "static_inputs" if node.op.name == "parameter" else "inputs"
|
|
2131
|
+
|
|
2132
|
+
|
|
2133
|
+
def _check_stream_ordering_for_inputs(program: object, analyzer: _Analyzer, /) -> None:
|
|
2134
|
+
for value in program.inputs:
|
|
2135
|
+
node = value._node
|
|
2136
|
+
name = _cstr(node.attr("name")) or ""
|
|
2137
|
+
if name in analyzer.temporal_lineages:
|
|
2138
|
+
analyzer.check_stream_ordering(node, _declaration_root(node), name)
|
|
2139
|
+
|
|
2140
|
+
|
|
2141
|
+
def analyze_program(
|
|
2142
|
+
program: object, runtime: object, mode: object, /
|
|
2143
|
+
) -> AnalysisResult:
|
|
2144
|
+
"""Analyze one program against one immutable capability snapshot."""
|
|
2145
|
+
|
|
2146
|
+
runtime_value = _require_runtime(runtime)
|
|
2147
|
+
mode_value = _require_mode(mode)
|
|
2148
|
+
analyzer, capabilities = _run(program, runtime_value, mode_value)
|
|
2149
|
+
return AnalysisResult(
|
|
2150
|
+
mode=mode_value,
|
|
2151
|
+
program_fingerprint=program.fingerprint,
|
|
2152
|
+
capability_session_id=capabilities.scope.session_id,
|
|
2153
|
+
capability_revision=capabilities.scope.revision,
|
|
2154
|
+
issues=analyzer.issues,
|
|
2155
|
+
)
|
|
2156
|
+
|
|
2157
|
+
|
|
2158
|
+
def explain_program(program: object, runtime: object, mode: object, /) -> str:
|
|
2159
|
+
"""Render deterministic analysis facts for one program."""
|
|
2160
|
+
|
|
2161
|
+
runtime_value = _require_runtime(runtime)
|
|
2162
|
+
mode_value = _require_mode(mode)
|
|
2163
|
+
analyzer, capabilities = _run(program, runtime_value, mode_value)
|
|
2164
|
+
lines = _explain_header(program, mode_value, capabilities)
|
|
2165
|
+
if program.inputs:
|
|
2166
|
+
lines.append(" inputs")
|
|
2167
|
+
lines.extend(_explain_input(value) for value in program.inputs)
|
|
2168
|
+
lines.append(" outputs")
|
|
2169
|
+
for output_name, value in program.outputs:
|
|
2170
|
+
lines.extend(_explain_output(output_name, value, analyzer))
|
|
2171
|
+
issues = analyzer.issues
|
|
2172
|
+
if not issues:
|
|
2173
|
+
from calc_flow.errors import CompileError
|
|
2174
|
+
from calc_flow.symbolic.lower import lower_program_document
|
|
2175
|
+
from calc_flow.symbolic.optimizer import explain_optimization
|
|
2176
|
+
|
|
2177
|
+
try:
|
|
2178
|
+
document = lower_program_document(program, runtime_value, mode_value)
|
|
2179
|
+
except CompileError:
|
|
2180
|
+
pass
|
|
2181
|
+
else:
|
|
2182
|
+
lines.extend(explain_optimization(document))
|
|
2183
|
+
if issues:
|
|
2184
|
+
lines.append(" issues")
|
|
2185
|
+
lines.extend(
|
|
2186
|
+
f" {issue.path}: {issue.code}: {issue.message}" for issue in issues
|
|
2187
|
+
)
|
|
2188
|
+
return "\n".join(lines)
|
|
2189
|
+
|
|
2190
|
+
|
|
2191
|
+
def _explain_header(
|
|
2192
|
+
program: object, mode: CompileMode, capabilities: RuntimeCapabilities, /
|
|
2193
|
+
) -> list[str]:
|
|
2194
|
+
return [
|
|
2195
|
+
f"program {program.name}",
|
|
2196
|
+
f" mode {mode}",
|
|
2197
|
+
f" fingerprint {program.fingerprint}",
|
|
2198
|
+
"capability session"
|
|
2199
|
+
f" {capabilities.scope.session_id} revision"
|
|
2200
|
+
f" {capabilities.scope.revision}",
|
|
2201
|
+
]
|
|
2202
|
+
|
|
2203
|
+
|
|
2204
|
+
def _explain_output(
|
|
2205
|
+
output_name: str, value: object, analyzer: _Analyzer, /
|
|
2206
|
+
) -> list[str]:
|
|
2207
|
+
from calc_flow.symbolic.expr import ArrayExpr, TableExpr
|
|
2208
|
+
|
|
2209
|
+
path = f"outputs.{output_name}"
|
|
2210
|
+
if isinstance(value, TableExpr):
|
|
2211
|
+
return _explain_table_output(output_name, analyzer.table(value._node, path))
|
|
2212
|
+
if isinstance(value, ArrayExpr):
|
|
2213
|
+
return _explain_array_output(output_name, analyzer.array(value._node, path))
|
|
2214
|
+
return []
|
|
2215
|
+
|
|
2216
|
+
|
|
2217
|
+
def _explain_input(value: object, /) -> str:
|
|
2218
|
+
node = value._node
|
|
2219
|
+
name = _cstr(node.attr("name")) or ""
|
|
2220
|
+
if node.op.name != "parameter":
|
|
2221
|
+
return (
|
|
2222
|
+
f" input {name} event_time"
|
|
2223
|
+
f" {_cstr(node.attr('event_time')) or 'none'} entity_by"
|
|
2224
|
+
f" {_render_names(_cstr_seq(node.attr('entity_by')))} sequence_by"
|
|
2225
|
+
f" {_render_names(_cstr_seq(node.attr('sequence_by')))}"
|
|
2226
|
+
)
|
|
2227
|
+
kind = node.attr("kind")
|
|
2228
|
+
if isinstance(kind, CEnum) and kind.variant == "array":
|
|
2229
|
+
return (
|
|
2230
|
+
f" static_input {name} array backend"
|
|
2231
|
+
f" {_cstr(node.attr('backend'))} dtype"
|
|
2232
|
+
f" {_ctype_str(node.attr('dtype'))} shape"
|
|
2233
|
+
f" {_render_shape(_parameter_shape(node))}"
|
|
2234
|
+
)
|
|
2235
|
+
return (
|
|
2236
|
+
f" static_input {name} table fields"
|
|
2237
|
+
f" {len(_schema_fields(node.attr('schema')))}"
|
|
2238
|
+
)
|
|
2239
|
+
|
|
2240
|
+
|
|
2241
|
+
def _explain_table_output(output_name: str, facts: TableFacts, /) -> list[str]:
|
|
2242
|
+
lines = [f" output {output_name} table"]
|
|
2243
|
+
lines.extend(
|
|
2244
|
+
f" field {field.name} {field.data_type}"
|
|
2245
|
+
f" nullable={'true' if field.nullable else 'false'}"
|
|
2246
|
+
for field in facts.schema
|
|
2247
|
+
)
|
|
2248
|
+
lines.append(f" state {_render_state(facts.state)}")
|
|
2249
|
+
return lines
|
|
2250
|
+
|
|
2251
|
+
|
|
2252
|
+
def _explain_array_output(output_name: str, facts: ArrayFacts, /) -> list[str]:
|
|
2253
|
+
lineage = facts.lineage if facts.lineage is not None else "none"
|
|
2254
|
+
return [
|
|
2255
|
+
f" output {output_name} array backend {facts.backend}"
|
|
2256
|
+
f" dtype {facts.dtype} shape {_render_shape(facts.shape)}"
|
|
2257
|
+
f" lineage {lineage}",
|
|
2258
|
+
f" state {_render_state(facts.state)}",
|
|
2259
|
+
]
|
|
2260
|
+
|
|
2261
|
+
|
|
2262
|
+
def _parameter_shape(node: Node, /) -> tuple[int | str, ...]:
|
|
2263
|
+
shape = node.attr("shape")
|
|
2264
|
+
if not isinstance(shape, CSeq):
|
|
2265
|
+
return ()
|
|
2266
|
+
return tuple(item.value for item in shape.items if isinstance(item, CInt))
|
|
2267
|
+
|
|
2268
|
+
|
|
2269
|
+
def _render_shape(shape: tuple[int | str, ...], /) -> str:
|
|
2270
|
+
return "(" + ", ".join(str(dimension) for dimension in shape) + ")"
|
|
2271
|
+
|
|
2272
|
+
|
|
2273
|
+
def _render_names(names: tuple[str, ...], /) -> str:
|
|
2274
|
+
return "[" + ", ".join(names) + "]"
|
|
2275
|
+
|
|
2276
|
+
|
|
2277
|
+
def _render_state(state: frozenset[str], /) -> str:
|
|
2278
|
+
if not state:
|
|
2279
|
+
return "stateless"
|
|
2280
|
+
return ", ".join(sorted(state))
|