sqlassert 0.1.2__py3-none-any.whl → 0.3.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. sqlassert/__init__.py +34 -9
  2. sqlassert/__main__.py +138 -0
  3. sqlassert/check.py +444 -0
  4. sqlassert/diagnostics.py +57 -0
  5. sqlassert/dialects.py +211 -0
  6. sqlassert/discovery/__init__.py +30 -0
  7. sqlassert/discovery/bigquery.py +71 -0
  8. sqlassert/discovery/databricks.py +18 -0
  9. sqlassert/discovery/duckdb.py +163 -0
  10. sqlassert/discovery/fetcher.py +402 -0
  11. sqlassert/discovery/information_schema.py +326 -0
  12. sqlassert/discovery/mysql.py +70 -0
  13. sqlassert/discovery/postgres.py +143 -0
  14. sqlassert/discovery/redshift.py +36 -0
  15. sqlassert/discovery/snowflake.py +63 -0
  16. sqlassert/discovery/trino.py +37 -0
  17. sqlassert/improvements.py +528 -0
  18. sqlassert/ir/__init__.py +56 -0
  19. sqlassert/ir/convert.py +2377 -0
  20. sqlassert/ir/model.py +676 -0
  21. sqlassert/main.py +57 -0
  22. sqlassert/properties.py +121 -0
  23. sqlassert/provenance.py +73 -0
  24. sqlassert/python_engine.py +299 -0
  25. sqlassert/reporting.py +102 -0
  26. sqlassert/rules/STYLEGUIDE.md +19 -0
  27. sqlassert/rules/__init__.py +11 -0
  28. sqlassert/rules/column_set.py +34 -0
  29. sqlassert/rules/domain.py +376 -0
  30. sqlassert/rules/equality.py +233 -0
  31. sqlassert/rules/exactly_one.py +75 -0
  32. sqlassert/rules/foreign_key.py +316 -0
  33. sqlassert/rules/join_cardinality.py +94 -0
  34. sqlassert/rules/nullability.py +418 -0
  35. sqlassert/rules/uniqueness.py +601 -0
  36. sqlassert/smells.py +306 -0
  37. sqlassert/sql_parse.py +974 -0
  38. sqlassert/suggest.py +339 -0
  39. sqlassert-0.3.1.dist-info/METADATA +725 -0
  40. sqlassert-0.3.1.dist-info/RECORD +44 -0
  41. {sqlassert-0.1.2.dist-info → sqlassert-0.3.1.dist-info}/WHEEL +1 -1
  42. sqlassert-0.3.1.dist-info/entry_points.txt +2 -0
  43. sqlassert-0.3.1.dist-info/licenses/LICENSE +21 -0
  44. sqlassert/unique.py +0 -605
  45. sqlassert-0.1.2.dist-info/METADATA +0 -173
  46. sqlassert-0.1.2.dist-info/RECORD +0 -6
  47. {sqlassert-0.1.2.dist-info → sqlassert-0.3.1.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,2377 @@
1
+ """Link a parsed SQL program into the framework-independent relational IR.
2
+
3
+ Declaration and scope discovery happens before lowering. This gives table,
4
+ view, and CTE references compiler-like symbol resolution: every occurrence is
5
+ a fresh Alias, while all occurrences share one memoized NamedRelation.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ from collections.abc import Callable, Iterable, Sequence, Set
11
+ from dataclasses import dataclass, field, replace
12
+ from functools import cache
13
+ from typing import Literal, assert_never
14
+
15
+ from sqlglot import Expr, exp
16
+
17
+ from sqlassert import diagnostics as diag
18
+ from sqlassert import ir
19
+ from sqlassert.dialects import DialectSemantics, semantics
20
+ from sqlassert.diagnostics import Diagnostic
21
+ from sqlassert.properties import (
22
+ EnumeratedDomain, ExactlyOneJoin, ExhaustiveCase, ForeignKey, FunctionalDependency, ManyToOneJoin, NonNullColumn, NonNullColumns, NonNullElements, NonNullUniqueSet,
23
+ OneToManyJoin, OneToOneJoin, Property, ReferentialCoverage, UniqueJoin,
24
+ UniqueSet,
25
+ )
26
+ from sqlassert.provenance import AttachmentSite, Origin, SQL
27
+ from sqlassert.sql_parse import (
28
+ JOIN_MARKER_NOUNS, MARKER_NOUNS, ColumnMarker, ParsedProgram, assertion_span, column_markers,
29
+ exhaustive_span, grouping, is_ambiguous_pipes, join_marker_kind, join_origin, literal_value, marker_position,
30
+ )
31
+
32
+
33
+ @dataclass(frozen=True)
34
+ class IrConversionResult:
35
+ program: ir.Program
36
+ knowledge: tuple[Property, ...]
37
+ diagnostics: tuple[Diagnostic, ...] = ()
38
+ warnings: tuple[Diagnostic, ...] = ()
39
+
40
+
41
+ @dataclass(eq=False)
42
+ class _Symbol:
43
+ name: str
44
+ role: ir.RelationRole
45
+ origin: Origin
46
+ body: exp.Query | None = None
47
+ statement: exp.Create | None = None
48
+ required_columns: set[str] = field(default_factory=set)
49
+ # Table-level constraints added after CREATE TABLE (ALTER TABLE, CREATE
50
+ # UNIQUE INDEX), shaped like the CREATE TABLE's own, and columns set NOT NULL.
51
+ constraints: list[Expr] = field(default_factory=list)
52
+ not_null: set[str] = field(default_factory=set)
53
+ # Each unique index with an expression key part: its parts and statement.
54
+ expression_keys: list[tuple[list[Expr], exp.Create]] = field(default_factory=list)
55
+
56
+
57
+ @dataclass(frozen=True)
58
+ class _Binding:
59
+ qualifier: str
60
+ name: str
61
+ column: ir.OutputColumn
62
+ # The schema (and catalog) of an unaliased table, as `s1` in `s1.t.k`.
63
+ schema: str = ""
64
+
65
+
66
+ @dataclass(frozen=True)
67
+ class _MarkerReference:
68
+ """A Foreign Key marker whose referencing columns are lowered, awaiting its referenced relation."""
69
+
70
+ marker: ColumnMarker
71
+ kind: Literal["foreign-key", "declared-foreign-key"]
72
+ origin: Origin
73
+ columns: tuple[ir.OutputColumn, ...]
74
+ target: _Symbol
75
+ position: int
76
+
77
+
78
+ @dataclass(frozen=True)
79
+ class _ConstraintReference:
80
+ """A `REFERENCES` constraint of a declared table onto a relation the program reaches."""
81
+
82
+ table: _Symbol
83
+ columns: tuple[str, ...]
84
+ target: _Symbol
85
+ target_columns: tuple[str, ...] | None
86
+ origin: Origin
87
+ # The `REFERENCES` or `FOREIGN KEY` constraint itself.
88
+ node: Expr
89
+
90
+
91
+ @dataclass(frozen=True)
92
+ class _Lowered:
93
+ relation: ir.RelationExpr
94
+ bindings: tuple[_Binding, ...]
95
+
96
+
97
+ @dataclass
98
+ class IrParser:
99
+ dialect: str
100
+ # Unenforced keys and foreign keys become Property Declarations, as `RELY` makes them.
101
+ trust_unenforced_keys: bool = True
102
+ _global_symbols: dict[str, _Symbol] = field(default_factory=dict)
103
+ _symbols: list[_Symbol] = field(default_factory=list)
104
+ _query_scopes: dict[int, dict[str, _Symbol]] = field(default_factory=dict)
105
+ _table_symbols: dict[int, _Symbol] = field(default_factory=dict)
106
+ _source_required: dict[int, set[str]] = field(default_factory=dict)
107
+ _source_expressions: dict[int, Expr] = field(default_factory=dict)
108
+ _named_relations: dict[_Symbol, ir.NamedRelation] = field(default_factory=dict)
109
+ _resolving: set[_Symbol] = field(default_factory=set)
110
+ _knowledge: list[Property] = field(default_factory=list)
111
+ _assertions: list[ir.Assertion] = field(default_factory=list)
112
+ _declarations: list[Property] = field(default_factory=list)
113
+ _diagnostics: list[Diagnostic] = field(default_factory=list)
114
+ _warnings: list[Diagnostic] = field(default_factory=list)
115
+ _handled_column_markers: set[int] = field(default_factory=set)
116
+ _handled_joins: set[int] = field(default_factory=set)
117
+ _handled_cases: set[int] = field(default_factory=set)
118
+ _reported_cycles: set[_Symbol] = field(default_factory=set)
119
+ _attachment_sites: list[tuple[AttachmentSite, ir.RelationExpr | ir.SimpleCase]] = field(default_factory=list)
120
+ _sited_cases: set[int] = field(default_factory=set)
121
+ # Each scalar subquery lowered as a join, by id, and the qualifier of the
122
+ # binding that holds its value.
123
+ _scalar_values: dict[int, str] = field(default_factory=dict)
124
+ _scalar_subqueries: list[tuple[AttachmentSite, ir.Join]] = field(default_factory=list)
125
+ _sort_keys: dict[int, tuple[ir.OutputColumn, ...] | None] = field(default_factory=dict)
126
+ _site_sort_keys: list[tuple[AttachmentSite, tuple[ir.OutputColumn, ...] | None]] = field(default_factory=list)
127
+ _set_operations: list[tuple[AttachmentSite, ir.SetOperation]] = field(default_factory=list)
128
+ # Columns made up for names a query uses: of a table with no CREATE, or an
129
+ # opaque relation (`_guessed`), and those a declared table does not have (`_absent`).
130
+ _guessed: set[ir.OutputColumn] = field(default_factory=set)
131
+ # A scalar subquery's columns, each bound to the scope that owns it.
132
+ _scoped_columns: dict[int, ir.OutputColumn] = field(default_factory=dict)
133
+ _absent: set[ir.OutputColumn] = field(default_factory=set)
134
+ # Each declared table column's type, as its CREATE TABLE writes it.
135
+ _declared_types: dict[ir.OutputColumn, exp.DataType] = field(default_factory=dict)
136
+ # The comparisons already checked for a zoned timestamp's text, by id.
137
+ _zone_checked: set[int] = field(default_factory=set)
138
+ # Each composite column or CAST a comparison reads, by id, with the
139
+ # comparison: a warning unless its elements are declared non-null.
140
+ _composite_comparisons: dict[int, tuple[Origin, str, str, ir.ScalarExpr]] = field(default_factory=dict)
141
+ _marker_targets: dict[int, _Symbol] = field(default_factory=dict)
142
+ _marker_references: list[_MarkerReference] = field(default_factory=list)
143
+ _constraint_references: list[_ConstraintReference] = field(default_factory=list)
144
+ # Constraints an ALTER TABLE ... NOT VALID added: rows already there were never checked.
145
+ _unvalidated: set[int] = field(default_factory=set)
146
+
147
+ @property
148
+ def semantics(self) -> DialectSemantics:
149
+ return semantics(self.dialect)
150
+
151
+ def parse(self, ast: ParsedProgram) -> IrConversionResult:
152
+ for statement in ast.create_statements:
153
+ match statement:
154
+ case exp.Create(kind="INDEX"):
155
+ self._declare_index(statement)
156
+ case exp.Create(kind="SCHEMA"):
157
+ pass # A schema states nothing about rows, like a plain index.
158
+ case exp.Create():
159
+ self._declare(statement)
160
+ case exp.Alter():
161
+ self._alter(statement)
162
+ case _:
163
+ raise TypeError(f"Unhandled schema statement: {type(statement).__name__}")
164
+
165
+ for symbol in tuple(self._symbols):
166
+ if symbol.role is ir.RelationRole.VIEW and symbol.body is not None:
167
+ self._discover_query(symbol.body, {})
168
+ if ast.root_select is not None:
169
+ self._discover_query(ast.root_select, {})
170
+ self._discover_constraint_references()
171
+ self._propagate_required_columns()
172
+
173
+ root = self._lower_query(ast.root_select) if ast.root_select is not None else None
174
+ # Every view is a query of its own: its markers are checked whether or
175
+ # not the root reads it, so a file of views (a project's models) needs
176
+ # no root SELECT.
177
+ for symbol in tuple(self._symbols):
178
+ if symbol.role is ir.RelationRole.VIEW and symbol not in self._named_relations:
179
+ self._resolve_symbol(symbol)
180
+ # Referenced relations are linked last: a marker on a view may name
181
+ # the view itself, or one that reads it.
182
+ self._link_constraint_references()
183
+ self._link_marker_references()
184
+ self._report_unanalyzed(ast)
185
+
186
+ named_relations = tuple(self._named_relations.get(symbol) or self._unresolved_named_relation(symbol) for symbol in self._symbols)
187
+ warnings = [*self._undeclared_table_warnings(), *self._warnings, *self._composite_warnings()]
188
+ return IrConversionResult(
189
+ ir.Program(named_relations=named_relations, root=root, assertions=tuple(self._assertions), declarations=tuple(self._declarations), attachment_sites=tuple(self._attachment_sites),
190
+ scalar_subqueries=tuple(self._scalar_subqueries), sort_keys=tuple(self._site_sort_keys),
191
+ set_operations=tuple(self._set_operations)),
192
+ tuple(self._knowledge), tuple(self._diagnostics), tuple(dict.fromkeys(warnings)),
193
+ )
194
+
195
+ def _undeclared_table_warnings(self) -> list[Diagnostic]:
196
+ return [
197
+ Diagnostic(
198
+ diag.UNDECLARED_TABLE,
199
+ f"table {symbol.name} has no CREATE TABLE in the program, so its keys, nullability and column types are unknown: "
200
+ "declare it with CREATE TABLE, or analyze with a database connection (analyze_database)",
201
+ symbol.origin,
202
+ )
203
+ for symbol in self._symbols
204
+ if symbol.role is ir.RelationRole.TABLE and symbol.statement is None
205
+ ]
206
+
207
+ # Declaration and symbol discovery ---------------------------------------
208
+
209
+ def _declare(self, statement: exp.Create) -> None:
210
+ kind = (statement.kind or "").upper()
211
+ if kind not in _SUPPORTED_CREATE_KINDS:
212
+ self._diagnostics.append(
213
+ Diagnostic(
214
+ diag.UNSUPPORTED_CREATE_STATEMENT,
215
+ f"create statement kind {kind or '<unknown>'} is not supported; only TABLE, VIEW, INDEX, and SCHEMA are analyzed: {statement.sql(dialect=self.dialect)}",
216
+ )
217
+ )
218
+ return
219
+
220
+ table = _created_table(statement)
221
+ if table is None:
222
+ return
223
+ name = _qualified_name(table)
224
+ key = name.lower()
225
+ origin = Origin(SQL, statement.sql(dialect=self.dialect))
226
+ if key in self._global_symbols:
227
+ self._diagnostics.append(Diagnostic(diag.DUPLICATE_DECLARATION, f"relation {name} is declared more than once", origin))
228
+ return
229
+
230
+ role = ir.RelationRole.TABLE if kind == "TABLE" else ir.RelationRole.VIEW
231
+ body = statement.args.get("expression")
232
+ query = _unwrap_query(body)
233
+ symbol = _Symbol(name=name, role=role, origin=origin, body=query, statement=statement)
234
+ self._global_symbols[key] = symbol
235
+ self._symbols.append(symbol)
236
+
237
+ def _declare_index(self, statement: exp.Create) -> None:
238
+ """A unique index over plain columns is a Unique Set, like UNIQUE; one
239
+ with an expression key part may be (`_expression_key`); a partial one,
240
+ or any other index, states nothing about the data."""
241
+ index = statement.this
242
+ table = index.args.get("table") if isinstance(index, exp.Index) else None
243
+ if not isinstance(table, exp.Table):
244
+ self._unsupported_create(statement, "an index without a table")
245
+ return
246
+ symbol = self._created_before(table, statement)
247
+ if symbol is None or not statement.args.get("unique"):
248
+ return
249
+ params = index.args.get("params")
250
+ columns = params.args.get("columns") or () if isinstance(params, exp.IndexParameters) else ()
251
+ plain = [
252
+ column.this for column in columns
253
+ if isinstance(column, exp.Ordered) and isinstance(column.this, exp.Column) and not column.this.table
254
+ ]
255
+ if not columns or params.args.get("where") is not None:
256
+ # Uniqueness of only some rows is not a Unique Set.
257
+ self._unsupported_create(statement, "a partial unique index")
258
+ return
259
+ if len(plain) != len(columns):
260
+ parts = [column.this if isinstance(column, exp.Ordered) else column for column in columns]
261
+ symbol.expression_keys.append(([part.unnest() for part in parts], statement))
262
+ return
263
+ schema = exp.Schema(expressions=[exp.to_identifier(column.name) for column in plain])
264
+ symbol.constraints.append(exp.UniqueColumnConstraint(this=schema))
265
+
266
+ def _alter(self, statement: exp.Alter) -> None:
267
+ """ALTER TABLE adding PRIMARY KEY, UNIQUE, or FOREIGN KEY constraints, or
268
+ setting columns NOT NULL. Any other action could invalidate what is
269
+ known of the table, so a statement with one states nothing."""
270
+ actions = statement.args.get("actions") or ()
271
+ added: list[Expr] = []
272
+ not_null: list[str] = []
273
+ for action in actions:
274
+ match action:
275
+ case exp.AddConstraint():
276
+ added.extend(_unwrap_constraints(action.expressions))
277
+ case exp.AlterColumn() if action.args.get("allow_null") is False and {
278
+ key for key, value in action.args.items() if value is not None and value is not False
279
+ } == {"this"}:
280
+ not_null.append(action.name)
281
+ case _:
282
+ added.append(action)
283
+ if not actions or not all(_is_added_constraint(constraint) for constraint in added):
284
+ self._diagnostics.append(Diagnostic(
285
+ diag.UNSUPPORTED_STATEMENT,
286
+ f"only ALTER TABLE adding PRIMARY KEY, UNIQUE, or FOREIGN KEY constraints, or setting NOT NULL, "
287
+ f"is analyzed: {statement.sql(dialect=self.dialect)}",
288
+ self._origin(statement),
289
+ ))
290
+ return
291
+ table = statement.this
292
+ symbol = self._created_before(table, statement) if isinstance(table, exp.Table) else None
293
+ if symbol is None:
294
+ return
295
+ symbol.constraints.extend(added)
296
+ symbol.not_null.update(not_null)
297
+ if statement.args.get("not_valid"):
298
+ self._unvalidated.update(id(constraint) for constraint in added)
299
+
300
+ def _created_before(self, table: exp.Table, statement: Expr) -> _Symbol | None:
301
+ """The table `statement` amends, which a CREATE TABLE earlier in the
302
+ program must declare; otherwise the statement is diagnosed."""
303
+ name = _qualified_name(table)
304
+ symbol = self._global_symbols.get(name.lower())
305
+ if symbol is None or symbol.role is not ir.RelationRole.TABLE or symbol.statement is None:
306
+ self._diagnostics.append(Diagnostic(
307
+ diag.CONSTRAINT_WITHOUT_TABLE,
308
+ f"{name} is not a table created earlier in the program, so this statement's constraints are ignored: "
309
+ f"{statement.sql(dialect=self.dialect)}",
310
+ self._origin(statement),
311
+ ))
312
+ return None
313
+ return symbol
314
+
315
+ def _unsupported_create(self, statement: exp.Create, what: str) -> None:
316
+ self._diagnostics.append(Diagnostic(
317
+ diag.UNSUPPORTED_CREATE_STATEMENT,
318
+ f"{what} is not analyzed: {statement.sql(dialect=self.dialect)}",
319
+ self._origin(statement),
320
+ ))
321
+
322
+ def _discover_query(self, query: exp.Query, inherited_scope: dict[str, _Symbol]) -> None:
323
+ if id(query) in self._query_scopes:
324
+ return
325
+
326
+ scope = dict(inherited_scope)
327
+ with_clause = _arg(query, "with")
328
+ if with_clause is not None:
329
+ for cte in with_clause.expressions:
330
+ body = _unwrap_query(cte.this)
331
+ symbol = _Symbol(
332
+ name=cte.alias,
333
+ role=ir.RelationRole.CTE,
334
+ origin=Origin(SQL, cte.sql(dialect=self.dialect)),
335
+ body=body,
336
+ )
337
+ scope[cte.alias.lower()] = symbol
338
+ self._symbols.append(symbol)
339
+
340
+ self._query_scopes[id(query)] = scope
341
+ if isinstance(query, exp.SetOperation):
342
+ # SQLGlot 30 wraps a nested set operation in a Subquery.
343
+ for side in (query.this, query.expression):
344
+ inner = _unwrap_query(side)
345
+ if inner is not None:
346
+ self._discover_query(inner, scope)
347
+ return
348
+ if not isinstance(query, exp.Select):
349
+ return
350
+
351
+ sources = _direct_sources(query)
352
+ by_qualifier: dict[str, list[Expr]] = {}
353
+ for source in sources:
354
+ self._source_required.setdefault(id(source), set())
355
+ self._source_expressions[id(source)] = source
356
+ qualifier = source.alias_or_name.lower()
357
+ if qualifier:
358
+ by_qualifier.setdefault(qualifier, []).append(source)
359
+ if isinstance(source, exp.Table):
360
+ symbol = self._symbol_for_table(source, scope)
361
+ self._table_symbols[id(source)] = symbol
362
+ elif isinstance(source, exp.Subquery):
363
+ body = _unwrap_query(source)
364
+ if body is not None:
365
+ self._discover_query(body, scope)
366
+
367
+ for column in query.find_all(exp.Column):
368
+ if column.table:
369
+ matching = by_qualifier.get(column.table.lower(), ())
370
+ for source in matching:
371
+ self._require(source, column.name)
372
+ elif len(sources) == 1 and column.find_ancestor(exp.Select) is query:
373
+ # An unqualified column in a nested query belongs to that query's scope.
374
+ self._require(sources[0], column.name)
375
+
376
+ for join in query.args.get("joins") or ():
377
+ for identifier in join.args.get("using") or ():
378
+ for source in sources:
379
+ self._require(source, identifier.name)
380
+
381
+ if len(sources) == 1 and _star_only(query):
382
+ for marker in column_markers(query):
383
+ for column in (*marker.columns, *marker.dependent):
384
+ self._require(sources[0], column)
385
+
386
+ for marker in column_markers(query):
387
+ if marker.referenced_relation is not None:
388
+ origin = Origin.marked(marker.referenced_relation, marker.span)
389
+ target = self._symbol_for_name(marker.referenced_relation, scope, origin)
390
+ self._marker_targets[id(marker)] = target
391
+ self._require_undeclared(target, marker.referenced_columns)
392
+
393
+ where = query.args.get("where")
394
+ for conjunct in _conjuncts(where.this) if where is not None else ():
395
+ matched = _subquery_conjunct(conjunct)
396
+ if matched is not None:
397
+ self._discover_query(matched[1], scope)
398
+ for expression in (*(_conjuncts(where.this) if where is not None else ()), *query.expressions):
399
+ for subquery in _scalar_subqueries(expression):
400
+ if isinstance(subquery.this, exp.Select):
401
+ self._discover_query(subquery.this, scope)
402
+
403
+ if with_clause is not None:
404
+ for cte in with_clause.expressions:
405
+ body = _unwrap_query(cte.this)
406
+ if body is not None:
407
+ self._discover_query(body, scope)
408
+
409
+ def _symbol_for_table(self, table: exp.Table, scope: dict[str, _Symbol]) -> _Symbol:
410
+ return self._symbol_for_name(_qualified_name(table), scope, Origin(SQL, table.sql(dialect=self.dialect)))
411
+
412
+ def _symbol_for_name(self, name: str, scope: dict[str, _Symbol], origin: Origin) -> _Symbol:
413
+ key = name.lower()
414
+ if key in scope:
415
+ return scope[key]
416
+ symbol = self._global_symbols.get(key)
417
+ if symbol is None:
418
+ symbol = _Symbol(name=name, role=ir.RelationRole.TABLE, origin=origin)
419
+ self._global_symbols[key] = symbol
420
+ self._symbols.append(symbol)
421
+ return symbol
422
+
423
+ def _require_undeclared(self, symbol: _Symbol, columns: Iterable[str]) -> None:
424
+ """Referenced columns become columns of an undeclared table only: a
425
+ declared relation's schema is never widened by a reference to it."""
426
+ if symbol.role is ir.RelationRole.TABLE and symbol.statement is None:
427
+ symbol.required_columns.update(columns)
428
+
429
+ def _discover_constraint_references(self) -> None:
430
+ """`REFERENCES` constraints onto relations the program reaches; one onto
431
+ a relation outside the program can never be used, so it is not linked."""
432
+ for symbol in tuple(self._symbols):
433
+ if symbol.role is not ir.RelationRole.TABLE or symbol.statement is None:
434
+ continue
435
+ for columns, table, target_columns, node in _references(symbol):
436
+ target = self._global_symbols.get(_qualified_name(table).lower())
437
+ if target is None or target.role is not ir.RelationRole.TABLE:
438
+ continue
439
+ self._require_undeclared(target, target_columns or ())
440
+ self._constraint_references.append(
441
+ _ConstraintReference(symbol, columns, target, target_columns, self._origin(node.find_ancestor(exp.ColumnDef) or node), node)
442
+ )
443
+
444
+ def _require(self, source: Expr, column: str) -> None:
445
+ self._source_required.setdefault(id(source), set()).add(column)
446
+ if isinstance(source, exp.Table):
447
+ self._table_symbols[id(source)].required_columns.add(column)
448
+
449
+ def _propagate_required_columns(self) -> None:
450
+ """Push requested output columns through star-only view/CTE/subquery bodies."""
451
+ changed = True
452
+ while changed:
453
+ before = sum(len(symbol.required_columns) for symbol in self._symbols) + sum(len(columns) for columns in self._source_required.values())
454
+ for symbol in self._symbols:
455
+ if isinstance(symbol.body, exp.Select) and _star_only(symbol.body):
456
+ sources = _direct_sources(symbol.body)
457
+ if len(sources) == 1:
458
+ for column in symbol.required_columns:
459
+ self._require(sources[0], column)
460
+ for source_id, columns in tuple(self._source_required.items()):
461
+ source = self._source_expressions[source_id]
462
+ body = _unwrap_query(source) if isinstance(source, exp.Subquery) else None
463
+ if isinstance(body, exp.Select) and _star_only(body):
464
+ nested_sources = _direct_sources(body)
465
+ if len(nested_sources) == 1:
466
+ for column in columns:
467
+ self._require(nested_sources[0], column)
468
+ after = sum(len(symbol.required_columns) for symbol in self._symbols) + sum(len(columns) for columns in self._source_required.values())
469
+ changed = after != before
470
+
471
+ # Lowering ---------------------------------------------------------------
472
+
473
+ def _lower_query(self, query: exp.Query) -> ir.RelationExpr | None:
474
+ """A Root Select, view, CTE, and FROM-subquery body all lower the same way."""
475
+ unwrapped = _unwrap_query(query)
476
+ if unwrapped is None:
477
+ return None
478
+ query = unwrapped
479
+ if isinstance(query, exp.SetOperation):
480
+ left = self._lower_query(query.this)
481
+ right = self._lower_query(query.expression)
482
+ if left is None:
483
+ left = self._opaque_relation(query.this)
484
+ if right is None:
485
+ right = self._opaque_relation(query.expression)
486
+ output_columns = _pass_output_columns(left.output_columns, self._origin(query))
487
+ relation = ir.SetOperation(
488
+ origin=self._origin(query), output_columns=output_columns, is_schema_complete=left.is_schema_complete and right.is_schema_complete,
489
+ operator=_SET_OPERATORS[type(query)], left=left, right=right, distinct=bool(query.args.get("distinct")),
490
+ by_name=bool(query.args.get("by_name")),
491
+ )
492
+ last = _last_select(query.this)
493
+ position = marker_position(last) if last is not None else None
494
+ if relation.operator != "union" and position is not None:
495
+ self._set_operations.append((AttachmentSite(relation.operator.upper(), position[1], position[0], True), relation))
496
+ return self._lower_limit(relation, query)
497
+ if not isinstance(query, exp.Select):
498
+ return None
499
+ return self._lower_select(query)
500
+
501
+ def _lower_select(self, select: exp.Select) -> ir.RelationExpr | None:
502
+ source = _from_source(select)
503
+ if source is None:
504
+ # No FROM: the select list is evaluated once, over one row of no
505
+ # columns, which an opaque relation limited to one row stands for.
506
+ origin = self._origin(select)
507
+ row = ir.OpaqueRelation(origin=origin, output_columns=(), is_schema_complete=True, description="no FROM")
508
+ one_row = ir.Limit(origin=origin, output_columns=(), is_schema_complete=True, input=row, count=1)
509
+ relation = self._lower_select_tail(_Lowered(one_row, ()), select)
510
+ elif any(join.args.get("method") == "NATURAL" for join in select.args.get("joins") or ()):
511
+ relation = self._opaque_relation(select, _selected_output_names(select))
512
+ else:
513
+ lowered = self._lower_source(source)
514
+ for join in select.args.get("joins") or ():
515
+ lowered = self._lower_join(lowered, join)
516
+ relation = self._lower_select_tail(lowered, select)
517
+ return self._finish_select(relation, select)
518
+
519
+ def _finish_select(self, relation: ir.RelationExpr, select: exp.Select) -> ir.RelationExpr:
520
+ relation = self._lower_limit(relation, select)
521
+ self._register_column_markers(relation, select)
522
+ self._add_attachment_site(select, _select_label(select), relation, trailing=True)
523
+ position = marker_position(select)
524
+ if position is not None:
525
+ offset, line = position
526
+ self._site_sort_keys.append((AttachmentSite(_select_label(select), line, offset, True), self._sort_keys.get(id(select))))
527
+ return relation
528
+
529
+ def _lower_limit(self, relation: ir.RelationExpr, query: exp.Query) -> ir.RelationExpr:
530
+ limit = query.args.get("limit")
531
+ if limit is None and query.args.get("offset") is None:
532
+ return relation
533
+ count = None
534
+ if limit is not None:
535
+ expression = limit.args.get("expression") if isinstance(limit, exp.Limit) else limit.args.get("count")
536
+ options = limit.args.get("limit_options")
537
+ hard_bound = options is None or not any(options.args.get(key) for key in ("percent", "with_ties"))
538
+ if hard_bound and isinstance(expression, exp.Literal) and expression.is_int:
539
+ count = int(expression.this)
540
+ origin = self._origin(query)
541
+ return ir.Limit(origin=origin, output_columns=_pass_output_columns(relation.output_columns, origin), is_schema_complete=relation.is_schema_complete, input=relation, count=count)
542
+
543
+ def _lower_source(self, source: Expr) -> _Lowered:
544
+ if isinstance(source, exp.Table):
545
+ symbol = self._table_symbols[id(source)]
546
+ alias_name = source.alias or source.name
547
+ # An unaliased table may be named with its schema: `s1.t.k`.
548
+ schema = "" if source.alias else _schema_path(source)
549
+ if symbol in self._resolving:
550
+ if symbol.role is ir.RelationRole.VIEW:
551
+ self._report_cycle(symbol, source)
552
+ recursive = self._recursive_relation(symbol, source)
553
+ return self._alias(recursive, alias_name, self._origin(source), schema)
554
+ return self._alias(self._resolve_symbol(symbol), alias_name, self._origin(source), schema)
555
+
556
+ if isinstance(source, exp.Subquery):
557
+ relation = self._lower_query(source)
558
+ if relation is not None:
559
+ # DuckDB allows an unaliased FROM subquery; nothing can qualify
560
+ # a column by its name, so a placeholder serves.
561
+ return self._alias(relation, source.alias or "subquery", self._origin(source))
562
+
563
+ opaque = self._opaque_relation(source, self._source_required.get(id(source), set()))
564
+ return self._alias(opaque, source.alias_or_name or "opaque", self._origin(source))
565
+
566
+ def _resolve_symbol(self, symbol: _Symbol) -> ir.NamedRelation:
567
+ existing = self._named_relations.get(symbol)
568
+ if existing is not None:
569
+ return existing
570
+
571
+ if symbol.role is ir.RelationRole.TABLE:
572
+ output_columns, complete = self._table_output_columns(symbol)
573
+ named = ir.NamedRelation(
574
+ origin=symbol.origin, output_columns=output_columns, is_schema_complete=complete, name=symbol.name, role=symbol.role
575
+ )
576
+ self._named_relations[symbol] = named
577
+ self._table_knowledge(named, symbol)
578
+ return named
579
+
580
+ self._resolving.add(symbol)
581
+ try:
582
+ body = self._lower_query(symbol.body) if symbol.body is not None else None
583
+ finally:
584
+ self._resolving.discard(symbol)
585
+ if body is None:
586
+ body = self._opaque_relation(symbol.body or exp.Table(this=symbol.name), symbol.required_columns)
587
+ output_columns = _pass_output_columns(body.output_columns, symbol.origin)
588
+ named = ir.NamedRelation(
589
+ origin=symbol.origin,
590
+ output_columns=output_columns,
591
+ is_schema_complete=body.is_schema_complete,
592
+ name=symbol.name,
593
+ role=symbol.role,
594
+ body=body,
595
+ )
596
+ self._named_relations[symbol] = named
597
+ return named
598
+
599
+ def _unresolved_named_relation(self, symbol: _Symbol) -> ir.NamedRelation:
600
+ output_columns, complete = self._table_output_columns(symbol) if symbol.role is ir.RelationRole.TABLE else (self._opaque_output_columns(symbol.required_columns, symbol.origin), False)
601
+ named = ir.NamedRelation(
602
+ origin=symbol.origin, output_columns=output_columns, is_schema_complete=complete, name=symbol.name, role=symbol.role
603
+ )
604
+ self._named_relations[symbol] = named
605
+ if symbol.role is ir.RelationRole.TABLE:
606
+ self._table_knowledge(named, symbol)
607
+ return named
608
+
609
+ def _table_output_columns(self, symbol: _Symbol) -> tuple[tuple[ir.OutputColumn, ...], bool]:
610
+ definitions = list(symbol.statement.this.find_all(exp.ColumnDef)) if symbol.statement is not None else []
611
+ names = [definition.name for definition in definitions]
612
+ seen = {name.lower() for name in names}
613
+ names.extend(column for column in sorted(symbol.required_columns, key=str.lower) if column.lower() not in seen)
614
+ complete = symbol.statement is not None and {column.lower() for column in symbol.required_columns} <= seen
615
+ columns = self._opaque_output_columns(names, symbol.origin)
616
+ by_name = {definition.name: definition for definition in definitions}
617
+ if symbol.statement is not None:
618
+ self._absent.update(column for column in columns if column.name not in by_name)
619
+ typed = tuple(
620
+ replace(
621
+ column, binary_text=self._is_binary_text(by_name[column.name]),
622
+ comparison_type=self._comparison_type(by_name[column.name]),
623
+ identity_comparison=self._identity_comparison(by_name[column.name]),
624
+ exact_numeric=self._exact_numeric(by_name[column.name]),
625
+ )
626
+ if column.name in by_name else column for column in columns
627
+ )
628
+ for column in typed:
629
+ kind = by_name[column.name].args.get("kind") if column.name in by_name else None
630
+ if isinstance(kind, exp.DataType):
631
+ self._declared_types[column] = kind
632
+ return typed, complete
633
+
634
+ def _comparison_type(self, definition: exp.ColumnDef) -> str | None:
635
+ kind = definition.args.get("kind")
636
+ if not isinstance(kind, exp.DataType):
637
+ return None
638
+ if kind.this in exp.DataType.TEXT_TYPES:
639
+ return kind.sql(dialect=self.dialect) if self._is_binary_text(definition) else None
640
+ if _collated(definition) or self._unscaled_decimal(kind):
641
+ return None
642
+ return kind.sql(dialect=self.dialect)
643
+
644
+ def _identity_comparison(self, definition: exp.ColumnDef) -> bool:
645
+ """Declared with a type whose equal values are identical (text only
646
+ when compared by identity, which `_comparison_type` already checks)."""
647
+ kind = definition.args.get("kind")
648
+ if not (self._comparison_type(definition) is not None and isinstance(kind, exp.DataType)):
649
+ return False
650
+ if kind.this == exp.DataType.Type.TIMESTAMPTZ:
651
+ return self.semantics.zoned_timestamp_is_instant
652
+ return kind.this in _IDENTITY_TYPES
653
+
654
+ def _exact_numeric(self, definition: exp.ColumnDef) -> bool:
655
+ kind = definition.args.get("kind")
656
+ return isinstance(kind, exp.DataType) and kind.this in _EXACT_NUMERIC_TYPES and not self._unscaled_decimal(kind)
657
+
658
+ def _unscaled_decimal(self, kind: exp.DataType) -> bool:
659
+ """A Postgres NUMERIC with no scale: 1.0 and 1.00 are equal values that differ."""
660
+ return self.semantics.unscaled_decimal and kind.this == exp.DataType.Type.DECIMAL and not kind.expressions
661
+
662
+ def _is_binary_text(self, definition: exp.ColumnDef) -> bool:
663
+ """Declared VARCHAR or TEXT, with no COLLATE, in a dialect whose
664
+ default collation is binary: compared by identity.
665
+
666
+ CHAR is not: it ignores trailing spaces when compared. Any COLLATE,
667
+ even a binary one, is not modelled (issue 49)."""
668
+ kind = definition.args.get("kind")
669
+ return (
670
+ isinstance(kind, exp.DataType) and kind.this in _TEXT_TYPES
671
+ and not _collated(definition) and self.semantics.text_is_binary
672
+ )
673
+
674
+ def _opaque_output_columns(self, names: Iterable[str], origin: Origin) -> tuple[ir.OutputColumn, ...]:
675
+ if isinstance(names, Set):
676
+ names = sorted(names, key=lambda name: (name.lower(), name))
677
+ columns = tuple(
678
+ ir.OutputColumn(
679
+ origin=origin,
680
+ name=name,
681
+ scalar_expr=ir.OpaqueExpression(origin=origin, description=f"source column {name}"),
682
+ )
683
+ for name in names
684
+ )
685
+ self._guessed.update(columns)
686
+ return columns
687
+
688
+ def _recursive_relation(self, symbol: _Symbol, source: Expr) -> ir.RecursiveRelation:
689
+ origin = self._origin(source)
690
+ return ir.RecursiveRelation(
691
+ origin=origin,
692
+ output_columns=self._opaque_output_columns(symbol.required_columns, origin),
693
+ is_schema_complete=False,
694
+ description=source.sql(dialect=self.dialect),
695
+ relation_name=symbol.name,
696
+ )
697
+
698
+ def _alias(self, source: ir.RelationExpr, name: str, origin: Origin, schema: str = "") -> _Lowered:
699
+ output_columns = _pass_output_columns(source.output_columns, origin)
700
+ alias = ir.Alias(origin=origin, output_columns=output_columns, is_schema_complete=source.is_schema_complete, source=source, name=name)
701
+ return _Lowered(alias, tuple(_Binding(name.lower(), output.name.lower(), output, schema) for output in output_columns))
702
+
703
+ def _lower_join(self, left: _Lowered, join: exp.Join) -> _Lowered:
704
+ right = self._lower_source(join.this)
705
+ marked = assertion_span(join)
706
+ origin = self._origin(join, marked.line if marked is not None else None)
707
+ kind = _join_kind(join, self.dialect)
708
+ if kind is None:
709
+ return self._opaque_join(left, right, join, origin)
710
+ predicate = self._lower_join_predicate(join, left, right)
711
+ if kind == "semi" or kind == "anti":
712
+ # RIGHT SEMI and RIGHT ANTI keep right rows: the same join, inputs swapped.
713
+ kept, other = (right, left) if (join.args.get("side") or "").lower() == "right" else (left, right)
714
+ filtered = self._filtering_join(kept, other.relation, kind, predicate, origin)
715
+ assert isinstance(filtered.relation, ir.Join)
716
+ self._join_marker(join, filtered.relation)
717
+ self._add_attachment_site(join, _join_label(join, self.dialect), filtered.relation, trailing=False)
718
+ return filtered
719
+ if _is_positional(join):
720
+ # A FULL JOIN on row position: an opaque condition, as no column holds it.
721
+ predicate = ir.OpaqueExpression(origin=origin, description="row position")
722
+ input_columns = left.relation.output_columns + right.relation.output_columns
723
+ output_columns = _pass_output_columns(input_columns, origin)
724
+ remapped = {source: output for source, output in zip(input_columns, output_columns)}
725
+ relation = ir.Join(
726
+ origin=origin,
727
+ output_columns=output_columns,
728
+ is_schema_complete=left.relation.is_schema_complete and right.relation.is_schema_complete,
729
+ kind=kind,
730
+ left=left.relation,
731
+ right=right.relation,
732
+ predicate=predicate,
733
+ )
734
+ bindings = tuple(
735
+ replace(binding, column=remapped[binding.column])
736
+ for binding in left.bindings + right.bindings
737
+ )
738
+ self._join_marker(join, relation)
739
+ self._add_attachment_site(join, _join_label(join, self.dialect), relation, trailing=False)
740
+ return _Lowered(relation, bindings)
741
+
742
+ def _opaque_join(self, left: _Lowered, right: _Lowered, join: exp.Join, origin: Origin) -> _Lowered:
743
+ """A join whose kind is not modelled: every column it outputs is opaque,
744
+ and a marker on it is reported as unanalyzed."""
745
+ description = join.sql(dialect=self.dialect)
746
+ input_columns = left.relation.output_columns + right.relation.output_columns
747
+ opaque = {
748
+ column: ir.OutputColumn(origin=origin, name=column.name, scalar_expr=ir.OpaqueExpression(origin=origin, description=description))
749
+ for column in input_columns
750
+ }
751
+ relation = ir.OpaqueRelation(origin=origin, output_columns=tuple(opaque.values()), is_schema_complete=False, description=description)
752
+ return _Lowered(relation, tuple(
753
+ replace(binding, column=opaque[binding.column]) for binding in left.bindings + right.bindings
754
+ ))
755
+
756
+ def _join_marker(self, join: exp.Join, relation: ir.Join) -> None:
757
+ kind = join_marker_kind(join)
758
+ if kind is not None:
759
+ self._handled_joins.add(id(join))
760
+ cardinality = {"unique-join": UniqueJoin, "many-to-one-join": ManyToOneJoin, "one-to-many-join": OneToManyJoin, "one-to-one-join": OneToOneJoin}
761
+ if kind in cardinality and relation.kind in {"semi", "anti"}:
762
+ # It keeps or drops left rows whole and outputs no right row: cardinality does not apply.
763
+ self._diagnostics.append(Diagnostic(
764
+ diag.CARDINALITY_ON_FILTERING_JOIN,
765
+ f"the {JOIN_MARKER_NOUNS[kind]} marker on line {join_origin(join, self.dialect).line} is on a {relation.kind} join, "
766
+ "which outputs no right rows: join cardinality does not apply to it",
767
+ join_origin(join, self.dialect),
768
+ ))
769
+ return
770
+ match kind:
771
+ case "unique-join" | "many-to-one-join" | "one-to-many-join" | "one-to-one-join":
772
+ self._assertions.append(ir.Assertion(origin=join_origin(join, self.dialect), property=cardinality[kind](join=relation)))
773
+ case "exactly-one-join":
774
+ self._assertions.append(
775
+ ir.Assertion(origin=join_origin(join, self.dialect), property=ExactlyOneJoin(join=relation))
776
+ )
777
+ case None:
778
+ pass
779
+ case _:
780
+ assert_never(kind)
781
+
782
+ def _lower_join_predicate(self, join: exp.Join, left: _Lowered, right: _Lowered) -> ir.ScalarExpr | None:
783
+ """The ON condition; USING is an AND of one equality per column."""
784
+ using = join.args.get("using")
785
+ if using:
786
+ return _and(tuple(
787
+ ir.Equality(
788
+ origin=self._origin(identifier, _source_line(identifier)),
789
+ left=self._lower_expression(exp.column(identifier.name), left.bindings),
790
+ right=self._lower_expression(exp.column(identifier.name), right.bindings),
791
+ )
792
+ for identifier in using
793
+ ), self._origin(join))
794
+ predicate = join.args.get("on")
795
+ if predicate is None:
796
+ return None
797
+ complete = left.relation.is_schema_complete and right.relation.is_schema_complete
798
+ return self._lower_condition(predicate, _conjuncts(predicate), left.bindings + right.bindings, schema_complete=complete)
799
+
800
+ def _lower_condition(
801
+ self, source: Expr, conjuncts: list[Expr], bindings: tuple[_Binding, ...], *, schema_complete: bool
802
+ ) -> ir.ScalarExpr:
803
+ """The AND of `conjuncts`, the top-level conjuncts of `source`."""
804
+ assert conjuncts, "a condition has at least one conjunct"
805
+ lowered = _and(
806
+ tuple(self._lower_conjunct(conjunct, bindings, schema_complete=schema_complete) for conjunct in conjuncts),
807
+ self._origin(source, _source_line(source)),
808
+ )
809
+ assert lowered is not None
810
+ return lowered
811
+
812
+ def _lower_conjunct(self, conjunct: Expr, bindings: tuple[_Binding, ...], *, schema_complete: bool) -> ir.ScalarExpr:
813
+ return self._lower_expression(conjunct, bindings, schema_complete=schema_complete, line=_source_line(conjunct))
814
+
815
+ def _lower_select_tail(self, lowered: _Lowered, select: exp.Select) -> ir.RelationExpr:
816
+ keys = grouping(select)
817
+ if keys is not None:
818
+ relation = self._lower_aggregate(lowered, select, keys)
819
+ elif select.args.get("distinct") is not None and select.args["distinct"].args.get("on") is not None:
820
+ relation = self._lower_distinct_on(lowered, select, select.args["distinct"].args["on"])
821
+ elif select.args.get("distinct") is not None:
822
+ relation = self._lower_distinct(lowered, select)
823
+ elif isinstance(select.args.get("qualify"), exp.Qualify):
824
+ relation = self._lower_qualify(lowered, select, select.args["qualify"])
825
+ else:
826
+ if select.args.get("where") is not None:
827
+ lowered = self._filter(lowered, select.args["where"])
828
+ if _star_only(select):
829
+ relation = lowered.relation
830
+ else:
831
+ lowered, _ = self._join_scalar_subqueries(lowered, select.expressions)
832
+ relation = self._project(lowered, select.expressions, self._origin(select))
833
+ if relation is None:
834
+ return self._opaque_relation(select, _selected_output_names(select))
835
+ self._sort_keys[id(select)] = self._resolve_order(select, lowered.bindings, relation)
836
+ return relation
837
+
838
+ def _resolve_order(
839
+ self, select: exp.Select, bindings: tuple[_Binding, ...], relation: ir.RelationExpr
840
+ ) -> tuple[ir.OutputColumn, ...] | None:
841
+ """Each ORDER BY key as a column of the stage it sorts (see
842
+ `Program.sort_keys`): a selected column by its name or position, else
843
+ the column a name binds, qualifier and schema included."""
844
+ order = select.args.get("order")
845
+ if order is None:
846
+ return ()
847
+ distinct = select.args.get("distinct")
848
+ if distinct is not None and distinct.args.get("on") is not None:
849
+ return None # Its ORDER BY picks the row each group keeps.
850
+ scope = relation.input if isinstance(relation, ir.Project) else relation
851
+ outputs = relation.output_columns
852
+ items = [item for item in select.expressions if not isinstance(item, exp.Star)]
853
+ keys: list[ir.OutputColumn] = []
854
+ for ordered in order.expressions:
855
+ expression = ordered.this if isinstance(ordered, exp.Ordered) else ordered
856
+ key: ir.OutputColumn | None = None
857
+ match expression:
858
+ case exp.Column(table="") if len(items) == len(outputs) and [item.alias_or_name.lower() for item in items].count(expression.name.lower()) == 1:
859
+ names = [item.alias_or_name.lower() for item in items]
860
+ key = _in_scope(outputs[names.index(expression.name.lower())], scope)
861
+ case exp.Literal(is_int=True) if len(items) == len(outputs) and 1 <= int(expression.this) <= len(outputs):
862
+ key = _in_scope(outputs[int(expression.this) - 1], scope)
863
+ case exp.Column():
864
+ schema, name = _schema_path(expression), expression.name.lower()
865
+ bound = [
866
+ binding for binding in bindings
867
+ if binding.name == name and (not expression.table or binding.qualifier == expression.table.lower())
868
+ and (not schema or binding.schema == schema)
869
+ ]
870
+ if len(bound) == 1:
871
+ key = _carrying(bound[0].column, scope)
872
+ case _:
873
+ pass
874
+ if key is None:
875
+ return None
876
+ keys.append(key)
877
+ return tuple(keys)
878
+
879
+ def _filter(self, lowered: _Lowered, where: exp.Where) -> _Lowered:
880
+ """WHERE: each `EXISTS`, `NOT EXISTS`, or `IN (subquery)` conjunct as a
881
+ semi or anti join (Paulley §4.4.2), the other conjuncts as one Filter."""
882
+ remaining = []
883
+ for conjunct in _conjuncts(where.this):
884
+ joined = self._subquery_join(lowered, conjunct)
885
+ if joined is None:
886
+ remaining.append(conjunct)
887
+ else:
888
+ lowered = joined
889
+ if not remaining:
890
+ return lowered
891
+ before = lowered.relation.output_columns
892
+ lowered, carried = self._join_scalar_subqueries(lowered, remaining)
893
+ origin = self._origin(where)
894
+ output_columns = _pass_output_columns(lowered.relation.output_columns, origin)
895
+ remapped = {source: output for source, output in zip(lowered.relation.output_columns, output_columns)}
896
+ complete = lowered.relation.is_schema_complete
897
+ relation = ir.Filter(
898
+ origin=origin, output_columns=output_columns, is_schema_complete=complete, input=lowered.relation,
899
+ predicate=self._lower_condition(where.this, remaining, lowered.bindings, schema_complete=complete),
900
+ )
901
+ bindings = tuple(
902
+ replace(binding, column=remapped[binding.column]) for binding in lowered.bindings
903
+ )
904
+ if all(carried[column] is column for column in before):
905
+ return _Lowered(relation, bindings)
906
+ # Scalar subqueries were joined in: keep only the columns from before,
907
+ # so their values stay inside this WHERE.
908
+ kept = tuple(remapped[carried[column]] for column in before)
909
+ project = ir.Project(
910
+ origin=origin, output_columns=_pass_output_columns(kept, origin), is_schema_complete=complete, input=relation,
911
+ )
912
+ projected = dict(zip(kept, project.output_columns))
913
+ return _Lowered(project, tuple(
914
+ replace(binding, column=projected[binding.column])
915
+ for binding in bindings if binding.column in projected
916
+ ))
917
+
918
+ def _correlated(self, select: exp.Select, inner: _Lowered, outer: _Lowered) -> bool | None:
919
+ """Whether `select` reads the outer scope, only in its WHERE; None when
920
+ it reads it anywhere else, or when a column may belong to either scope."""
921
+ where = select.args.get("where")
922
+ correlated = False
923
+ for column in select.find_all(exp.Column):
924
+ found = self._scope_of(column, inner, outer)
925
+ if found is None:
926
+ return None
927
+ scope, binding = found
928
+ self._scoped_columns[id(column)] = binding.column
929
+ if scope == "outer":
930
+ if where is None or column.find_ancestor(exp.Where) is not where:
931
+ return None # Only a WHERE becomes the dependent join's ON.
932
+ correlated = True
933
+ return correlated
934
+
935
+ def _scope_of(
936
+ self, column: exp.Column, inner: _Lowered, outer: _Lowered
937
+ ) -> tuple[Literal["inner", "outer"], _Binding] | None:
938
+ """The scope a subquery's column belongs to, the innermost that has it,
939
+ and its binding there. None when that is unknown (a table with no
940
+ CREATE may have the column) or the name binds more than one column."""
941
+ name = column.name.lower()
942
+
943
+ def one(scope: Literal["inner", "outer"], bindings: list[_Binding]) -> tuple[Literal["inner", "outer"], _Binding] | None:
944
+ return (scope, bindings[0]) if len(bindings) == 1 else None
945
+
946
+ if column.table:
947
+ qualifier, schema = column.table.lower(), _schema_path(column)
948
+ for scope, bindings in (("inner", inner.bindings), ("outer", outer.bindings)):
949
+ if any(binding.qualifier == qualifier and (not schema or binding.schema == schema) for binding in bindings):
950
+ return one(scope, [
951
+ binding for binding in bindings
952
+ if binding.qualifier == qualifier and (not schema or binding.schema == schema) and binding.name == name
953
+ ])
954
+ return None
955
+ inner_named = [binding for binding in inner.bindings if binding.name == name and self._owns(binding) != "no"]
956
+ outer_named = [binding for binding in outer.bindings if binding.name == name and self._owns(binding) != "no"]
957
+ owners = {self._owns(binding) for binding in inner_named}
958
+ if "yes" in owners:
959
+ return one("inner", inner_named)
960
+ outer_has = bool(outer_named)
961
+ if "unknown" in owners or (not inner_named and not inner.relation.is_schema_complete and not any(binding.name == name for binding in inner.bindings)):
962
+ if not outer_has:
963
+ return one("inner", inner_named)
964
+ self._warnings.append(Diagnostic(
965
+ diag.AMBIGUOUS_COLUMN,
966
+ f"column {column.name} in a subquery may belong to its own FROM or to the outer query, "
967
+ "since some of its tables have no declared columns: qualify it, declare those tables with CREATE TABLE, "
968
+ "or analyze with a database connection (analyze_database)",
969
+ self._origin(column, _source_line(column)),
970
+ ))
971
+ return None
972
+ return one("outer", outer_named) if outer_has else None
973
+
974
+ def _owns(self, binding: _Binding) -> Literal["yes", "no", "unknown"]:
975
+ """Whether the relation behind `binding` has its column: a declared or
976
+ computed column it has, one its declared table lacks it does not, and
977
+ one made up for a table with no CREATE it may have."""
978
+ base = binding.column
979
+ while isinstance(base.scalar_expr, ir.ColumnRef):
980
+ base = base.scalar_expr.column
981
+ if base in self._absent:
982
+ return "no"
983
+ return "unknown" if base in self._guessed else "yes"
984
+
985
+ def _join_scalar_subqueries(
986
+ self, lowered: _Lowered, expressions: Sequence[Expr]
987
+ ) -> tuple[_Lowered, dict[ir.OutputColumn, ir.OutputColumn]]:
988
+ """Each scalar subquery in `expressions` joined onto `lowered`, and
989
+ where each of `lowered`'s columns went."""
990
+ carried = {column: column for column in lowered.relation.output_columns}
991
+ for expression in expressions:
992
+ for subquery in _scalar_subqueries(expression):
993
+ joined = self._scalar_subquery_join(lowered, subquery)
994
+ if joined is None:
995
+ continue
996
+ lowered, remapped = joined
997
+ carried = {column: remapped[current] for column, current in carried.items()}
998
+ return lowered, carried
999
+
1000
+ def _scalar_subquery_join(
1001
+ self, lowered: _Lowered, subquery: exp.Subquery
1002
+ ) -> tuple[_Lowered, dict[ir.OutputColumn, ir.OutputColumn]] | None:
1003
+ """A scalar subquery with body B as a left join of `lowered` with B: no
1004
+ row makes it NULL, which is a left join's padding. More than one row
1005
+ is a runtime error, so the join is many to one, or the query fails.
1006
+
1007
+ Uncorrelated, B is lowered whole under an implied LIMIT 1, sound since
1008
+ it changes the result only when the query would fail: its value is
1009
+ the same on every row. Correlated, B's WHERE becomes the ON of a
1010
+ dependent join (as for EXISTS), so only a plain select-from-where
1011
+ whose value is one of its columns is lowered. Anything else, or a
1012
+ column that may belong to either scope, stays opaque."""
1013
+ select = subquery.this
1014
+ if not isinstance(select, exp.Select) or len(select.expressions) != 1 or _star_only(select):
1015
+ return None
1016
+ source = _from_source(select)
1017
+ if source is None or select.args.get("with") or any(
1018
+ join.args.get("method") == "NATURAL" for join in select.args.get("joins") or ()
1019
+ ) or any(isinstance(node, (exp.Subquery, exp.Exists, exp.Query)) and node is not select for node in select.walk()):
1020
+ return None
1021
+ right = self._lower_source(source)
1022
+ for join in select.args.get("joins") or ():
1023
+ right = self._lower_join(right, join)
1024
+ correlated = self._correlated(select, right, lowered)
1025
+ origin = self._origin(subquery)
1026
+ if correlated is None:
1027
+ return None
1028
+ if not correlated:
1029
+ body = self._finish_select(self._lower_select_tail(right, select), select)
1030
+ right_relation: ir.RelationExpr = ir.Limit(
1031
+ origin=origin, output_columns=_pass_output_columns(body.output_columns, origin),
1032
+ is_schema_complete=body.is_schema_complete, input=body, count=1,
1033
+ )
1034
+ value = right_relation.output_columns[0]
1035
+ predicate = None
1036
+ else:
1037
+ if grouping(select) is not None or any(select.args.get(key) for key in ("having", "qualify", "distinct", "limit", "offset", "windows")):
1038
+ return None
1039
+ item = select.expressions[0]
1040
+ lowered_value = self._lower_expression(
1041
+ item.this if isinstance(item, exp.Alias) else item, right.bindings, schema_complete=right.relation.is_schema_complete,
1042
+ )
1043
+ if not (isinstance(lowered_value, ir.ColumnRef) and lowered_value.column in right.relation.output_columns):
1044
+ return None
1045
+ inner = select.args.get("where")
1046
+ predicate = None if inner is None else self._lower_condition(
1047
+ inner.this, _conjuncts(inner.this), lowered.bindings + right.bindings,
1048
+ schema_complete=lowered.relation.is_schema_complete and right.relation.is_schema_complete,
1049
+ )
1050
+ right_relation, value = right.relation, lowered_value.column
1051
+ input_columns = lowered.relation.output_columns + right_relation.output_columns
1052
+ output_columns = _pass_output_columns(input_columns, origin)
1053
+ remapped = dict(zip(input_columns, output_columns))
1054
+ join = ir.Join(
1055
+ origin=origin, output_columns=output_columns,
1056
+ is_schema_complete=lowered.relation.is_schema_complete and right_relation.is_schema_complete,
1057
+ kind="left", left=lowered.relation, right=right_relation, predicate=predicate, scalar_subquery=True,
1058
+ )
1059
+ position = marker_position(select)
1060
+ if position is not None:
1061
+ self._scalar_subqueries.append((AttachmentSite("scalar subquery", position[1], position[0], True), join))
1062
+ qualifier = f"{_SCALAR_QUALIFIER}{len(self._scalar_values)}"
1063
+ self._scalar_values[id(subquery)] = qualifier
1064
+ bindings = (
1065
+ *(replace(binding, column=remapped[binding.column]) for binding in lowered.bindings),
1066
+ _Binding(qualifier, _SCALAR_QUALIFIER, remapped[value]),
1067
+ )
1068
+ return _Lowered(join, bindings), remapped
1069
+
1070
+ def _subquery_join(self, lowered: _Lowered, conjunct: Expr) -> _Lowered | None:
1071
+ """`EXISTS (SELECT ... FROM r WHERE p)` as a semi join of r on p, and
1072
+ `NOT EXISTS` as an anti join; `p` may read the outer row (a dependent
1073
+ join, so the IR stays a DAG). `x IN (SELECT a ...)` as a semi join on
1074
+ `x = a`. Not `NOT IN`: a NULL among the values makes it reject every row."""
1075
+ matched = _subquery_conjunct(conjunct)
1076
+ if matched is None:
1077
+ return None
1078
+ kind, select, operand = matched
1079
+ if operand is None:
1080
+ source = _from_source(select)
1081
+ if source is None or grouping(select) is not None or any(
1082
+ select.args.get(key) for key in ("having", "qualify", "distinct", "limit", "offset", "with", "windows")
1083
+ ) or any(join.args.get("method") == "NATURAL" for join in select.args.get("joins") or ()):
1084
+ # Grouped by nothing (`grouping` is []), it returns one row whatever its WHERE: EXISTS is always true.
1085
+ return None
1086
+ right = self._lower_source(source)
1087
+ for join in select.args.get("joins") or ():
1088
+ right = self._lower_join(right, join)
1089
+ inner = select.args.get("where")
1090
+ predicate = None if inner is None else self._lower_condition(
1091
+ inner.this, _conjuncts(inner.this), lowered.bindings + right.bindings,
1092
+ schema_complete=lowered.relation.is_schema_complete and right.relation.is_schema_complete,
1093
+ )
1094
+ relation = right.relation
1095
+ else:
1096
+ subquery = self._lower_query(select)
1097
+ if subquery is None or len(subquery.output_columns) != 1:
1098
+ return None
1099
+ relation = subquery
1100
+ predicate = ir.Equality(
1101
+ origin=self._origin(conjunct, _source_line(conjunct)),
1102
+ left=self._lower_expression(operand, lowered.bindings, schema_complete=lowered.relation.is_schema_complete),
1103
+ right=ir.ColumnRef(origin=subquery.origin, column=subquery.output_columns[0]),
1104
+ )
1105
+ return self._filtering_join(lowered, relation, kind, predicate, self._origin(conjunct))
1106
+
1107
+ def _filtering_join(
1108
+ self, left: _Lowered, right: ir.RelationExpr, kind: ir.JoinKind, predicate: ir.ScalarExpr | None, origin: Origin
1109
+ ) -> _Lowered:
1110
+ """A semi or anti join: it keeps or drops left rows, and outputs only theirs."""
1111
+ output_columns = _pass_output_columns(left.relation.output_columns, origin)
1112
+ remapped = {source: output for source, output in zip(left.relation.output_columns, output_columns)}
1113
+ relation = ir.Join(
1114
+ origin=origin, output_columns=output_columns, is_schema_complete=left.relation.is_schema_complete,
1115
+ kind=kind, left=left.relation, right=right, predicate=predicate,
1116
+ )
1117
+ return _Lowered(relation, tuple(replace(binding, column=remapped[binding.column]) for binding in left.bindings))
1118
+
1119
+ def _project(self, lowered: _Lowered, items: list[Expr], origin: Origin) -> ir.Project:
1120
+ output_columns = tuple(
1121
+ ir.OutputColumn(
1122
+ origin=self._origin(item),
1123
+ name=item.alias_or_name,
1124
+ scalar_expr=self._lower_expression(
1125
+ item.this if isinstance(item, exp.Alias) else item,
1126
+ lowered.bindings,
1127
+ schema_complete=lowered.relation.is_schema_complete,
1128
+ ),
1129
+ )
1130
+ for item in items
1131
+ )
1132
+ return ir.Project(origin=origin, output_columns=output_columns, is_schema_complete=True, input=lowered.relation)
1133
+
1134
+ def _lower_aggregate(self, lowered: _Lowered, select: exp.Select, keys: list[Expr]) -> ir.Aggregate | None:
1135
+ """`select` grouped by `keys`; [] is one group, one row however many rows its input has."""
1136
+ group = select.args.get("group")
1137
+ if select.args.get("having") is not None or (
1138
+ group is not None and any(group.args.get(key) for key in ("grouping_sets", "rollup", "cube", "totals"))
1139
+ ):
1140
+ return None
1141
+ if select.args.get("where") is not None:
1142
+ lowered = self._filter(lowered, select.args["where"])
1143
+
1144
+ grouped_items: set[int] = set()
1145
+ for group_expression in keys:
1146
+ item = _matching_grouping_output(group_expression, select.expressions)
1147
+ if item is None:
1148
+ return None
1149
+ grouped_items.add(id(item))
1150
+
1151
+ output_columns = self._aggregate_output_columns(select.expressions, lowered, grouped_items)
1152
+ grouping_outputs: list[ir.OutputColumn] = []
1153
+ for group_expression in keys:
1154
+ item = _matching_grouping_output(group_expression, select.expressions)
1155
+ grouping_outputs.append(output_columns[select.expressions.index(item)])
1156
+ calls = tuple(
1157
+ ir.AggregateCall(
1158
+ origin=self._origin(call), description=call.sql(dialect=self.dialect),
1159
+ arguments=tuple(
1160
+ self._lower_expression(argument, lowered.bindings, schema_complete=lowered.relation.is_schema_complete)
1161
+ for argument in _aggregate_arguments(call)
1162
+ ),
1163
+ )
1164
+ for item in select.expressions for call in item.find_all(exp.AggFunc)
1165
+ if call.find_ancestor(exp.Window, exp.Select) is select
1166
+ )
1167
+ return ir.Aggregate(
1168
+ origin=self._origin(select), output_columns=output_columns, is_schema_complete=True, input=lowered.relation,
1169
+ grouping_outputs=tuple(grouping_outputs), calls=calls,
1170
+ )
1171
+
1172
+ def _lower_distinct(self, lowered: _Lowered, select: exp.Select) -> ir.Distinct | None:
1173
+ distinct = select.args.get("distinct")
1174
+ if distinct is None or select.args.get("having") is not None or _star_only(select):
1175
+ return None
1176
+ if select.args.get("where") is not None:
1177
+ lowered = self._filter(lowered, select.args["where"])
1178
+ return ir.Distinct(
1179
+ origin=self._origin(select), output_columns=self._selected_output_columns(select.expressions, lowered),
1180
+ is_schema_complete=True, input=lowered.relation,
1181
+ )
1182
+
1183
+ def _lower_qualify(self, lowered: _Lowered, select: exp.Select, qualify: exp.Qualify) -> ir.QualifyByPartition | None:
1184
+ window = _row_number_equals_one(qualify.this)
1185
+ if window is None:
1186
+ return None
1187
+ order = window.args.get("order")
1188
+ return self._one_row_per_partition(
1189
+ lowered, select, window.args.get("partition_by") or (), order.expressions if order else (),
1190
+ )
1191
+
1192
+ def _lower_distinct_on(self, lowered: _Lowered, select: exp.Select, on: Expr) -> ir.QualifyByPartition | None:
1193
+ """`DISTINCT ON (k...)` keeps one row per distinct `k...`, like
1194
+ `QUALIFY ROW_NUMBER() OVER (PARTITION BY k...) = 1`. Its ORDER BY only
1195
+ picks which row survives, and may name outputs, so it is not lowered."""
1196
+ if select.args.get("qualify") is not None:
1197
+ return None
1198
+ keys = on.expressions if isinstance(on, exp.Tuple) else [on]
1199
+ return self._one_row_per_partition(lowered, select, keys, ())
1200
+
1201
+ def _one_row_per_partition(
1202
+ self, lowered: _Lowered, select: exp.Select, partitions: Sequence[Expr], order: Sequence[exp.Ordered],
1203
+ ) -> ir.QualifyByPartition | None:
1204
+ """Every partition key must be a selected output: the Unique Set is stated on outputs."""
1205
+ if not partitions or select.args.get("having") is not None or _star_only(select):
1206
+ return None
1207
+ if select.args.get("where") is not None:
1208
+ lowered = self._filter(lowered, select.args["where"])
1209
+
1210
+ output_columns = self._selected_output_columns(select.expressions, lowered)
1211
+ partition_outputs: list[ir.OutputColumn] = []
1212
+ for partition in partitions:
1213
+ item = _matching_output(partition, select.expressions)
1214
+ if item is None:
1215
+ return None
1216
+ partition_outputs.append(output_columns[select.expressions.index(item)])
1217
+ ordering = tuple(
1218
+ self._lower_expression(ordered.this, lowered.bindings, schema_complete=lowered.relation.is_schema_complete)
1219
+ for ordered in order
1220
+ )
1221
+ return ir.QualifyByPartition(
1222
+ origin=self._origin(select), output_columns=output_columns, is_schema_complete=True, input=lowered.relation,
1223
+ partition_outputs=tuple(partition_outputs), ordering=ordering,
1224
+ )
1225
+
1226
+ def _selected_output_columns(self, items: list[Expr], lowered: _Lowered) -> tuple[ir.OutputColumn, ...]:
1227
+ return tuple(
1228
+ ir.OutputColumn(
1229
+ origin=self._origin(item),
1230
+ name=item.alias_or_name,
1231
+ scalar_expr=self._lower_expression(
1232
+ item.this if isinstance(item, exp.Alias) else item,
1233
+ lowered.bindings,
1234
+ schema_complete=lowered.relation.is_schema_complete,
1235
+ ),
1236
+ )
1237
+ for item in items
1238
+ )
1239
+
1240
+ def _aggregate_output_columns(self, items: list[Expr], source: _Lowered, grouped_items: set[int]) -> tuple[ir.OutputColumn, ...]:
1241
+ output_columns: list[ir.OutputColumn] = []
1242
+ for item in items:
1243
+ expression = item.this if isinstance(item, exp.Alias) else item
1244
+ lowered = self._lower_expression(expression, source.bindings, schema_complete=source.relation.is_schema_complete)
1245
+ if id(item) not in grouped_items and _is_bare_aggregate_expression(expression):
1246
+ # Grouped by nothing (no grouped items), the one row exists even
1247
+ # over no input rows, so a bare column, where a dialect allows
1248
+ # one, may be NULL.
1249
+ lowered = (
1250
+ ir.AnyAggregate(origin=self._origin(expression), input=lowered) if grouped_items
1251
+ else ir.OpaqueExpression(origin=self._origin(expression), description=expression.sql(dialect=self.dialect))
1252
+ )
1253
+ output_columns.append(ir.OutputColumn(origin=self._origin(item), name=item.alias_or_name, scalar_expr=lowered))
1254
+ return tuple(output_columns)
1255
+
1256
+ def _lower_expression(
1257
+ self, expression: Expr, bindings: tuple[_Binding, ...], *, schema_complete: bool = False, line: int | None = None
1258
+ ) -> ir.ScalarExpr:
1259
+ """An unqualified column resolves by name across several sources only
1260
+ when every source's schema is complete: otherwise the one visible match
1261
+ might be shadowed by a column of a source whose columns are unknown.
1262
+
1263
+ A Builtin becomes a Call; any other function or operator an
1264
+ OpaquePureFunction of its lowered arguments, unless its value is not a
1265
+ function of them (`_NOT_A_FUNCTION_OF_ITS_COLUMNS`)."""
1266
+ origin = self._origin(expression, line)
1267
+
1268
+ def lower(node: Expr) -> ir.ScalarExpr:
1269
+ return self._lower_expression(node, bindings, schema_complete=schema_complete)
1270
+
1271
+ if isinstance(expression, exp.Subquery) and id(expression) in self._scalar_values:
1272
+ qualifier = self._scalar_values[id(expression)]
1273
+ value = next((binding.column for binding in bindings if binding.qualifier == qualifier), None)
1274
+ if value is not None:
1275
+ return ir.ColumnRef(origin=origin, column=value)
1276
+ if isinstance(expression, exp.Column) and id(expression) in self._scoped_columns:
1277
+ column = self._scoped_columns[id(expression)]
1278
+ if any(binding.column is column for binding in bindings):
1279
+ return ir.ColumnRef(origin=origin, column=column)
1280
+ if isinstance(expression, exp.Column):
1281
+ if expression.table:
1282
+ schema = _schema_path(expression)
1283
+ matches = [
1284
+ binding for binding in bindings
1285
+ if binding.qualifier == expression.table.lower() and binding.name == expression.name.lower() and (not schema or binding.schema == schema)
1286
+ ]
1287
+ elif schema_complete or len({binding.qualifier for binding in bindings if not binding.qualifier.startswith(_SCALAR_QUALIFIER)}) == 1:
1288
+ matches = [binding for binding in bindings if binding.name == expression.name.lower()]
1289
+ else:
1290
+ matches = []
1291
+ sources = ", ".join(dict.fromkeys(binding.qualifier for binding in bindings))
1292
+ self._warnings.append(Diagnostic(
1293
+ diag.AMBIGUOUS_COLUMN,
1294
+ f"column {expression.name} is unqualified and more than one source may own it ({sources}), "
1295
+ "since some have no declared columns: qualify it, declare those tables with CREATE TABLE, "
1296
+ "or analyze with a database connection (analyze_database)",
1297
+ origin,
1298
+ ))
1299
+ if len(matches) == 1:
1300
+ return ir.ColumnRef(origin=origin, column=matches[0].column)
1301
+ return ir.OpaqueExpression(origin=origin, description=expression.sql(dialect=self.dialect))
1302
+ if isinstance(expression, _COMPARISONS):
1303
+ self._check_zoned_text(expression, lower, self._origin(expression, line or _source_line(expression)))
1304
+ description = expression.sql(dialect=self.dialect)
1305
+ quantified = isinstance(expression, exp.Binary) and any(
1306
+ isinstance(operand, (exp.Any, exp.All)) for operand in (expression.this, expression.expression)
1307
+ ) # `n > ANY (...)` is FALSE over no rows, even for a NULL `n`.
1308
+ match expression:
1309
+ case exp.Paren():
1310
+ return self._lower_expression(expression.this, bindings, schema_complete=schema_complete, line=line)
1311
+ case exp.Null():
1312
+ return ir.NullLiteral(origin=origin)
1313
+ case exp.Count():
1314
+ return ir.Count(origin=origin, description=description)
1315
+ case exp.Coalesce():
1316
+ return ir.Coalesce(origin=origin, arguments=tuple(lower(argument) for argument in (expression.this, *expression.expressions)))
1317
+ case exp.Case() if expression.this is not None:
1318
+ return self._lower_case(expression, bindings, schema_complete=schema_complete)
1319
+ case exp.And():
1320
+ return self._lower_condition(expression, _conjuncts(expression), bindings, schema_complete=schema_complete)
1321
+ case exp.Or() if is_ambiguous_pipes(expression):
1322
+ pipes = f"{expression.this.sql(dialect=self.dialect)} || {expression.expression.sql(dialect=self.dialect)}"
1323
+ self._warnings.append(Diagnostic(
1324
+ diag.AMBIGUOUS_OPERATOR,
1325
+ f"`||` is OR in {self.dialect}, or concatenation under sql_mode PIPES_AS_CONCAT, which also groups "
1326
+ f"its operands differently, so {pipes} is an unknown function of its columns: write OR or CONCAT()",
1327
+ self._origin(expression, line or _source_line(expression)),
1328
+ ))
1329
+ return ir.OpaquePureFunction(origin=origin, arguments=tuple(lower(child) for child in expression.iter_expressions()), description=pipes)
1330
+ case exp.Or():
1331
+ return ir.Or(origin=origin, parts=tuple(
1332
+ self._lower_condition(branch, _conjuncts(branch), bindings, schema_complete=schema_complete)
1333
+ for branch in _disjuncts(expression)
1334
+ ))
1335
+ case exp.EQ() if not quantified:
1336
+ return ir.Equality(origin=origin, left=lower(expression.this), right=lower(expression.expression))
1337
+ case exp.Is(expression=exp.Null()):
1338
+ return ir.IsNull(origin=origin, operand=lower(expression.this))
1339
+ case exp.Not(this=exp.Is(expression=exp.Null()) as test):
1340
+ return ir.IsNotNull(origin=origin, operand=lower(test.this))
1341
+ case exp.In() if expression.expressions and not any(expression.args.get(key) for key in ("query", "unnest", "field")):
1342
+ return ir.InList(origin=origin, operand=lower(expression.this), values=tuple(lower(value) for value in expression.expressions))
1343
+ case _:
1344
+ pass
1345
+ value = literal_value(expression)
1346
+ if value == ir.LiteralValue(type="string", text="") and self.semantics.empty_string_is_null:
1347
+ return ir.NullLiteral(origin=origin)
1348
+ if value is not None or isinstance(expression, exp.Literal):
1349
+ return ir.Constant(origin=origin, value=value)
1350
+ builtin = dialect_builtins(self.semantics).get(type(expression))
1351
+ if builtin is not None and not quantified:
1352
+ function, keys = builtin
1353
+ if type(expression) is exp.Cast and function.total is not None and self._is_json(expression.this, lower):
1354
+ # `CAST(JSON 'null' AS BIGINT)` is NULL.
1355
+ function = replace(function, total=None)
1356
+ values = [value for key in keys for value in _as_list(expression.args[key])]
1357
+ if function.accepts(len(values)):
1358
+ arguments = tuple(lower(value) for value in values)
1359
+ composites = [
1360
+ (value, argument) for value, argument in zip(values, arguments)
1361
+ if function.name in _ORDER_COMPARISONS and function.total is not None and self._is_composite(value, lower)
1362
+ ]
1363
+ if composites:
1364
+ function, arguments = self._composite_comparison(expression, function, arguments, composites, self._origin(expression, line or _source_line(expression)))
1365
+ return ir.Call(origin=origin, function=function, arguments=arguments, description=description)
1366
+ if isinstance(expression, _NOT_A_FUNCTION_OF_ITS_COLUMNS):
1367
+ return ir.OpaqueExpression(origin=origin, description=description)
1368
+ return ir.OpaquePureFunction(origin=origin, arguments=tuple(lower(child) for child in expression.iter_expressions()), description=description)
1369
+
1370
+ def _check_zoned_text(self, comparison: Expr, lower: Callable[[Expr], ir.ScalarExpr], origin: Origin) -> None:
1371
+ """Warn of a comparison with a CAST of a zoned timestamp, where it keeps
1372
+ its zone: equal instants cast to different text (or dates, or local
1373
+ times), so the comparison depends on the zone each value was given."""
1374
+ if self.semantics.zoned_timestamp_is_instant or id(comparison) in self._zone_checked:
1375
+ return
1376
+ self._zone_checked.add(id(comparison))
1377
+ operands = [comparison.this, *_as_list(comparison.args.get("expression") or []), *comparison.expressions]
1378
+ operands += [comparison.args[key] for key in ("low", "high") if comparison.args.get(key) is not None]
1379
+ for operand in operands:
1380
+ while isinstance(operand, exp.Paren):
1381
+ operand = operand.this
1382
+ if not isinstance(operand, exp.Cast) or operand.to.this in _ZONED_TYPES or not isinstance(operand.this, exp.Column):
1383
+ continue
1384
+ kind = self._declared_type(lower(operand.this))
1385
+ if kind is None or kind.this not in _ZONED_TYPES:
1386
+ continue
1387
+ cast = operand.sql(dialect=self.dialect)
1388
+ self._warnings.append(Diagnostic(
1389
+ diag.ZONED_TIMESTAMP_CAST,
1390
+ f"{cast} is compared, but a {kind.sql(dialect=self.dialect)} keeps the zone it was given in {self.dialect}, "
1391
+ "so equal instants cast to different values and the comparison depends on each value's zone: "
1392
+ "compare the timestamps themselves, or convert to one zone first (AT TIME ZONE 'UTC')",
1393
+ origin,
1394
+ ))
1395
+
1396
+ def _composite_comparison(
1397
+ self, comparison: Expr, function: ir.Builtin, arguments: tuple[ir.ScalarExpr, ...],
1398
+ composites: list[tuple[Expr, ir.ScalarExpr]], origin: Origin,
1399
+ ) -> tuple[ir.Builtin, tuple[ir.ScalarExpr, ...]]:
1400
+ """A comparison of composites (ARRAYs, ROWs, MAPs), non-null only when
1401
+ no element is NULL either: `ARRAY[1, NULL] <> ARRAY[1, NULL]` is NULL in
1402
+ Trino. So it also takes each composite operand's Elements, as a
1403
+ non-strict, total argument. A composite column or CAST, whose elements
1404
+ the SQL does not show, is noted for a warning."""
1405
+ assert function.total is not None
1406
+ extra = len(composites)
1407
+ widened = ir.Builtin(
1408
+ name=function.name, strict=function.strict + (False,) * extra, total=function.total + (True,) * extra, injective=False,
1409
+ )
1410
+ elements = tuple(ir.Elements(origin=argument.origin, operand=argument) for _, argument in composites)
1411
+ for value, argument in composites:
1412
+ while isinstance(value, exp.Paren):
1413
+ value = value.this
1414
+ if isinstance(value, (exp.Column, exp.Cast)) and id(value) not in self._composite_comparisons:
1415
+ self._composite_comparisons[id(value)] = (origin, comparison.sql(dialect=self.dialect), value.sql(dialect=self.dialect), argument)
1416
+ return widened, arguments + elements
1417
+
1418
+ def _composite_warnings(self) -> list[Diagnostic]:
1419
+ """A warning for each composite operand whose elements no NOT NULL
1420
+ ELEMENTS declaration covers, on it or on a column it passes on from."""
1421
+ declared = {column for item in self._declarations if isinstance(item, NonNullElements) for column in item.columns}
1422
+ warnings = []
1423
+ for origin, comparison, operand, argument in self._composite_comparisons.values():
1424
+ expression, covered = argument, False
1425
+ while isinstance(expression, ir.ColumnRef) and not covered:
1426
+ covered = expression.column in declared
1427
+ expression = expression.column.scalar_expr
1428
+ if not covered:
1429
+ warnings.append(Diagnostic(
1430
+ diag.COMPOSITE_COMPARISON,
1431
+ f"{comparison} compares {operand}, which may hold a NULL element, and a comparison of composites with "
1432
+ "a NULL element may be NULL (it is in Trino), so it is not taken to be non-null: if its elements are "
1433
+ "never NULL, say so with /**DECLARE NOT NULL ELEMENTS(...)**/ on a Select Expression that outputs it",
1434
+ origin,
1435
+ ))
1436
+ return warnings
1437
+
1438
+ def _is_composite(self, node: Expr, lower: Callable[[Expr], ir.ScalarExpr]) -> bool:
1439
+ """Whether `node`'s value is known to be an ARRAY, ROW or MAP. A column
1440
+ of unknown type is taken to be a scalar: composites are rare."""
1441
+ match node:
1442
+ case exp.Paren() | exp.Nullif():
1443
+ return self._is_composite(node.this, lower)
1444
+ case exp.Cast():
1445
+ return node.to.this in _COMPOSITE_TYPES
1446
+ case exp.Column():
1447
+ kind = self._declared_type(lower(node))
1448
+ return kind is not None and kind.this in _COMPOSITE_TYPES
1449
+ case exp.Array() | exp.Struct() | exp.Map() | exp.VarMap():
1450
+ return True
1451
+ case exp.Coalesce():
1452
+ return any(self._is_composite(each, lower) for each in (node.this, *node.expressions))
1453
+ case exp.Case():
1454
+ arms = [arm.args["true"] for arm in node.args.get("ifs") or ()]
1455
+ default = node.args.get("default")
1456
+ return any(self._is_composite(each, lower) for each in (*arms, *([default] if default is not None else [])))
1457
+ case exp.If():
1458
+ false = node.args.get("false")
1459
+ return any(self._is_composite(each, lower) for each in (node.args["true"], *([false] if false is not None else [])))
1460
+ case _:
1461
+ return False
1462
+
1463
+ def _is_json(self, node: Expr, lower: Callable[[Expr], ir.ScalarExpr]) -> bool:
1464
+ """Whether `node`'s value is known to be JSON (or a VARIANT), whose
1465
+ `null` a CAST turns into NULL. A column of unknown type is taken not to be."""
1466
+ match node:
1467
+ case exp.Paren() | exp.Nullif():
1468
+ return self._is_json(node.this, lower)
1469
+ case exp.Cast():
1470
+ return node.to.this in _JSON_TYPES
1471
+ case exp.Column():
1472
+ kind = self._declared_type(lower(node))
1473
+ return kind is not None and kind.this in _JSON_TYPES
1474
+ case exp.ParseJSON() | exp.JSONExtract() | exp.JSONBExtract() | exp.JSONObject() | exp.JSONArray() | exp.JSONObjectAgg() | exp.JSONArrayAgg():
1475
+ return True
1476
+ case exp.Coalesce():
1477
+ return any(self._is_json(each, lower) for each in (node.this, *node.expressions))
1478
+ case exp.Case():
1479
+ arms = [arm.args["true"] for arm in node.args.get("ifs") or ()]
1480
+ default = node.args.get("default")
1481
+ return any(self._is_json(each, lower) for each in (*arms, *([default] if default is not None else [])))
1482
+ case exp.If():
1483
+ false = node.args.get("false")
1484
+ return any(self._is_json(each, lower) for each in (node.args["true"], *([false] if false is not None else [])))
1485
+ case _:
1486
+ return False
1487
+
1488
+ def _declared_type(self, expression: ir.ScalarExpr) -> exp.DataType | None:
1489
+ """The type a CREATE TABLE declares for the column `expression` reads,
1490
+ through every column that passes it on."""
1491
+ while isinstance(expression, ir.ColumnRef):
1492
+ column = expression.column
1493
+ if column in self._declared_types:
1494
+ return self._declared_types[column]
1495
+ expression = column.scalar_expr
1496
+ return None
1497
+
1498
+ def _lower_case(self, case: exp.Case, bindings: tuple[_Binding, ...], *, schema_complete: bool) -> ir.SimpleCase:
1499
+ """A simple CASE, and the assertion its marker states, once per CASE."""
1500
+
1501
+ def lower(expression: Expr) -> ir.ScalarExpr:
1502
+ return self._lower_expression(expression, bindings, schema_complete=schema_complete)
1503
+
1504
+ arms = case.args.get("ifs") or ()
1505
+ default = case.args.get("default")
1506
+ lowered = ir.SimpleCase(
1507
+ origin=self._origin(case),
1508
+ operand=lower(case.this),
1509
+ whens=tuple(lower(arm.this) for arm in arms),
1510
+ thens=tuple(lower(arm.args["true"]) for arm in arms),
1511
+ default=lower(default) if default is not None else ir.NullLiteral(origin=self._origin(case)),
1512
+ )
1513
+ if id(case) not in self._sited_cases:
1514
+ self._sited_cases.add(id(case))
1515
+ self._add_attachment_site(case, f"CASE {case.this.sql(dialect=self.dialect)}", lowered, trailing=False)
1516
+ span = exhaustive_span(case)
1517
+ if span is not None and id(case) not in self._handled_cases:
1518
+ self._handled_cases.add(id(case))
1519
+ origin = Origin.marked(case.sql(dialect=self.dialect), span)
1520
+ self._assertions.append(ir.Assertion(origin=origin, property=ExhaustiveCase(case=lowered)))
1521
+ return lowered
1522
+
1523
+ def _opaque_relation(self, source: Expr, names=()) -> ir.OpaqueRelation:
1524
+ origin = self._origin(source)
1525
+ return ir.OpaqueRelation(
1526
+ origin=origin,
1527
+ output_columns=self._opaque_output_columns(names, origin),
1528
+ is_schema_complete=False,
1529
+ description=source.sql(dialect=self.dialect),
1530
+ )
1531
+
1532
+ # Assertions and diagnostics --------------------------------------------
1533
+
1534
+ def _register_column_markers(self, relation: ir.RelationExpr, select: exp.Select) -> None:
1535
+ by_name = {output.name.lower(): output for output in relation.output_columns}
1536
+ for marker in column_markers(select):
1537
+ kind, columns, line = marker.kind, marker.columns, marker.span.line
1538
+ self._handled_column_markers.add(id(marker))
1539
+ origin = Origin.marked(select.sql(dialect=self.dialect), marker.span)
1540
+ unknown = [column for column in (*columns, *marker.dependent) if column.lower() not in by_name]
1541
+ if unknown and relation.is_schema_complete:
1542
+ self._diagnostics.append(
1543
+ Diagnostic(
1544
+ diag.UNKNOWN_ASSERTED_COLUMN,
1545
+ f"the {MARKER_NOUNS[kind]} on line {line} names {', '.join(unknown)}, which is not among this Select Expression's own output columns",
1546
+ origin,
1547
+ )
1548
+ )
1549
+ continue
1550
+ if unknown:
1551
+ # The column may exist in a source whose columns are unknown,
1552
+ # so it is neither absent nor something to state a Property of.
1553
+ self._diagnostics.append(
1554
+ Diagnostic(
1555
+ diag.UNRESOLVED_ASSERTED_COLUMN,
1556
+ f"the {MARKER_NOUNS[kind]} on line {line} names {', '.join(unknown)}, which cannot be resolved: "
1557
+ "this Select Expression reads a relation whose columns are unknown (declare it with CREATE TABLE)",
1558
+ origin,
1559
+ )
1560
+ )
1561
+ continue
1562
+ members = frozenset(by_name[column.lower()] for column in columns)
1563
+ dependent = frozenset(by_name[column.lower()] for column in marker.dependent)
1564
+ match kind:
1565
+ case "foreign-key" | "declared-foreign-key":
1566
+ self._marker_references.append(_MarkerReference(
1567
+ marker, kind, origin, tuple(by_name[column.lower()] for column in columns),
1568
+ self._marker_targets[id(marker)], len(self._assertions),
1569
+ ))
1570
+ case "unique":
1571
+ self._assertions.append(ir.Assertion(origin=origin, property=UniqueSet(columns=members)))
1572
+ case "key":
1573
+ self._assertions.append(ir.Assertion(origin=origin, property=NonNullUniqueSet(columns=members)))
1574
+ case "not-null":
1575
+ self._assertions.append(ir.Assertion(origin=origin, property=NonNullColumns(columns=members)))
1576
+ case "declared-unique":
1577
+ self._declarations.append(UniqueSet(columns=members))
1578
+ case "declared-key":
1579
+ # Non-nullness is declared per column too, so it outlives
1580
+ # projections that drop part of a composite key.
1581
+ self._declarations.append(NonNullUniqueSet(columns=members))
1582
+ self._declarations.extend(NonNullColumn(column=column) for column in members)
1583
+ case "declared-not-null":
1584
+ self._declarations.extend(NonNullColumn(column=column) for column in members)
1585
+ case "not-null-elements":
1586
+ self._assertions.append(ir.Assertion(origin=origin, property=NonNullElements(columns=members)))
1587
+ case "declared-not-null-elements":
1588
+ # One per column, so each column's elements are known on their own.
1589
+ self._declarations.extend(NonNullElements(columns=frozenset({column})) for column in members)
1590
+ case "determines":
1591
+ property = FunctionalDependency(determinant=members, dependent=dependent)
1592
+ self._assertions.append(ir.Assertion(origin=origin, property=property))
1593
+ case "declared-determines":
1594
+ self._declarations.append(FunctionalDependency(determinant=members, dependent=dependent))
1595
+ case "enum" | "declared-enum":
1596
+ column, = members
1597
+ values = frozenset(marker.values)
1598
+ domain = EnumeratedDomain(column=column, values=values)
1599
+ if kind == "enum":
1600
+ self._assertions.append(ir.Assertion(origin=origin, property=domain))
1601
+ else:
1602
+ self._declarations.append(domain)
1603
+ case _:
1604
+ assert_never(kind)
1605
+
1606
+ def _add_attachment_site(
1607
+ self, node: exp.Join | exp.Select | exp.Case, label: str, relation: ir.RelationExpr | ir.SimpleCase, *, trailing: bool
1608
+ ) -> None:
1609
+ position = marker_position(node)
1610
+ if position is not None:
1611
+ offset, line = position
1612
+ self._attachment_sites.append((AttachmentSite(label, line, offset, trailing), relation))
1613
+
1614
+ def _link_marker_references(self) -> None:
1615
+ """Lower each Foreign Key marker's referenced relation, then state the
1616
+ marker's Property; assertions keep the place their marker was written."""
1617
+ placed: list[tuple[int, ir.Assertion]] = []
1618
+ while self._marker_references:
1619
+ pending = self._marker_references.pop(0)
1620
+ target = self._resolve_symbol(pending.target)
1621
+ marker = pending.marker
1622
+ referenced = self._columns_named(target, marker.referenced_columns)
1623
+ if referenced is None:
1624
+ self._diagnostics.append(Diagnostic(
1625
+ diag.UNKNOWN_REFERENCED_COLUMN,
1626
+ f"the {MARKER_NOUNS[marker.kind]} on line {marker.span.line} references "
1627
+ f"{target.name}({', '.join(marker.referenced_columns)}), but {target.name} has no column "
1628
+ f"{', '.join(_missing(target, marker.referenced_columns))}",
1629
+ pending.origin,
1630
+ ))
1631
+ continue
1632
+ match pending.kind:
1633
+ case "foreign-key":
1634
+ placed.append((pending.position, ir.Assertion(
1635
+ origin=pending.origin, property=ForeignKey(columns=pending.columns, referenced_columns=referenced),
1636
+ )))
1637
+ case "declared-foreign-key":
1638
+ self._declarations.append(ReferentialCoverage(columns=pending.columns, referenced_columns=referenced))
1639
+ case _:
1640
+ assert_never(pending.kind)
1641
+ # Descending, so earlier positions stay valid; ties keep written order.
1642
+ for position, assertion in reversed(placed):
1643
+ self._assertions.insert(position, assertion)
1644
+
1645
+ def _link_constraint_references(self) -> None:
1646
+ for reference in self._constraint_references:
1647
+ table = self._resolve_symbol(reference.table)
1648
+ target = self._resolve_symbol(reference.target)
1649
+ target_names = reference.target_columns
1650
+ if target_names is None:
1651
+ if reference.target.statement is None:
1652
+ continue # An undeclared table's primary key is unknown.
1653
+ target_names = _primary_key(reference.target)
1654
+ if target_names is None:
1655
+ self._diagnostics.append(Diagnostic(
1656
+ diag.UNKNOWN_REFERENCED_COLUMN,
1657
+ f"a foreign key of {table.name} references {target.name} without columns, "
1658
+ f"and {target.name} declares no primary key",
1659
+ reference.origin,
1660
+ ))
1661
+ continue
1662
+ columns = self._columns_named(table, reference.columns)
1663
+ if columns is None or len(reference.columns) != len(target_names):
1664
+ self._diagnostics.append(Diagnostic(
1665
+ diag.INVALID_FOREIGN_KEY_CONSTRAINT,
1666
+ f"a foreign key of {table.name} must pair columns of its own with equally many columns of "
1667
+ f"{target.name}: ({', '.join(reference.columns)}) references ({', '.join(target_names)})",
1668
+ reference.origin,
1669
+ ))
1670
+ continue
1671
+ referenced = self._columns_named(target, target_names)
1672
+ if referenced is None:
1673
+ self._diagnostics.append(Diagnostic(
1674
+ diag.UNKNOWN_REFERENCED_COLUMN,
1675
+ f"a foreign key of {table.name} references {target.name}({', '.join(target_names)}), "
1676
+ f"but {target.name} has no column {', '.join(_missing(target, target_names))}",
1677
+ reference.origin,
1678
+ ))
1679
+ continue
1680
+ trust = self._key_trust(reference.node, reference.table, reference.columns)
1681
+ if trust is not None:
1682
+ (self._knowledge if trust == "enforced" else self._declarations).append(
1683
+ ReferentialCoverage(columns=columns, referenced_columns=referenced)
1684
+ )
1685
+
1686
+ def _columns_named(self, relation: ir.RelationExpr, names: Iterable[str]) -> tuple[ir.OutputColumn, ...] | None:
1687
+ by_name = {column.name.lower(): column for column in relation.output_columns}
1688
+ columns = [by_name.get(name.lower()) for name in names]
1689
+ found = tuple(column for column in columns if column is not None)
1690
+ return found if len(found) == len(columns) else None
1691
+
1692
+ def _report_cycle(self, symbol: _Symbol, table: exp.Table) -> None:
1693
+ if symbol in self._reported_cycles:
1694
+ return
1695
+ self._reported_cycles.add(symbol)
1696
+ self._diagnostics.append(
1697
+ Diagnostic(
1698
+ diag.RECURSIVE_VIEW_DEFINITION,
1699
+ f"view {symbol.name.lower()} is defined recursively, which is not supported",
1700
+ Origin(SQL, table.sql(dialect=self.dialect)),
1701
+ )
1702
+ )
1703
+
1704
+ def _report_unanalyzed(self, ast: ParsedProgram) -> None:
1705
+ for join in _asserted_joins(ast):
1706
+ origin, kind = join_origin(join, self.dialect), join_marker_kind(join)
1707
+ if id(join) in self._handled_joins:
1708
+ continue
1709
+ assert kind is not None, "every asserted join records its marker kind"
1710
+ self._diagnostics.append(
1711
+ Diagnostic(
1712
+ diag.UNANALYZED_ASSERTION,
1713
+ f"the {JOIN_MARKER_NOUNS[kind]} assertion on line {origin.line} "
1714
+ "is in a part of the program this analysis does not model",
1715
+ origin,
1716
+ )
1717
+ )
1718
+ for case in _asserted_cases(ast):
1719
+ if id(case) in self._handled_cases:
1720
+ continue
1721
+ span = exhaustive_span(case)
1722
+ assert span is not None, "every asserted case records its marker span"
1723
+ self._diagnostics.append(
1724
+ Diagnostic(
1725
+ diag.UNANALYZED_ASSERTION,
1726
+ f"the exhaustive assertion on line {span.line} is in a part of the program this analysis does not model",
1727
+ Origin.marked(case.sql(dialect=self.dialect), span),
1728
+ )
1729
+ )
1730
+ for select in _asserted_selects(ast):
1731
+ for marker in column_markers(select):
1732
+ if id(marker) in self._handled_column_markers:
1733
+ continue
1734
+ self._diagnostics.append(
1735
+ Diagnostic(
1736
+ diag.UNANALYZED_ASSERTION,
1737
+ f"the {MARKER_NOUNS[marker.kind]} on line {marker.span.line} is in a part of the program this analysis does not model",
1738
+ Origin.marked(select.sql(dialect=self.dialect), marker.span),
1739
+ )
1740
+ )
1741
+
1742
+ def _table_knowledge(self, relation: ir.NamedRelation, symbol: _Symbol) -> None:
1743
+ """A declared table's ENUM types, NOT NULL columns and keys: Knowledge
1744
+ where the dialect enforces them, Property Declarations where they are
1745
+ trusted (`_key_trust`)."""
1746
+ statement = symbol.statement
1747
+ if statement is None:
1748
+ return
1749
+ columns = {column.name.lower(): column for column in relation.output_columns}
1750
+ non_null = {column.lower() for column in symbol.not_null}
1751
+ # Each key's columns, its constraint, and whether it is a primary key.
1752
+ keys: list[tuple[tuple[str, ...], Expr, bool]] = []
1753
+ for definition in statement.this.find_all(exp.ColumnDef):
1754
+ domain = _enum_type_values(definition, self.semantics)
1755
+ if domain is not None and definition.name.lower() in columns:
1756
+ self._knowledge.append(EnumeratedDomain(column=columns[definition.name.lower()], values=domain))
1757
+ for constraint in definition.constraints:
1758
+ match constraint.args.get("kind"):
1759
+ case exp.NotNullColumnConstraint() as not_null if not not_null.args.get("allow_null"):
1760
+ non_null.add(definition.name.lower())
1761
+ case exp.PrimaryKeyColumnConstraint() as key:
1762
+ keys.append(((definition.name,), key, True))
1763
+ case exp.UniqueColumnConstraint() as key:
1764
+ keys.append(((definition.name,), key, False))
1765
+ case _:
1766
+ pass # Other column constraints state nothing about keys or NULL.
1767
+ for constraint in _table_constraints(symbol):
1768
+ constrained = _table_level_unique_columns(constraint)
1769
+ if constrained is not None:
1770
+ keys.append((constrained, constraint, isinstance(constraint, exp.PrimaryKey)))
1771
+ self._knowledge.extend(NonNullColumn(column=columns[name]) for name in sorted(non_null) if name in columns)
1772
+ for column_names, constraint, primary in keys:
1773
+ members = tuple(columns[name.lower()] for name in column_names if name.lower() in columns)
1774
+ if len(members) != len(column_names):
1775
+ continue
1776
+ trust = self._key_trust(constraint, symbol, column_names)
1777
+ if trust is None:
1778
+ continue
1779
+ facts: list[Property] = [UniqueSet(columns=frozenset(members))]
1780
+ if primary and (trust == "declared" or self.semantics.primary_key_not_null):
1781
+ facts.extend(NonNullColumn(column=member) for member in members if member.name.lower() not in non_null)
1782
+ (self._knowledge if trust == "enforced" else self._declarations).extend(facts)
1783
+ for parts, index in symbol.expression_keys:
1784
+ members = self._expression_key(relation, parts, index)
1785
+ if members is None:
1786
+ continue
1787
+ names = tuple(member.name for member in members)
1788
+ trust = self._key_trust(exp.UniqueColumnConstraint(), symbol, names)
1789
+ if trust is not None:
1790
+ (self._knowledge if trust == "enforced" else self._declarations).append(UniqueSet(columns=frozenset(members)))
1791
+
1792
+ def _expression_key(
1793
+ self, relation: ir.NamedRelation, parts: list[Expr], index: exp.Create
1794
+ ) -> tuple[ir.OutputColumn, ...] | None:
1795
+ """The columns a unique index over `parts` makes a Unique Set, or None
1796
+ (diagnosed) when it makes none.
1797
+
1798
+ Two rows equal and non-null on the columns the parts read would give
1799
+ equal index keys, which the index forbids, when every part is a pure
1800
+ function of its columns (equal inputs, equal keys: so only over columns
1801
+ whose equal values are identical; under a case-insensitive collation
1802
+ `'A' = 'a'` yet their bytes differ) and total in them (a NULL key part
1803
+ exempts the row: `nullif(email, '')` lets every '' through). No
1804
+ injectivity is needed: `lower(email)` unique makes `email` unique."""
1805
+ bindings = tuple(_Binding(relation.name.lower(), column.name.lower(), column) for column in relation.output_columns)
1806
+ found: dict[ir.OutputColumn, None] = {}
1807
+ for part in parts:
1808
+ lowered = self._lower_expression(part, bindings, schema_complete=relation.is_schema_complete)
1809
+ inputs = ir.determinants(lowered)
1810
+ text = part.sql(dialect=self.dialect)
1811
+ if inputs is None:
1812
+ problem = f"{text} is not a function of the table's columns"
1813
+ elif not ir.total(lowered):
1814
+ problem = f"{text} may be NULL where its columns are not, which exempts the row from the index"
1815
+ elif not all(column in relation.output_columns and column.identity_comparison for column in inputs):
1816
+ problem = f"{text} reads a column whose equal values may differ (a float, or text under a collation)"
1817
+ else:
1818
+ found.update(dict.fromkeys(inputs))
1819
+ continue
1820
+ self._unsupported_create(index, f"a unique index whose key states no Unique Set ({problem})")
1821
+ return None
1822
+ return tuple(found)
1823
+
1824
+ def _key_trust(self, constraint: Expr, symbol: _Symbol, columns: tuple[str, ...]) -> Literal["enforced", "declared"] | None:
1825
+ """Whether a key or foreign key constraint holds: enforced by the
1826
+ database (Knowledge), trusted by its author (a Property Declaration:
1827
+ marked `RELY`, or any unenforced one under `trust_unenforced_keys`),
1828
+ or neither, which a warning reports. An unenforced one is never
1829
+ Knowledge, so never proves anything by itself (ADR 0005)."""
1830
+ assert symbol.statement is not None
1831
+ foreign = isinstance(constraint, exp.ForeignKey | exp.Reference)
1832
+ reference = constraint.args.get("reference") if isinstance(constraint, exp.ForeignKey) else constraint
1833
+ options = {str(option).upper() for option in (reference.args.get("options") or ())} if isinstance(reference, Expr) else set()
1834
+ meaning = self.semantics
1835
+ reason = None
1836
+ if "NOT ENFORCED" in options:
1837
+ reason = "it is declared NOT ENFORCED"
1838
+ elif id(constraint) in self._unvalidated:
1839
+ reason = "ALTER TABLE ... NOT VALID added it, so rows already in the table were never checked"
1840
+ elif "DEFERRABLE" in options or "INITIALLY DEFERRED" in options:
1841
+ reason = "it is DEFERRABLE, so a transaction may see it violated"
1842
+ elif (meaning.foreign_keys if foreign else meaning.unique_keys) != "enforced":
1843
+ reason = f"{self.dialect} does not enforce it"
1844
+ elif isinstance(constraint, exp.Reference) and not meaning.inline_references:
1845
+ reason = f"{self.dialect} before 9.0 ignores a column-level REFERENCES"
1846
+ elif foreign and meaning.foreign_key_engines is not None and (engine := _engine(symbol)) not in meaning.foreign_key_engines:
1847
+ reason = (
1848
+ f"ENGINE={engine} does not enforce foreign keys" if engine is not None
1849
+ else "its table names no ENGINE, and the server's default_storage_engine may not enforce foreign keys"
1850
+ )
1851
+ if reason is None:
1852
+ return "enforced"
1853
+ if ("RELY" in options and meaning.rely) or self.trust_unenforced_keys:
1854
+ return "declared"
1855
+ kind = "foreign key" if foreign else "primary key" if isinstance(constraint, exp.PrimaryKey | exp.PrimaryKeyColumnConstraint) else "unique constraint"
1856
+ fixes = ["mark it RELY"] if meaning.rely else []
1857
+ if foreign and meaning.foreign_key_engines is not None and _engine(symbol) is None:
1858
+ fixes.append("write ENGINE=InnoDB")
1859
+ fixes.append("analyze with trust_unenforced_keys=True")
1860
+ self._warnings.append(Diagnostic(
1861
+ diag.UNENFORCED_CONSTRAINT,
1862
+ f"the {kind} ({', '.join(columns)}) of {symbol.name} is not Knowledge, since {reason}: "
1863
+ f"if the data satisfies it, {' or '.join(fixes)}",
1864
+ self._origin(constraint, _source_line(constraint) or _source_line(symbol.statement)),
1865
+ ))
1866
+ return None
1867
+
1868
+ def _origin(self, node: Expr, line: int | None = None) -> Origin:
1869
+ return Origin(SQL, node.sql(dialect=self.dialect), line)
1870
+
1871
+
1872
+ def _pass_output_columns(inputs: tuple[ir.OutputColumn, ...], origin: Origin) -> tuple[ir.OutputColumn, ...]:
1873
+ return tuple(
1874
+ ir.OutputColumn(origin=origin, name=column.name, scalar_expr=ir.ColumnRef(origin=origin, column=column))
1875
+ for column in inputs
1876
+ )
1877
+
1878
+
1879
+ def _source_line(node: Expr) -> int | None:
1880
+ """The first source line of `node`, from the positions SQLGlot keeps on its tokens."""
1881
+ lines = [descendant.meta["line"] for descendant in node.walk() if "line" in descendant.meta]
1882
+ return min(lines, default=None)
1883
+
1884
+
1885
+ def _subquery_conjunct(conjunct: Expr) -> tuple[ir.JoinKind, exp.Select, Expr | None] | None:
1886
+ """A WHERE conjunct lowered as a filtering join: its kind, the subquery,
1887
+ and the IN operand (None for EXISTS)."""
1888
+ match conjunct:
1889
+ case exp.Exists(this=exp.Select() as select):
1890
+ return "semi", select, None
1891
+ case exp.Not(this=exp.Exists(this=exp.Select() as select)):
1892
+ return "anti", select, None
1893
+ case exp.In(this=operand, args={"query": exp.Subquery() as subquery}) if not isinstance(operand, exp.Tuple):
1894
+ body = _unwrap_query(subquery)
1895
+ return ("semi", body, operand) if isinstance(body, exp.Select) else None
1896
+ case _:
1897
+ return None
1898
+
1899
+
1900
+ @cache
1901
+ def dialect_builtins(meaning: DialectSemantics) -> dict[type[Expr], tuple[ir.Builtin, tuple[str, ...]]]:
1902
+ """Each modelled Builtin, by the SQLGlot node it lowers from (exactly that
1903
+ type: TRY_CAST, a subclass of CAST, may be NULL for a value), with the
1904
+ argument keys its parameters come from, a list key in place of a variadic
1905
+ Builtin's repeated parameter, as a dialect's semantics define it
1906
+ (`dialects.py`; `scripts/certify_dialects.py` probes each Builtin fact).
1907
+
1908
+ Comparisons, LIKE, NOT, BETWEEN and the bitwise operators are never NULL
1909
+ for non-null arguments where a failed implicit cast raises (`casts_raise`;
1910
+ in Spark without ANSI mode `1 < 'x'` is NULL). Arithmetic and CAST only
1911
+ where they raise an error rather than return NULL (`checked_arithmetic`:
1912
+ SQLite turns a NaN into NULL, MySQL a bad date CAST), and division and
1913
+ remainder only where division by zero raises too (`checked_division`).
1914
+ A CAST from JSON is not total anywhere (`convert._is_json`): JSON `null`
1915
+ is a value that casts to NULL. `+`, `-` and negation are injective where exact arithmetic overflows into
1916
+ an error. `||`, `concat()` and `concat_ws()` as each dialect treats NULL;
1917
+ `||` not at all where a setting decides what it means (MySQL: OR, or
1918
+ concatenation)."""
1919
+ checked = meaning.checked_arithmetic
1920
+ casts = meaning.casts_raise
1921
+
1922
+ def builtin(name: str, arity: int, *, total: bool = True, injective: bool = False) -> ir.Builtin:
1923
+ return ir.Builtin(name=name, strict=(True,) * arity, total=(True,) * arity if total else None, injective=injective)
1924
+
1925
+ binary = {
1926
+ exp.NEQ: "<>", exp.GT: ">", exp.GTE: ">=", exp.LT: "<", exp.LTE: "<=", exp.Like: "LIKE", exp.ILike: "ILIKE",
1927
+ exp.BitwiseAnd: "&", exp.BitwiseOr: "|", exp.BitwiseXor: "^", exp.BitwiseLeftShift: "<<", exp.BitwiseRightShift: ">>",
1928
+ }
1929
+ division = {exp.Div: "/", exp.IntDiv: "//", exp.Mod: "%"}
1930
+ builtins: dict[type[Expr], tuple[ir.Builtin, tuple[str, ...]]] = {
1931
+ **{node: (builtin(name, 2, total=casts), ("this", "expression")) for node, name in binary.items()},
1932
+ exp.Mul: (builtin("*", 2, total=checked), ("this", "expression")),
1933
+ **{node: (builtin(name, 2, total=meaning.checked_division), ("this", "expression")) for node, name in division.items()},
1934
+ **{node: (builtin(name, 2, total=checked, injective=checked), ("this", "expression")) for node, name in {exp.Add: "+", exp.Sub: "-"}.items()},
1935
+ exp.Neg: (builtin("unary -", 1, total=checked, injective=checked), ("this",)),
1936
+ exp.Not: (builtin("NOT", 1, total=casts), ("this",)),
1937
+ exp.BitwiseNot: (builtin("~", 1, total=casts), ("this",)),
1938
+ exp.Cast: (builtin("CAST", 1, total=checked), ("this",)),
1939
+ # `1 BETWEEN NULL AND 0` is FALSE.
1940
+ exp.Between: (ir.Builtin(name="BETWEEN", strict=(True, False, False), total=(True, True, True) if casts else None, injective=False), ("this", "low", "high")),
1941
+ }
1942
+ match meaning.pipes:
1943
+ case "strict":
1944
+ builtins[exp.DPipe] = (builtin("||", 2), ("this", "expression"))
1945
+ case "lenient":
1946
+ builtins[exp.DPipe] = (ir.Builtin(name="||", strict=(False, False), total=(True, True), injective=False), ("this", "expression"))
1947
+ case "ambiguous" | None:
1948
+ pass
1949
+ case _:
1950
+ assert_never(meaning.pipes)
1951
+ match meaning.concat:
1952
+ case "strict":
1953
+ builtins[exp.Concat] = (
1954
+ ir.BuiltinVariadic(name="concat", strict=(), total=(), injective=False, repeated_strict=True, repeated_total=True), ("expressions",),
1955
+ )
1956
+ case "skips-null":
1957
+ # `concat(NULL)` is ''.
1958
+ builtins[exp.Concat] = (
1959
+ ir.BuiltinVariadic(name="concat", strict=(), total=(), injective=False, repeated_strict=False, repeated_total=False), ("expressions",),
1960
+ )
1961
+ case None:
1962
+ pass
1963
+ case _:
1964
+ assert_never(meaning.concat)
1965
+ match meaning.concat_ws:
1966
+ case "strict":
1967
+ builtins[exp.ConcatWs] = (
1968
+ ir.BuiltinVariadic(name="concat_ws", strict=(True,), total=(True,), injective=False, repeated_strict=True, repeated_total=True),
1969
+ ("expressions",),
1970
+ )
1971
+ case "skips-null":
1972
+ # NULL values are skipped: `concat_ws(',', NULL)` is ''.
1973
+ builtins[exp.ConcatWs] = (
1974
+ ir.BuiltinVariadic(name="concat_ws", strict=(True,), total=(True,), injective=False, repeated_strict=False, repeated_total=False),
1975
+ ("expressions",),
1976
+ )
1977
+ case None:
1978
+ pass
1979
+ case _:
1980
+ assert_never(meaning.concat_ws)
1981
+ return builtins
1982
+
1983
+
1984
+ def _as_list(value: Expr | list[Expr]) -> list[Expr]:
1985
+ return value if isinstance(value, list) else [value]
1986
+
1987
+
1988
+ def _disjuncts(predicate: Expr) -> list[Expr]:
1989
+ """The operands of a predicate's top-level ORs, parentheses removed."""
1990
+ while isinstance(predicate, exp.Paren):
1991
+ predicate = predicate.this
1992
+ if isinstance(predicate, exp.Or) and not is_ambiguous_pipes(predicate):
1993
+ return _disjuncts(predicate.this) + _disjuncts(predicate.expression)
1994
+ return [predicate]
1995
+
1996
+
1997
+ def _and(parts: tuple[ir.ScalarExpr, ...], origin: Origin) -> ir.ScalarExpr | None:
1998
+ """The AND of `parts`: None for none, the part itself for one."""
1999
+ match parts:
2000
+ case ():
2001
+ return None
2002
+ case (part,):
2003
+ return part
2004
+ case _:
2005
+ return ir.And(origin=origin, parts=parts)
2006
+
2007
+
2008
+ def _conjuncts(predicate: Expr) -> list[Expr]:
2009
+ """The operands of a predicate's top-level ANDs, parentheses removed."""
2010
+ while isinstance(predicate, exp.Paren):
2011
+ predicate = predicate.this
2012
+ if isinstance(predicate, exp.And):
2013
+ return _conjuncts(predicate.this) + _conjuncts(predicate.expression)
2014
+ return [predicate]
2015
+
2016
+
2017
+ def _direct_sources(select: exp.Select) -> list[Expr]:
2018
+ source = _from_source(select)
2019
+ return ([] if source is None else [source]) + [join.this for join in select.args.get("joins") or ()]
2020
+
2021
+
2022
+ def _asserted_joins(ast: ParsedProgram) -> list[exp.Join]:
2023
+ statements = (*ast.create_statements, ast.root_select)
2024
+ return [
2025
+ join for statement in statements if statement is not None
2026
+ for join in statement.find_all(exp.Join) if assertion_span(join) is not None
2027
+ ]
2028
+
2029
+
2030
+ def _asserted_selects(ast: ParsedProgram) -> list[exp.Select]:
2031
+ statements = (*ast.create_statements, ast.root_select)
2032
+ return [
2033
+ select for statement in statements if statement is not None
2034
+ for select in statement.find_all(exp.Select) if column_markers(select)
2035
+ ]
2036
+
2037
+
2038
+
2039
+ def _asserted_cases(ast: ParsedProgram) -> list[exp.Case]:
2040
+ statements = (*ast.create_statements, ast.root_select)
2041
+ return [
2042
+ case for statement in statements if statement is not None
2043
+ for case in statement.find_all(exp.Case) if exhaustive_span(case) is not None
2044
+ ]
2045
+
2046
+
2047
+ def _table_level_unique_columns(constraint: Expr) -> tuple[str, ...] | None:
2048
+ if isinstance(constraint, exp.PrimaryKey):
2049
+ return tuple(identifier.name for identifier in constraint.expressions)
2050
+ if isinstance(constraint, exp.UniqueColumnConstraint) and isinstance(constraint.this, exp.Schema):
2051
+ return tuple(identifier.name for identifier in constraint.this.expressions)
2052
+ return None
2053
+
2054
+
2055
+ def _table_constraints(symbol: _Symbol) -> list[Expr]:
2056
+ """A declared table's table-level constraints, named or not: its CREATE
2057
+ TABLE's, then those added later."""
2058
+ assert symbol.statement is not None
2059
+ schema = symbol.statement.this
2060
+ own = _unwrap_constraints(schema.expressions) if isinstance(schema, exp.Schema) else []
2061
+ return own + symbol.constraints
2062
+
2063
+
2064
+ def _unwrap_constraints(constraints: Iterable[Expr]) -> list[Expr]:
2065
+ """`CONSTRAINT name ...` is the constraints it names."""
2066
+ return [
2067
+ inner for constraint in constraints
2068
+ for inner in (constraint.expressions if isinstance(constraint, exp.Constraint) else (constraint,))
2069
+ ]
2070
+
2071
+
2072
+ def _is_added_constraint(constraint: Expr) -> bool:
2073
+ return isinstance(constraint, exp.PrimaryKey | exp.ForeignKey) or _table_level_unique_columns(constraint) is not None
2074
+
2075
+
2076
+ def _references(symbol: _Symbol) -> list[tuple[tuple[str, ...], exp.Table, tuple[str, ...] | None, Expr]]:
2077
+ """Every `REFERENCES` constraint of a table, column-level and table-level,
2078
+ as `(columns, referenced table, referenced columns, node)`. Referenced
2079
+ columns are None when omitted, meaning the referenced primary key."""
2080
+ found = []
2081
+ assert symbol.statement is not None
2082
+ for definition in symbol.statement.this.find_all(exp.ColumnDef):
2083
+ for constraint in definition.constraints:
2084
+ reference = constraint.args.get("kind")
2085
+ if isinstance(reference, exp.Reference):
2086
+ found.append(((definition.name,), *_reference_target(reference), reference))
2087
+ for foreign_key in _table_constraints(symbol):
2088
+ if not isinstance(foreign_key, exp.ForeignKey):
2089
+ continue
2090
+ reference = foreign_key.args.get("reference")
2091
+ if isinstance(reference, exp.Reference):
2092
+ columns = tuple(identifier.name for identifier in foreign_key.expressions)
2093
+ found.append((columns, *_reference_target(reference), foreign_key))
2094
+ return found
2095
+
2096
+
2097
+ def _reference_target(reference: exp.Reference) -> tuple[exp.Table, tuple[str, ...] | None]:
2098
+ target = reference.this
2099
+ if isinstance(target, exp.Schema):
2100
+ return target.this, tuple(identifier.name for identifier in target.expressions)
2101
+ return target, None
2102
+
2103
+
2104
+ def _primary_key(symbol: _Symbol) -> tuple[str, ...] | None:
2105
+ assert symbol.statement is not None
2106
+ for definition in symbol.statement.this.find_all(exp.ColumnDef):
2107
+ if any(isinstance(constraint.args.get("kind"), exp.PrimaryKeyColumnConstraint) for constraint in definition.constraints):
2108
+ return (definition.name,)
2109
+ for constraint in _table_constraints(symbol):
2110
+ if isinstance(constraint, exp.PrimaryKey):
2111
+ return tuple(identifier.name for identifier in constraint.expressions)
2112
+ return None
2113
+
2114
+
2115
+ def _missing(relation: ir.RelationExpr, names: Iterable[str]) -> list[str]:
2116
+ known = {column.name.lower() for column in relation.output_columns}
2117
+ return [name for name in names if name.lower() not in known]
2118
+
2119
+
2120
+ _SUPPORTED_CREATE_KINDS = {"TABLE", "VIEW"}
2121
+
2122
+ _TEXT_TYPES = {exp.DataType.Type.VARCHAR, exp.DataType.Type.TEXT}
2123
+
2124
+
2125
+ def _collated(definition: exp.ColumnDef) -> bool:
2126
+ """Whether a column is declared with a `COLLATE`."""
2127
+ return any(isinstance(constraint.args.get("kind"), exp.CollateColumnConstraint) for constraint in definition.constraints)
2128
+
2129
+
2130
+ def _created_table(statement: exp.Create) -> exp.Table | None:
2131
+ target = statement.this
2132
+ if isinstance(target, exp.Schema):
2133
+ target = target.this
2134
+ return target if isinstance(target, exp.Table) else None
2135
+
2136
+
2137
+ def _qualified_name(table: exp.Table) -> str:
2138
+ return ".".join(part for part in (table.catalog, table.db, table.name) if part)
2139
+
2140
+
2141
+ def _unwrap_query(body: Expr | None) -> exp.Query | None:
2142
+ while isinstance(body, exp.Subquery):
2143
+ body = body.this
2144
+ return body if isinstance(body, exp.Query) else None
2145
+
2146
+
2147
+ def _arg(node: Expr, name: str) -> Expr | None:
2148
+ return node.args.get(f"{name}_") or node.args.get(name)
2149
+
2150
+
2151
+ def _from_source(select: exp.Query) -> Expr | None:
2152
+ source = _arg(select, "from")
2153
+ return source.this if isinstance(source, exp.From) else None
2154
+
2155
+
2156
+ # A scalar subquery's value binding: no SQL name can match it.
2157
+ _SCALAR_QUALIFIER = "\0scalar"
2158
+
2159
+
2160
+ def _scalar_subqueries(expression: Expr) -> list[exp.Subquery]:
2161
+ """The scalar subqueries `expression` evaluates itself: not one under
2162
+ EXISTS, IN, ANY or ALL, nor inside another subquery."""
2163
+ found = []
2164
+ for node in expression.walk(prune=lambda node: isinstance(node, (exp.Subquery, exp.Query, exp.Exists))):
2165
+ if isinstance(node, exp.Subquery) and not isinstance(node.parent, (exp.In, exp.Exists, exp.Any, exp.All)):
2166
+ found.append(node)
2167
+ return found
2168
+
2169
+
2170
+ def _in_scope(column: ir.OutputColumn, scope: ir.RelationExpr) -> ir.OutputColumn | None:
2171
+ """`column` as a column of `scope`: itself, or the input column it passes on."""
2172
+ if column in scope.output_columns:
2173
+ return column
2174
+ match column.scalar_expr:
2175
+ case ir.ColumnRef(column=upstream) if upstream in scope.output_columns:
2176
+ return upstream
2177
+ case _:
2178
+ return None
2179
+
2180
+
2181
+ def _carrying(column: ir.OutputColumn, scope: ir.RelationExpr) -> ir.OutputColumn | None:
2182
+ """The one column of `scope` that passes `column` on, through any stages."""
2183
+ found = []
2184
+ for candidate in scope.output_columns:
2185
+ current = candidate
2186
+ while current is not column and isinstance(current.scalar_expr, ir.ColumnRef):
2187
+ current = current.scalar_expr.column
2188
+ if current is column:
2189
+ found.append(candidate)
2190
+ return found[0] if len(found) == 1 else None
2191
+
2192
+
2193
+ def _schema_path(node: exp.Table | exp.Column) -> str:
2194
+ """The catalog and schema a table or column is written with, as `s1` or `c.s1`."""
2195
+ return ".".join(part for part in (node.catalog, node.db) if part).lower()
2196
+
2197
+
2198
+ def _star_only(select: exp.Select) -> bool:
2199
+ return len(select.expressions) == 1 and isinstance(select.expressions[0], exp.Star)
2200
+
2201
+
2202
+ def _selected_output_names(select: exp.Select) -> tuple[str, ...]:
2203
+ return tuple(item.alias_or_name for item in select.expressions if not isinstance(item, exp.Star))
2204
+
2205
+
2206
+ def _matching_output(group_expression: Expr, items: list[Expr]) -> Expr | None:
2207
+ return next(
2208
+ (item for item in items if (item.this if isinstance(item, exp.Alias) else item) == group_expression),
2209
+ None,
2210
+ )
2211
+
2212
+
2213
+ def _matching_grouping_output(group_expression: Expr, items: list[Expr]) -> Expr | None:
2214
+ if isinstance(group_expression, exp.Literal) and group_expression.is_int:
2215
+ position = int(group_expression.this)
2216
+ return items[position - 1] if 1 <= position <= len(items) else None
2217
+ return _matching_output(group_expression, items)
2218
+
2219
+
2220
+ def _is_bare_aggregate_expression(expression: Expr) -> bool:
2221
+ return any(isinstance(node, exp.Column) for node in expression.walk()) and not any(
2222
+ isinstance(node, exp.AggFunc) for node in expression.walk()
2223
+ )
2224
+
2225
+
2226
+ def _row_number_equals_one(predicate: Expr) -> exp.Window | None:
2227
+ if not isinstance(predicate, exp.EQ):
2228
+ return None
2229
+ left, right = predicate.this, predicate.expression
2230
+ window = left if isinstance(left, exp.Window) else right if isinstance(right, exp.Window) else None
2231
+ literal = right if window is left else left
2232
+ if window is None or not isinstance(window.this, exp.RowNumber):
2233
+ return None
2234
+ if not isinstance(literal, exp.Literal) or literal.is_string or literal.this != "1":
2235
+ return None
2236
+ return window
2237
+
2238
+
2239
+ def _last_select(query: Expr) -> exp.Select | None:
2240
+ """The Select a set operation's left input ends with."""
2241
+ while True:
2242
+ unwrapped = _unwrap_query(query) if isinstance(query, exp.Query) else None
2243
+ match unwrapped:
2244
+ case exp.SetOperation():
2245
+ query = unwrapped.expression
2246
+ case exp.Select():
2247
+ return unwrapped
2248
+ case _:
2249
+ return None
2250
+
2251
+
2252
+ _SET_OPERATORS: dict[type[Expr], ir.SetOperator] = {exp.Union: "union", exp.Intersect: "intersect", exp.Except: "except"}
2253
+
2254
+
2255
+ # Types whose equal values are identical: no collation (text has a comparison
2256
+ # type only when compared by identity), no signed zero or NaN, no two
2257
+ # spellings of one interval.
2258
+ _IDENTITY_TYPES = {
2259
+ *exp.DataType.INTEGER_TYPES, *exp.DataType.TEXT_TYPES,
2260
+ exp.DataType.Type.DECIMAL, exp.DataType.Type.BOOLEAN, exp.DataType.Type.DATE, exp.DataType.Type.TIME,
2261
+ exp.DataType.Type.TIMESTAMP, exp.DataType.Type.UUID,
2262
+ }
2263
+ # TIMESTAMPTZ is too, where it holds an instant only (`zoned_timestamp_is_instant`).
2264
+
2265
+ # Types whose values keep the zone they were given, where a TIMESTAMP WITH
2266
+ # TIME ZONE is not an instant only: a CAST to another type reads that zone.
2267
+ _ZONED_TYPES = {exp.DataType.Type.TIMESTAMPTZ, exp.DataType.Type.TIMETZ}
2268
+
2269
+ # Types whose `null` (JSON's, a VARIANT's) is a non-null value that a CAST turns into NULL.
2270
+ _JSON_TYPES = {exp.DataType.Type.JSON, exp.DataType.Type.JSONB, exp.DataType.Type.VARIANT, exp.DataType.Type.OBJECT}
2271
+
2272
+ # Composite types: a comparison of their values may be NULL for a NULL element.
2273
+ _COMPOSITE_TYPES = {exp.DataType.Type.ARRAY, exp.DataType.Type.LIST, exp.DataType.Type.STRUCT, exp.DataType.Type.MAP, exp.DataType.Type.NESTED}
2274
+
2275
+ # The Builtins that compare two values, which compare composites element by element.
2276
+ _ORDER_COMPARISONS = {"<>", ">", ">=", "<", "<=", "BETWEEN"}
2277
+
2278
+ # Comparisons, whose operands a zoned timestamp's text may be.
2279
+ _COMPARISONS = (exp.EQ, exp.NEQ, exp.GT, exp.GTE, exp.LT, exp.LTE, exp.NullSafeEQ, exp.NullSafeNEQ, exp.In, exp.Like, exp.ILike, exp.Between)
2280
+
2281
+ _EXACT_NUMERIC_TYPES = {*exp.DataType.INTEGER_TYPES, exp.DataType.Type.DECIMAL}
2282
+
2283
+ # Nodes whose value is not a function of the columns an expression reads.
2284
+ _NOT_A_FUNCTION_OF_ITS_COLUMNS = (
2285
+ exp.Anonymous, exp.Rand, exp.Randn, exp.Uuid, exp.AggFunc, exp.Window, exp.Query, exp.Subquery,
2286
+ exp.CurrentDate, exp.CurrentDatetime, exp.CurrentTime, exp.CurrentTimestamp, exp.Placeholder, exp.Parameter, exp.Star,
2287
+ )
2288
+
2289
+
2290
+ def _enum_type_values(definition: exp.ColumnDef, meaning: DialectSemantics) -> frozenset[ir.LiteralValue] | None:
2291
+ """The values of a column declared `ENUM('a', ...)`: its members, with
2292
+ their trailing spaces removed where the dialect does (MySQL)."""
2293
+ kind = definition.args.get("kind")
2294
+ if not (isinstance(kind, exp.DataType) and kind.this == exp.DataType.Type.ENUM and kind.expressions):
2295
+ return None
2296
+ values = [literal_value(member) for member in kind.expressions]
2297
+ if any(value is None or value.type != "string" for value in values):
2298
+ return None
2299
+ members = {value for value in values if value is not None}
2300
+ if meaning.enum_strips_trailing_spaces:
2301
+ members = {replace(value, text=value.text.rstrip(" ")) for value in members}
2302
+ return frozenset(members)
2303
+
2304
+
2305
+ def _engine(symbol: _Symbol) -> str | None:
2306
+ """The storage ENGINE a CREATE TABLE names (lower-cased), if any."""
2307
+ assert symbol.statement is not None
2308
+ engine = symbol.statement.find(exp.EngineProperty)
2309
+ return engine.name.lower() if engine is not None else None
2310
+
2311
+
2312
+ def _aggregate_arguments(call: exp.AggFunc) -> list[Expr]:
2313
+ """The value arguments of an aggregate call: none for `count(*)`, the
2314
+ listed ones under DISTINCT."""
2315
+ this = call.this
2316
+ if this is None or isinstance(this, exp.Star):
2317
+ return []
2318
+ if isinstance(this, exp.Distinct):
2319
+ return list(this.expressions)
2320
+ return [this, *call.expressions]
2321
+
2322
+
2323
+ def _is_positional(join: exp.Join) -> bool:
2324
+ return (join.args.get("method") or "").lower() == "positional"
2325
+
2326
+
2327
+ def _join_kind(join: exp.Join, dialect: str) -> ir.JoinKind | None:
2328
+ """SEMI, ANTI and CROSS whatever the side; otherwise the side, else INNER
2329
+ (`STRAIGHT_JOIN`, and ASOF, which keeps a subset of the inner join's rows).
2330
+ None for a join whose semantics are not modelled: it lowers opaque."""
2331
+ if _is_positional(join):
2332
+ return "full"
2333
+ kind, side = (join.args.get("kind") or "").lower(), (join.args.get("side") or "").lower()
2334
+ if dialect == "clickhouse" and (side or kind not in {"", "inner", "cross"}):
2335
+ # TODO: check ClickHouse semantics. Outer joins pad with the column
2336
+ # type's default, not NULL, unless join_use_nulls is set; SEMI and ANTI
2337
+ # joins keep the right columns; ANY and ASOF pick one match.
2338
+ return None
2339
+ match kind, side:
2340
+ case ("semi" | "anti" | "cross") as kind, _:
2341
+ return kind
2342
+ case "" | "inner" | "outer" | "straight_join", ("left" | "right" | "full") as side:
2343
+ return side
2344
+ case "outer", "":
2345
+ return "full"
2346
+ case "" | "inner" | "straight_join", "":
2347
+ return "inner"
2348
+ case _:
2349
+ # TODO: check the semantics of each dialect's other join kinds
2350
+ # (ClickHouse ARRAY JOIN, ANY, ...) before modelling one.
2351
+ return None
2352
+
2353
+
2354
+ def _join_label(join: exp.Join, dialect: str) -> str:
2355
+ keywords = " ".join(part for part in (join.side, join.kind, "JOIN") if part)
2356
+ source = join.this
2357
+ if isinstance(source, exp.Subquery):
2358
+ return f"{keywords} subquery {source.alias}".rstrip()
2359
+ return f"{keywords} {source.sql(dialect=dialect)}"
2360
+
2361
+
2362
+ def _select_label(select: exp.Select) -> str:
2363
+ node = select.parent
2364
+ while node is not None:
2365
+ match node:
2366
+ case exp.CTE():
2367
+ return f"CTE {node.alias}"
2368
+ case exp.Create():
2369
+ table = node.this.this if isinstance(node.this, exp.Schema) else node.this
2370
+ return f"view {table.name}"
2371
+ case exp.Subquery() if node.alias:
2372
+ return f"subquery {node.alias}"
2373
+ case exp.Subquery():
2374
+ return "subquery"
2375
+ case _:
2376
+ node = node.parent
2377
+ return "Root Select"