sqlassert 0.1.2__py3-none-any.whl → 0.3.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sqlassert/__init__.py +34 -9
- sqlassert/__main__.py +138 -0
- sqlassert/check.py +444 -0
- sqlassert/diagnostics.py +57 -0
- sqlassert/dialects.py +211 -0
- sqlassert/discovery/__init__.py +30 -0
- sqlassert/discovery/bigquery.py +71 -0
- sqlassert/discovery/databricks.py +18 -0
- sqlassert/discovery/duckdb.py +163 -0
- sqlassert/discovery/fetcher.py +402 -0
- sqlassert/discovery/information_schema.py +326 -0
- sqlassert/discovery/mysql.py +70 -0
- sqlassert/discovery/postgres.py +143 -0
- sqlassert/discovery/redshift.py +36 -0
- sqlassert/discovery/snowflake.py +63 -0
- sqlassert/discovery/trino.py +37 -0
- sqlassert/improvements.py +528 -0
- sqlassert/ir/__init__.py +56 -0
- sqlassert/ir/convert.py +2377 -0
- sqlassert/ir/model.py +676 -0
- sqlassert/main.py +57 -0
- sqlassert/properties.py +121 -0
- sqlassert/provenance.py +73 -0
- sqlassert/python_engine.py +299 -0
- sqlassert/reporting.py +102 -0
- sqlassert/rules/STYLEGUIDE.md +19 -0
- sqlassert/rules/__init__.py +11 -0
- sqlassert/rules/column_set.py +34 -0
- sqlassert/rules/domain.py +376 -0
- sqlassert/rules/equality.py +233 -0
- sqlassert/rules/exactly_one.py +75 -0
- sqlassert/rules/foreign_key.py +316 -0
- sqlassert/rules/join_cardinality.py +94 -0
- sqlassert/rules/nullability.py +418 -0
- sqlassert/rules/uniqueness.py +601 -0
- sqlassert/smells.py +306 -0
- sqlassert/sql_parse.py +974 -0
- sqlassert/suggest.py +339 -0
- sqlassert-0.3.1.dist-info/METADATA +725 -0
- sqlassert-0.3.1.dist-info/RECORD +44 -0
- {sqlassert-0.1.2.dist-info → sqlassert-0.3.1.dist-info}/WHEEL +1 -1
- sqlassert-0.3.1.dist-info/entry_points.txt +2 -0
- sqlassert-0.3.1.dist-info/licenses/LICENSE +21 -0
- sqlassert/unique.py +0 -605
- sqlassert-0.1.2.dist-info/METADATA +0 -173
- sqlassert-0.1.2.dist-info/RECORD +0 -6
- {sqlassert-0.1.2.dist-info → sqlassert-0.3.1.dist-info}/top_level.txt +0 -0
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
"""Trino: the standard discovery, each catalog's own `information_schema`, over the `trino` client.
|
|
2
|
+
|
|
3
|
+
Trino federates catalogs, each with its own `catalog.information_schema`;
|
|
4
|
+
a name may be qualified by any catalog. Its `information_schema` has
|
|
5
|
+
`TABLES`, `COLUMNS` and `VIEWS` but no constraint views: Trino has no key
|
|
6
|
+
constraints, and does not surface those a connector's source enforces, so a
|
|
7
|
+
Trino catalog reports NOT NULLs and no keys. That loses proofs, never
|
|
8
|
+
soundness. Parameters are `?` (the client's qmark style).
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
from collections.abc import Sequence
|
|
14
|
+
|
|
15
|
+
from sqlassert.discovery.fetcher import DatabaseForeignKey, DatabaseMetadataError, DatabaseRelationMetadata
|
|
16
|
+
from sqlassert.discovery.information_schema import InformationSchemaKnowledgeFetcher, Key, quoted
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class TrinoKnowledgeFetcher(InformationSchemaKnowledgeFetcher):
|
|
20
|
+
type_dialect = "trino"
|
|
21
|
+
paramstyle = "qmark"
|
|
22
|
+
cross_catalog = True
|
|
23
|
+
|
|
24
|
+
def session(self, scope: DatabaseRelationMetadata | None) -> tuple[str, tuple[str, ...]]:
|
|
25
|
+
catalog, path = super().session(scope)
|
|
26
|
+
if catalog is None: # The client may connect without a catalog.
|
|
27
|
+
raise DatabaseMetadataError("the Trino session has no current catalog: connect with catalog=")
|
|
28
|
+
return catalog, path
|
|
29
|
+
|
|
30
|
+
def information_schema(self, catalog: str, schema: str) -> str:
|
|
31
|
+
return f"{quoted(catalog)}.information_schema"
|
|
32
|
+
|
|
33
|
+
def keys(self, catalog: str, schema: str, tables: Sequence[str]) -> list[Key]:
|
|
34
|
+
return []
|
|
35
|
+
|
|
36
|
+
def foreign_keys(self, catalog: str, schema: str, tables: Sequence[str]) -> list[tuple[str, DatabaseForeignKey]]:
|
|
37
|
+
return []
|
|
@@ -0,0 +1,528 @@
|
|
|
1
|
+
"""High-level suggestions: native SQL improvements justified by a proof.
|
|
2
|
+
|
|
3
|
+
Each Improvement names a rewrite that leaves the program's result unchanged,
|
|
4
|
+
as a multiset, and the evidence that it does:
|
|
5
|
+
|
|
6
|
+
- An outer join that never keeps a padded row is an inner join: it matches
|
|
7
|
+
exactly one row, or a null-rejecting conjunct later in the same FROM and
|
|
8
|
+
WHERE removes every row it pads (padding promotion), directly or through a
|
|
9
|
+
later LEFT or FULL JOIN that then pads the same rows.
|
|
10
|
+
- A join none of whose columns is used outside its own ON is redundant when
|
|
11
|
+
it keeps every left row once (a unique LEFT JOIN, an Exactly One JOIN), and
|
|
12
|
+
only filters when it is a unique inner join (a `WHERE EXISTS`).
|
|
13
|
+
- A `DISTINCT` over rows that already differ, NULL equal to NULL, on the
|
|
14
|
+
selected columns; a `GROUP BY` column the remaining grouping columns
|
|
15
|
+
strictly determine.
|
|
16
|
+
- A CTE nothing references.
|
|
17
|
+
- A `WHERE EXISTS` or `IN` subquery (or a SEMI JOIN) every row passes: its
|
|
18
|
+
semi join is proved Exactly One, so it keeps every row once.
|
|
19
|
+
- A `GROUP BY` whose grouping columns are a key of its input, NULL equal to
|
|
20
|
+
NULL: each group is one row, so it is a projection.
|
|
21
|
+
- An `EXCEPT` whose left input, or an `INTERSECT` either of whose inputs,
|
|
22
|
+
has no two rows alike: `EXCEPT ALL` and `INTERSECT ALL` give the same rows
|
|
23
|
+
without removing duplicates, where the dialect has them.
|
|
24
|
+
|
|
25
|
+
Proofs come from the rules' internals (null sources, strict closure) as well
|
|
26
|
+
as `Arena.prove`, since none of these facts is a Property. Whether a column
|
|
27
|
+
is used is read from the syntax: the IR keeps neither ORDER BY nor the
|
|
28
|
+
columns inside opaque expressions. Anything the checks cannot see whole --
|
|
29
|
+
a padded row leaving its own Select Expression, a join whose right side's
|
|
30
|
+
columns are unknown -- is left alone.
|
|
31
|
+
"""
|
|
32
|
+
|
|
33
|
+
from __future__ import annotations
|
|
34
|
+
|
|
35
|
+
from collections.abc import Iterator
|
|
36
|
+
from dataclasses import dataclass
|
|
37
|
+
from enum import Enum
|
|
38
|
+
from typing import ClassVar
|
|
39
|
+
|
|
40
|
+
from sqlglot import Expr, exp
|
|
41
|
+
|
|
42
|
+
from sqlassert import ir
|
|
43
|
+
from sqlassert.dialects import semantics
|
|
44
|
+
from sqlassert.properties import ExactlyOneJoin, ManyToOneJoin
|
|
45
|
+
from sqlassert.provenance import AttachmentSite
|
|
46
|
+
from sqlassert.python_engine import Arena, Result
|
|
47
|
+
from sqlassert.rules.column_set import RowIdentity
|
|
48
|
+
from sqlassert.rules.nullability import Padding, _null_rejected, _null_sources, _on_rejected
|
|
49
|
+
from sqlassert.rules.uniqueness import _closure, _reaches_row_identity
|
|
50
|
+
from sqlassert.sql_parse import ParsedProgram, grouping, marker_position
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
class Level(Enum):
|
|
54
|
+
"""How directly a suggestion improves the SQL; HIGH ranks first."""
|
|
55
|
+
|
|
56
|
+
HIGH = "high"
|
|
57
|
+
MID = "mid"
|
|
58
|
+
LOW = "low"
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
class Rewrite(Enum):
|
|
62
|
+
INNER_JOIN = "INNER JOIN"
|
|
63
|
+
REDUNDANT_JOIN = "Redundant Join"
|
|
64
|
+
FILTER_JOIN = "Filter-Only Join"
|
|
65
|
+
REDUNDANT_DISTINCT = "Redundant DISTINCT"
|
|
66
|
+
REDUNDANT_GROUPING = "Redundant GROUP BY"
|
|
67
|
+
GROUPING_PROJECTION = "GROUP BY of one row per group"
|
|
68
|
+
REDUNDANT_SORT_KEY = "Redundant ORDER BY key"
|
|
69
|
+
REDUNDANT_FILTER = "Redundant EXISTS"
|
|
70
|
+
SET_OPERATION_ALL = "EXCEPT ALL or INTERSECT ALL"
|
|
71
|
+
UNUSED_CTE = "Unused CTE"
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
@dataclass(frozen=True)
|
|
75
|
+
class Improvement:
|
|
76
|
+
"""A result-preserving rewrite of the SQL at one site, with its proof."""
|
|
77
|
+
|
|
78
|
+
site: AttachmentSite
|
|
79
|
+
rewrite: Rewrite
|
|
80
|
+
message: str
|
|
81
|
+
evidence: Result[bool]
|
|
82
|
+
level: ClassVar[Level] = Level.HIGH
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def improvements(ast: ParsedProgram, program: ir.Program, arena: Arena, dialect: str) -> list[Improvement]:
|
|
86
|
+
"""Every Improvement for `program`, in source order. `arena` must be active."""
|
|
87
|
+
syntax = _syntax_at_sites(ast)
|
|
88
|
+
parents: dict[ir.RelationExpr, list[ir.RelationExpr]] = {}
|
|
89
|
+
for relation in program.relations():
|
|
90
|
+
for child in ir.children(relation):
|
|
91
|
+
if isinstance(child, ir.RelationExpr):
|
|
92
|
+
parents.setdefault(child, []).append(relation)
|
|
93
|
+
found: list[Improvement] = []
|
|
94
|
+
sort_keys = dict(program.sort_keys)
|
|
95
|
+
for site, relation in program.attachment_sites:
|
|
96
|
+
if isinstance(relation, ir.SimpleCase):
|
|
97
|
+
continue # A CASE has nothing to rewrite yet.
|
|
98
|
+
if relation not in parents and relation is not program.root:
|
|
99
|
+
# Unreachable: lowered inside a Select that is opaque as a whole.
|
|
100
|
+
# (Every reachable relation but the root is some relation's input.)
|
|
101
|
+
continue
|
|
102
|
+
node = syntax.get((site.offset, site.trailing))
|
|
103
|
+
if isinstance(relation, ir.Limit):
|
|
104
|
+
relation = relation.input
|
|
105
|
+
if site.trailing:
|
|
106
|
+
found.extend(_redundant_subquery_filters(arena, site, relation))
|
|
107
|
+
if site.trailing and isinstance(node, exp.Select):
|
|
108
|
+
found.extend(_redundant_sort_keys(site, relation, node, sort_keys.get(site), dialect))
|
|
109
|
+
match relation:
|
|
110
|
+
case ir.Join() if isinstance(node, exp.Join):
|
|
111
|
+
removed = _redundant_join(arena, site, relation, node, dialect)
|
|
112
|
+
inner = None if removed else _inner_join(arena, site, relation, parents)
|
|
113
|
+
found.extend(item for item in (removed, inner) if item is not None)
|
|
114
|
+
case ir.Distinct():
|
|
115
|
+
found.extend(_redundant_distinct(site, relation))
|
|
116
|
+
case ir.Aggregate() if isinstance(node, exp.Select):
|
|
117
|
+
projection = _grouping_is_projection(site, relation, node, dialect)
|
|
118
|
+
found.extend([projection] if projection else _redundant_grouping(site, relation, node, dialect))
|
|
119
|
+
case _:
|
|
120
|
+
pass # Other Select Expressions have nothing to rewrite yet.
|
|
121
|
+
found.extend(_unused_ctes(ast))
|
|
122
|
+
if semantics(dialect).set_operations_all:
|
|
123
|
+
found.extend(item for site, operation in program.set_operations if (item := _set_operation_all(site, operation)) is not None)
|
|
124
|
+
return sorted(found, key=lambda item: item.site.offset)
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def _redundant_subquery_filters(arena: Arena, site: AttachmentSite, relation: ir.RelationExpr) -> list[Improvement]:
|
|
128
|
+
"""The semi joins of this Select (its WHERE subqueries, issue 23, or an
|
|
129
|
+
explicit SEMI JOIN) that match every row exactly once, so filter nothing."""
|
|
130
|
+
found: list[Improvement] = []
|
|
131
|
+
while True:
|
|
132
|
+
match relation:
|
|
133
|
+
case ir.Project(input=source) | ir.Aggregate(input=source) | ir.Distinct(input=source) | ir.QualifyByPartition(input=source) | ir.Filter(input=source):
|
|
134
|
+
relation = source
|
|
135
|
+
case ir.Join(kind="semi" | "anti", left=source):
|
|
136
|
+
if relation.kind == "semi":
|
|
137
|
+
proof = arena.prove(ExactlyOneJoin(join=relation))
|
|
138
|
+
if proof.value:
|
|
139
|
+
message = f"every row matches exactly one row, so {relation.origin.detail} filters nothing: delete it"
|
|
140
|
+
found.append(Improvement(site, Rewrite.REDUNDANT_FILTER, message, proof))
|
|
141
|
+
relation = source
|
|
142
|
+
case _:
|
|
143
|
+
return found
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def _syntax_at_sites(ast: ParsedProgram) -> dict[tuple[int, bool], exp.Join | exp.Select]:
|
|
147
|
+
"""Each Join and Select by its Attachment Site's (offset, trailing)."""
|
|
148
|
+
index: dict[tuple[int, bool], exp.Join | exp.Select] = {}
|
|
149
|
+
for node in _nodes(ast, exp.Join, exp.Select):
|
|
150
|
+
position = marker_position(node)
|
|
151
|
+
if position is not None:
|
|
152
|
+
index[(position[0], isinstance(node, exp.Select))] = node
|
|
153
|
+
return index
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def _nodes(ast: ParsedProgram, *types: type[Expr]) -> Iterator:
|
|
157
|
+
for statement in (*ast.create_statements, ast.root_select):
|
|
158
|
+
if statement is not None:
|
|
159
|
+
yield from statement.find_all(*types)
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
# Outer joins that cannot pad ------------------------------------------------------
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def _inner_join(
|
|
166
|
+
arena: Arena, site: AttachmentSite, join: ir.Join, parents: dict[ir.RelationExpr, list[ir.RelationExpr]]
|
|
167
|
+
) -> Improvement | None:
|
|
168
|
+
"""An outer join none of whose padded rows reaches the result.
|
|
169
|
+
|
|
170
|
+
Only the joins above it in the same FROM and its own WHERE are searched:
|
|
171
|
+
within one Select Expression nothing between them looks at a row as a
|
|
172
|
+
whole, while past it a LIMIT, window, or aggregate may.
|
|
173
|
+
"""
|
|
174
|
+
if join.kind not in {"left", "right"}:
|
|
175
|
+
return None
|
|
176
|
+
keyword = f"{join.kind.upper()} JOIN"
|
|
177
|
+
if join.kind == "left":
|
|
178
|
+
exactly_one = arena.prove(ExactlyOneJoin(join=join))
|
|
179
|
+
if exactly_one.value:
|
|
180
|
+
message = f"every left row matches exactly one right row, so the {keyword} pads nothing: write JOIN"
|
|
181
|
+
return Improvement(site, Rewrite.INNER_JOIN, message, exactly_one)
|
|
182
|
+
# Joins that pad every row this one pads, each with why (issue 33): a
|
|
183
|
+
# later LEFT or FULL JOIN whose ON rejects NULL in a column one of them
|
|
184
|
+
# pads finds no match for those rows, so it pads them too.
|
|
185
|
+
# Keyed by join and the side it pads: a FULL JOIN pads each side on different rows.
|
|
186
|
+
chained: dict[tuple[ir.Join, bool], list[Result[bool]]] = {(join, join.kind == "right"): []}
|
|
187
|
+
|
|
188
|
+
def padded_by(column: ir.OutputColumn) -> tuple[ir.Join, bool] | None:
|
|
189
|
+
return next((
|
|
190
|
+
(source.join, source.left) for source in _null_sources(column).value
|
|
191
|
+
if isinstance(source, Padding) and (source.join, source.left) in chained
|
|
192
|
+
), None)
|
|
193
|
+
|
|
194
|
+
relation: ir.RelationExpr = join
|
|
195
|
+
while len(parents.get(relation, ())) == 1:
|
|
196
|
+
parent, = parents[relation]
|
|
197
|
+
match parent:
|
|
198
|
+
case ir.Join(kind="anti"):
|
|
199
|
+
rejected, clause = {}, ""
|
|
200
|
+
case ir.Join(kind=kind):
|
|
201
|
+
# A semi join (a WHERE EXISTS or IN) keeps only rows its ON holds on.
|
|
202
|
+
described = "WHERE EXISTS or IN subquery" if kind == "semi" else f"later {kind.upper()} JOIN"
|
|
203
|
+
rejected, clause = _on_rejected(parent), f"the ON condition of a {described}"
|
|
204
|
+
case ir.Filter():
|
|
205
|
+
rejected, clause = _null_rejected(parent.conjuncts), "the WHERE clause"
|
|
206
|
+
case _:
|
|
207
|
+
return None
|
|
208
|
+
for column, conjunct in rejected.items():
|
|
209
|
+
via = padded_by(column)
|
|
210
|
+
if via is not None:
|
|
211
|
+
how = f"is padded by the {keyword}" if via[0] is join else f"is NULL on every row the {keyword} pads"
|
|
212
|
+
padded = Result(True, f"{column.name} {how}", (join, column), chained[via])
|
|
213
|
+
evidence = Result(
|
|
214
|
+
True, f"{clause} rejects NULL in {column.name}, so every row the {keyword} pads is removed",
|
|
215
|
+
(conjunct, column), [padded],
|
|
216
|
+
)
|
|
217
|
+
message = f"{clause} rejects NULL in {column.name}, so no padded row survives: write JOIN"
|
|
218
|
+
return Improvement(site, Rewrite.INNER_JOIN, message, evidence)
|
|
219
|
+
if isinstance(parent, ir.Join) and parent.kind in {"left", "full"}:
|
|
220
|
+
for column, conjunct in _null_rejected(parent.conjuncts).items():
|
|
221
|
+
via = padded_by(column)
|
|
222
|
+
# The left rows it cannot match keep a padded right side.
|
|
223
|
+
if via is not None and column in parent.left.output_columns and (parent, False) not in chained:
|
|
224
|
+
reason = f"its ON rejects NULL in {column.name}, so the later {parent.kind.upper()} JOIN pads every row the {keyword} pads"
|
|
225
|
+
chained[(parent, False)] = [*chained[via], Result(True, reason, (conjunct, column))]
|
|
226
|
+
if isinstance(parent, ir.Filter):
|
|
227
|
+
return None
|
|
228
|
+
relation = parent
|
|
229
|
+
return None
|
|
230
|
+
|
|
231
|
+
|
|
232
|
+
# Redundant ORDER BY keys -----------------------------------------------------------
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
def _redundant_sort_keys(
|
|
236
|
+
site: AttachmentSite, relation: ir.RelationExpr, select: exp.Select, keys: tuple[ir.OutputColumn, ...] | None, dialect: str
|
|
237
|
+
) -> list[Improvement]:
|
|
238
|
+
"""Each ORDER BY key the keys before it determine, NULL equal to NULL
|
|
239
|
+
(Paulley §3, reducing the sort list): rows tied on the earlier keys agree
|
|
240
|
+
on it, so it never breaks a tie. Only while the earlier keys compare by
|
|
241
|
+
identity: a collation or a float ties values its equality tells apart.
|
|
242
|
+
Silent when a key is an expression or a name that does not resolve to one
|
|
243
|
+
column, and for DISTINCT ON, whose ORDER BY picks the rows it keeps."""
|
|
244
|
+
from sqlassert.smells import _ties_identical
|
|
245
|
+
|
|
246
|
+
if not keys:
|
|
247
|
+
return []
|
|
248
|
+
items = select.args["order"].expressions
|
|
249
|
+
scope = relation.input if isinstance(relation, ir.Project) else relation
|
|
250
|
+
found = []
|
|
251
|
+
for position in range(1, len(keys)):
|
|
252
|
+
if not all(_ties_identical(key) for key in keys[:position]):
|
|
253
|
+
break
|
|
254
|
+
closure = _closure(scope, frozenset(keys[:position]), frozenset())
|
|
255
|
+
key = keys[position]
|
|
256
|
+
if key not in closure.value.columns:
|
|
257
|
+
continue
|
|
258
|
+
written = [(item.this if isinstance(item, exp.Ordered) else item).sql(dialect=dialect) for item in items]
|
|
259
|
+
message = f"({', '.join(written[:position])}) determine {written[position]}, NULL equal to NULL, so it never breaks a tie: drop it from ORDER BY"
|
|
260
|
+
evidence = Result(True, "The earlier ORDER BY keys determine this one", (key,), [closure.members[key]])
|
|
261
|
+
found.append(Improvement(site, Rewrite.REDUNDANT_SORT_KEY, message, evidence))
|
|
262
|
+
return found
|
|
263
|
+
|
|
264
|
+
|
|
265
|
+
# Redundant joins ------------------------------------------------------------------
|
|
266
|
+
|
|
267
|
+
|
|
268
|
+
def _redundant_join(arena: Arena, site: AttachmentSite, join: ir.Join, node: exp.Join, dialect: str) -> Improvement | None:
|
|
269
|
+
if join.kind not in {"inner", "left"} or _right_side_used(join, node):
|
|
270
|
+
return None
|
|
271
|
+
source = node.this.sql(dialect=dialect)
|
|
272
|
+
unused = f"no column of {source} is used outside its ON"
|
|
273
|
+
if join.kind == "left":
|
|
274
|
+
unique = arena.prove(ManyToOneJoin(join=join))
|
|
275
|
+
if unique.value:
|
|
276
|
+
message = f"at most one match per row and {unused}: delete the join"
|
|
277
|
+
return Improvement(site, Rewrite.REDUNDANT_JOIN, message, unique)
|
|
278
|
+
return None
|
|
279
|
+
exactly_one = arena.prove(ExactlyOneJoin(join=join))
|
|
280
|
+
if exactly_one.value:
|
|
281
|
+
message = f"exactly one match per row and {unused}: delete the join"
|
|
282
|
+
return Improvement(site, Rewrite.REDUNDANT_JOIN, message, exactly_one)
|
|
283
|
+
unique = arena.prove(ManyToOneJoin(join=join))
|
|
284
|
+
if not unique.value or _right_or_full_join_follows(node):
|
|
285
|
+
# A filter moved to WHERE would act after a later RIGHT or FULL JOIN,
|
|
286
|
+
# which then no longer pads the rows this join used to drop first.
|
|
287
|
+
return None
|
|
288
|
+
on = node.args.get("on")
|
|
289
|
+
condition = f" WHERE {on.sql(dialect=dialect)}" if on is not None else ""
|
|
290
|
+
message = (
|
|
291
|
+
f"at most one match per row and {unused}, so it only filters: "
|
|
292
|
+
f"write WHERE EXISTS (SELECT 1 FROM {source}{condition})"
|
|
293
|
+
)
|
|
294
|
+
return Improvement(site, Rewrite.FILTER_JOIN, message, unique)
|
|
295
|
+
|
|
296
|
+
|
|
297
|
+
def _right_side_used(join: ir.Join, node: exp.Join) -> bool:
|
|
298
|
+
"""Whether anything in the join's Select Expression but its own ON may
|
|
299
|
+
read a column of its right side. Unknown counts as used."""
|
|
300
|
+
select = node.parent
|
|
301
|
+
name = node.this.alias_or_name.lower()
|
|
302
|
+
if not isinstance(select, exp.Select) or not name or not join.right.is_schema_complete or node.args.get("using"):
|
|
303
|
+
return True
|
|
304
|
+
if node.args.get("method"):
|
|
305
|
+
return True # NATURAL: the condition is implicit.
|
|
306
|
+
columns = {column.name.lower() for column in join.right.output_columns}
|
|
307
|
+
if any(isinstance(item, exp.Star) for item in select.expressions):
|
|
308
|
+
return True
|
|
309
|
+
on = node.args.get("on")
|
|
310
|
+
for column in select.find_all(exp.Column):
|
|
311
|
+
if on is not None and _within(column, on):
|
|
312
|
+
continue
|
|
313
|
+
if column.table.lower() == name if column.table else column.name.lower() in columns:
|
|
314
|
+
return True
|
|
315
|
+
return any(
|
|
316
|
+
identifier.name.lower() in columns
|
|
317
|
+
for other in select.args.get("joins") or ()
|
|
318
|
+
for identifier in other.args.get("using") or ()
|
|
319
|
+
)
|
|
320
|
+
|
|
321
|
+
|
|
322
|
+
def _right_or_full_join_follows(node: exp.Join) -> bool:
|
|
323
|
+
joins = list(node.parent.args.get("joins") or ()) if node.parent is not None else []
|
|
324
|
+
later = joins[next(index for index, other in enumerate(joins) if other is node) + 1:]
|
|
325
|
+
return any((other.side or "").upper() in {"RIGHT", "FULL"} for other in later)
|
|
326
|
+
|
|
327
|
+
|
|
328
|
+
def _within(node: Expr, ancestor: Expr) -> bool:
|
|
329
|
+
current: Expr | None = node
|
|
330
|
+
while current is not None:
|
|
331
|
+
if current is ancestor:
|
|
332
|
+
return True
|
|
333
|
+
current = current.parent
|
|
334
|
+
return False
|
|
335
|
+
|
|
336
|
+
|
|
337
|
+
# DISTINCT and GROUP BY ------------------------------------------------------------
|
|
338
|
+
|
|
339
|
+
|
|
340
|
+
def _redundant_distinct(site: AttachmentSite, distinct: ir.Distinct) -> list[Improvement]:
|
|
341
|
+
"""DISTINCT over input rows no two of which agree on the selected
|
|
342
|
+
columns, NULL equal to NULL: a strict key of the input."""
|
|
343
|
+
selected = {
|
|
344
|
+
column.scalar_expr.column: column.name
|
|
345
|
+
for column in distinct.output_columns if isinstance(column.scalar_expr, ir.ColumnRef)
|
|
346
|
+
}
|
|
347
|
+
key = _strict_key(distinct.input, tuple(selected))
|
|
348
|
+
if key is None:
|
|
349
|
+
return []
|
|
350
|
+
evidence = _reaches_row_identity(_closure(distinct.input, frozenset(key), frozenset()))
|
|
351
|
+
names = ", ".join(selected[column] for column in key)
|
|
352
|
+
message = f"no two rows agree on ({names}), NULL equal to NULL, so DISTINCT removes nothing: delete DISTINCT"
|
|
353
|
+
return [Improvement(site, Rewrite.REDUNDANT_DISTINCT, message, evidence)]
|
|
354
|
+
|
|
355
|
+
|
|
356
|
+
def _strict_key(relation: ir.RelationExpr, columns: tuple[ir.OutputColumn, ...]) -> tuple[ir.OutputColumn, ...] | None:
|
|
357
|
+
"""A minimal subset of `columns` that strictly determines a row of `relation`, if any."""
|
|
358
|
+
|
|
359
|
+
def reaches(members: tuple[ir.OutputColumn, ...]) -> bool:
|
|
360
|
+
return RowIdentity(relation) in _closure(relation, frozenset(members), frozenset()).value.columns
|
|
361
|
+
|
|
362
|
+
if not columns or not reaches(columns):
|
|
363
|
+
return None
|
|
364
|
+
for column in columns:
|
|
365
|
+
rest = tuple(other for other in columns if other is not column)
|
|
366
|
+
if rest and reaches(rest):
|
|
367
|
+
return _strict_key(relation, rest)
|
|
368
|
+
return columns
|
|
369
|
+
|
|
370
|
+
|
|
371
|
+
def _grouping_is_projection(site: AttachmentSite, aggregate: ir.Aggregate, node: exp.Select, dialect: str) -> Improvement | None:
|
|
372
|
+
"""A GROUP BY whose grouping columns no two input rows agree on, NULL
|
|
373
|
+
equal to NULL (GROUP BY puts NULLs in one group): every group is one row,
|
|
374
|
+
so the query is a projection, each aggregate its value over that row.
|
|
375
|
+
An empty input gives no rows either way. Silent when a grouping
|
|
376
|
+
expression is not a column, or an aggregate has no one-row form here
|
|
377
|
+
(`_one_row`), or is read outside the select list (ORDER BY, QUALIFY, a
|
|
378
|
+
window over it)."""
|
|
379
|
+
expressions = grouping(node) or []
|
|
380
|
+
outputs = aggregate.grouping_outputs
|
|
381
|
+
if not outputs or len(expressions) != len(outputs) or not all(isinstance(column.scalar_expr, ir.ColumnRef) for column in outputs):
|
|
382
|
+
return None
|
|
383
|
+
if any(
|
|
384
|
+
call.find_ancestor(exp.Select) is node and not any(_within(call, item) for item in node.expressions)
|
|
385
|
+
for call in node.find_all(exp.AggFunc)
|
|
386
|
+
) or node.args.get("qualify") is not None or any(item.find(exp.Window) for item in node.expressions):
|
|
387
|
+
return None
|
|
388
|
+
keys = frozenset(column.scalar_expr.column for column in outputs if isinstance(column.scalar_expr, ir.ColumnRef))
|
|
389
|
+
evidence = _reaches_row_identity(_closure(aggregate.input, keys, frozenset()))
|
|
390
|
+
if not evidence.value:
|
|
391
|
+
return None
|
|
392
|
+
items = []
|
|
393
|
+
for item in node.expressions:
|
|
394
|
+
rewritten = _one_row(item.copy(), dialect)
|
|
395
|
+
if rewritten is None:
|
|
396
|
+
return None
|
|
397
|
+
items.append(rewritten)
|
|
398
|
+
names = ", ".join(expression.sql(dialect=dialect) for expression in expressions)
|
|
399
|
+
message = (
|
|
400
|
+
f"no two rows agree on ({names}), NULL equal to NULL, so every group is one row: "
|
|
401
|
+
f"drop GROUP BY and write SELECT {', '.join(items)}"
|
|
402
|
+
)
|
|
403
|
+
return Improvement(site, Rewrite.GROUPING_PROJECTION, message, evidence)
|
|
404
|
+
|
|
405
|
+
|
|
406
|
+
def _one_row(item: Expr, dialect: str) -> str | None:
|
|
407
|
+
"""`item` with each aggregate in it replaced by its value over one row,
|
|
408
|
+
or None when an aggregate has no such form here: count(*) is 1, count(x)
|
|
409
|
+
is 1 unless x is NULL, sum, avg, min, max, any_value, bool_and and bool_or
|
|
410
|
+
are x. With or without DISTINCT, IGNORE or RESPECT NULLS; never with
|
|
411
|
+
FILTER, nor a count of several values."""
|
|
412
|
+
for call in list(item.find_all(exp.AggFunc)):
|
|
413
|
+
target: Expr = call
|
|
414
|
+
while isinstance(target.parent, exp.IgnoreNulls | exp.RespectNulls):
|
|
415
|
+
target = target.parent
|
|
416
|
+
if isinstance(target.parent, exp.Filter) or any(value for key, value in call.args.items() if key not in {"this", "big_int"}):
|
|
417
|
+
return None
|
|
418
|
+
argument = call.this
|
|
419
|
+
if isinstance(argument, exp.Distinct):
|
|
420
|
+
if len(argument.expressions) != 1:
|
|
421
|
+
return None
|
|
422
|
+
argument = argument.expressions[0]
|
|
423
|
+
match call:
|
|
424
|
+
case exp.Count() if argument is None or isinstance(argument, exp.Star):
|
|
425
|
+
value: Expr = exp.Literal.number(1)
|
|
426
|
+
case exp.Count():
|
|
427
|
+
value = exp.Case(ifs=[exp.If(this=exp.Is(this=argument.copy(), expression=exp.null()), true=exp.Literal.number(0))], default=exp.Literal.number(1))
|
|
428
|
+
case exp.Sum() | exp.Avg() | exp.Min() | exp.Max() | exp.AnyValue() | exp.LogicalAnd() | exp.LogicalOr() if isinstance(argument, Expr) and not isinstance(argument, exp.Star | exp.Order):
|
|
429
|
+
value = argument.copy()
|
|
430
|
+
case _:
|
|
431
|
+
return None
|
|
432
|
+
if target is item:
|
|
433
|
+
item = value
|
|
434
|
+
else:
|
|
435
|
+
target.replace(value)
|
|
436
|
+
return item.sql(dialect=dialect)
|
|
437
|
+
|
|
438
|
+
|
|
439
|
+
def _redundant_grouping(site: AttachmentSite, aggregate: ir.Aggregate, node: exp.Select, dialect: str) -> list[Improvement]:
|
|
440
|
+
"""Each grouping column the grouping columns kept so far strictly determine.
|
|
441
|
+
|
|
442
|
+
One column always stays: grouping by nothing turns an empty input into one row.
|
|
443
|
+
"""
|
|
444
|
+
expressions = grouping(node) or []
|
|
445
|
+
if len(expressions) != len(aggregate.grouping_outputs):
|
|
446
|
+
return []
|
|
447
|
+
# Grouping outputs are lowered in GROUP BY order.
|
|
448
|
+
written = {column: expression.sql(dialect=dialect) for column, expression in zip(aggregate.grouping_outputs, expressions)}
|
|
449
|
+
kept = list(aggregate.grouping_outputs)
|
|
450
|
+
found = []
|
|
451
|
+
for grouped in aggregate.grouping_outputs:
|
|
452
|
+
others = [other for other in kept if other is not grouped]
|
|
453
|
+
if not others or not isinstance(grouped.scalar_expr, ir.ColumnRef):
|
|
454
|
+
continue
|
|
455
|
+
determinant = frozenset(other.scalar_expr.column for other in others if isinstance(other.scalar_expr, ir.ColumnRef))
|
|
456
|
+
closure = _closure(aggregate.input, determinant, frozenset())
|
|
457
|
+
source = grouped.scalar_expr.column
|
|
458
|
+
if source not in closure.value.columns:
|
|
459
|
+
continue
|
|
460
|
+
kept.remove(grouped)
|
|
461
|
+
names = ", ".join(written[other] for other in others)
|
|
462
|
+
alias = exp.to_identifier(grouped.name).sql(dialect=dialect)
|
|
463
|
+
selected = node.expressions[aggregate.output_columns.index(grouped)]
|
|
464
|
+
expression = selected.this if isinstance(selected, exp.Alias) else selected
|
|
465
|
+
message = (
|
|
466
|
+
f"({names}) determine {written[grouped]}, NULL equal to NULL: drop {written[grouped]} from GROUP BY "
|
|
467
|
+
f"and select any_value({expression.sql(dialect=dialect)}) AS {alias}"
|
|
468
|
+
)
|
|
469
|
+
found.append(Improvement(site, Rewrite.REDUNDANT_GROUPING, message, closure.members[source]))
|
|
470
|
+
return found
|
|
471
|
+
|
|
472
|
+
|
|
473
|
+
# EXCEPT and INTERSECT over inputs without duplicates --------------------------------
|
|
474
|
+
|
|
475
|
+
|
|
476
|
+
def _set_operation_all(site: AttachmentSite, operation: ir.SetOperation) -> Improvement | None:
|
|
477
|
+
"""`A EXCEPT B` keeps each distinct row of A that B lacks; `EXCEPT ALL`
|
|
478
|
+
keeps it as often as A has it more than B. When A has no two rows alike,
|
|
479
|
+
NULL equal to NULL as a set operation compares them, both keep each such
|
|
480
|
+
row once. `INTERSECT ALL` keeps a row as often as the input with fewer of
|
|
481
|
+
it, so a duplicate-free input on either side suffices."""
|
|
482
|
+
if not operation.distinct or operation.operator == "union":
|
|
483
|
+
return None
|
|
484
|
+
keyword = operation.operator.upper()
|
|
485
|
+
sides = [("left", operation.left)] + ([("right", operation.right)] if operation.operator == "intersect" else [])
|
|
486
|
+
for side, relation in sides:
|
|
487
|
+
evidence = _reaches_row_identity(_closure(relation, frozenset(relation.output_columns), frozenset()))
|
|
488
|
+
if evidence.value:
|
|
489
|
+
message = f"no two rows of the {side} input are alike, NULL equal to NULL, so {keyword} removes no duplicates: write {keyword} ALL"
|
|
490
|
+
return Improvement(site, Rewrite.SET_OPERATION_ALL, message, evidence)
|
|
491
|
+
return None
|
|
492
|
+
|
|
493
|
+
|
|
494
|
+
# Unused CTEs ----------------------------------------------------------------------
|
|
495
|
+
|
|
496
|
+
|
|
497
|
+
def _unused_ctes(ast: ParsedProgram) -> list[Improvement]:
|
|
498
|
+
"""CTEs no query of their WITH reaches, directly or through a used CTE.
|
|
499
|
+
|
|
500
|
+
A name is matched wherever it appears as a table, shadowed or not, so a
|
|
501
|
+
CTE is only ever kept by mistake.
|
|
502
|
+
"""
|
|
503
|
+
found = []
|
|
504
|
+
for with_ in _nodes(ast, exp.With):
|
|
505
|
+
query = with_.parent
|
|
506
|
+
ctes = {cte.alias_or_name.lower(): cte for cte in with_.expressions}
|
|
507
|
+
live = {name for name in _table_names(query, skip=with_) if name in ctes}
|
|
508
|
+
pending = list(live)
|
|
509
|
+
while pending:
|
|
510
|
+
for name in _table_names(ctes[pending.pop()].this):
|
|
511
|
+
if name in ctes and name not in live:
|
|
512
|
+
live.add(name)
|
|
513
|
+
pending.append(name)
|
|
514
|
+
for name, cte in ctes.items():
|
|
515
|
+
body = cte.this.find(exp.Select)
|
|
516
|
+
position = marker_position(body) if body is not None else None
|
|
517
|
+
if name in live or position is None:
|
|
518
|
+
continue
|
|
519
|
+
offset, line = position
|
|
520
|
+
site = AttachmentSite(f"CTE {cte.alias_or_name}", line, offset, True)
|
|
521
|
+
evidence = Result(True, f"No query of its WITH clause references CTE {cte.alias_or_name}")
|
|
522
|
+
message = f"CTE {cte.alias_or_name} is never referenced, so it never runs and its markers go unchecked: delete it"
|
|
523
|
+
found.append(Improvement(site, Rewrite.UNUSED_CTE, message, evidence))
|
|
524
|
+
return found
|
|
525
|
+
|
|
526
|
+
|
|
527
|
+
def _table_names(node: Expr, skip: Expr | None = None) -> set[str]:
|
|
528
|
+
return {table.name.lower() for table in node.find_all(exp.Table) if skip is None or not _within(table, skip)}
|
sqlassert/ir/__init__.py
ADDED
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
"""Relational intermediate representation."""
|
|
2
|
+
|
|
3
|
+
from sqlassert.ir.model import Aggregate, AggregateCall, Alias, And, AnyAggregate, AnyRelation, AnyScalar, Assertion, Builtin, BuiltinVariadic, Call, Coalesce, ColumnRef, Constant, Count, Distinct, Elements, Equality, Filter, InList, IsNotNull, IsNull, Join, JoinKind, Limit, NamedRelation, Node, NullLiteral, OpaqueExpression, OpaquePureFunction, OpaqueRelation, Or, OutputColumn, Program, Project, QualifyByPartition, RecursiveRelation, RelationExpr, RelationRole, ScalarExpr, SetOperation, SimpleCase, LiteralValue, SetOperator, children, conjuncts, determinants, injective_input, strict_inputs, total
|
|
4
|
+
|
|
5
|
+
__all__ = [
|
|
6
|
+
"Aggregate",
|
|
7
|
+
"AggregateCall",
|
|
8
|
+
"Alias",
|
|
9
|
+
"And",
|
|
10
|
+
"AnyAggregate",
|
|
11
|
+
"AnyRelation",
|
|
12
|
+
"AnyScalar",
|
|
13
|
+
"Builtin",
|
|
14
|
+
"BuiltinVariadic",
|
|
15
|
+
"Call",
|
|
16
|
+
"Assertion",
|
|
17
|
+
"Coalesce",
|
|
18
|
+
"ColumnRef",
|
|
19
|
+
"Constant",
|
|
20
|
+
"Count",
|
|
21
|
+
"Distinct",
|
|
22
|
+
"Elements",
|
|
23
|
+
"Equality",
|
|
24
|
+
"Filter",
|
|
25
|
+
"IsNotNull",
|
|
26
|
+
"IsNull",
|
|
27
|
+
"InList",
|
|
28
|
+
"Join",
|
|
29
|
+
"JoinKind",
|
|
30
|
+
"Limit",
|
|
31
|
+
"LiteralValue",
|
|
32
|
+
"NamedRelation",
|
|
33
|
+
"Node",
|
|
34
|
+
"NullLiteral",
|
|
35
|
+
"OpaqueExpression",
|
|
36
|
+
"OpaquePureFunction",
|
|
37
|
+
"OpaqueRelation",
|
|
38
|
+
"Or",
|
|
39
|
+
"OutputColumn",
|
|
40
|
+
"Program",
|
|
41
|
+
"Project",
|
|
42
|
+
"QualifyByPartition",
|
|
43
|
+
"RecursiveRelation",
|
|
44
|
+
"RelationExpr",
|
|
45
|
+
"RelationRole",
|
|
46
|
+
"ScalarExpr",
|
|
47
|
+
"SetOperation",
|
|
48
|
+
"SetOperator",
|
|
49
|
+
"SimpleCase",
|
|
50
|
+
"children",
|
|
51
|
+
"conjuncts",
|
|
52
|
+
"determinants",
|
|
53
|
+
"injective_input",
|
|
54
|
+
"strict_inputs",
|
|
55
|
+
"total",
|
|
56
|
+
]
|