sqlassert 0.1.2__py3-none-any.whl → 0.3.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sqlassert/__init__.py +34 -9
- sqlassert/__main__.py +138 -0
- sqlassert/check.py +444 -0
- sqlassert/diagnostics.py +57 -0
- sqlassert/dialects.py +211 -0
- sqlassert/discovery/__init__.py +30 -0
- sqlassert/discovery/bigquery.py +71 -0
- sqlassert/discovery/databricks.py +18 -0
- sqlassert/discovery/duckdb.py +163 -0
- sqlassert/discovery/fetcher.py +402 -0
- sqlassert/discovery/information_schema.py +326 -0
- sqlassert/discovery/mysql.py +70 -0
- sqlassert/discovery/postgres.py +143 -0
- sqlassert/discovery/redshift.py +36 -0
- sqlassert/discovery/snowflake.py +63 -0
- sqlassert/discovery/trino.py +37 -0
- sqlassert/improvements.py +528 -0
- sqlassert/ir/__init__.py +56 -0
- sqlassert/ir/convert.py +2377 -0
- sqlassert/ir/model.py +676 -0
- sqlassert/main.py +57 -0
- sqlassert/properties.py +121 -0
- sqlassert/provenance.py +73 -0
- sqlassert/python_engine.py +299 -0
- sqlassert/reporting.py +102 -0
- sqlassert/rules/STYLEGUIDE.md +19 -0
- sqlassert/rules/__init__.py +11 -0
- sqlassert/rules/column_set.py +34 -0
- sqlassert/rules/domain.py +376 -0
- sqlassert/rules/equality.py +233 -0
- sqlassert/rules/exactly_one.py +75 -0
- sqlassert/rules/foreign_key.py +316 -0
- sqlassert/rules/join_cardinality.py +94 -0
- sqlassert/rules/nullability.py +418 -0
- sqlassert/rules/uniqueness.py +601 -0
- sqlassert/smells.py +306 -0
- sqlassert/sql_parse.py +974 -0
- sqlassert/suggest.py +339 -0
- sqlassert-0.3.1.dist-info/METADATA +725 -0
- sqlassert-0.3.1.dist-info/RECORD +44 -0
- {sqlassert-0.1.2.dist-info → sqlassert-0.3.1.dist-info}/WHEEL +1 -1
- sqlassert-0.3.1.dist-info/entry_points.txt +2 -0
- sqlassert-0.3.1.dist-info/licenses/LICENSE +21 -0
- sqlassert/unique.py +0 -605
- sqlassert-0.1.2.dist-info/METADATA +0 -173
- sqlassert-0.1.2.dist-info/RECORD +0 -6
- {sqlassert-0.1.2.dist-info → sqlassert-0.3.1.dist-info}/top_level.txt +0 -0
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
"""Intermediate values used by column-set property functions."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from dataclasses import dataclass, field
|
|
6
|
+
|
|
7
|
+
from sqlassert import ir
|
|
8
|
+
from sqlassert.python_engine import Result
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
@dataclass(frozen=True)
|
|
12
|
+
class RowIdentity:
|
|
13
|
+
"""Tells apart every row of `relation`, duplicates included (Paulley's ι).
|
|
14
|
+
|
|
15
|
+
Analysis only: never in the IR, in SQL, or shown to authors.
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
relation: ir.RelationExpr
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
type Attribute = ir.OutputColumn | RowIdentity
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
@dataclass(eq=False)
|
|
25
|
+
class ColumnSet:
|
|
26
|
+
relation: ir.RelationExpr
|
|
27
|
+
columns: set[Attribute]
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
@dataclass(eq=False)
|
|
31
|
+
class Closure(ColumnSet):
|
|
32
|
+
"""A closure, with the closures it took in the relation's inputs."""
|
|
33
|
+
|
|
34
|
+
inputs: tuple[Result[Closure], ...] = field(default=())
|
|
@@ -0,0 +1,376 @@
|
|
|
1
|
+
"""Enumerated Domains: the finite set a column's non-null values come from.
|
|
2
|
+
|
|
3
|
+
A column's own domain comes from Knowledge about it and from its expression:
|
|
4
|
+
a literal is its own value, NULL contributes none, and `coalesce` takes one of
|
|
5
|
+
its arguments' values. A pass-through column keeps its source's domain, which
|
|
6
|
+
a strict `WHERE` or `ON` conjunct `col = expr` or `col IN (...)` narrows to the
|
|
7
|
+
values on its other side. Its domain is then the intersection of the own
|
|
8
|
+
domains of every column in its equality group: on every row they carry one
|
|
9
|
+
value, or are all NULL. An unbounded domain is `None`.
|
|
10
|
+
|
|
11
|
+
NULL is outside every domain, so padding and NULL literals never widen one.
|
|
12
|
+
|
|
13
|
+
Members are typed literals, and covering is by identity: a value is covered
|
|
14
|
+
only by an identical literal, so numeric 1 never covers string '1', nor 1.0.
|
|
15
|
+
Intersecting is not by identity, because a column is compared with a literal
|
|
16
|
+
after implicit casts, so two distinct literals can both equal its value: a
|
|
17
|
+
FLOAT equals 0.1 and 0.100000001, a VARCHAR '1.4' equals 1 and 1.4, an
|
|
18
|
+
INTEGER '01' and '1', and a case-insensitive VARCHAR 'a' and 'A'. A meet
|
|
19
|
+
therefore keeps every value that may equal one on each other side
|
|
20
|
+
(`_may_coincide`), which only ever keeps more than the true intersection.
|
|
21
|
+
Distinct strings are distinct values only where the column is text compared
|
|
22
|
+
by identity (`_binary_text`).
|
|
23
|
+
|
|
24
|
+
A simple CASE takes one of its THEN values, or its ELSE value unless ELSE is
|
|
25
|
+
unreachable: the CASE is exhaustive and its operand non-null. Exhaustiveness
|
|
26
|
+
covers by identity too: an arm covers a domain member only when it is that
|
|
27
|
+
literal, so the operand compared with the arm is the very comparison that
|
|
28
|
+
puts its value in the domain.
|
|
29
|
+
"""
|
|
30
|
+
|
|
31
|
+
from __future__ import annotations
|
|
32
|
+
|
|
33
|
+
from decimal import Decimal
|
|
34
|
+
from fractions import Fraction
|
|
35
|
+
from typing import assert_never
|
|
36
|
+
|
|
37
|
+
from sqlassert import ir
|
|
38
|
+
from sqlassert.properties import EnumeratedDomain
|
|
39
|
+
from sqlassert.python_engine import Result, Stratum, analysis, any_of, current_arena, unknown
|
|
40
|
+
from sqlassert.rules.equality import _equal_columns, _same_type
|
|
41
|
+
from sqlassert.rules.nullability import non_null_column
|
|
42
|
+
|
|
43
|
+
# A finite set of possible non-null values, or None when any value is possible.
|
|
44
|
+
type Domain = frozenset[ir.LiteralValue] | None
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
@analysis(Stratum.DOMAIN)
|
|
48
|
+
def enumerated_domain(column: ir.OutputColumn, values: frozenset[ir.LiteralValue]) -> Result[bool]:
|
|
49
|
+
domain = _domain(column)
|
|
50
|
+
if domain.value is None:
|
|
51
|
+
return Result(False, f"No finite domain is known for {column.name}", (column,), [domain], kind="unknown")
|
|
52
|
+
uncovered = domain.value - values
|
|
53
|
+
if uncovered:
|
|
54
|
+
return Result(
|
|
55
|
+
False, f"{column.name} may be {_render(uncovered)}, which the assertion does not cover",
|
|
56
|
+
(column,), [domain], kind="unknown",
|
|
57
|
+
)
|
|
58
|
+
return Result(True, "Every possible value is listed", (column,), [domain])
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
@analysis(Stratum.DOMAIN)
|
|
62
|
+
def exhaustive_case(case: ir.SimpleCase) -> Result[bool]:
|
|
63
|
+
operand = _operand_name(case.operand)
|
|
64
|
+
domain = _expression_domain(case.operand)
|
|
65
|
+
if domain.value is None:
|
|
66
|
+
return Result(False, f"No finite domain is known for {operand}", (case,), [domain], kind="unknown")
|
|
67
|
+
missing = domain.value - _listed(case)
|
|
68
|
+
if missing:
|
|
69
|
+
return Result(False, f"{operand} may be {_render(missing)}, which no WHEN arm lists", (case,), [domain], kind="unknown")
|
|
70
|
+
return Result(True, f"Every possible value of {operand} has a WHEN arm", (case,), [domain])
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def _listed(case: ir.SimpleCase) -> frozenset[ir.LiteralValue]:
|
|
74
|
+
"""The literals a WHEN arm is exactly."""
|
|
75
|
+
listed: set[ir.LiteralValue] = set()
|
|
76
|
+
for when in case.whens:
|
|
77
|
+
match when:
|
|
78
|
+
case ir.Constant(value=ir.LiteralValue() as value):
|
|
79
|
+
listed.add(value)
|
|
80
|
+
case _:
|
|
81
|
+
pass # Another arm covers no member by identity.
|
|
82
|
+
return frozenset(listed)
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def _operand_name(operand: ir.ScalarExpr) -> str:
|
|
86
|
+
match operand:
|
|
87
|
+
case ir.ColumnRef(column=column):
|
|
88
|
+
return column.name
|
|
89
|
+
case _:
|
|
90
|
+
return "the CASE operand"
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def _else_unreachable(case: ir.SimpleCase) -> Result[bool]:
|
|
94
|
+
"""No row reaches ELSE: every non-null operand value has an arm, and the operand is non-null."""
|
|
95
|
+
match case.operand:
|
|
96
|
+
case ir.ColumnRef(column=column):
|
|
97
|
+
present = non_null_column(column)
|
|
98
|
+
case ir.Constant(value=ir.LiteralValue()):
|
|
99
|
+
present = Result(True, "A non-null literal", (case.operand,))
|
|
100
|
+
case _:
|
|
101
|
+
present = unknown("Nullability of the CASE operand is not inferred", case.operand)
|
|
102
|
+
return exhaustive_case(case) & present
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
@analysis(Stratum.DOMAIN)
|
|
106
|
+
def _domain(column: ir.OutputColumn) -> Result[Domain]:
|
|
107
|
+
"""`column`'s domain: its own, intersected with every equal column's."""
|
|
108
|
+
group = _equal_columns(column)
|
|
109
|
+
relation = column.get_relation(current_arena().program)
|
|
110
|
+
members = [member for member in relation.output_columns if member in group.value]
|
|
111
|
+
owns = [_own_domain(member) for member in members]
|
|
112
|
+
known = [own for own in owns if own.value is not None]
|
|
113
|
+
if not known:
|
|
114
|
+
return Result(None, f"No finite domain is known for {column.name}", (column,), owns)
|
|
115
|
+
value = _meet([own.value for own in known if own.value is not None], _binary_text(column))
|
|
116
|
+
premises = [group, *known] if len(members) > 1 else known
|
|
117
|
+
return Result(value, f"{column.name} is one of {_render(value)}", (column,), premises)
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
@analysis(Stratum.DOMAIN)
|
|
121
|
+
def _own_domain(column: ir.OutputColumn) -> Result[Domain]:
|
|
122
|
+
"""`column`'s domain from Knowledge about it and from how its relation computes it."""
|
|
123
|
+
arena = current_arena()
|
|
124
|
+
sources: list[Result[Domain]] = [_computed_domain(column.get_relation(arena.program), column)]
|
|
125
|
+
for item in arena.knowledge:
|
|
126
|
+
match item:
|
|
127
|
+
case EnumeratedDomain(column=known, values=values) if known is column:
|
|
128
|
+
sources.append(Result(values, f"{column.name} is known to be one of {_render(values)}", (column,), [any_of(arena.accepted_evidence(item))]))
|
|
129
|
+
case _:
|
|
130
|
+
pass # Other Knowledge says nothing about values.
|
|
131
|
+
return _intersection(column, sources)
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def _computed_domain(relation: ir.RelationExpr, column: ir.OutputColumn) -> Result[Domain]:
|
|
135
|
+
relation = relation.closed
|
|
136
|
+
match relation:
|
|
137
|
+
case ir.NamedRelation(body=None) | ir.OpaqueRelation():
|
|
138
|
+
return Result(None, f"{column.name} is not enumerated", (column,))
|
|
139
|
+
case ir.SetOperation():
|
|
140
|
+
return _set_operation_domain(relation, column)
|
|
141
|
+
case ir.Filter() | ir.Join():
|
|
142
|
+
return _intersection(column, [_expression_domain(column.scalar_expr), *_narrowings(relation, column)])
|
|
143
|
+
case ir.Alias() | ir.NamedRelation() | ir.Project() | ir.Limit() | ir.Aggregate() | ir.Distinct() | ir.QualifyByPartition():
|
|
144
|
+
return _expression_domain(column.scalar_expr)
|
|
145
|
+
case _:
|
|
146
|
+
assert_never(relation)
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def _set_operation_domain(relation: ir.SetOperation, column: ir.OutputColumn) -> Result[Domain]:
|
|
150
|
+
"""A UNION row comes from either input, an INTERSECT row equals a row of
|
|
151
|
+
both, an EXCEPT row is a left input row (Paulley §3.2.5-3.2.6). Inputs
|
|
152
|
+
pair by position; a UNION of mixed value types is not enumerated, since
|
|
153
|
+
the output type may convert one."""
|
|
154
|
+
left = _expression_domain(column.scalar_expr)
|
|
155
|
+
arms = relation.arms(column)
|
|
156
|
+
if relation.operator == "except" or arms is None and relation.operator == "intersect":
|
|
157
|
+
return left
|
|
158
|
+
if arms is None:
|
|
159
|
+
return Result(None, "The UNION's inputs are not paired by position", (relation, column))
|
|
160
|
+
right = _domain(arms[1])
|
|
161
|
+
if relation.operator == "intersect":
|
|
162
|
+
# The row carries the left value, equal to a right one under the left
|
|
163
|
+
# column's comparison: a string bounds it only if that is identity.
|
|
164
|
+
if right.value is not None and any(member.type == "string" for member in right.value) and not _binary_text(arms[0]):
|
|
165
|
+
return left
|
|
166
|
+
return _intersection(column, [left, right])
|
|
167
|
+
if left.value is None or right.value is None:
|
|
168
|
+
return Result(None, f"An input of the UNION does not enumerate {column.name}", (relation, column), [left, right])
|
|
169
|
+
value = left.value | right.value
|
|
170
|
+
if len({member.type for member in value}) > 1:
|
|
171
|
+
return Result(None, f"The UNION's inputs give {column.name} values of different types", (relation, column), [left, right])
|
|
172
|
+
return Result(value, f"{column.name} is one of either input's values: {_render(value)}", (relation, column), [left, right])
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
def _expression_domain(expression: ir.ScalarExpr) -> Result[Domain]:
|
|
176
|
+
expression = expression.closed
|
|
177
|
+
match expression:
|
|
178
|
+
case ir.ColumnRef(column=source):
|
|
179
|
+
return _domain(source)
|
|
180
|
+
case ir.Constant(value=ir.LiteralValue() as value):
|
|
181
|
+
return Result(frozenset({value}), f"The literal {value.sql()}", (expression,))
|
|
182
|
+
case ir.NullLiteral():
|
|
183
|
+
return Result(frozenset(), "NULL is outside every domain", (expression,))
|
|
184
|
+
case ir.Coalesce(arguments=arguments):
|
|
185
|
+
# The first non-null argument, so one of theirs.
|
|
186
|
+
domains = [_expression_domain(argument) for argument in arguments]
|
|
187
|
+
if any(domain.value is None for domain in domains):
|
|
188
|
+
return Result(None, "A coalesce argument is not enumerated", (expression,), domains)
|
|
189
|
+
value = frozenset().union(*(domain.value for domain in domains if domain.value is not None))
|
|
190
|
+
return Result(value, "Coalesce takes one of its arguments' values", (expression,), domains)
|
|
191
|
+
case ir.AnyAggregate(input=value):
|
|
192
|
+
# Groups are never empty, so the value comes from some input row.
|
|
193
|
+
return _expression_domain(value)
|
|
194
|
+
case ir.SimpleCase(thens=thens, default=default):
|
|
195
|
+
arms = [_expression_domain(then) for then in thens]
|
|
196
|
+
unreachable = _else_unreachable(expression)
|
|
197
|
+
if not unreachable.value:
|
|
198
|
+
arms.append(_expression_domain(default))
|
|
199
|
+
if any(arm.value is None for arm in arms):
|
|
200
|
+
return Result(None, "A CASE arm is not enumerated", (expression,), arms)
|
|
201
|
+
value = frozenset().union(*(arm.value for arm in arms if arm.value is not None))
|
|
202
|
+
premises = [*arms, unreachable] if unreachable.value else arms
|
|
203
|
+
return Result(value, "A CASE takes one of its arms' values", (expression,), premises)
|
|
204
|
+
case ir.Constant() | ir.OpaqueExpression() | ir.OpaquePureFunction() | ir.Call() | ir.Count() | ir.Elements() | ir.Equality() | ir.InList() | ir.IsNull() | ir.IsNotNull() | ir.And() | ir.Or():
|
|
205
|
+
return Result(None, "Values of this expression are not enumerated", (expression,))
|
|
206
|
+
case _:
|
|
207
|
+
assert_never(expression)
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
def _narrowings(relation: ir.Filter | ir.Join, column: ir.OutputColumn) -> list[Result[Domain]]:
|
|
211
|
+
"""Domains a conjunct true on every row carrying `column`'s value bounds it to.
|
|
212
|
+
|
|
213
|
+
`WHERE` and an inner join's ON hold on every row; an outer join's ON holds
|
|
214
|
+
on the matched rows only, which pad nothing on the side it pads.
|
|
215
|
+
"""
|
|
216
|
+
match column.scalar_expr:
|
|
217
|
+
case ir.ColumnRef(column=source):
|
|
218
|
+
pass
|
|
219
|
+
case _:
|
|
220
|
+
raise TypeError(f"{type(relation).__name__} column {column.name} is not a pass-through")
|
|
221
|
+
match relation:
|
|
222
|
+
case ir.Filter(conjuncts=conjuncts):
|
|
223
|
+
clause = "WHERE"
|
|
224
|
+
case ir.Join(kind="inner" | "cross" | "semi", conjuncts=conjuncts):
|
|
225
|
+
clause = "ON"
|
|
226
|
+
case ir.Join(kind="left", right=padded, conjuncts=conjuncts) | ir.Join(kind="right", left=padded, conjuncts=conjuncts) if source in padded.output_columns:
|
|
227
|
+
clause = "ON"
|
|
228
|
+
case _:
|
|
229
|
+
return []
|
|
230
|
+
narrowings: list[Result[Domain]] = []
|
|
231
|
+
for conjunct in conjuncts:
|
|
232
|
+
bound = _conjunct_bound(conjunct, source)
|
|
233
|
+
if bound is not None and bound.value is not None:
|
|
234
|
+
narrowings.append(Result(bound.value, f"The {clause} conjunct bounds {column.name} to {_render(bound.value)}", (conjunct,), [bound]))
|
|
235
|
+
return narrowings
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
def _conjunct_bound(conjunct: ir.ScalarExpr, source: ir.OutputColumn) -> Result[Domain] | None:
|
|
239
|
+
"""The values `source` takes on rows where `conjunct` is TRUE, or None
|
|
240
|
+
when the conjunct does not bound it.
|
|
241
|
+
|
|
242
|
+
An AND bounds it to what each of its bounding parts allows; an OR to the
|
|
243
|
+
union of its parts' bounds, when every part bounds it. A bound is never an
|
|
244
|
+
unknown domain.
|
|
245
|
+
"""
|
|
246
|
+
match conjunct:
|
|
247
|
+
case ir.Equality(left=ir.ColumnRef(column=first), right=other) if first is source:
|
|
248
|
+
bounds = [_bound_domain(source, other)]
|
|
249
|
+
case ir.Equality(left=other, right=ir.ColumnRef(column=second)) if second is source:
|
|
250
|
+
bounds = [_bound_domain(source, other)]
|
|
251
|
+
case ir.InList(operand=ir.ColumnRef(column=operand), values=values) if operand is source:
|
|
252
|
+
bounds = [_bound_domain(source, value) for value in values]
|
|
253
|
+
case ir.And(parts=parts):
|
|
254
|
+
found = [bound for part in parts if (bound := _conjunct_bound(part, source)) is not None]
|
|
255
|
+
# Distinct literals may compare equal (issue 29), so parts meet as domains do.
|
|
256
|
+
return _intersection(source, found) if found else None
|
|
257
|
+
case ir.Or(parts=parts):
|
|
258
|
+
bounds = [_conjunct_bound(part, source) for part in parts]
|
|
259
|
+
if any(bound is None for bound in bounds):
|
|
260
|
+
return None # A part that does not bound the column lets any value through.
|
|
261
|
+
case _:
|
|
262
|
+
return None # Other conjuncts do not bound this column.
|
|
263
|
+
if any(bound is None or bound.value is None for bound in bounds):
|
|
264
|
+
return None
|
|
265
|
+
value = frozenset().union(*(bound.value for bound in bounds if bound is not None and bound.value is not None))
|
|
266
|
+
return Result(value, f"{source.name} is in {_render(value)} on every row this conjunct keeps", (conjunct,), [bound for bound in bounds if bound is not None])
|
|
267
|
+
|
|
268
|
+
|
|
269
|
+
def _bound_domain(column: ir.OutputColumn, expression: ir.ScalarExpr) -> Result[Domain]:
|
|
270
|
+
match expression:
|
|
271
|
+
case ir.Constant() | ir.NullLiteral():
|
|
272
|
+
return _expression_domain(expression)
|
|
273
|
+
case ir.ColumnRef(column=source) if _same_type(column, source):
|
|
274
|
+
return _expression_domain(expression)
|
|
275
|
+
case _:
|
|
276
|
+
return Result(None, "The comparison may cast its operands, so no value domain is transferred", (expression,))
|
|
277
|
+
|
|
278
|
+
|
|
279
|
+
def _intersection(column: ir.OutputColumn, sources: list[Result[Domain]]) -> Result[Domain]:
|
|
280
|
+
known = [source for source in sources if source.value is not None]
|
|
281
|
+
if not known:
|
|
282
|
+
return Result(None, f"No finite domain is known for {column.name}", (column,), sources)
|
|
283
|
+
if len(known) == 1:
|
|
284
|
+
return known[0]
|
|
285
|
+
value = _meet([source.value for source in known if source.value is not None], _binary_text(column))
|
|
286
|
+
return Result(value, f"{column.name} is one of {_render(value)}", (column,), known)
|
|
287
|
+
|
|
288
|
+
|
|
289
|
+
def _meet(domains: list[frozenset[ir.LiteralValue]], binary_text: bool) -> frozenset[ir.LiteralValue]:
|
|
290
|
+
"""The first domain's values that may equal a value of every other one.
|
|
291
|
+
|
|
292
|
+
A column's value equals some member of each domain, so the member of the
|
|
293
|
+
first it equals may coincide with one of every other and is kept.
|
|
294
|
+
"""
|
|
295
|
+
first, *others = domains
|
|
296
|
+
return frozenset(value for value in first if all(any(_may_coincide(value, other, binary_text) for other in domain) for domain in others))
|
|
297
|
+
|
|
298
|
+
|
|
299
|
+
def _binary_text(column: ir.OutputColumn) -> bool:
|
|
300
|
+
"""Whether `column` and every column equated with it, here or upstream,
|
|
301
|
+
are text compared by identity, so carry one string value.
|
|
302
|
+
|
|
303
|
+
Only matching comparison types merge Equality Groups. A set operation's
|
|
304
|
+
type may widen its left input's, so its columns are not followed.
|
|
305
|
+
"""
|
|
306
|
+
relation = column.get_relation(current_arena().program)
|
|
307
|
+
for member in _equal_columns(column).value:
|
|
308
|
+
match member.scalar_expr:
|
|
309
|
+
case ir.ColumnRef(column=source) if not isinstance(relation, ir.SetOperation):
|
|
310
|
+
if not _binary_text(source):
|
|
311
|
+
return False
|
|
312
|
+
case _:
|
|
313
|
+
if not member.binary_text:
|
|
314
|
+
return False
|
|
315
|
+
return True
|
|
316
|
+
|
|
317
|
+
|
|
318
|
+
# A FLOAT keeps 24 significant bits: two numbers rounding to one FLOAT differ
|
|
319
|
+
# by at most its precision relative to their size, with margin for a literal
|
|
320
|
+
# rounded through DOUBLE first, or are both too small to tell from zero.
|
|
321
|
+
_FLOAT_RELATIVE = Fraction(1, 2**22)
|
|
322
|
+
_FLOAT_ABSOLUTE = Fraction(1, 2**148)
|
|
323
|
+
|
|
324
|
+
|
|
325
|
+
def _may_coincide(first: ir.LiteralValue, second: ir.LiteralValue, binary_text: bool) -> bool:
|
|
326
|
+
"""Whether some column value may equal both literals.
|
|
327
|
+
|
|
328
|
+
Literals of different types may: SQL casts one side, so '1' and 1 both
|
|
329
|
+
equal INTEGER 1. Distinct booleans never do. Distinct strings may unless
|
|
330
|
+
the column is text compared by identity: an INTEGER equals '01' and '1',
|
|
331
|
+
a case-insensitive one 'a' and 'A'. Distinct numbers may when one value
|
|
332
|
+
rounds to both.
|
|
333
|
+
"""
|
|
334
|
+
if first == second or first.type != second.type:
|
|
335
|
+
return True
|
|
336
|
+
match first.type:
|
|
337
|
+
case "string":
|
|
338
|
+
return not binary_text
|
|
339
|
+
case "boolean":
|
|
340
|
+
return False
|
|
341
|
+
case "numeric":
|
|
342
|
+
return _numbers_may_coincide(first.text, second.text)
|
|
343
|
+
case _:
|
|
344
|
+
assert_never(first.type)
|
|
345
|
+
|
|
346
|
+
|
|
347
|
+
def _numbers_may_coincide(first: str, second: str) -> bool:
|
|
348
|
+
gap = abs(_number(first) - _number(second))
|
|
349
|
+
# A string is cast to an exact literal's type, rounding half away from
|
|
350
|
+
# zero to its scale: one value rounds to both when the cells overlap.
|
|
351
|
+
if gap < _half_step(first) + _half_step(second):
|
|
352
|
+
return True
|
|
353
|
+
# A FLOAT or DOUBLE is compared at its own precision.
|
|
354
|
+
return gap <= max(abs(_number(first)), abs(_number(second))) * _FLOAT_RELATIVE or gap <= _FLOAT_ABSOLUTE
|
|
355
|
+
|
|
356
|
+
|
|
357
|
+
def _half_step(text: str) -> Fraction:
|
|
358
|
+
"""Half the rounding step of a numeric literal's SQL type: an integer's
|
|
359
|
+
is one, a decimal's its last digit, and an exponent form is a DOUBLE."""
|
|
360
|
+
if "e" in text:
|
|
361
|
+
return Fraction(0)
|
|
362
|
+
_, _, decimals = text.partition(".")
|
|
363
|
+
return Fraction(1, 2 * 10 ** len(decimals))
|
|
364
|
+
|
|
365
|
+
|
|
366
|
+
def _number(text: str) -> Fraction:
|
|
367
|
+
return Fraction(Decimal(text))
|
|
368
|
+
|
|
369
|
+
|
|
370
|
+
def _render(values: frozenset[ir.LiteralValue]) -> str:
|
|
371
|
+
return ", ".join(value.sql() for value in sorted(values, key=_order)) if values else "no non-null value"
|
|
372
|
+
|
|
373
|
+
|
|
374
|
+
def _order(value: ir.LiteralValue) -> tuple[str, Fraction, str]:
|
|
375
|
+
"""Booleans, then numbers by value, then strings: a fixed reading order."""
|
|
376
|
+
return value.type, _number(value.text) if value.type == "numeric" else Fraction(0), value.text
|
|
@@ -0,0 +1,233 @@
|
|
|
1
|
+
"""Equality groups: the output columns of a relation that carry one value.
|
|
2
|
+
|
|
3
|
+
A strict equality -- a `WHERE` or inner-join `ON` conjunct `a = b` -- keeps
|
|
4
|
+
only rows on which `a` and `b` are non-null and equal. Equality groups are a
|
|
5
|
+
union-find over those equalities, per relation, over its output columns: an
|
|
6
|
+
input's groups pass through every column that passes an input column through,
|
|
7
|
+
then the relation's own strict equalities merge them. On every row a group's
|
|
8
|
+
members are equal, or all NULL where an outer join padded them (padding nulls
|
|
9
|
+
a whole side). An outer join's ON never merges: it holds on matched rows only,
|
|
10
|
+
and a padded row keeps the preserved side's value.
|
|
11
|
+
|
|
12
|
+
A functional dependency `a ↔ b` also holds for an injective `a = f(b)`, whose
|
|
13
|
+
values differ; a group says the values are the same. Facts about values read
|
|
14
|
+
it (Enumerated Domains, Foreign Keys through an equality join); uniqueness
|
|
15
|
+
never does.
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
from dataclasses import dataclass
|
|
21
|
+
from typing import assert_never
|
|
22
|
+
|
|
23
|
+
from sqlassert import ir
|
|
24
|
+
from sqlassert.python_engine import Result, Stratum, analysis, current_arena
|
|
25
|
+
|
|
26
|
+
type _Group = tuple[ir.OutputColumn, ...]
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
@dataclass(frozen=True)
|
|
30
|
+
class EqualityGroups:
|
|
31
|
+
"""A partition of `relation`'s output columns into value-equal groups.
|
|
32
|
+
|
|
33
|
+
Each group lists its columns in output order, and groups are ordered by
|
|
34
|
+
their first column; a column no equality reaches is a group of its own.
|
|
35
|
+
"""
|
|
36
|
+
|
|
37
|
+
relation: ir.RelationExpr
|
|
38
|
+
groups: tuple[_Group, ...]
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
@analysis(Stratum.EQUALITY_GROUP)
|
|
42
|
+
def _equal_columns(column: ir.OutputColumn) -> Result[frozenset[ir.OutputColumn]]:
|
|
43
|
+
"""Every column of `column`'s relation that carries its value, itself included."""
|
|
44
|
+
groups = _equality_groups(column.get_relation(current_arena().program))
|
|
45
|
+
group, = (group for group in groups.value.groups if column in group)
|
|
46
|
+
return Result(frozenset(group), "Equal columns", (column,), [groups.members[group]])
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
@analysis(Stratum.EQUALITY_GROUP)
|
|
50
|
+
def _equality_groups(relation: ir.RelationExpr) -> Result[EqualityGroups]:
|
|
51
|
+
"""`relation`'s equality groups; each group's member cites the equalities that merged it."""
|
|
52
|
+
roots = {column: column for column in relation.output_columns}
|
|
53
|
+
evidence: dict[ir.OutputColumn, list[Result[bool]]] = {column: [] for column in relation.output_columns}
|
|
54
|
+
|
|
55
|
+
def find(column: ir.OutputColumn) -> ir.OutputColumn:
|
|
56
|
+
while roots[column] is not column:
|
|
57
|
+
column = roots[column]
|
|
58
|
+
return column
|
|
59
|
+
|
|
60
|
+
def merge(columns: list[ir.OutputColumn], reason: Result[bool]) -> None:
|
|
61
|
+
found = list(dict.fromkeys(find(column) for column in columns))
|
|
62
|
+
if len(found) < 2:
|
|
63
|
+
return # Already one group: the first justification suffices.
|
|
64
|
+
root, *others = found
|
|
65
|
+
for other in others:
|
|
66
|
+
roots[other] = root
|
|
67
|
+
evidence[root].extend(evidence.pop(other))
|
|
68
|
+
evidence[root].append(reason)
|
|
69
|
+
|
|
70
|
+
outputs: dict[ir.OutputColumn, list[ir.OutputColumn]] = {}
|
|
71
|
+
for source in _inputs(relation):
|
|
72
|
+
passed = _pass_through(relation, source)
|
|
73
|
+
outputs.update(passed)
|
|
74
|
+
groups = _equality_groups(source)
|
|
75
|
+
for group in groups.value.groups:
|
|
76
|
+
own = [column for upstream in group for column in passed.get(upstream, ())]
|
|
77
|
+
merge(own, Result(True, "Equal in the input", (), [groups.members[group]]))
|
|
78
|
+
|
|
79
|
+
conjuncts, clause = _strict_conjuncts(relation)
|
|
80
|
+
for conjunct in conjuncts:
|
|
81
|
+
match conjunct:
|
|
82
|
+
case ir.Equality(left=ir.ColumnRef(column=first), right=ir.ColumnRef(column=second)) if first in outputs and second in outputs and _same_type(first, second):
|
|
83
|
+
merge([*outputs[first], *outputs[second]], Result(True, f"{clause} equality", (conjunct,)))
|
|
84
|
+
case _:
|
|
85
|
+
pass # Other conjuncts only remove rows.
|
|
86
|
+
|
|
87
|
+
grouped: dict[ir.OutputColumn, list[ir.OutputColumn]] = {}
|
|
88
|
+
for column in relation.output_columns:
|
|
89
|
+
grouped.setdefault(find(column), []).append(column)
|
|
90
|
+
members: dict[_Group, Result[bool]] = {}
|
|
91
|
+
for root, columns in grouped.items():
|
|
92
|
+
group = tuple(columns)
|
|
93
|
+
members[group] = (
|
|
94
|
+
Result(True, "Equal columns", group, evidence[root]) if evidence[root]
|
|
95
|
+
else Result(True, "A column equals itself", group)
|
|
96
|
+
)
|
|
97
|
+
value = EqualityGroups(relation, tuple(members))
|
|
98
|
+
return Result(value, "Equality groups", (relation,), list(members.values()), members=members)
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def _same_type(first: ir.OutputColumn, second: ir.OutputColumn) -> bool:
|
|
102
|
+
"""Equalities are value identities only for known, matching comparison types."""
|
|
103
|
+
first_type = _comparison_type(first).value
|
|
104
|
+
return first is second or first_type is not None and first_type == _comparison_type(second).value
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
@analysis(Stratum.EQUALITY_GROUP)
|
|
108
|
+
def _comparison_type(column: ir.OutputColumn) -> Result[str | None]:
|
|
109
|
+
relation = column.get_relation(current_arena().program)
|
|
110
|
+
match column.scalar_expr:
|
|
111
|
+
case ir.ColumnRef(column=source) if not isinstance(relation, ir.SetOperation):
|
|
112
|
+
return _comparison_type(source)
|
|
113
|
+
case _:
|
|
114
|
+
return Result(column.comparison_type, "Declared comparison type", (column,))
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
@analysis(Stratum.EQUALITY_GROUP)
|
|
118
|
+
def _identity_comparison(column: ir.OutputColumn) -> Result[bool]:
|
|
119
|
+
"""Whether equal values of `column` are identical, from its declared type."""
|
|
120
|
+
relation = column.get_relation(current_arena().program)
|
|
121
|
+
match column.scalar_expr:
|
|
122
|
+
case ir.ColumnRef(column=source) if not isinstance(relation, ir.SetOperation):
|
|
123
|
+
return _identity_comparison(source)
|
|
124
|
+
case _:
|
|
125
|
+
return Result(column.identity_comparison, "Declared type compares by identity", (column,))
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def _pins(column: ir.OutputColumn, value: ir.ScalarExpr) -> bool:
|
|
129
|
+
"""Whether `column = value` leaves `column` one value for each value of
|
|
130
|
+
`value`, whatever casts the comparison adds.
|
|
131
|
+
|
|
132
|
+
A column of a non-text type whose equal values are identical is: the
|
|
133
|
+
other side is cast to its type, or it is cast to a wider type, which
|
|
134
|
+
loses nothing. Text compared by identity is only when the other side is
|
|
135
|
+
such text too, or a string literal: compared with a number it is cast to
|
|
136
|
+
the number (`'01'` and `'1'` both equal 1). Anything else is not: a
|
|
137
|
+
collation or a float equates different values, and of an unknown type
|
|
138
|
+
nothing is known."""
|
|
139
|
+
if _binary_text(column).value:
|
|
140
|
+
match value:
|
|
141
|
+
case ir.Constant(value=ir.LiteralValue(type="string")):
|
|
142
|
+
return True
|
|
143
|
+
case ir.ColumnRef(column=other):
|
|
144
|
+
return _binary_text(other).value
|
|
145
|
+
case _:
|
|
146
|
+
return False
|
|
147
|
+
return _identity_comparison(column).value or _exact_arithmetic(column)
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
def _exact_arithmetic(column: ir.OutputColumn) -> bool:
|
|
151
|
+
"""Whether `column` is an integer or decimal it computes: a `count`, or
|
|
152
|
+
exact arithmetic over an integer or decimal (`id + 7`)."""
|
|
153
|
+
program = current_arena().program
|
|
154
|
+
while isinstance(column.scalar_expr, ir.ColumnRef) and not isinstance(column.get_relation(program), ir.SetOperation):
|
|
155
|
+
column = column.scalar_expr.column
|
|
156
|
+
if isinstance(column.scalar_expr, ir.Count):
|
|
157
|
+
return True
|
|
158
|
+
if not isinstance(column.scalar_expr, ir.Call):
|
|
159
|
+
return False
|
|
160
|
+
source = ir.injective_input(column.scalar_expr)
|
|
161
|
+
return source is not None and exact_numeric(source).value
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
@analysis(Stratum.EQUALITY_GROUP)
|
|
165
|
+
def _binary_text(column: ir.OutputColumn) -> Result[bool]:
|
|
166
|
+
"""Whether `column` is text compared by identity, from its declared type."""
|
|
167
|
+
relation = column.get_relation(current_arena().program)
|
|
168
|
+
match column.scalar_expr:
|
|
169
|
+
case ir.ColumnRef(column=source) if not isinstance(relation, ir.SetOperation):
|
|
170
|
+
return _binary_text(source)
|
|
171
|
+
case _:
|
|
172
|
+
return Result(column.binary_text, "Declared as text compared by identity", (column,))
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
@analysis(Stratum.EQUALITY_GROUP)
|
|
176
|
+
def exact_numeric(column: ir.OutputColumn) -> Result[bool]:
|
|
177
|
+
"""Whether `column` is an integer or decimal, from its declared type."""
|
|
178
|
+
relation = column.get_relation(current_arena().program)
|
|
179
|
+
match column.scalar_expr:
|
|
180
|
+
case ir.ColumnRef(column=source) if not isinstance(relation, ir.SetOperation):
|
|
181
|
+
return exact_numeric(source)
|
|
182
|
+
case _:
|
|
183
|
+
return Result(column.exact_numeric, "Declared an integer or decimal", (column,))
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
def _inputs(relation: ir.RelationExpr) -> tuple[ir.RelationExpr, ...]:
|
|
187
|
+
"""The inputs whose groups hold on `relation`'s pass-through columns.
|
|
188
|
+
|
|
189
|
+
Each output row of these takes its pass-through values from one row of
|
|
190
|
+
each listed input (or, for a padded side, NULL in every column).
|
|
191
|
+
"""
|
|
192
|
+
relation = relation.closed
|
|
193
|
+
match relation:
|
|
194
|
+
case ir.Alias(source=source) | ir.NamedRelation(body=ir.RelationExpr() as source) | ir.Filter(input=source) | ir.Project(input=source) | ir.Limit(input=source) | ir.Aggregate(input=source) | ir.Distinct(input=source) | ir.QualifyByPartition(input=source):
|
|
195
|
+
return (source,)
|
|
196
|
+
case ir.Join(kind="inner" | "cross" | "left" | "right" | "full", left=left, right=right):
|
|
197
|
+
return (left, right)
|
|
198
|
+
case ir.Join(left=left):
|
|
199
|
+
# A semi or anti join keeps left rows; its right columns have no row.
|
|
200
|
+
return (left,)
|
|
201
|
+
case ir.SetOperation(operator="intersect" | "except", left=left):
|
|
202
|
+
# Every output row equals some row of the left input.
|
|
203
|
+
return (left,)
|
|
204
|
+
case ir.SetOperation() | ir.NamedRelation() | ir.OpaqueRelation():
|
|
205
|
+
# A UNION row may come from either branch; the right one is not tracked.
|
|
206
|
+
return ()
|
|
207
|
+
case _:
|
|
208
|
+
assert_never(relation)
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
def _strict_conjuncts(relation: ir.RelationExpr) -> tuple[tuple[ir.ScalarExpr, ...], str]:
|
|
212
|
+
"""The conjuncts true on every row of `relation`, and the clause they come from."""
|
|
213
|
+
match relation:
|
|
214
|
+
case ir.Filter(conjuncts=conjuncts):
|
|
215
|
+
return conjuncts, "WHERE"
|
|
216
|
+
case ir.Join(kind="inner" | "cross" | "semi", conjuncts=conjuncts):
|
|
217
|
+
# A semi join's on each output row with its matched right row.
|
|
218
|
+
return conjuncts, "ON"
|
|
219
|
+
case _:
|
|
220
|
+
# An outer join's ON holds on matched rows only; an anti join's on no output row.
|
|
221
|
+
return (), ""
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
def _pass_through(relation: ir.RelationExpr, source: ir.RelationExpr) -> dict[ir.OutputColumn, list[ir.OutputColumn]]:
|
|
225
|
+
"""Each column of `source` that `relation` passes through, with the columns passing it."""
|
|
226
|
+
passed: dict[ir.OutputColumn, list[ir.OutputColumn]] = {}
|
|
227
|
+
for column in relation.output_columns:
|
|
228
|
+
match column.scalar_expr:
|
|
229
|
+
case ir.ColumnRef(column=upstream) if upstream in source.output_columns:
|
|
230
|
+
passed.setdefault(upstream, []).append(column)
|
|
231
|
+
case _:
|
|
232
|
+
pass # A computed column starts its own group.
|
|
233
|
+
return passed
|