sql-diff-explainer 0.1.4__py3-none-any.whl → 0.1.5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sql_diff_explainer/__init__.py +29 -1
- sql_diff_explainer/analysis.py +804 -0
- sql_diff_explainer/cli.py +143 -32
- sql_diff_explainer/core.py +91 -1
- sql_diff_explainer/reports.py +194 -1
- {sql_diff_explainer-0.1.4.dist-info → sql_diff_explainer-0.1.5.dist-info}/METADATA +75 -6
- sql_diff_explainer-0.1.5.dist-info/RECORD +10 -0
- sql_diff_explainer-0.1.4.dist-info/RECORD +0 -9
- {sql_diff_explainer-0.1.4.dist-info → sql_diff_explainer-0.1.5.dist-info}/WHEEL +0 -0
- {sql_diff_explainer-0.1.4.dist-info → sql_diff_explainer-0.1.5.dist-info}/entry_points.txt +0 -0
- {sql_diff_explainer-0.1.4.dist-info → sql_diff_explainer-0.1.5.dist-info}/licenses/LICENSE +0 -0
|
@@ -0,0 +1,804 @@
|
|
|
1
|
+
"""Static analysis helpers for query contracts, lineage, checks, and migrations."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from dataclasses import dataclass, field
|
|
6
|
+
from decimal import Decimal, InvalidOperation
|
|
7
|
+
from typing import TYPE_CHECKING, Any
|
|
8
|
+
|
|
9
|
+
from .core import SqlglotParser, SQLParseError, UnsupportedSQL
|
|
10
|
+
|
|
11
|
+
if TYPE_CHECKING:
|
|
12
|
+
from .core import SQLChange
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
@dataclass(frozen=True)
|
|
16
|
+
class SchemaColumn:
|
|
17
|
+
"""One statically identifiable output column."""
|
|
18
|
+
|
|
19
|
+
name: str
|
|
20
|
+
position: int
|
|
21
|
+
expression: str
|
|
22
|
+
data_type: str | None
|
|
23
|
+
source: str | None
|
|
24
|
+
|
|
25
|
+
def to_dict(self) -> dict[str, object]:
|
|
26
|
+
return {
|
|
27
|
+
"name": self.name,
|
|
28
|
+
"position": self.position,
|
|
29
|
+
"expression": self.expression,
|
|
30
|
+
"type": self.data_type,
|
|
31
|
+
"source": self.source,
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
@dataclass(frozen=True)
|
|
36
|
+
class SchemaChange:
|
|
37
|
+
"""A change to a query's output-column contract."""
|
|
38
|
+
|
|
39
|
+
kind: str
|
|
40
|
+
before: SchemaColumn | None
|
|
41
|
+
after: SchemaColumn | None
|
|
42
|
+
|
|
43
|
+
def to_dict(self) -> dict[str, object]:
|
|
44
|
+
return {
|
|
45
|
+
"kind": self.kind,
|
|
46
|
+
"before": self.before.to_dict() if self.before else None,
|
|
47
|
+
"after": self.after.to_dict() if self.after else None,
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
@dataclass
|
|
52
|
+
class SchemaContractDiff:
|
|
53
|
+
"""Static comparison of the output columns from two simple SELECT queries."""
|
|
54
|
+
|
|
55
|
+
supported: bool = True
|
|
56
|
+
changes: list[SchemaChange] = field(default_factory=list)
|
|
57
|
+
unsupported_reason: str | None = None
|
|
58
|
+
|
|
59
|
+
def to_dict(self) -> dict[str, object]:
|
|
60
|
+
return {
|
|
61
|
+
"supported": self.supported,
|
|
62
|
+
"unsupported_reason": self.unsupported_reason,
|
|
63
|
+
"changes": [change.to_dict() for change in self.changes],
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
@dataclass(frozen=True)
|
|
68
|
+
class OutputLineage:
|
|
69
|
+
"""Source columns and expression contributing to one query output."""
|
|
70
|
+
|
|
71
|
+
name: str
|
|
72
|
+
expression: str
|
|
73
|
+
sources: tuple[str, ...]
|
|
74
|
+
|
|
75
|
+
def to_dict(self) -> dict[str, object]:
|
|
76
|
+
return {
|
|
77
|
+
"name": self.name,
|
|
78
|
+
"expression": self.expression,
|
|
79
|
+
"sources": list(self.sources),
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
@dataclass(frozen=True)
|
|
84
|
+
class LineageChange:
|
|
85
|
+
"""A changed output lineage between two query versions."""
|
|
86
|
+
|
|
87
|
+
kind: str
|
|
88
|
+
before: OutputLineage | None
|
|
89
|
+
after: OutputLineage | None
|
|
90
|
+
|
|
91
|
+
def to_dict(self) -> dict[str, object]:
|
|
92
|
+
return {
|
|
93
|
+
"kind": self.kind,
|
|
94
|
+
"before": self.before.to_dict() if self.before else None,
|
|
95
|
+
"after": self.after.to_dict() if self.after else None,
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
@dataclass
|
|
100
|
+
class CTELineageDiff:
|
|
101
|
+
"""Changed source-to-output lineage for a pair of supported SELECT queries."""
|
|
102
|
+
|
|
103
|
+
supported: bool = True
|
|
104
|
+
changes: list[LineageChange] = field(default_factory=list)
|
|
105
|
+
before: list[OutputLineage] = field(default_factory=list)
|
|
106
|
+
after: list[OutputLineage] = field(default_factory=list)
|
|
107
|
+
unsupported_reason: str | None = None
|
|
108
|
+
|
|
109
|
+
def to_dict(self) -> dict[str, object]:
|
|
110
|
+
return {
|
|
111
|
+
"supported": self.supported,
|
|
112
|
+
"unsupported_reason": self.unsupported_reason,
|
|
113
|
+
"before": [item.to_dict() for item in self.before],
|
|
114
|
+
"after": [item.to_dict() for item in self.after],
|
|
115
|
+
"changes": [change.to_dict() for change in self.changes],
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
@dataclass(frozen=True)
|
|
120
|
+
class ValidationSuggestion:
|
|
121
|
+
"""An advisory validation check; SQL is returned but never executed."""
|
|
122
|
+
|
|
123
|
+
kind: str
|
|
124
|
+
description: str
|
|
125
|
+
sql: str | None
|
|
126
|
+
checklist: tuple[str, ...]
|
|
127
|
+
unsupported_reason: str | None = None
|
|
128
|
+
|
|
129
|
+
def to_dict(self) -> dict[str, object]:
|
|
130
|
+
return {
|
|
131
|
+
"kind": self.kind,
|
|
132
|
+
"description": self.description,
|
|
133
|
+
"advisory": True,
|
|
134
|
+
"executed": False,
|
|
135
|
+
"sql": self.sql,
|
|
136
|
+
"checklist": list(self.checklist),
|
|
137
|
+
"unsupported_reason": self.unsupported_reason,
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
@dataclass(frozen=True)
|
|
142
|
+
class MigrationOperation:
|
|
143
|
+
"""One recognized operation in an ALTER TABLE migration."""
|
|
144
|
+
|
|
145
|
+
kind: str
|
|
146
|
+
table: str
|
|
147
|
+
column: str
|
|
148
|
+
before: str | None
|
|
149
|
+
after: str | None
|
|
150
|
+
statement: str
|
|
151
|
+
risk: str
|
|
152
|
+
|
|
153
|
+
def to_dict(self) -> dict[str, str | None]:
|
|
154
|
+
return {
|
|
155
|
+
"kind": self.kind,
|
|
156
|
+
"table": self.table,
|
|
157
|
+
"column": self.column,
|
|
158
|
+
"before": self.before,
|
|
159
|
+
"after": self.after,
|
|
160
|
+
"statement": self.statement,
|
|
161
|
+
"risk": self.risk,
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
@dataclass(frozen=True)
|
|
166
|
+
class UnsupportedMigrationStatement:
|
|
167
|
+
"""A migration statement or operation outside the supported ALTER subset."""
|
|
168
|
+
|
|
169
|
+
statement: str
|
|
170
|
+
reason: str
|
|
171
|
+
|
|
172
|
+
def to_dict(self) -> dict[str, str]:
|
|
173
|
+
return {"statement": self.statement, "reason": self.reason}
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
@dataclass
|
|
177
|
+
class MigrationAnalysisResult:
|
|
178
|
+
"""Recognized ALTER TABLE operations and explicit unsupported statements."""
|
|
179
|
+
|
|
180
|
+
operations: list[MigrationOperation] = field(default_factory=list)
|
|
181
|
+
unsupported: list[UnsupportedMigrationStatement] = field(default_factory=list)
|
|
182
|
+
|
|
183
|
+
@property
|
|
184
|
+
def supported(self) -> bool:
|
|
185
|
+
return not self.unsupported
|
|
186
|
+
|
|
187
|
+
@property
|
|
188
|
+
def risk(self) -> str:
|
|
189
|
+
if self.unsupported:
|
|
190
|
+
return "high"
|
|
191
|
+
severity = {"low": 0, "medium": 1, "high": 2}
|
|
192
|
+
return max(
|
|
193
|
+
(operation.risk for operation in self.operations),
|
|
194
|
+
key=severity.__getitem__,
|
|
195
|
+
default="low",
|
|
196
|
+
)
|
|
197
|
+
|
|
198
|
+
@property
|
|
199
|
+
def summary(self) -> str:
|
|
200
|
+
if not self.operations and not self.unsupported:
|
|
201
|
+
return "No ALTER TABLE operations detected."
|
|
202
|
+
return (
|
|
203
|
+
f"{len(self.operations)} migration operation(s), "
|
|
204
|
+
f"{len(self.unsupported)} unsupported statement/operation(s); "
|
|
205
|
+
f"overall risk is {self.risk}."
|
|
206
|
+
)
|
|
207
|
+
|
|
208
|
+
def to_dict(self) -> dict[str, object]:
|
|
209
|
+
return {
|
|
210
|
+
"summary": self.summary,
|
|
211
|
+
"risk": self.risk,
|
|
212
|
+
"supported": self.supported,
|
|
213
|
+
"operations": [operation.to_dict() for operation in self.operations],
|
|
214
|
+
"unsupported": [item.to_dict() for item in self.unsupported],
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
def _query_select(tree: Any, purpose: str) -> Any:
|
|
219
|
+
from sqlglot import exp
|
|
220
|
+
|
|
221
|
+
if not isinstance(tree, exp.Select):
|
|
222
|
+
raise UnsupportedSQL(f"{purpose} supports a single SELECT query, not set operations")
|
|
223
|
+
return tree
|
|
224
|
+
|
|
225
|
+
|
|
226
|
+
def _projection_expression(projection: Any) -> Any:
|
|
227
|
+
from sqlglot import exp
|
|
228
|
+
|
|
229
|
+
return projection.this if isinstance(projection, exp.Alias) else projection
|
|
230
|
+
|
|
231
|
+
|
|
232
|
+
def _projection_name(projection: Any) -> str:
|
|
233
|
+
from sqlglot import exp
|
|
234
|
+
|
|
235
|
+
name = projection.alias_or_name
|
|
236
|
+
if name:
|
|
237
|
+
return str(name)
|
|
238
|
+
expression = _projection_expression(projection)
|
|
239
|
+
if isinstance(expression, exp.Column):
|
|
240
|
+
return expression.name
|
|
241
|
+
raise UnsupportedSQL("Output expressions require explicit aliases for analysis")
|
|
242
|
+
|
|
243
|
+
|
|
244
|
+
def _projection_type(expression: Any, dialect: str | None) -> str | None:
|
|
245
|
+
from sqlglot import exp
|
|
246
|
+
|
|
247
|
+
if isinstance(expression, (exp.Cast, exp.TryCast)):
|
|
248
|
+
return expression.args["to"].sql(dialect=dialect or None)
|
|
249
|
+
if isinstance(expression, exp.Literal):
|
|
250
|
+
if expression.is_string:
|
|
251
|
+
return "TEXT"
|
|
252
|
+
literal = str(expression.this)
|
|
253
|
+
try:
|
|
254
|
+
Decimal(literal)
|
|
255
|
+
except InvalidOperation:
|
|
256
|
+
return None
|
|
257
|
+
return "DOUBLE" if any(marker in literal.lower() for marker in (".", "e")) else "INT"
|
|
258
|
+
if isinstance(expression, (exp.Boolean, exp.Null)):
|
|
259
|
+
return "BOOLEAN" if isinstance(expression, exp.Boolean) else "NULL"
|
|
260
|
+
return None
|
|
261
|
+
|
|
262
|
+
|
|
263
|
+
def _projection_source(expression: Any, dialect: str | None) -> str | None:
|
|
264
|
+
from sqlglot import exp
|
|
265
|
+
|
|
266
|
+
if isinstance(expression, exp.Column):
|
|
267
|
+
return expression.sql(dialect=dialect or None)
|
|
268
|
+
if isinstance(expression, (exp.Cast, exp.TryCast)) and isinstance(
|
|
269
|
+
expression.this, exp.Column
|
|
270
|
+
):
|
|
271
|
+
return expression.this.sql(dialect=dialect or None)
|
|
272
|
+
return None
|
|
273
|
+
|
|
274
|
+
|
|
275
|
+
def _schema_columns(tree: Any, dialect: str | None) -> list[SchemaColumn]:
|
|
276
|
+
from sqlglot import exp
|
|
277
|
+
|
|
278
|
+
select = _query_select(tree, "Output-schema contracts")
|
|
279
|
+
if any(isinstance(item, exp.Star) for item in select.expressions) or any(
|
|
280
|
+
isinstance(item, exp.Column) and isinstance(item.this, exp.Star)
|
|
281
|
+
for item in select.expressions
|
|
282
|
+
):
|
|
283
|
+
raise UnsupportedSQL("Output-schema contracts require explicit projections, not *")
|
|
284
|
+
columns = []
|
|
285
|
+
seen: set[str] = set()
|
|
286
|
+
for index, projection in enumerate(select.expressions):
|
|
287
|
+
name = _projection_name(projection)
|
|
288
|
+
if name.casefold() in seen:
|
|
289
|
+
raise UnsupportedSQL(f"Output-schema contracts require unique output names: {name}")
|
|
290
|
+
seen.add(name.casefold())
|
|
291
|
+
expression = _projection_expression(projection)
|
|
292
|
+
columns.append(
|
|
293
|
+
SchemaColumn(
|
|
294
|
+
name=name,
|
|
295
|
+
position=index,
|
|
296
|
+
expression=expression.sql(dialect=dialect or None),
|
|
297
|
+
data_type=_projection_type(expression, dialect),
|
|
298
|
+
source=_projection_source(expression, dialect),
|
|
299
|
+
)
|
|
300
|
+
)
|
|
301
|
+
return columns
|
|
302
|
+
|
|
303
|
+
|
|
304
|
+
def _unsupported_contract(exc: UnsupportedSQL) -> SchemaContractDiff:
|
|
305
|
+
return SchemaContractDiff(supported=False, unsupported_reason=str(exc))
|
|
306
|
+
|
|
307
|
+
|
|
308
|
+
def compare_output_schema(
|
|
309
|
+
before: str,
|
|
310
|
+
after: str,
|
|
311
|
+
*,
|
|
312
|
+
dialect: str | None = None,
|
|
313
|
+
) -> SchemaContractDiff:
|
|
314
|
+
"""Compare output names and statically-known types from simple SELECT queries.
|
|
315
|
+
|
|
316
|
+
Types are reported only when SQL contains a direct cast or literal whose
|
|
317
|
+
type can be identified without a database catalog.
|
|
318
|
+
"""
|
|
319
|
+
parser = SqlglotParser()
|
|
320
|
+
before_tree = parser.parse(before, dialect)
|
|
321
|
+
after_tree = parser.parse(after, dialect)
|
|
322
|
+
try:
|
|
323
|
+
before_columns = _schema_columns(before_tree, dialect)
|
|
324
|
+
after_columns = _schema_columns(after_tree, dialect)
|
|
325
|
+
except UnsupportedSQL as exc:
|
|
326
|
+
return _unsupported_contract(exc)
|
|
327
|
+
|
|
328
|
+
before_remaining = list(before_columns)
|
|
329
|
+
after_remaining = list(after_columns)
|
|
330
|
+
pairs: list[tuple[SchemaColumn, SchemaColumn]] = []
|
|
331
|
+
changes: list[SchemaChange] = []
|
|
332
|
+
|
|
333
|
+
for old in before_columns:
|
|
334
|
+
match = next(
|
|
335
|
+
(new for new in after_remaining if new.name.casefold() == old.name.casefold()),
|
|
336
|
+
None,
|
|
337
|
+
)
|
|
338
|
+
if match is not None:
|
|
339
|
+
pairs.append((old, match))
|
|
340
|
+
before_remaining.remove(old)
|
|
341
|
+
after_remaining.remove(match)
|
|
342
|
+
|
|
343
|
+
for old in list(before_remaining):
|
|
344
|
+
identity = old.source or old.expression
|
|
345
|
+
matching = [
|
|
346
|
+
new
|
|
347
|
+
for new in after_remaining
|
|
348
|
+
if (new.source or new.expression) == identity
|
|
349
|
+
]
|
|
350
|
+
if len(matching) == 1:
|
|
351
|
+
new = matching[0]
|
|
352
|
+
pairs.append((old, new))
|
|
353
|
+
before_remaining.remove(old)
|
|
354
|
+
after_remaining.remove(new)
|
|
355
|
+
|
|
356
|
+
for old, new in pairs:
|
|
357
|
+
renamed = old.name.casefold() != new.name.casefold()
|
|
358
|
+
if renamed:
|
|
359
|
+
changes.append(SchemaChange("renamed", old, new))
|
|
360
|
+
if old.position != new.position:
|
|
361
|
+
changes.append(SchemaChange("reordered", old, new))
|
|
362
|
+
type_changed = (
|
|
363
|
+
old.data_type is not None
|
|
364
|
+
and new.data_type is not None
|
|
365
|
+
and old.data_type.casefold() != new.data_type.casefold()
|
|
366
|
+
)
|
|
367
|
+
expression_changed = old.expression != new.expression and not (
|
|
368
|
+
renamed and old.source is not None and old.source == new.source
|
|
369
|
+
)
|
|
370
|
+
if type_changed:
|
|
371
|
+
changes.append(SchemaChange("type_changed", old, new))
|
|
372
|
+
elif expression_changed and not renamed:
|
|
373
|
+
changes.append(SchemaChange("changed", old, new))
|
|
374
|
+
|
|
375
|
+
changes.extend(SchemaChange("removed", old, None) for old in before_remaining)
|
|
376
|
+
changes.extend(SchemaChange("added", None, new) for new in after_remaining)
|
|
377
|
+
return SchemaContractDiff(changes=changes)
|
|
378
|
+
|
|
379
|
+
|
|
380
|
+
def _lineage_select(
|
|
381
|
+
select: Any,
|
|
382
|
+
ctes: dict[str, dict[str, OutputLineage]],
|
|
383
|
+
dialect: str | None,
|
|
384
|
+
) -> list[OutputLineage]:
|
|
385
|
+
from sqlglot import exp
|
|
386
|
+
|
|
387
|
+
if not isinstance(select, exp.Select):
|
|
388
|
+
raise UnsupportedSQL("CTE lineage supports SELECT CTE bodies and a SELECT output")
|
|
389
|
+
if select.find(exp.Subquery):
|
|
390
|
+
raise UnsupportedSQL("Subqueries are not supported for CTE lineage")
|
|
391
|
+
|
|
392
|
+
sources: dict[str, tuple[str, dict[str, OutputLineage] | None]] = {}
|
|
393
|
+
source_nodes = []
|
|
394
|
+
from_clause = select.args.get("from_")
|
|
395
|
+
if from_clause is not None:
|
|
396
|
+
source_nodes.append(from_clause.this)
|
|
397
|
+
source_nodes.extend(from_clause.expressions or [])
|
|
398
|
+
source_nodes.extend(join.this for join in select.args.get("joins", []))
|
|
399
|
+
for source in source_nodes:
|
|
400
|
+
if not isinstance(source, exp.Table):
|
|
401
|
+
raise UnsupportedSQL("CTE lineage supports direct table and CTE sources only")
|
|
402
|
+
name = source.name
|
|
403
|
+
alias = source.alias_or_name
|
|
404
|
+
cte_outputs = ctes.get(name.casefold())
|
|
405
|
+
if cte_outputs is not None:
|
|
406
|
+
rendered = name
|
|
407
|
+
else:
|
|
408
|
+
source_table = source.copy()
|
|
409
|
+
source_table.set("alias", None)
|
|
410
|
+
rendered = source_table.sql(dialect=dialect or None)
|
|
411
|
+
if alias.casefold() in sources:
|
|
412
|
+
raise UnsupportedSQL(f"CTE lineage cannot resolve duplicate source alias: {alias}")
|
|
413
|
+
sources[alias.casefold()] = (rendered, cte_outputs)
|
|
414
|
+
|
|
415
|
+
outputs: list[OutputLineage] = []
|
|
416
|
+
seen: set[str] = set()
|
|
417
|
+
for projection in select.expressions:
|
|
418
|
+
expression = _projection_expression(projection)
|
|
419
|
+
if isinstance(expression, exp.Star) or (
|
|
420
|
+
isinstance(expression, exp.Column) and isinstance(expression.this, exp.Star)
|
|
421
|
+
):
|
|
422
|
+
raise UnsupportedSQL("CTE lineage requires explicit projections, not *")
|
|
423
|
+
name = _projection_name(projection)
|
|
424
|
+
if name.casefold() in seen:
|
|
425
|
+
raise UnsupportedSQL(f"CTE lineage requires unique output names: {name}")
|
|
426
|
+
seen.add(name.casefold())
|
|
427
|
+
inherited: set[str] = set()
|
|
428
|
+
for column in expression.find_all(exp.Column):
|
|
429
|
+
qualifier = column.table.casefold()
|
|
430
|
+
if qualifier:
|
|
431
|
+
source = sources.get(qualifier)
|
|
432
|
+
if source is None:
|
|
433
|
+
raise UnsupportedSQL(
|
|
434
|
+
f"CTE lineage cannot resolve source qualifier: {column.table}"
|
|
435
|
+
)
|
|
436
|
+
elif len(sources) == 1:
|
|
437
|
+
source = next(iter(sources.values()))
|
|
438
|
+
elif not sources:
|
|
439
|
+
continue
|
|
440
|
+
else:
|
|
441
|
+
raise UnsupportedSQL(
|
|
442
|
+
f"CTE lineage cannot resolve unqualified column: {column.name}"
|
|
443
|
+
)
|
|
444
|
+
table_name, cte_outputs = source
|
|
445
|
+
if cte_outputs is not None:
|
|
446
|
+
upstream = cte_outputs.get(column.name.casefold())
|
|
447
|
+
if upstream is None:
|
|
448
|
+
raise UnsupportedSQL(
|
|
449
|
+
f"CTE lineage cannot resolve {table_name}.{column.name}"
|
|
450
|
+
)
|
|
451
|
+
inherited.update(upstream.sources)
|
|
452
|
+
else:
|
|
453
|
+
inherited.add(f"{table_name}.{column.name}")
|
|
454
|
+
outputs.append(
|
|
455
|
+
OutputLineage(
|
|
456
|
+
name=name,
|
|
457
|
+
expression=expression.sql(dialect=dialect or None),
|
|
458
|
+
sources=tuple(sorted(inherited)),
|
|
459
|
+
)
|
|
460
|
+
)
|
|
461
|
+
return outputs
|
|
462
|
+
|
|
463
|
+
|
|
464
|
+
def _query_lineage(sql: str, dialect: str | None) -> list[OutputLineage]:
|
|
465
|
+
from sqlglot import exp
|
|
466
|
+
|
|
467
|
+
tree = SqlglotParser().parse(sql, dialect)
|
|
468
|
+
if not isinstance(tree, exp.Select):
|
|
469
|
+
raise UnsupportedSQL("CTE lineage supports a single SELECT query")
|
|
470
|
+
with_clause = tree.args.get("with_")
|
|
471
|
+
ctes: dict[str, dict[str, OutputLineage]] = {}
|
|
472
|
+
if with_clause is not None:
|
|
473
|
+
if with_clause.args.get("recursive"):
|
|
474
|
+
raise UnsupportedSQL("Recursive CTE lineage is not supported")
|
|
475
|
+
for cte in with_clause.expressions:
|
|
476
|
+
alias = cte.alias
|
|
477
|
+
if not alias:
|
|
478
|
+
raise UnsupportedSQL("CTE lineage requires named CTEs")
|
|
479
|
+
if cte.this.args.get("with_"):
|
|
480
|
+
raise UnsupportedSQL("Nested WITH clauses are not supported for CTE lineage")
|
|
481
|
+
alias_node = cte.args.get("alias")
|
|
482
|
+
if alias_node is not None and alias_node.args.get("columns"):
|
|
483
|
+
raise UnsupportedSQL("CTE column alias lists are not supported")
|
|
484
|
+
outputs = _lineage_select(cte.this, ctes, dialect)
|
|
485
|
+
ctes[alias.casefold()] = {item.name.casefold(): item for item in outputs}
|
|
486
|
+
return _lineage_select(tree, ctes, dialect)
|
|
487
|
+
|
|
488
|
+
|
|
489
|
+
def trace_cte_lineage(
|
|
490
|
+
before: str,
|
|
491
|
+
after: str,
|
|
492
|
+
*,
|
|
493
|
+
dialect: str | None = None,
|
|
494
|
+
) -> CTELineageDiff:
|
|
495
|
+
"""Trace source-column lineage through supported CTE chains and compare outputs.
|
|
496
|
+
|
|
497
|
+
Unsupported query shapes are returned explicitly in ``unsupported_reason``.
|
|
498
|
+
"""
|
|
499
|
+
try:
|
|
500
|
+
before_outputs = _query_lineage(before, dialect)
|
|
501
|
+
after_outputs = _query_lineage(after, dialect)
|
|
502
|
+
except UnsupportedSQL as exc:
|
|
503
|
+
return CTELineageDiff(supported=False, unsupported_reason=str(exc))
|
|
504
|
+
|
|
505
|
+
before_by_name = {item.name.casefold(): item for item in before_outputs}
|
|
506
|
+
after_by_name = {item.name.casefold(): item for item in after_outputs}
|
|
507
|
+
changes = []
|
|
508
|
+
output_names = list(before_by_name)
|
|
509
|
+
output_names.extend(name for name in after_by_name if name not in before_by_name)
|
|
510
|
+
for name in output_names:
|
|
511
|
+
old = before_by_name.get(name)
|
|
512
|
+
new = after_by_name.get(name)
|
|
513
|
+
if old is None:
|
|
514
|
+
changes.append(LineageChange("added", None, new))
|
|
515
|
+
elif new is None:
|
|
516
|
+
changes.append(LineageChange("removed", old, None))
|
|
517
|
+
elif old.expression != new.expression or old.sources != new.sources:
|
|
518
|
+
changes.append(LineageChange("changed", old, new))
|
|
519
|
+
return CTELineageDiff(
|
|
520
|
+
changes=changes,
|
|
521
|
+
before=before_outputs,
|
|
522
|
+
after=after_outputs,
|
|
523
|
+
)
|
|
524
|
+
|
|
525
|
+
|
|
526
|
+
def generate_validation_suggestions(
|
|
527
|
+
before: str,
|
|
528
|
+
after: str,
|
|
529
|
+
changes: list[SQLChange],
|
|
530
|
+
contract: SchemaContractDiff,
|
|
531
|
+
*,
|
|
532
|
+
dialect: str | None = None,
|
|
533
|
+
) -> list[ValidationSuggestion]:
|
|
534
|
+
"""Create advisory row-difference and duplicate checks for relevant changes."""
|
|
535
|
+
if not changes:
|
|
536
|
+
return []
|
|
537
|
+
|
|
538
|
+
kinds = {change.category for change in changes}
|
|
539
|
+
suggestions: list[ValidationSuggestion] = []
|
|
540
|
+
row_changes = {
|
|
541
|
+
"where",
|
|
542
|
+
"join",
|
|
543
|
+
"select",
|
|
544
|
+
"group_by",
|
|
545
|
+
"aggregation",
|
|
546
|
+
"cte_lineage",
|
|
547
|
+
"distinct",
|
|
548
|
+
"limit",
|
|
549
|
+
"offset",
|
|
550
|
+
"schema_contract",
|
|
551
|
+
}
|
|
552
|
+
if kinds & row_changes:
|
|
553
|
+
compatible = contract.supported and not any(
|
|
554
|
+
change.kind in {"added", "removed", "renamed", "reordered", "type_changed"}
|
|
555
|
+
for change in contract.changes
|
|
556
|
+
)
|
|
557
|
+
if compatible:
|
|
558
|
+
before_query = before.strip().rstrip(";")
|
|
559
|
+
after_query = after.strip().rstrip(";")
|
|
560
|
+
row_sql = (
|
|
561
|
+
"WITH before_result AS (\n"
|
|
562
|
+
f"{before_query}\n"
|
|
563
|
+
"), after_result AS (\n"
|
|
564
|
+
f"{after_query}\n"
|
|
565
|
+
")\n"
|
|
566
|
+
"SELECT 'before_only' AS diff_side, diff.* FROM (\n"
|
|
567
|
+
" SELECT * FROM before_result\n"
|
|
568
|
+
" EXCEPT\n"
|
|
569
|
+
" SELECT * FROM after_result\n"
|
|
570
|
+
") AS diff\n"
|
|
571
|
+
"UNION ALL\n"
|
|
572
|
+
"SELECT 'after_only' AS diff_side, diff.* FROM (\n"
|
|
573
|
+
" SELECT * FROM after_result\n"
|
|
574
|
+
" EXCEPT\n"
|
|
575
|
+
" SELECT * FROM before_result\n"
|
|
576
|
+
") AS diff;"
|
|
577
|
+
)
|
|
578
|
+
suggestions.append(
|
|
579
|
+
ValidationSuggestion(
|
|
580
|
+
"unmatched_rows",
|
|
581
|
+
"Compare rows present in only one query result.",
|
|
582
|
+
row_sql,
|
|
583
|
+
(
|
|
584
|
+
"Confirm both query outputs have compatible column order and types.",
|
|
585
|
+
"Review unmatched rows against intended filtering and join behavior.",
|
|
586
|
+
),
|
|
587
|
+
)
|
|
588
|
+
)
|
|
589
|
+
else:
|
|
590
|
+
if not contract.supported:
|
|
591
|
+
reason = contract.unsupported_reason or "Output schema could not be analyzed."
|
|
592
|
+
elif any(
|
|
593
|
+
change.kind in {"added", "removed", "renamed", "reordered"}
|
|
594
|
+
for change in contract.changes
|
|
595
|
+
):
|
|
596
|
+
reason = "Output columns differ; align names and order before comparing rows."
|
|
597
|
+
else:
|
|
598
|
+
reason = "Output types changed; align types before comparing rows."
|
|
599
|
+
suggestions.append(
|
|
600
|
+
ValidationSuggestion(
|
|
601
|
+
"unmatched_rows",
|
|
602
|
+
"Review rows affected by the query change.",
|
|
603
|
+
None,
|
|
604
|
+
(
|
|
605
|
+
"Compare before/after row counts and representative keys.",
|
|
606
|
+
"Review unmatched rows after aligning output columns and types.",
|
|
607
|
+
),
|
|
608
|
+
reason,
|
|
609
|
+
)
|
|
610
|
+
)
|
|
611
|
+
|
|
612
|
+
if kinds & {"join", "group_by", "aggregation", "distinct"}:
|
|
613
|
+
if contract.supported:
|
|
614
|
+
after_columns = _schema_columns(SqlglotParser().parse(after, dialect), dialect)
|
|
615
|
+
if after_columns:
|
|
616
|
+
duplicate_query = after
|
|
617
|
+
description = "Find duplicate rows in the updated query output."
|
|
618
|
+
distinct_added = any(
|
|
619
|
+
change.category == "distinct"
|
|
620
|
+
and change.before == "false"
|
|
621
|
+
and change.after == "true"
|
|
622
|
+
for change in changes
|
|
623
|
+
)
|
|
624
|
+
if distinct_added:
|
|
625
|
+
duplicate_query = before
|
|
626
|
+
description = "Check duplicates in the input before DISTINCT was added."
|
|
627
|
+
duplicate_columns = _schema_columns(
|
|
628
|
+
SqlglotParser().parse(duplicate_query, dialect), dialect
|
|
629
|
+
)
|
|
630
|
+
column_sql = [
|
|
631
|
+
f"result.{_quote_identifier(column.name, dialect)}"
|
|
632
|
+
for column in duplicate_columns
|
|
633
|
+
]
|
|
634
|
+
grouped = ", ".join(column_sql)
|
|
635
|
+
duplicate_sql = (
|
|
636
|
+
"SELECT "
|
|
637
|
+
f"{grouped}, COUNT(*) AS duplicate_count\n"
|
|
638
|
+
f"FROM (\n{duplicate_query.strip().rstrip(';')}\n) AS result\n"
|
|
639
|
+
f"GROUP BY {grouped}\n"
|
|
640
|
+
"HAVING COUNT(*) > 1;"
|
|
641
|
+
)
|
|
642
|
+
suggestions.append(
|
|
643
|
+
ValidationSuggestion(
|
|
644
|
+
"duplicate_rows",
|
|
645
|
+
description,
|
|
646
|
+
duplicate_sql,
|
|
647
|
+
(
|
|
648
|
+
"Confirm whether duplicates are valid at the intended output grain.",
|
|
649
|
+
"Inspect join keys and grouping columns if duplicates are unexpected.",
|
|
650
|
+
),
|
|
651
|
+
)
|
|
652
|
+
)
|
|
653
|
+
else:
|
|
654
|
+
suggestions.append(
|
|
655
|
+
ValidationSuggestion(
|
|
656
|
+
"duplicate_rows",
|
|
657
|
+
"Review duplicate behavior in the updated query.",
|
|
658
|
+
None,
|
|
659
|
+
("Check duplicate counts at the intended output grain.",),
|
|
660
|
+
"No explicit output columns were found.",
|
|
661
|
+
)
|
|
662
|
+
)
|
|
663
|
+
else:
|
|
664
|
+
suggestions.append(
|
|
665
|
+
ValidationSuggestion(
|
|
666
|
+
"duplicate_rows",
|
|
667
|
+
"Review duplicate behavior in the updated query.",
|
|
668
|
+
None,
|
|
669
|
+
("Check duplicate counts at the intended output grain.",),
|
|
670
|
+
contract.unsupported_reason,
|
|
671
|
+
)
|
|
672
|
+
)
|
|
673
|
+
return suggestions
|
|
674
|
+
|
|
675
|
+
|
|
676
|
+
def _quote_identifier(name: str, dialect: str | None) -> str:
|
|
677
|
+
from sqlglot import exp
|
|
678
|
+
|
|
679
|
+
return exp.Column(this=exp.to_identifier(name)).sql(dialect=dialect or None)
|
|
680
|
+
|
|
681
|
+
|
|
682
|
+
def _identifier_name(node: Any) -> str:
|
|
683
|
+
from sqlglot import exp
|
|
684
|
+
|
|
685
|
+
if isinstance(node, exp.Column):
|
|
686
|
+
return node.name
|
|
687
|
+
if isinstance(node, exp.Identifier):
|
|
688
|
+
return str(node.this)
|
|
689
|
+
return node.sql()
|
|
690
|
+
|
|
691
|
+
|
|
692
|
+
def analyze_migration(
|
|
693
|
+
sql: str,
|
|
694
|
+
*,
|
|
695
|
+
dialect: str | None = None,
|
|
696
|
+
) -> MigrationAnalysisResult:
|
|
697
|
+
"""Analyze common column operations in a multi-statement ALTER TABLE script.
|
|
698
|
+
|
|
699
|
+
Only add/drop/rename column, change column type, and nullability changes
|
|
700
|
+
are interpreted. All other statements and ALTER actions are reported as
|
|
701
|
+
unsupported instead of being silently ignored.
|
|
702
|
+
"""
|
|
703
|
+
import sqlglot
|
|
704
|
+
from sqlglot import exp
|
|
705
|
+
|
|
706
|
+
if not sql.strip():
|
|
707
|
+
raise SQLParseError("SQL input is empty")
|
|
708
|
+
try:
|
|
709
|
+
statements = sqlglot.parse(sql, read=dialect or None)
|
|
710
|
+
except Exception as exc:
|
|
711
|
+
raise SQLParseError(f"Unable to parse migration SQL: {exc}") from exc
|
|
712
|
+
|
|
713
|
+
result = MigrationAnalysisResult()
|
|
714
|
+
for statement in statements:
|
|
715
|
+
if statement is None:
|
|
716
|
+
continue
|
|
717
|
+
statement_text = statement.sql(dialect=dialect or None)
|
|
718
|
+
if not isinstance(statement, exp.Alter) or statement.args.get("kind") != "TABLE":
|
|
719
|
+
result.unsupported.append(
|
|
720
|
+
UnsupportedMigrationStatement(
|
|
721
|
+
statement_text,
|
|
722
|
+
"Only ALTER TABLE statements are supported by migration analysis.",
|
|
723
|
+
)
|
|
724
|
+
)
|
|
725
|
+
continue
|
|
726
|
+
table = statement.this.sql(dialect=dialect or None)
|
|
727
|
+
actions = statement.args.get("actions") or []
|
|
728
|
+
if not actions:
|
|
729
|
+
result.unsupported.append(
|
|
730
|
+
UnsupportedMigrationStatement(statement_text, "No ALTER TABLE actions found.")
|
|
731
|
+
)
|
|
732
|
+
continue
|
|
733
|
+
for action in actions:
|
|
734
|
+
operation: MigrationOperation | None = None
|
|
735
|
+
if isinstance(action, exp.ColumnDef):
|
|
736
|
+
operation = MigrationOperation(
|
|
737
|
+
"column_added",
|
|
738
|
+
table,
|
|
739
|
+
_identifier_name(action.this),
|
|
740
|
+
None,
|
|
741
|
+
action.args.get("kind").sql(dialect=dialect or None)
|
|
742
|
+
if action.args.get("kind") is not None
|
|
743
|
+
else None,
|
|
744
|
+
statement_text,
|
|
745
|
+
"medium",
|
|
746
|
+
)
|
|
747
|
+
elif isinstance(action, exp.Drop) and str(action.args.get("kind")).upper() == "COLUMN":
|
|
748
|
+
for column in action.args.get("expressions") or action.args.get("tables") or []:
|
|
749
|
+
result.operations.append(
|
|
750
|
+
MigrationOperation(
|
|
751
|
+
"column_removed",
|
|
752
|
+
table,
|
|
753
|
+
_identifier_name(column),
|
|
754
|
+
None,
|
|
755
|
+
None,
|
|
756
|
+
statement_text,
|
|
757
|
+
"high",
|
|
758
|
+
)
|
|
759
|
+
)
|
|
760
|
+
continue
|
|
761
|
+
elif isinstance(action, exp.RenameColumn):
|
|
762
|
+
operation = MigrationOperation(
|
|
763
|
+
"column_renamed",
|
|
764
|
+
table,
|
|
765
|
+
_identifier_name(action.this),
|
|
766
|
+
_identifier_name(action.this),
|
|
767
|
+
_identifier_name(action.args["to"]),
|
|
768
|
+
statement_text,
|
|
769
|
+
"high",
|
|
770
|
+
)
|
|
771
|
+
elif isinstance(action, exp.AlterColumn):
|
|
772
|
+
column = _identifier_name(action.this)
|
|
773
|
+
data_type = action.args.get("dtype")
|
|
774
|
+
if data_type is not None:
|
|
775
|
+
operation = MigrationOperation(
|
|
776
|
+
"column_type_changed",
|
|
777
|
+
table,
|
|
778
|
+
column,
|
|
779
|
+
None,
|
|
780
|
+
data_type.sql(dialect=dialect or None),
|
|
781
|
+
statement_text,
|
|
782
|
+
"medium",
|
|
783
|
+
)
|
|
784
|
+
elif action.args.get("allow_null") is not None:
|
|
785
|
+
nullable = bool(action.args["allow_null"])
|
|
786
|
+
operation = MigrationOperation(
|
|
787
|
+
"nullability_changed",
|
|
788
|
+
table,
|
|
789
|
+
column,
|
|
790
|
+
"NOT NULL" if nullable else "NULL",
|
|
791
|
+
"NULL" if nullable else "NOT NULL",
|
|
792
|
+
statement_text,
|
|
793
|
+
"medium",
|
|
794
|
+
)
|
|
795
|
+
if operation is None:
|
|
796
|
+
result.unsupported.append(
|
|
797
|
+
UnsupportedMigrationStatement(
|
|
798
|
+
statement_text,
|
|
799
|
+
f"Unsupported ALTER TABLE action: {action.sql(dialect=dialect or None)}",
|
|
800
|
+
)
|
|
801
|
+
)
|
|
802
|
+
else:
|
|
803
|
+
result.operations.append(operation)
|
|
804
|
+
return result
|