sql-code-graph 1.4.0__py3-none-any.whl → 1.4.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {sql_code_graph-1.4.0.dist-info → sql_code_graph-1.4.3.dist-info}/METADATA +1 -1
- {sql_code_graph-1.4.0.dist-info → sql_code_graph-1.4.3.dist-info}/RECORD +12 -12
- sqlcg/__init__.py +1 -1
- sqlcg/cli/commands/analyze.py +8 -1
- sqlcg/core/queries.py +4 -0
- sqlcg/core/queries.sql +60 -0
- sqlcg/parsers/ansi_parser.py +7 -2
- sqlcg/parsers/base.py +82 -0
- sqlcg/server/models.py +6 -0
- sqlcg/server/tools.py +169 -27
- {sql_code_graph-1.4.0.dist-info → sql_code_graph-1.4.3.dist-info}/WHEEL +0 -0
- {sql_code_graph-1.4.0.dist-info → sql_code_graph-1.4.3.dist-info}/entry_points.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: sql-code-graph
|
|
3
|
-
Version: 1.4.
|
|
3
|
+
Version: 1.4.3
|
|
4
4
|
Summary: SQL code graph analyzer and lineage tracer
|
|
5
5
|
Project-URL: Homepage, https://github.com/Warhorze/sql-code-graph
|
|
6
6
|
Project-URL: Repository, https://github.com/Warhorze/sql-code-graph
|
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
sqlcg/__init__.py,sha256
|
|
1
|
+
sqlcg/__init__.py,sha256=-pjjoaMnukW9b0-s_OOYNjy46alw_PTB6xzvfSyWjew,115
|
|
2
2
|
sqlcg/__main__.py,sha256=1YoFLcqEgTwYq1J3TbUwpkdG0zeeLIf2fJvwWI-CLFU,109
|
|
3
3
|
sqlcg/cli/__init__.py,sha256=W8fD0LpMq2xm_5WKGNMvJh2WBL1ho5E8hUeAqXQYT1g,28
|
|
4
4
|
sqlcg/cli/main.py,sha256=WmdTjsOlz1ozi2Y3Aq4ezR_FCRl-Lc1YOKw3_d48dlY,1650
|
|
5
5
|
sqlcg/cli/commands/__init__.py,sha256=oSHtr6VD-jNubOjuCQyZj2tBppjMEpQDh-IGQ8of9eA,30
|
|
6
|
-
sqlcg/cli/commands/analyze.py,sha256=
|
|
6
|
+
sqlcg/cli/commands/analyze.py,sha256=_aC5ML3w7YdLi7DL3TFS9OiCEIipuNZxWR6S4peTcn4,12154
|
|
7
7
|
sqlcg/cli/commands/db.py,sha256=TMhPCHRWSo8YmlNnSxmGxrR0_1r3K4e0Bn8unZTkvU4,7248
|
|
8
8
|
sqlcg/cli/commands/find.py,sha256=p5Vyyx-VBk8YDWYQN16UhECh7PIeMaEyCPEcUFcRFlM,2598
|
|
9
9
|
sqlcg/cli/commands/gain.py,sha256=SJU1c51a7MgNbZItqQnaBfOWGnV5xpXQctbew5Dr9BE,9062
|
|
@@ -22,8 +22,8 @@ sqlcg/core/freshness.py,sha256=F9jWn2cbFs60jA9ta8KrT6MghD0mzI7SqqKs_Op9AeU,4577
|
|
|
22
22
|
sqlcg/core/graph_db.py,sha256=eLYdmiqPWrZHtKLFcKgD2aiWZHFU3wrwz6Y0A3d9NcE,8633
|
|
23
23
|
sqlcg/core/jobs.py,sha256=Je-fCdSKRgiSsv1W8SgNAlp36a7t7-pJZ-qKPbka9OE,3298
|
|
24
24
|
sqlcg/core/queries.cypher,sha256=cvPOVe5GUOzJN4bxUvDxNI--xIIP8gm42TR-gUnea4U,4685
|
|
25
|
-
sqlcg/core/queries.py,sha256=
|
|
26
|
-
sqlcg/core/queries.sql,sha256=
|
|
25
|
+
sqlcg/core/queries.py,sha256=DeGrpsh-N8CczapKW5Iw_CPBMOxtWZfusMOhpN4Qn_c,3319
|
|
26
|
+
sqlcg/core/queries.sql,sha256=yRITKvjnWY0e1Sa8me0yFe_FkkIBsk3BQn2l0iqOJ7Q,10768
|
|
27
27
|
sqlcg/core/schema.cypher,sha256=rK5QMhSrzZhuj73NeNXGX6oM-rPPPvxFjex0fEyUvkQ,2859
|
|
28
28
|
sqlcg/core/schema.py,sha256=7fKf314ueIV7-tIkQQUS4O6H-OhlFKFVFFvGFqw_5Xk,1476
|
|
29
29
|
sqlcg/indexer/__init__.py,sha256=Wh20Unz2OHs1oIyWLrpurPAasF0BET2g4iXtNk7mh2U,56
|
|
@@ -40,8 +40,8 @@ sqlcg/lineage/schema_resolver.py,sha256=iXt6LYF6UVWsGUpcfbmjmGn9wCgXl721lTGf_8Aa
|
|
|
40
40
|
sqlcg/metrics/__init__.py,sha256=hLJ6wm4St8qqYwKh3o9QG7lcEt1BEYM31ccqO9tGpIg,133
|
|
41
41
|
sqlcg/metrics/store.py,sha256=KuDtxvyAgug9_KtiSCpvgKM2VZM7VSaI3D11uMLjJJk,10604
|
|
42
42
|
sqlcg/parsers/__init__.py,sha256=AamA8wBbDZV9_zEtZCI4Hyen5UAVKHmBwjTghTt2PZE,785
|
|
43
|
-
sqlcg/parsers/ansi_parser.py,sha256=
|
|
44
|
-
sqlcg/parsers/base.py,sha256=
|
|
43
|
+
sqlcg/parsers/ansi_parser.py,sha256=UNousZcTgHmgDzbvqF77hPf8RmTqwz4oo1rk9OMGWz4,17570
|
|
44
|
+
sqlcg/parsers/base.py,sha256=N6uqQWHO2lpzTIWgPqzewAXmij0ikZdNmNVJkIQ8Mr0,54889
|
|
45
45
|
sqlcg/parsers/bigquery_parser.py,sha256=mOnWTfXB_Dp4JwFE1PVYOB6CDPf5nYE0Dea8kJCl9uQ,2827
|
|
46
46
|
sqlcg/parsers/postgres_parser.py,sha256=lYfUpQY6j4Qm7ndXBtXbgPoGzYqYddWt5YeFnWKdA6I,946
|
|
47
47
|
sqlcg/parsers/registry.py,sha256=LXy1F6rqQI6VdxpRvZg_tNpoEucW3mXZHYBMlMONbX4,1496
|
|
@@ -50,18 +50,18 @@ sqlcg/parsers/tsql_parser.py,sha256=RRj1pACtAk2tLTDaFWRYF67a0IDvaf5A1YQXWIz0bpQ,
|
|
|
50
50
|
sqlcg/server/__init__.py,sha256=n4wuNE7xyJIJxJZBtmtdccCMQfvTdF-IqIaZVbC4FC4,35
|
|
51
51
|
sqlcg/server/control.py,sha256=qUcztb_aDhL-_X_Nq4q6uGx17cUlbLnI6vUpoZsEjbo,4506
|
|
52
52
|
sqlcg/server/exceptions.py,sha256=EONw34icOByCTpppSQrvQBW6asc4hfqaGDCAFjv96II,469
|
|
53
|
-
sqlcg/server/models.py,sha256=
|
|
53
|
+
sqlcg/server/models.py,sha256=plnBVoV6y6JuU2IHzez0yHfsd5QZ_o977UCacS7A5NA,19693
|
|
54
54
|
sqlcg/server/noise_filter.py,sha256=Ats2FFzmzFKqvQPWvlUzK8mY9pzlUhL4m1s8P_HNnvI,6335
|
|
55
55
|
sqlcg/server/read_client.py,sha256=4v1OOl12HCHp6J14HrcWQApSixvJ0wwE8UIIZ0pzLyw,7808
|
|
56
56
|
sqlcg/server/server.py,sha256=A1soT-hYnAX78X5j97nIjw9oIZCxBejjV6p_tRDIK38,24335
|
|
57
57
|
sqlcg/server/skill.py,sha256=EDvmEgl-LQwCxZ9Lca4lt8zCtkk2cCgh9GLYlGgDA64,12828
|
|
58
|
-
sqlcg/server/tools.py,sha256=
|
|
58
|
+
sqlcg/server/tools.py,sha256=oelzyVydELcD0ePYQMiIdZPqm-GolxD4YxtKUtGSeTU,76744
|
|
59
59
|
sqlcg/server/writer.py,sha256=HH8pW6k1c2pbY4b6g2N87Tx4gNCezYqTLNTlAmsHg2Y,18522
|
|
60
60
|
sqlcg/utils/__init__.py,sha256=--iqt5ThTXmT8Wz7da8hs3n0zDfYPl8P-z5OgRJ_77E,154
|
|
61
61
|
sqlcg/utils/hashing.py,sha256=H25-sYfxHKb3_IERFnHyAIYNiXN470Oqo5sJT_D3YOA,438
|
|
62
62
|
sqlcg/utils/ignore.py,sha256=wJjwa0mjnQ_xJExOUxk25y00g065XmmzJapqV3ifD5o,1151
|
|
63
63
|
sqlcg/utils/logging.py,sha256=u0fCmYsLj9o81vawm3xZTHaw68GQYVm7JxG-gP81u8A,840
|
|
64
|
-
sql_code_graph-1.4.
|
|
65
|
-
sql_code_graph-1.4.
|
|
66
|
-
sql_code_graph-1.4.
|
|
67
|
-
sql_code_graph-1.4.
|
|
64
|
+
sql_code_graph-1.4.3.dist-info/METADATA,sha256=a-9uSmKrh5FDurLJg_LFgkaNH8K1_ke-engcyBzhues,14085
|
|
65
|
+
sql_code_graph-1.4.3.dist-info/WHEEL,sha256=mffPy8wBnZQn2VnJUU5jE99KsxaSfiyMHV9Yt0aLVxs,87
|
|
66
|
+
sql_code_graph-1.4.3.dist-info/entry_points.txt,sha256=Wfe49sVzV9p4eVFGo5RxcV-frr3HOP0yzzst8JBxQLQ,46
|
|
67
|
+
sql_code_graph-1.4.3.dist-info/RECORD,,
|
sqlcg/__init__.py
CHANGED
sqlcg/cli/commands/analyze.py
CHANGED
|
@@ -139,6 +139,7 @@ def upstream( # noqa: B008
|
|
|
139
139
|
console.print("[red]Error: --depth must be between 1 and 100[/red]")
|
|
140
140
|
raise typer.Exit(1)
|
|
141
141
|
|
|
142
|
+
ref = ref.lower() # graph keys are lowercased at index time (C2 normalization)
|
|
142
143
|
sql = _upstream_sql(depth, include_intermediate)
|
|
143
144
|
results = run_read_routed(sql, {"ref": ref})
|
|
144
145
|
if not results and len(ref.split(".")) >= 3:
|
|
@@ -175,6 +176,7 @@ def downstream( # noqa: B008
|
|
|
175
176
|
console.print("[red]Error: --depth must be between 1 and 100[/red]")
|
|
176
177
|
raise typer.Exit(1)
|
|
177
178
|
|
|
179
|
+
ref = ref.lower() # graph keys are lowercased at index time (C2 normalization)
|
|
178
180
|
sql = _downstream_sql(depth, include_intermediate)
|
|
179
181
|
results = run_read_routed(sql, {"ref": ref})
|
|
180
182
|
if not results and len(ref.split(".")) >= 3:
|
|
@@ -291,7 +293,12 @@ def unused(
|
|
|
291
293
|
|
|
292
294
|
|
|
293
295
|
def _bare_ref(ref: str) -> str:
|
|
294
|
-
"""Strip schema prefix from a ref string, keeping table.column.
|
|
296
|
+
"""Strip schema prefix from a ref string, keeping table.column.
|
|
297
|
+
|
|
298
|
+
Lowercases defensively so this is safe to call even if the caller did not
|
|
299
|
+
first fold the ref — graph keys are lowercased at index time (C2 normalization).
|
|
300
|
+
"""
|
|
301
|
+
ref = ref.lower()
|
|
295
302
|
parts = ref.split(".")
|
|
296
303
|
if len(parts) >= 3:
|
|
297
304
|
return ".".join(parts[1:])
|
sqlcg/core/queries.py
CHANGED
|
@@ -54,9 +54,13 @@ COUNT_STAR_SOURCES_QUERY = _Q["COUNT_STAR_SOURCES"]
|
|
|
54
54
|
COUNT_STAR_EXPANSIONS_QUERY = _Q["COUNT_STAR_EXPANSIONS"]
|
|
55
55
|
FIND_DEFINITION_QUERY = _Q["FIND_DEFINITION"]
|
|
56
56
|
GET_TABLE_DEFINING_FILES_QUERY = _Q["GET_TABLE_DEFINING_FILES"]
|
|
57
|
+
GET_PRODUCER_FILES_FOR_TABLE_QUERY = _Q["GET_PRODUCER_FILES_FOR_TABLE"]
|
|
57
58
|
GET_TABLE_DIRECT_UPSTREAMS_QUERY = _Q["GET_TABLE_DIRECT_UPSTREAMS"]
|
|
58
59
|
GET_COLUMNS_FOR_TABLE_QUERY = _Q["GET_COLUMNS_FOR_TABLE"]
|
|
59
60
|
GET_TABLES_DEFINED_IN_FILE_QUERY = _Q["GET_TABLES_DEFINED_IN_FILE"]
|
|
61
|
+
GET_TARGET_TABLES_FOR_FILE_QUERY = _Q["GET_TARGET_TABLES_FOR_FILE"]
|
|
62
|
+
GET_TABLE_ADJACENCY_FOR_COLUMNS_QUERY = _Q["GET_TABLE_ADJACENCY_FOR_COLUMNS"]
|
|
63
|
+
GET_TABLE_KINDS_BATCH_QUERY = _Q["GET_TABLE_KINDS_BATCH"]
|
|
60
64
|
ANALYZE_UNUSED_TABLES_QUERY = _Q["ANALYZE_UNUSED_TABLES"]
|
|
61
65
|
HUB_RANKING_QUERY = _Q["HUB_RANKING"]
|
|
62
66
|
DEPENDENT_FILES_OF_TABLES_QUERY = _Q["DEPENDENT_FILES_OF_TABLES"]
|
sqlcg/core/queries.sql
CHANGED
|
@@ -172,6 +172,19 @@ JOIN "DEFINED_IN" di ON di.src_key = t.qualified
|
|
|
172
172
|
JOIN "File" f ON f.path = di.dst_key
|
|
173
173
|
WHERE t.qualified = ?
|
|
174
174
|
|
|
175
|
+
-- GET_PRODUCER_FILES_FOR_TABLE
|
|
176
|
+
-- ETL INSERT...SELECT producers populate a table without a DEFINED_IN edge
|
|
177
|
+
-- (that edge is DDL-only). Resolve SqlQuery.target_table -> QUERY_DEFINED_IN -> File
|
|
178
|
+
-- so get_definition / get_change_scope can also surface "populated here" producer
|
|
179
|
+
-- files, not just "defined here" DDL files (mirror of GET_TARGET_TABLES_FOR_FILE,
|
|
180
|
+
-- the reverse-direction lookup). table_qualified is stored lowercase.
|
|
181
|
+
-- params: [table_qualified]
|
|
182
|
+
SELECT DISTINCT f.path AS file_path
|
|
183
|
+
FROM "SqlQuery" q
|
|
184
|
+
JOIN "QUERY_DEFINED_IN" qdi ON qdi.src_key = q.id
|
|
185
|
+
JOIN "File" f ON f.path = qdi.dst_key
|
|
186
|
+
WHERE q.target_table = ?
|
|
187
|
+
|
|
175
188
|
-- GET_TABLE_DIRECT_UPSTREAMS
|
|
176
189
|
-- params: [table_qualified, table_qualified]
|
|
177
190
|
SELECT DISTINCT src.qualified AS upstream_table, f.path AS in_file
|
|
@@ -198,6 +211,53 @@ FROM "SqlTable" t
|
|
|
198
211
|
JOIN "DEFINED_IN" di ON di.src_key = t.qualified
|
|
199
212
|
WHERE di.dst_key = ?
|
|
200
213
|
|
|
214
|
+
-- GET_TARGET_TABLES_FOR_FILE
|
|
215
|
+
-- ETL INSERT...SELECT producers populate a table without a DEFINED_IN edge
|
|
216
|
+
-- (that edge is DDL-only). Resolve query->file QUERY_DEFINED_IN -> SqlQuery.target_table
|
|
217
|
+
-- so diff_impact can also see "populated here" producers, not just "defined here" DDL.
|
|
218
|
+
-- params: [file_path]
|
|
219
|
+
SELECT DISTINCT q.target_table AS table_qualified
|
|
220
|
+
FROM "SqlQuery" q
|
|
221
|
+
JOIN "QUERY_DEFINED_IN" qdi ON qdi.src_key = q.id
|
|
222
|
+
WHERE qdi.dst_key = ?
|
|
223
|
+
AND q.target_table <> ''
|
|
224
|
+
|
|
225
|
+
-- GET_TABLE_ADJACENCY_FOR_COLUMNS
|
|
226
|
+
-- Aggregate table-level producer->consumer adjacency derived from COLUMN_LINEAGE,
|
|
227
|
+
-- restricted to a closure's column-id set (Option A — issue #38 backfill fix).
|
|
228
|
+
-- Replaces the N x GET_TABLE_DIRECT_UPSTREAMS loop with a single query: CTE-wrapped
|
|
229
|
+
-- INSERT...SELECT statements emit no SELECTS_FROM adjacency at all (the real source
|
|
230
|
+
-- table is nested in the CTE child scope and never surfaces into the statement's
|
|
231
|
+
-- top-level sources). The raw COLUMN_LINEAGE topology bridges producer -> consumer
|
|
232
|
+
-- via a TWO-HOP path through the synthetic cte/derived node (producer -> cte ->
|
|
233
|
+
-- consumer, not a parallel direct edge) — this query returns ALL rolled-up table
|
|
234
|
+
-- pairs (including synthetic endpoints); the caller contracts synthetic-node hops
|
|
235
|
+
-- into direct real-table adjacency in one pass over this small edge set (still
|
|
236
|
+
-- ONCE per closure, never per-table/per-column). Also returns each endpoint's
|
|
237
|
+
-- SqlTable.kind so the caller can identify synthetic nodes without a second query.
|
|
238
|
+
-- params: [col_ids, col_ids]
|
|
239
|
+
SELECT DISTINCT
|
|
240
|
+
src.table_qualified AS upstream_table,
|
|
241
|
+
ut.kind AS upstream_kind,
|
|
242
|
+
dst.table_qualified AS downstream_table,
|
|
243
|
+
dt.kind AS downstream_kind
|
|
244
|
+
FROM "COLUMN_LINEAGE" cl
|
|
245
|
+
JOIN "SqlColumn" src ON src.id = cl.src_key
|
|
246
|
+
JOIN "SqlColumn" dst ON dst.id = cl.dst_key
|
|
247
|
+
LEFT JOIN "SqlTable" ut ON ut.qualified = src.table_qualified
|
|
248
|
+
LEFT JOIN "SqlTable" dt ON dt.qualified = dst.table_qualified
|
|
249
|
+
WHERE cl.src_key = ANY(?)
|
|
250
|
+
AND cl.dst_key = ANY(?)
|
|
251
|
+
AND src.table_qualified <> dst.table_qualified
|
|
252
|
+
|
|
253
|
+
-- GET_TABLE_KINDS_BATCH
|
|
254
|
+
-- Batch lookup of SqlTable.kind for a set of qualified names — the authoritative
|
|
255
|
+
-- marker for synthetic cte/derived nodes (issue #38 synthetic-node-leak fix).
|
|
256
|
+
-- params: [table_qualifieds]
|
|
257
|
+
SELECT t.qualified AS table_qualified, t.kind AS kind
|
|
258
|
+
FROM "SqlTable" t
|
|
259
|
+
WHERE t.qualified = ANY(?)
|
|
260
|
+
|
|
201
261
|
-- ANALYZE_UNUSED_TABLES
|
|
202
262
|
-- params: []
|
|
203
263
|
SELECT t.qualified AS table_qualified
|
sqlcg/parsers/ansi_parser.py
CHANGED
|
@@ -404,14 +404,19 @@ class AnsiParser(SqlParser):
|
|
|
404
404
|
create_stmt: CREATE expression (must be a TABLE create)
|
|
405
405
|
|
|
406
406
|
Returns:
|
|
407
|
-
List of column names in source order (as they appear in the SQL)
|
|
407
|
+
List of column names in source order (as they appear in the SQL),
|
|
408
|
+
normalized to lowercase with surrounding double-quotes stripped —
|
|
409
|
+
identical to ColumnRef.__post_init__ — so DDL columns share node
|
|
410
|
+
identity with lowercase ETL lineage columns (issue #38 Phase 5).
|
|
408
411
|
"""
|
|
409
412
|
columns: list[str] = []
|
|
410
413
|
# Find all ColumnDef nodes in the CREATE statement
|
|
411
414
|
for col_def in create_stmt.find_all(exp.ColumnDef):
|
|
412
415
|
col_name = col_def.this.name if col_def.this else None
|
|
413
416
|
if col_name:
|
|
414
|
-
|
|
417
|
+
if len(col_name) >= 2 and col_name[0] == '"' and col_name[-1] == '"':
|
|
418
|
+
col_name = col_name[1:-1]
|
|
419
|
+
columns.append(col_name.lower())
|
|
415
420
|
return columns
|
|
416
421
|
|
|
417
422
|
@staticmethod
|
sqlcg/parsers/base.py
CHANGED
|
@@ -515,6 +515,32 @@ class SqlParser(ABC):
|
|
|
515
515
|
_walk(root)
|
|
516
516
|
return edges
|
|
517
517
|
|
|
518
|
+
def _table_node_to_ref(self, table_node: Any) -> "TableRef | None":
|
|
519
|
+
"""Convert a sqlglot exp.Table AST node to a TableRef.
|
|
520
|
+
|
|
521
|
+
Used by the #49 mis-bind override to enumerate candidate source tables
|
|
522
|
+
from a CTE body's FROM/JOIN once per CTE body (before the per-projection
|
|
523
|
+
loop). Does NOT call qualify/build_scope — preserves the O(1)-per-body
|
|
524
|
+
perf invariant.
|
|
525
|
+
|
|
526
|
+
Schema aliases are applied via _apply_table_alias so the emitted edges
|
|
527
|
+
carry the canonical (post-alias) table identity.
|
|
528
|
+
|
|
529
|
+
Args:
|
|
530
|
+
table_node: sqlglot.expressions.Table AST node
|
|
531
|
+
|
|
532
|
+
Returns:
|
|
533
|
+
TableRef with catalog/db/name extracted and alias-resolved, or None
|
|
534
|
+
if the alias resolution returns None (treated as an unresolvable ref).
|
|
535
|
+
"""
|
|
536
|
+
return self._apply_table_alias(
|
|
537
|
+
TableRef(
|
|
538
|
+
catalog=table_node.catalog if table_node.catalog else None,
|
|
539
|
+
db=table_node.db if table_node.db else None,
|
|
540
|
+
name=table_node.name if table_node.name else str(table_node),
|
|
541
|
+
)
|
|
542
|
+
)
|
|
543
|
+
|
|
518
544
|
def _lineage_node_to_table_ref(self, node: Any) -> "TableRef | None":
|
|
519
545
|
"""Extract a TableRef from a sqlglot LineageNode's source attribute.
|
|
520
546
|
|
|
@@ -987,6 +1013,20 @@ class SqlParser(ABC):
|
|
|
987
1013
|
# string for every column rather than re-serializing O(N_cols) times.
|
|
988
1014
|
cte_body_sql = cte_body.sql(dialect=self.DIALECT)
|
|
989
1015
|
|
|
1016
|
+
# Compute the candidate source-table set ONCE per CTE body
|
|
1017
|
+
# (before the per-projection loop) — never inside it.
|
|
1018
|
+
# Uses find_all(exp.Table) on the already-built AST; does NOT
|
|
1019
|
+
# call qualify/build_scope, preserving O(1) per CTE body.
|
|
1020
|
+
# This set is reused across all projections to detect ambiguity
|
|
1021
|
+
# (#49 mis-bind override).
|
|
1022
|
+
cte_source_tables: list[TableRef] = [
|
|
1023
|
+
ref
|
|
1024
|
+
for t in cte_body.find_all(exp.Table)
|
|
1025
|
+
if t.name # skip anonymous / subquery placeholders
|
|
1026
|
+
for ref in (self._table_node_to_ref(t),)
|
|
1027
|
+
if ref is not None
|
|
1028
|
+
]
|
|
1029
|
+
|
|
990
1030
|
# For each projection in the CTE, extract lineage.
|
|
991
1031
|
# Only iterate projections from left branch for column names, but pass
|
|
992
1032
|
# entire Union body to sg_lineage so sqlglot resolves both branches.
|
|
@@ -1010,6 +1050,48 @@ class SqlParser(ABC):
|
|
|
1010
1050
|
)
|
|
1011
1051
|
if not cte_col_name or cte_col_name == "*":
|
|
1012
1052
|
continue
|
|
1053
|
+
|
|
1054
|
+
# #49 mis-bind override: detect bare (unqualified) columns
|
|
1055
|
+
# in a ≥2-table CTE body.
|
|
1056
|
+
#
|
|
1057
|
+
# sqlglot's sg_lineage (called with no schema and no scope)
|
|
1058
|
+
# re-qualifies the CTE body internally and mis-binds bare
|
|
1059
|
+
# columns to the last-joined table — emitting a confident
|
|
1060
|
+
# WRONG edge (confirmed live: bare `m` from fact_daily was
|
|
1061
|
+
# bound to dim_time). This is not a missing edge; it is an
|
|
1062
|
+
# incorrect one.
|
|
1063
|
+
#
|
|
1064
|
+
# Override: when any bare column appears inside the projection
|
|
1065
|
+
# expression AND the CTE body has ≥2 source tables, skip
|
|
1066
|
+
# sg_lineage for this projection and instead emit one
|
|
1067
|
+
# CTE_PROJECTION_AMBIGUOUS edge per candidate source table
|
|
1068
|
+
# (confidence=0.5). Over-attribution is the safe failure mode
|
|
1069
|
+
# for impact analysis; a wrong single edge is not acceptable.
|
|
1070
|
+
#
|
|
1071
|
+
# Single-table bodies: no ambiguity; existing path unchanged.
|
|
1072
|
+
# Qualified columns in any body: no bare columns; existing path.
|
|
1073
|
+
bare_cols_in_expr = [
|
|
1074
|
+
c
|
|
1075
|
+
for c in cte_col_expr.find_all(exp.Column)
|
|
1076
|
+
if not c.table # bare = no qualifier
|
|
1077
|
+
]
|
|
1078
|
+
if bare_cols_in_expr and len(cte_source_tables) >= 2:
|
|
1079
|
+
# Emit one ambiguous edge per candidate source table.
|
|
1080
|
+
# bare_col.name is the column name to attribute;
|
|
1081
|
+
# for aggregates/CASE the bare col name is the leaf.
|
|
1082
|
+
bare_col_name = bare_cols_in_expr[0].name or cte_col_name
|
|
1083
|
+
dst_col_ref = ColumnRef(cte_dst_table, cte_col_name)
|
|
1084
|
+
for src_tbl in cte_source_tables:
|
|
1085
|
+
edges.append(
|
|
1086
|
+
LineageEdge(
|
|
1087
|
+
src=ColumnRef(src_tbl, bare_col_name),
|
|
1088
|
+
dst=dst_col_ref,
|
|
1089
|
+
transform="CTE_PROJECTION_AMBIGUOUS",
|
|
1090
|
+
confidence=0.5,
|
|
1091
|
+
)
|
|
1092
|
+
)
|
|
1093
|
+
continue # skip sg_lineage for this projection
|
|
1094
|
+
|
|
1013
1095
|
try:
|
|
1014
1096
|
# No schema: resolver.as_dict() {table:[cols]} triggers
|
|
1015
1097
|
# sqlglot nesting-level errors on fresh string parses.
|
sqlcg/server/models.py
CHANGED
|
@@ -242,6 +242,12 @@ class DefinitionResult(BaseModel):
|
|
|
242
242
|
duplicate_ddl: bool = Field(
|
|
243
243
|
False, description="True when the same table is defined in more than one file."
|
|
244
244
|
)
|
|
245
|
+
producer_files: list[str] = Field(
|
|
246
|
+
default_factory=list,
|
|
247
|
+
description="Files whose INSERT...SELECT populates this table (via "
|
|
248
|
+
"QUERY_DEFINED_IN -> SqlQuery.target_table). Often the more useful answer "
|
|
249
|
+
"to 'where do I change this table's logic?' than the DDL file alone.",
|
|
250
|
+
)
|
|
245
251
|
noise_excluded: list[str] = Field(
|
|
246
252
|
default_factory=list,
|
|
247
253
|
description="Definition file paths that were flagged as backup/noise.",
|
sqlcg/server/tools.py
CHANGED
|
@@ -21,10 +21,14 @@ from sqlcg.core.queries import (
|
|
|
21
21
|
FIND_TABLE_USAGES_QUERY,
|
|
22
22
|
GET_COLUMNS_FOR_TABLE_QUERY,
|
|
23
23
|
GET_DOWNSTREAM_DEPENDENCIES_QUERY,
|
|
24
|
+
GET_PRODUCER_FILES_FOR_TABLE_QUERY,
|
|
25
|
+
GET_TABLE_ADJACENCY_FOR_COLUMNS_QUERY,
|
|
24
26
|
GET_TABLE_DEFINING_FILES_QUERY,
|
|
25
27
|
GET_TABLE_DIRECT_UPSTREAMS_QUERY,
|
|
28
|
+
GET_TABLE_KINDS_BATCH_QUERY,
|
|
26
29
|
GET_TABLES_DEFINED_IN_FILE_QUERY,
|
|
27
30
|
GET_TABLES_EXTERNAL_CONSUMERS_BATCH_QUERY,
|
|
31
|
+
GET_TARGET_TABLES_FOR_FILE_QUERY,
|
|
28
32
|
GET_UPSTREAM_DEPENDENCIES_QUERY,
|
|
29
33
|
HUB_RANKING_QUERY,
|
|
30
34
|
INDEX_REPO_FILES_QUERY,
|
|
@@ -396,6 +400,51 @@ def _rollup_to_tables(col_ids: list[str]) -> list[str]:
|
|
|
396
400
|
return tables
|
|
397
401
|
|
|
398
402
|
|
|
403
|
+
# Synthetic SqlTable.kind values that must never surface as rebuildable tables —
|
|
404
|
+
# the authoritative marker per indexer.py (kind="cte" / kind="derived"); see
|
|
405
|
+
# _exclude_synthetic_tables. Keying on kind (not the alias string) avoids dropping
|
|
406
|
+
# a real table that happens to share a name with a CTE alias.
|
|
407
|
+
_SYNTHETIC_TABLE_KINDS = frozenset({"cte", "derived"})
|
|
408
|
+
|
|
409
|
+
|
|
410
|
+
def _exclude_synthetic_tables(db: GraphBackend, tables: list[str]) -> tuple[list[str], list[str]]:
|
|
411
|
+
"""Split *tables* into (real, synthetic) using the authoritative SqlTable.kind.
|
|
412
|
+
|
|
413
|
+
Synthetic CTE/derived nodes (``kind in {"cte", "derived"}``) are emitted into
|
|
414
|
+
the graph for lineage-tracing purposes but must never surface as rebuildable
|
|
415
|
+
tables in get_change_scope / get_backfill_order / diff_impact / scope_change
|
|
416
|
+
(issue #38 synthetic-node leak). Exclusion is keyed on ``kind`` — the
|
|
417
|
+
authoritative marker set in indexer.py — never on the alias string, so a real
|
|
418
|
+
table that happens to share a name with a CTE alias is never dropped.
|
|
419
|
+
|
|
420
|
+
Args:
|
|
421
|
+
db: Graph backend.
|
|
422
|
+
tables: Candidate table_qualified strings (already noise-filtered).
|
|
423
|
+
|
|
424
|
+
Returns:
|
|
425
|
+
(real_tables, synthetic_excluded) — both order-preserving, deduplicated.
|
|
426
|
+
"""
|
|
427
|
+
if not tables:
|
|
428
|
+
return [], []
|
|
429
|
+
|
|
430
|
+
rows = db.run_read(GET_TABLE_KINDS_BATCH_QUERY, {"table_qualifieds": tables})
|
|
431
|
+
synthetic_ids = {
|
|
432
|
+
row["table_qualified"] for row in rows if row["kind"] in _SYNTHETIC_TABLE_KINDS
|
|
433
|
+
}
|
|
434
|
+
|
|
435
|
+
if not synthetic_ids:
|
|
436
|
+
return tables, []
|
|
437
|
+
|
|
438
|
+
real: list[str] = []
|
|
439
|
+
synthetic: list[str] = []
|
|
440
|
+
for t in tables:
|
|
441
|
+
if t in synthetic_ids:
|
|
442
|
+
synthetic.append(t)
|
|
443
|
+
else:
|
|
444
|
+
real.append(t)
|
|
445
|
+
return real, synthetic
|
|
446
|
+
|
|
447
|
+
|
|
399
448
|
def _dedup_preserve_order(items: list[str]) -> list[str]:
|
|
400
449
|
"""Deduplicate a list while preserving first-seen order."""
|
|
401
450
|
out: list[str] = []
|
|
@@ -421,30 +470,84 @@ def _risk_label(affected_table_count: int) -> str:
|
|
|
421
470
|
return "high"
|
|
422
471
|
|
|
423
472
|
|
|
424
|
-
def _kahn_topological_sort(
|
|
473
|
+
def _kahn_topological_sort(
|
|
474
|
+
affected_tables: list[str], db: GraphBackend, closure_col_ids: list[str]
|
|
475
|
+
) -> tuple[list[str], bool]:
|
|
425
476
|
"""Order affected tables so producers come before consumers (rebuild order).
|
|
426
477
|
|
|
427
|
-
Builds
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
478
|
+
Builds adjacency from the **COLUMN_LINEAGE closure** (Option A — issue #38),
|
|
479
|
+
not from ``SELECTS_FROM``: CTE-wrapped ``INSERT ... WITH cte AS (...) SELECT
|
|
480
|
+
... FROM cte`` statements emit no ``SELECTS_FROM`` adjacency at all (the real
|
|
481
|
+
source table is nested in the CTE's child scope and never surfaces into the
|
|
482
|
+
statement's top-level ``sources``), which previously left every table at
|
|
483
|
+
indegree 0 and degraded ordering to alphabetical.
|
|
484
|
+
|
|
485
|
+
``COLUMN_LINEAGE`` *does* carry the producer -> consumer connectivity, but —
|
|
486
|
+
contrary to an earlier (unreproduced) assumption of a parallel direct edge —
|
|
487
|
+
it bridges producer and consumer via a **two-hop path through the synthetic
|
|
488
|
+
cte/derived node** (``producer -> cte -> consumer``), not a parallel direct
|
|
489
|
+
edge bypassing it. Rolling the raw closure up to table level therefore yields
|
|
490
|
+
edges *to and from* synthetic nodes, not real producer -> consumer adjacency
|
|
491
|
+
directly. This function **contracts** those synthetic-node hops in one pass
|
|
492
|
+
over the (small) rolled-up edge set — ``real -> synthetic -> ... -> real``
|
|
493
|
+
collapses to ``real -> real`` — yielding the true causal adjacency. Still
|
|
494
|
+
**one** aggregate query + one contraction pass per closure (never
|
|
495
|
+
per-table/per-column; replaces the old N x GET_TABLE_DIRECT_UPSTREAMS loop).
|
|
496
|
+
|
|
497
|
+
Runs Kahn's algorithm over the contracted adjacency. Returns (ordered_tables,
|
|
498
|
+
had_cycle). On a cycle the acyclic prefix is emitted in topological order
|
|
499
|
+
and the remaining cyclic tables are appended in input order — no exception
|
|
500
|
+
is raised.
|
|
501
|
+
|
|
502
|
+
Args:
|
|
503
|
+
affected_tables: Noise-filtered, synthetic-excluded table set to order.
|
|
504
|
+
db: Graph backend.
|
|
505
|
+
closure_col_ids: Full column-id set of the closure (start columns plus
|
|
506
|
+
their transitive downstream columns) — both endpoints of every
|
|
507
|
+
producer -> consumer hop (including synthetic bridges) must be
|
|
508
|
+
present here for the adjacency query to find the connecting path.
|
|
433
509
|
"""
|
|
434
510
|
table_set = set(affected_tables)
|
|
435
511
|
successors: dict[str, set[str]] = {t: set() for t in affected_tables}
|
|
436
512
|
indegree: dict[str, int] = {t: 0 for t in affected_tables}
|
|
437
513
|
|
|
438
|
-
|
|
514
|
+
if closure_col_ids:
|
|
439
515
|
rows = db.run_read(
|
|
440
|
-
|
|
441
|
-
{"
|
|
516
|
+
GET_TABLE_ADJACENCY_FOR_COLUMNS_QUERY,
|
|
517
|
+
{"col_ids": closure_col_ids, "col_ids2": closure_col_ids},
|
|
442
518
|
)
|
|
519
|
+
# Raw rolled-up adjacency, including synthetic (cte/derived) endpoints.
|
|
520
|
+
raw_successors: dict[str, set[str]] = {}
|
|
521
|
+
synthetic_nodes: set[str] = set()
|
|
443
522
|
for row in rows:
|
|
444
|
-
src = row["upstream_table"]
|
|
445
|
-
if src
|
|
446
|
-
|
|
447
|
-
|
|
523
|
+
src, dst = row["upstream_table"], row["downstream_table"]
|
|
524
|
+
if src == dst:
|
|
525
|
+
continue
|
|
526
|
+
raw_successors.setdefault(src, set()).add(dst)
|
|
527
|
+
if row["upstream_kind"] in _SYNTHETIC_TABLE_KINDS:
|
|
528
|
+
synthetic_nodes.add(src)
|
|
529
|
+
if row["downstream_kind"] in _SYNTHETIC_TABLE_KINDS:
|
|
530
|
+
synthetic_nodes.add(dst)
|
|
531
|
+
|
|
532
|
+
# Contract synthetic-node hops: from each real producer, BFS through
|
|
533
|
+
# any chain of synthetic nodes to find the real consumers it reaches —
|
|
534
|
+
# `real -> synthetic [-> synthetic ...] -> real` collapses to `real -> real`.
|
|
535
|
+
for src in table_set:
|
|
536
|
+
seen_chain: set[str] = set()
|
|
537
|
+
frontier = deque(raw_successors.get(src, ()))
|
|
538
|
+
while frontier:
|
|
539
|
+
node = frontier.popleft()
|
|
540
|
+
if node in seen_chain:
|
|
541
|
+
continue
|
|
542
|
+
seen_chain.add(node)
|
|
543
|
+
if node in table_set:
|
|
544
|
+
if node != src and node not in successors[src]:
|
|
545
|
+
successors[src].add(node)
|
|
546
|
+
indegree[node] += 1
|
|
547
|
+
elif node in synthetic_nodes:
|
|
548
|
+
frontier.extend(raw_successors.get(node, ()))
|
|
549
|
+
# Real nodes outside table_set (e.g. noise-filtered) are dead ends —
|
|
550
|
+
# do not traverse through them; only synthetic bridges are contracted.
|
|
448
551
|
|
|
449
552
|
# Sort the zero-indegree frontier for deterministic output.
|
|
450
553
|
ready: deque[str] = deque(sorted(t for t in affected_tables if indegree[t] == 0))
|
|
@@ -842,7 +945,10 @@ def find_definition(table_qualified: str) -> DefinitionResult:
|
|
|
842
945
|
with _open_backend() as db:
|
|
843
946
|
_assert_indexed(db)
|
|
844
947
|
|
|
845
|
-
|
|
948
|
+
target = table_qualified.lower()
|
|
949
|
+
rows = db.run_read(FIND_DEFINITION_QUERY, {"table_qualified": target})
|
|
950
|
+
producer_rows = db.run_read(GET_PRODUCER_FILES_FOR_TABLE_QUERY, {"table_qualified": target})
|
|
951
|
+
producer_files = _dedup_preserve_order([r["file_path"] for r in producer_rows])
|
|
846
952
|
noise_filter = NoiseFilter.from_config()
|
|
847
953
|
|
|
848
954
|
duplicate_ddl = len(rows) > 1
|
|
@@ -874,6 +980,7 @@ def find_definition(table_qualified: str) -> DefinitionResult:
|
|
|
874
980
|
table_qualified=table_qualified,
|
|
875
981
|
definitions=definitions,
|
|
876
982
|
duplicate_ddl=duplicate_ddl,
|
|
983
|
+
producer_files=producer_files,
|
|
877
984
|
noise_excluded=noise_excluded,
|
|
878
985
|
hint=hint,
|
|
879
986
|
)
|
|
@@ -905,7 +1012,10 @@ def get_change_scope(table_qualified: str) -> ChangeScopeResult:
|
|
|
905
1012
|
noise_filter = NoiseFilter.from_config()
|
|
906
1013
|
|
|
907
1014
|
def_rows = db.run_read(GET_TABLE_DEFINING_FILES_QUERY, {"table_qualified": target})
|
|
908
|
-
|
|
1015
|
+
producer_rows = db.run_read(GET_PRODUCER_FILES_FOR_TABLE_QUERY, {"table_qualified": target})
|
|
1016
|
+
defining_files = _dedup_preserve_order(
|
|
1017
|
+
[r["file_path"] for r in def_rows] + [r["file_path"] for r in producer_rows]
|
|
1018
|
+
)
|
|
909
1019
|
|
|
910
1020
|
up_rows = db.run_read(
|
|
911
1021
|
GET_TABLE_DIRECT_UPSTREAMS_QUERY,
|
|
@@ -921,7 +1031,13 @@ def get_change_scope(table_qualified: str) -> ChangeScopeResult:
|
|
|
921
1031
|
affected_cols, _depth, truncated = _affected_columns_closure(db, start_cols, max_depth=None)
|
|
922
1032
|
|
|
923
1033
|
affected_tables_all = [t for t in _rollup_to_tables(affected_cols) if t != target]
|
|
924
|
-
|
|
1034
|
+
affected_tables_noise_filtered, noise_excluded = noise_filter.filter_nodes(
|
|
1035
|
+
affected_tables_all
|
|
1036
|
+
)
|
|
1037
|
+
affected_tables, synthetic_excluded = _exclude_synthetic_tables(
|
|
1038
|
+
db, affected_tables_noise_filtered
|
|
1039
|
+
)
|
|
1040
|
+
noise_excluded = [*noise_excluded, *synthetic_excluded]
|
|
925
1041
|
kept_tables = set(affected_tables)
|
|
926
1042
|
affected_columns = [c for c in affected_cols if c.rsplit(".", 1)[0] in kept_tables]
|
|
927
1043
|
|
|
@@ -985,11 +1101,21 @@ def get_backfill_order(table_qualified: str) -> BackfillOrderResult:
|
|
|
985
1101
|
affected_cols, _depth, truncated = _affected_columns_closure(db, start_cols, max_depth=None)
|
|
986
1102
|
|
|
987
1103
|
affected_tables_all = [t for t in _rollup_to_tables(affected_cols) if t != target]
|
|
988
|
-
|
|
1104
|
+
affected_tables_noise_filtered, noise_excluded = noise_filter.filter_nodes(
|
|
1105
|
+
affected_tables_all
|
|
1106
|
+
)
|
|
1107
|
+
affected_tables, synthetic_excluded = _exclude_synthetic_tables(
|
|
1108
|
+
db, affected_tables_noise_filtered
|
|
1109
|
+
)
|
|
1110
|
+
noise_excluded = [*noise_excluded, *synthetic_excluded]
|
|
989
1111
|
kept_tables = set(affected_tables)
|
|
990
1112
|
affected_columns = [c for c in affected_cols if c.rsplit(".", 1)[0] in kept_tables]
|
|
991
1113
|
|
|
992
|
-
|
|
1114
|
+
# Option A (issue #38): adjacency is derived from the same COLUMN_LINEAGE
|
|
1115
|
+
# closure as membership — pass the full id set (start + affected) so the
|
|
1116
|
+
# direct producer->consumer edge endpoints are both present.
|
|
1117
|
+
closure_col_ids = _dedup_preserve_order([*start_cols, *affected_cols])
|
|
1118
|
+
order, had_cycle = _kahn_topological_sort(affected_tables, db, closure_col_ids)
|
|
993
1119
|
|
|
994
1120
|
hint = None
|
|
995
1121
|
if had_cycle:
|
|
@@ -1044,19 +1170,26 @@ def diff_impact(changed_files: list[str]) -> DiffImpactResult:
|
|
|
1044
1170
|
fp = Path(file_path)
|
|
1045
1171
|
if not fp.is_absolute():
|
|
1046
1172
|
fp = root / fp
|
|
1047
|
-
|
|
1048
|
-
|
|
1049
|
-
|
|
1050
|
-
|
|
1051
|
-
seen_changed
|
|
1052
|
-
|
|
1173
|
+
for query in (GET_TABLES_DEFINED_IN_FILE_QUERY, GET_TARGET_TABLES_FOR_FILE_QUERY):
|
|
1174
|
+
rows = db.run_read(query, {"file_path": str(fp)})
|
|
1175
|
+
for row in rows:
|
|
1176
|
+
tq = row["table_qualified"]
|
|
1177
|
+
if tq and tq not in seen_changed:
|
|
1178
|
+
seen_changed.add(tq)
|
|
1179
|
+
changed_tables.append(tq)
|
|
1053
1180
|
|
|
1054
1181
|
all_affected_cols: list[str] = []
|
|
1055
1182
|
affected_seen: set[str] = set()
|
|
1183
|
+
all_start_cols: list[str] = []
|
|
1184
|
+
start_seen: set[str] = set()
|
|
1056
1185
|
truncated = False
|
|
1057
1186
|
for tq in changed_tables:
|
|
1058
1187
|
col_rows = db.run_read(GET_COLUMNS_FOR_TABLE_QUERY, {"table_qualified": tq})
|
|
1059
1188
|
start_cols = [r["col_id"] for r in col_rows]
|
|
1189
|
+
for col in start_cols:
|
|
1190
|
+
if col not in start_seen:
|
|
1191
|
+
start_seen.add(col)
|
|
1192
|
+
all_start_cols.append(col)
|
|
1060
1193
|
cols, _depth, tr = _affected_columns_closure(db, start_cols, max_depth=None)
|
|
1061
1194
|
truncated = truncated or tr
|
|
1062
1195
|
for col in cols:
|
|
@@ -1067,9 +1200,18 @@ def diff_impact(changed_files: list[str]) -> DiffImpactResult:
|
|
|
1067
1200
|
affected_tables_all = [
|
|
1068
1201
|
t for t in _rollup_to_tables(all_affected_cols) if t not in seen_changed
|
|
1069
1202
|
]
|
|
1070
|
-
|
|
1203
|
+
affected_tables_noise_filtered, noise_excluded = noise_filter.filter_nodes(
|
|
1204
|
+
affected_tables_all
|
|
1205
|
+
)
|
|
1206
|
+
affected_tables, synthetic_excluded = _exclude_synthetic_tables(
|
|
1207
|
+
db, affected_tables_noise_filtered
|
|
1208
|
+
)
|
|
1209
|
+
noise_excluded = [*noise_excluded, *synthetic_excluded]
|
|
1071
1210
|
presentation_facing = [t for t in affected_tables if any(t.startswith(p) for p in prefixes)]
|
|
1072
|
-
|
|
1211
|
+
# Option A (issue #38): same closure-derived adjacency as get_backfill_order —
|
|
1212
|
+
# union of every changed table's start columns plus the unioned affected set.
|
|
1213
|
+
closure_col_ids = _dedup_preserve_order([*all_start_cols, *all_affected_cols])
|
|
1214
|
+
order, had_cycle = _kahn_topological_sort(affected_tables, db, closure_col_ids)
|
|
1073
1215
|
|
|
1074
1216
|
# Resolve external consumers for the blast radius in a single BATCH query (not per-table).
|
|
1075
1217
|
external_consumers: list[str] = []
|
|
File without changes
|
|
File without changes
|