sql-code-graph 1.4.0__py3-none-any.whl → 1.4.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: sql-code-graph
3
- Version: 1.4.0
3
+ Version: 1.4.3
4
4
  Summary: SQL code graph analyzer and lineage tracer
5
5
  Project-URL: Homepage, https://github.com/Warhorze/sql-code-graph
6
6
  Project-URL: Repository, https://github.com/Warhorze/sql-code-graph
@@ -1,9 +1,9 @@
1
- sqlcg/__init__.py,sha256=spIaM-bJwyRvB5fw8VYDKkrB5Lz7cKAgRuvBBgKYX3g,115
1
+ sqlcg/__init__.py,sha256=-pjjoaMnukW9b0-s_OOYNjy46alw_PTB6xzvfSyWjew,115
2
2
  sqlcg/__main__.py,sha256=1YoFLcqEgTwYq1J3TbUwpkdG0zeeLIf2fJvwWI-CLFU,109
3
3
  sqlcg/cli/__init__.py,sha256=W8fD0LpMq2xm_5WKGNMvJh2WBL1ho5E8hUeAqXQYT1g,28
4
4
  sqlcg/cli/main.py,sha256=WmdTjsOlz1ozi2Y3Aq4ezR_FCRl-Lc1YOKw3_d48dlY,1650
5
5
  sqlcg/cli/commands/__init__.py,sha256=oSHtr6VD-jNubOjuCQyZj2tBppjMEpQDh-IGQ8of9eA,30
6
- sqlcg/cli/commands/analyze.py,sha256=YG5N1iYG-ierDgAJ36mFpQ4BMf2yJ4YF-UV7Rx-bmYA,11793
6
+ sqlcg/cli/commands/analyze.py,sha256=_aC5ML3w7YdLi7DL3TFS9OiCEIipuNZxWR6S4peTcn4,12154
7
7
  sqlcg/cli/commands/db.py,sha256=TMhPCHRWSo8YmlNnSxmGxrR0_1r3K4e0Bn8unZTkvU4,7248
8
8
  sqlcg/cli/commands/find.py,sha256=p5Vyyx-VBk8YDWYQN16UhECh7PIeMaEyCPEcUFcRFlM,2598
9
9
  sqlcg/cli/commands/gain.py,sha256=SJU1c51a7MgNbZItqQnaBfOWGnV5xpXQctbew5Dr9BE,9062
@@ -22,8 +22,8 @@ sqlcg/core/freshness.py,sha256=F9jWn2cbFs60jA9ta8KrT6MghD0mzI7SqqKs_Op9AeU,4577
22
22
  sqlcg/core/graph_db.py,sha256=eLYdmiqPWrZHtKLFcKgD2aiWZHFU3wrwz6Y0A3d9NcE,8633
23
23
  sqlcg/core/jobs.py,sha256=Je-fCdSKRgiSsv1W8SgNAlp36a7t7-pJZ-qKPbka9OE,3298
24
24
  sqlcg/core/queries.cypher,sha256=cvPOVe5GUOzJN4bxUvDxNI--xIIP8gm42TR-gUnea4U,4685
25
- sqlcg/core/queries.py,sha256=jtZR6caLpF0WqqkjncYTlf5L1GN6DZweoM5dNOAB8OY,3043
26
- sqlcg/core/queries.sql,sha256=75EHoHA5hKz-Xs8g-CCtDXbf3lYFEw_fxghjYU48gUM,7794
25
+ sqlcg/core/queries.py,sha256=DeGrpsh-N8CczapKW5Iw_CPBMOxtWZfusMOhpN4Qn_c,3319
26
+ sqlcg/core/queries.sql,sha256=yRITKvjnWY0e1Sa8me0yFe_FkkIBsk3BQn2l0iqOJ7Q,10768
27
27
  sqlcg/core/schema.cypher,sha256=rK5QMhSrzZhuj73NeNXGX6oM-rPPPvxFjex0fEyUvkQ,2859
28
28
  sqlcg/core/schema.py,sha256=7fKf314ueIV7-tIkQQUS4O6H-OhlFKFVFFvGFqw_5Xk,1476
29
29
  sqlcg/indexer/__init__.py,sha256=Wh20Unz2OHs1oIyWLrpurPAasF0BET2g4iXtNk7mh2U,56
@@ -40,8 +40,8 @@ sqlcg/lineage/schema_resolver.py,sha256=iXt6LYF6UVWsGUpcfbmjmGn9wCgXl721lTGf_8Aa
40
40
  sqlcg/metrics/__init__.py,sha256=hLJ6wm4St8qqYwKh3o9QG7lcEt1BEYM31ccqO9tGpIg,133
41
41
  sqlcg/metrics/store.py,sha256=KuDtxvyAgug9_KtiSCpvgKM2VZM7VSaI3D11uMLjJJk,10604
42
42
  sqlcg/parsers/__init__.py,sha256=AamA8wBbDZV9_zEtZCI4Hyen5UAVKHmBwjTghTt2PZE,785
43
- sqlcg/parsers/ansi_parser.py,sha256=mGZvijMOMQ4i1BybpwU29a8jnIGViefhy9fxzkSpsRM,17193
44
- sqlcg/parsers/base.py,sha256=IiOkVsm6jz9-48RqDCXiW-UXAraNxQ4pKXvSA7aolnA,49907
43
+ sqlcg/parsers/ansi_parser.py,sha256=UNousZcTgHmgDzbvqF77hPf8RmTqwz4oo1rk9OMGWz4,17570
44
+ sqlcg/parsers/base.py,sha256=N6uqQWHO2lpzTIWgPqzewAXmij0ikZdNmNVJkIQ8Mr0,54889
45
45
  sqlcg/parsers/bigquery_parser.py,sha256=mOnWTfXB_Dp4JwFE1PVYOB6CDPf5nYE0Dea8kJCl9uQ,2827
46
46
  sqlcg/parsers/postgres_parser.py,sha256=lYfUpQY6j4Qm7ndXBtXbgPoGzYqYddWt5YeFnWKdA6I,946
47
47
  sqlcg/parsers/registry.py,sha256=LXy1F6rqQI6VdxpRvZg_tNpoEucW3mXZHYBMlMONbX4,1496
@@ -50,18 +50,18 @@ sqlcg/parsers/tsql_parser.py,sha256=RRj1pACtAk2tLTDaFWRYF67a0IDvaf5A1YQXWIz0bpQ,
50
50
  sqlcg/server/__init__.py,sha256=n4wuNE7xyJIJxJZBtmtdccCMQfvTdF-IqIaZVbC4FC4,35
51
51
  sqlcg/server/control.py,sha256=qUcztb_aDhL-_X_Nq4q6uGx17cUlbLnI6vUpoZsEjbo,4506
52
52
  sqlcg/server/exceptions.py,sha256=EONw34icOByCTpppSQrvQBW6asc4hfqaGDCAFjv96II,469
53
- sqlcg/server/models.py,sha256=l7ORy6sbtzBW1y3qVaeLwEukbyAgBkz9S5VIm2q4b24,19378
53
+ sqlcg/server/models.py,sha256=plnBVoV6y6JuU2IHzez0yHfsd5QZ_o977UCacS7A5NA,19693
54
54
  sqlcg/server/noise_filter.py,sha256=Ats2FFzmzFKqvQPWvlUzK8mY9pzlUhL4m1s8P_HNnvI,6335
55
55
  sqlcg/server/read_client.py,sha256=4v1OOl12HCHp6J14HrcWQApSixvJ0wwE8UIIZ0pzLyw,7808
56
56
  sqlcg/server/server.py,sha256=A1soT-hYnAX78X5j97nIjw9oIZCxBejjV6p_tRDIK38,24335
57
57
  sqlcg/server/skill.py,sha256=EDvmEgl-LQwCxZ9Lca4lt8zCtkk2cCgh9GLYlGgDA64,12828
58
- sqlcg/server/tools.py,sha256=Rdl5SXAuePydk8Z12JVx-lNQeAqm3k9R-z0Q4DstCkw,69743
58
+ sqlcg/server/tools.py,sha256=oelzyVydELcD0ePYQMiIdZPqm-GolxD4YxtKUtGSeTU,76744
59
59
  sqlcg/server/writer.py,sha256=HH8pW6k1c2pbY4b6g2N87Tx4gNCezYqTLNTlAmsHg2Y,18522
60
60
  sqlcg/utils/__init__.py,sha256=--iqt5ThTXmT8Wz7da8hs3n0zDfYPl8P-z5OgRJ_77E,154
61
61
  sqlcg/utils/hashing.py,sha256=H25-sYfxHKb3_IERFnHyAIYNiXN470Oqo5sJT_D3YOA,438
62
62
  sqlcg/utils/ignore.py,sha256=wJjwa0mjnQ_xJExOUxk25y00g065XmmzJapqV3ifD5o,1151
63
63
  sqlcg/utils/logging.py,sha256=u0fCmYsLj9o81vawm3xZTHaw68GQYVm7JxG-gP81u8A,840
64
- sql_code_graph-1.4.0.dist-info/METADATA,sha256=rjzA7ons9zPtU1rsKtlhdx3z3qJUrVHL7tYfdg1HcrE,14085
65
- sql_code_graph-1.4.0.dist-info/WHEEL,sha256=mffPy8wBnZQn2VnJUU5jE99KsxaSfiyMHV9Yt0aLVxs,87
66
- sql_code_graph-1.4.0.dist-info/entry_points.txt,sha256=Wfe49sVzV9p4eVFGo5RxcV-frr3HOP0yzzst8JBxQLQ,46
67
- sql_code_graph-1.4.0.dist-info/RECORD,,
64
+ sql_code_graph-1.4.3.dist-info/METADATA,sha256=a-9uSmKrh5FDurLJg_LFgkaNH8K1_ke-engcyBzhues,14085
65
+ sql_code_graph-1.4.3.dist-info/WHEEL,sha256=mffPy8wBnZQn2VnJUU5jE99KsxaSfiyMHV9Yt0aLVxs,87
66
+ sql_code_graph-1.4.3.dist-info/entry_points.txt,sha256=Wfe49sVzV9p4eVFGo5RxcV-frr3HOP0yzzst8JBxQLQ,46
67
+ sql_code_graph-1.4.3.dist-info/RECORD,,
sqlcg/__init__.py CHANGED
@@ -1,5 +1,5 @@
1
1
  """SQL Code Graph - SQL lineage and dependency analysis tool."""
2
2
 
3
- __version__ = "1.4.0"
3
+ __version__ = "1.4.3"
4
4
 
5
5
  __all__ = ["__version__"]
@@ -139,6 +139,7 @@ def upstream( # noqa: B008
139
139
  console.print("[red]Error: --depth must be between 1 and 100[/red]")
140
140
  raise typer.Exit(1)
141
141
 
142
+ ref = ref.lower() # graph keys are lowercased at index time (C2 normalization)
142
143
  sql = _upstream_sql(depth, include_intermediate)
143
144
  results = run_read_routed(sql, {"ref": ref})
144
145
  if not results and len(ref.split(".")) >= 3:
@@ -175,6 +176,7 @@ def downstream( # noqa: B008
175
176
  console.print("[red]Error: --depth must be between 1 and 100[/red]")
176
177
  raise typer.Exit(1)
177
178
 
179
+ ref = ref.lower() # graph keys are lowercased at index time (C2 normalization)
178
180
  sql = _downstream_sql(depth, include_intermediate)
179
181
  results = run_read_routed(sql, {"ref": ref})
180
182
  if not results and len(ref.split(".")) >= 3:
@@ -291,7 +293,12 @@ def unused(
291
293
 
292
294
 
293
295
  def _bare_ref(ref: str) -> str:
294
- """Strip schema prefix from a ref string, keeping table.column."""
296
+ """Strip schema prefix from a ref string, keeping table.column.
297
+
298
+ Lowercases defensively so this is safe to call even if the caller did not
299
+ first fold the ref — graph keys are lowercased at index time (C2 normalization).
300
+ """
301
+ ref = ref.lower()
295
302
  parts = ref.split(".")
296
303
  if len(parts) >= 3:
297
304
  return ".".join(parts[1:])
sqlcg/core/queries.py CHANGED
@@ -54,9 +54,13 @@ COUNT_STAR_SOURCES_QUERY = _Q["COUNT_STAR_SOURCES"]
54
54
  COUNT_STAR_EXPANSIONS_QUERY = _Q["COUNT_STAR_EXPANSIONS"]
55
55
  FIND_DEFINITION_QUERY = _Q["FIND_DEFINITION"]
56
56
  GET_TABLE_DEFINING_FILES_QUERY = _Q["GET_TABLE_DEFINING_FILES"]
57
+ GET_PRODUCER_FILES_FOR_TABLE_QUERY = _Q["GET_PRODUCER_FILES_FOR_TABLE"]
57
58
  GET_TABLE_DIRECT_UPSTREAMS_QUERY = _Q["GET_TABLE_DIRECT_UPSTREAMS"]
58
59
  GET_COLUMNS_FOR_TABLE_QUERY = _Q["GET_COLUMNS_FOR_TABLE"]
59
60
  GET_TABLES_DEFINED_IN_FILE_QUERY = _Q["GET_TABLES_DEFINED_IN_FILE"]
61
+ GET_TARGET_TABLES_FOR_FILE_QUERY = _Q["GET_TARGET_TABLES_FOR_FILE"]
62
+ GET_TABLE_ADJACENCY_FOR_COLUMNS_QUERY = _Q["GET_TABLE_ADJACENCY_FOR_COLUMNS"]
63
+ GET_TABLE_KINDS_BATCH_QUERY = _Q["GET_TABLE_KINDS_BATCH"]
60
64
  ANALYZE_UNUSED_TABLES_QUERY = _Q["ANALYZE_UNUSED_TABLES"]
61
65
  HUB_RANKING_QUERY = _Q["HUB_RANKING"]
62
66
  DEPENDENT_FILES_OF_TABLES_QUERY = _Q["DEPENDENT_FILES_OF_TABLES"]
sqlcg/core/queries.sql CHANGED
@@ -172,6 +172,19 @@ JOIN "DEFINED_IN" di ON di.src_key = t.qualified
172
172
  JOIN "File" f ON f.path = di.dst_key
173
173
  WHERE t.qualified = ?
174
174
 
175
+ -- GET_PRODUCER_FILES_FOR_TABLE
176
+ -- ETL INSERT...SELECT producers populate a table without a DEFINED_IN edge
177
+ -- (that edge is DDL-only). Resolve SqlQuery.target_table -> QUERY_DEFINED_IN -> File
178
+ -- so get_definition / get_change_scope can also surface "populated here" producer
179
+ -- files, not just "defined here" DDL files (mirror of GET_TARGET_TABLES_FOR_FILE,
180
+ -- the reverse-direction lookup). table_qualified is stored lowercase.
181
+ -- params: [table_qualified]
182
+ SELECT DISTINCT f.path AS file_path
183
+ FROM "SqlQuery" q
184
+ JOIN "QUERY_DEFINED_IN" qdi ON qdi.src_key = q.id
185
+ JOIN "File" f ON f.path = qdi.dst_key
186
+ WHERE q.target_table = ?
187
+
175
188
  -- GET_TABLE_DIRECT_UPSTREAMS
176
189
  -- params: [table_qualified, table_qualified]
177
190
  SELECT DISTINCT src.qualified AS upstream_table, f.path AS in_file
@@ -198,6 +211,53 @@ FROM "SqlTable" t
198
211
  JOIN "DEFINED_IN" di ON di.src_key = t.qualified
199
212
  WHERE di.dst_key = ?
200
213
 
214
+ -- GET_TARGET_TABLES_FOR_FILE
215
+ -- ETL INSERT...SELECT producers populate a table without a DEFINED_IN edge
216
+ -- (that edge is DDL-only). Resolve query->file QUERY_DEFINED_IN -> SqlQuery.target_table
217
+ -- so diff_impact can also see "populated here" producers, not just "defined here" DDL.
218
+ -- params: [file_path]
219
+ SELECT DISTINCT q.target_table AS table_qualified
220
+ FROM "SqlQuery" q
221
+ JOIN "QUERY_DEFINED_IN" qdi ON qdi.src_key = q.id
222
+ WHERE qdi.dst_key = ?
223
+ AND q.target_table <> ''
224
+
225
+ -- GET_TABLE_ADJACENCY_FOR_COLUMNS
226
+ -- Aggregate table-level producer->consumer adjacency derived from COLUMN_LINEAGE,
227
+ -- restricted to a closure's column-id set (Option A — issue #38 backfill fix).
228
+ -- Replaces the N x GET_TABLE_DIRECT_UPSTREAMS loop with a single query: CTE-wrapped
229
+ -- INSERT...SELECT statements emit no SELECTS_FROM adjacency at all (the real source
230
+ -- table is nested in the CTE child scope and never surfaces into the statement's
231
+ -- top-level sources). The raw COLUMN_LINEAGE topology bridges producer -> consumer
232
+ -- via a TWO-HOP path through the synthetic cte/derived node (producer -> cte ->
233
+ -- consumer, not a parallel direct edge) — this query returns ALL rolled-up table
234
+ -- pairs (including synthetic endpoints); the caller contracts synthetic-node hops
235
+ -- into direct real-table adjacency in one pass over this small edge set (still
236
+ -- ONCE per closure, never per-table/per-column). Also returns each endpoint's
237
+ -- SqlTable.kind so the caller can identify synthetic nodes without a second query.
238
+ -- params: [col_ids, col_ids]
239
+ SELECT DISTINCT
240
+ src.table_qualified AS upstream_table,
241
+ ut.kind AS upstream_kind,
242
+ dst.table_qualified AS downstream_table,
243
+ dt.kind AS downstream_kind
244
+ FROM "COLUMN_LINEAGE" cl
245
+ JOIN "SqlColumn" src ON src.id = cl.src_key
246
+ JOIN "SqlColumn" dst ON dst.id = cl.dst_key
247
+ LEFT JOIN "SqlTable" ut ON ut.qualified = src.table_qualified
248
+ LEFT JOIN "SqlTable" dt ON dt.qualified = dst.table_qualified
249
+ WHERE cl.src_key = ANY(?)
250
+ AND cl.dst_key = ANY(?)
251
+ AND src.table_qualified <> dst.table_qualified
252
+
253
+ -- GET_TABLE_KINDS_BATCH
254
+ -- Batch lookup of SqlTable.kind for a set of qualified names — the authoritative
255
+ -- marker for synthetic cte/derived nodes (issue #38 synthetic-node-leak fix).
256
+ -- params: [table_qualifieds]
257
+ SELECT t.qualified AS table_qualified, t.kind AS kind
258
+ FROM "SqlTable" t
259
+ WHERE t.qualified = ANY(?)
260
+
201
261
  -- ANALYZE_UNUSED_TABLES
202
262
  -- params: []
203
263
  SELECT t.qualified AS table_qualified
@@ -404,14 +404,19 @@ class AnsiParser(SqlParser):
404
404
  create_stmt: CREATE expression (must be a TABLE create)
405
405
 
406
406
  Returns:
407
- List of column names in source order (as they appear in the SQL)
407
+ List of column names in source order (as they appear in the SQL),
408
+ normalized to lowercase with surrounding double-quotes stripped —
409
+ identical to ColumnRef.__post_init__ — so DDL columns share node
410
+ identity with lowercase ETL lineage columns (issue #38 Phase 5).
408
411
  """
409
412
  columns: list[str] = []
410
413
  # Find all ColumnDef nodes in the CREATE statement
411
414
  for col_def in create_stmt.find_all(exp.ColumnDef):
412
415
  col_name = col_def.this.name if col_def.this else None
413
416
  if col_name:
414
- columns.append(col_name)
417
+ if len(col_name) >= 2 and col_name[0] == '"' and col_name[-1] == '"':
418
+ col_name = col_name[1:-1]
419
+ columns.append(col_name.lower())
415
420
  return columns
416
421
 
417
422
  @staticmethod
sqlcg/parsers/base.py CHANGED
@@ -515,6 +515,32 @@ class SqlParser(ABC):
515
515
  _walk(root)
516
516
  return edges
517
517
 
518
+ def _table_node_to_ref(self, table_node: Any) -> "TableRef | None":
519
+ """Convert a sqlglot exp.Table AST node to a TableRef.
520
+
521
+ Used by the #49 mis-bind override to enumerate candidate source tables
522
+ from a CTE body's FROM/JOIN once per CTE body (before the per-projection
523
+ loop). Does NOT call qualify/build_scope — preserves the O(1)-per-body
524
+ perf invariant.
525
+
526
+ Schema aliases are applied via _apply_table_alias so the emitted edges
527
+ carry the canonical (post-alias) table identity.
528
+
529
+ Args:
530
+ table_node: sqlglot.expressions.Table AST node
531
+
532
+ Returns:
533
+ TableRef with catalog/db/name extracted and alias-resolved, or None
534
+ if the alias resolution returns None (treated as an unresolvable ref).
535
+ """
536
+ return self._apply_table_alias(
537
+ TableRef(
538
+ catalog=table_node.catalog if table_node.catalog else None,
539
+ db=table_node.db if table_node.db else None,
540
+ name=table_node.name if table_node.name else str(table_node),
541
+ )
542
+ )
543
+
518
544
  def _lineage_node_to_table_ref(self, node: Any) -> "TableRef | None":
519
545
  """Extract a TableRef from a sqlglot LineageNode's source attribute.
520
546
 
@@ -987,6 +1013,20 @@ class SqlParser(ABC):
987
1013
  # string for every column rather than re-serializing O(N_cols) times.
988
1014
  cte_body_sql = cte_body.sql(dialect=self.DIALECT)
989
1015
 
1016
+ # Compute the candidate source-table set ONCE per CTE body
1017
+ # (before the per-projection loop) — never inside it.
1018
+ # Uses find_all(exp.Table) on the already-built AST; does NOT
1019
+ # call qualify/build_scope, preserving O(1) per CTE body.
1020
+ # This set is reused across all projections to detect ambiguity
1021
+ # (#49 mis-bind override).
1022
+ cte_source_tables: list[TableRef] = [
1023
+ ref
1024
+ for t in cte_body.find_all(exp.Table)
1025
+ if t.name # skip anonymous / subquery placeholders
1026
+ for ref in (self._table_node_to_ref(t),)
1027
+ if ref is not None
1028
+ ]
1029
+
990
1030
  # For each projection in the CTE, extract lineage.
991
1031
  # Only iterate projections from left branch for column names, but pass
992
1032
  # entire Union body to sg_lineage so sqlglot resolves both branches.
@@ -1010,6 +1050,48 @@ class SqlParser(ABC):
1010
1050
  )
1011
1051
  if not cte_col_name or cte_col_name == "*":
1012
1052
  continue
1053
+
1054
+ # #49 mis-bind override: detect bare (unqualified) columns
1055
+ # in a ≥2-table CTE body.
1056
+ #
1057
+ # sqlglot's sg_lineage (called with no schema and no scope)
1058
+ # re-qualifies the CTE body internally and mis-binds bare
1059
+ # columns to the last-joined table — emitting a confident
1060
+ # WRONG edge (confirmed live: bare `m` from fact_daily was
1061
+ # bound to dim_time). This is not a missing edge; it is an
1062
+ # incorrect one.
1063
+ #
1064
+ # Override: when any bare column appears inside the projection
1065
+ # expression AND the CTE body has ≥2 source tables, skip
1066
+ # sg_lineage for this projection and instead emit one
1067
+ # CTE_PROJECTION_AMBIGUOUS edge per candidate source table
1068
+ # (confidence=0.5). Over-attribution is the safe failure mode
1069
+ # for impact analysis; a wrong single edge is not acceptable.
1070
+ #
1071
+ # Single-table bodies: no ambiguity; existing path unchanged.
1072
+ # Qualified columns in any body: no bare columns; existing path.
1073
+ bare_cols_in_expr = [
1074
+ c
1075
+ for c in cte_col_expr.find_all(exp.Column)
1076
+ if not c.table # bare = no qualifier
1077
+ ]
1078
+ if bare_cols_in_expr and len(cte_source_tables) >= 2:
1079
+ # Emit one ambiguous edge per candidate source table.
1080
+ # bare_col.name is the column name to attribute;
1081
+ # for aggregates/CASE the bare col name is the leaf.
1082
+ bare_col_name = bare_cols_in_expr[0].name or cte_col_name
1083
+ dst_col_ref = ColumnRef(cte_dst_table, cte_col_name)
1084
+ for src_tbl in cte_source_tables:
1085
+ edges.append(
1086
+ LineageEdge(
1087
+ src=ColumnRef(src_tbl, bare_col_name),
1088
+ dst=dst_col_ref,
1089
+ transform="CTE_PROJECTION_AMBIGUOUS",
1090
+ confidence=0.5,
1091
+ )
1092
+ )
1093
+ continue # skip sg_lineage for this projection
1094
+
1013
1095
  try:
1014
1096
  # No schema: resolver.as_dict() {table:[cols]} triggers
1015
1097
  # sqlglot nesting-level errors on fresh string parses.
sqlcg/server/models.py CHANGED
@@ -242,6 +242,12 @@ class DefinitionResult(BaseModel):
242
242
  duplicate_ddl: bool = Field(
243
243
  False, description="True when the same table is defined in more than one file."
244
244
  )
245
+ producer_files: list[str] = Field(
246
+ default_factory=list,
247
+ description="Files whose INSERT...SELECT populates this table (via "
248
+ "QUERY_DEFINED_IN -> SqlQuery.target_table). Often the more useful answer "
249
+ "to 'where do I change this table's logic?' than the DDL file alone.",
250
+ )
245
251
  noise_excluded: list[str] = Field(
246
252
  default_factory=list,
247
253
  description="Definition file paths that were flagged as backup/noise.",
sqlcg/server/tools.py CHANGED
@@ -21,10 +21,14 @@ from sqlcg.core.queries import (
21
21
  FIND_TABLE_USAGES_QUERY,
22
22
  GET_COLUMNS_FOR_TABLE_QUERY,
23
23
  GET_DOWNSTREAM_DEPENDENCIES_QUERY,
24
+ GET_PRODUCER_FILES_FOR_TABLE_QUERY,
25
+ GET_TABLE_ADJACENCY_FOR_COLUMNS_QUERY,
24
26
  GET_TABLE_DEFINING_FILES_QUERY,
25
27
  GET_TABLE_DIRECT_UPSTREAMS_QUERY,
28
+ GET_TABLE_KINDS_BATCH_QUERY,
26
29
  GET_TABLES_DEFINED_IN_FILE_QUERY,
27
30
  GET_TABLES_EXTERNAL_CONSUMERS_BATCH_QUERY,
31
+ GET_TARGET_TABLES_FOR_FILE_QUERY,
28
32
  GET_UPSTREAM_DEPENDENCIES_QUERY,
29
33
  HUB_RANKING_QUERY,
30
34
  INDEX_REPO_FILES_QUERY,
@@ -396,6 +400,51 @@ def _rollup_to_tables(col_ids: list[str]) -> list[str]:
396
400
  return tables
397
401
 
398
402
 
403
+ # Synthetic SqlTable.kind values that must never surface as rebuildable tables —
404
+ # the authoritative marker per indexer.py (kind="cte" / kind="derived"); see
405
+ # _exclude_synthetic_tables. Keying on kind (not the alias string) avoids dropping
406
+ # a real table that happens to share a name with a CTE alias.
407
+ _SYNTHETIC_TABLE_KINDS = frozenset({"cte", "derived"})
408
+
409
+
410
+ def _exclude_synthetic_tables(db: GraphBackend, tables: list[str]) -> tuple[list[str], list[str]]:
411
+ """Split *tables* into (real, synthetic) using the authoritative SqlTable.kind.
412
+
413
+ Synthetic CTE/derived nodes (``kind in {"cte", "derived"}``) are emitted into
414
+ the graph for lineage-tracing purposes but must never surface as rebuildable
415
+ tables in get_change_scope / get_backfill_order / diff_impact / scope_change
416
+ (issue #38 synthetic-node leak). Exclusion is keyed on ``kind`` — the
417
+ authoritative marker set in indexer.py — never on the alias string, so a real
418
+ table that happens to share a name with a CTE alias is never dropped.
419
+
420
+ Args:
421
+ db: Graph backend.
422
+ tables: Candidate table_qualified strings (already noise-filtered).
423
+
424
+ Returns:
425
+ (real_tables, synthetic_excluded) — both order-preserving, deduplicated.
426
+ """
427
+ if not tables:
428
+ return [], []
429
+
430
+ rows = db.run_read(GET_TABLE_KINDS_BATCH_QUERY, {"table_qualifieds": tables})
431
+ synthetic_ids = {
432
+ row["table_qualified"] for row in rows if row["kind"] in _SYNTHETIC_TABLE_KINDS
433
+ }
434
+
435
+ if not synthetic_ids:
436
+ return tables, []
437
+
438
+ real: list[str] = []
439
+ synthetic: list[str] = []
440
+ for t in tables:
441
+ if t in synthetic_ids:
442
+ synthetic.append(t)
443
+ else:
444
+ real.append(t)
445
+ return real, synthetic
446
+
447
+
399
448
  def _dedup_preserve_order(items: list[str]) -> list[str]:
400
449
  """Deduplicate a list while preserving first-seen order."""
401
450
  out: list[str] = []
@@ -421,30 +470,84 @@ def _risk_label(affected_table_count: int) -> str:
421
470
  return "high"
422
471
 
423
472
 
424
- def _kahn_topological_sort(affected_tables: list[str], db: GraphBackend) -> tuple[list[str], bool]:
473
+ def _kahn_topological_sort(
474
+ affected_tables: list[str], db: GraphBackend, closure_col_ids: list[str]
475
+ ) -> tuple[list[str], bool]:
425
476
  """Order affected tables so producers come before consumers (rebuild order).
426
477
 
427
- Builds an adjacency list from SELECTS_FROM edges *within* the affected set
428
- (a table's producing query selecting from another affected table is an edge
429
- producer -> consumer) and runs Kahn's algorithm. Returns
430
- (ordered_tables, had_cycle). On a cycle the acyclic prefix is emitted in
431
- topological order and the remaining cyclic tables are appended in input
432
- order no exception is raised.
478
+ Builds adjacency from the **COLUMN_LINEAGE closure** (Option A — issue #38),
479
+ not from ``SELECTS_FROM``: CTE-wrapped ``INSERT ... WITH cte AS (...) SELECT
480
+ ... FROM cte`` statements emit no ``SELECTS_FROM`` adjacency at all (the real
481
+ source table is nested in the CTE's child scope and never surfaces into the
482
+ statement's top-level ``sources``), which previously left every table at
483
+ indegree 0 and degraded ordering to alphabetical.
484
+
485
+ ``COLUMN_LINEAGE`` *does* carry the producer -> consumer connectivity, but —
486
+ contrary to an earlier (unreproduced) assumption of a parallel direct edge —
487
+ it bridges producer and consumer via a **two-hop path through the synthetic
488
+ cte/derived node** (``producer -> cte -> consumer``), not a parallel direct
489
+ edge bypassing it. Rolling the raw closure up to table level therefore yields
490
+ edges *to and from* synthetic nodes, not real producer -> consumer adjacency
491
+ directly. This function **contracts** those synthetic-node hops in one pass
492
+ over the (small) rolled-up edge set — ``real -> synthetic -> ... -> real``
493
+ collapses to ``real -> real`` — yielding the true causal adjacency. Still
494
+ **one** aggregate query + one contraction pass per closure (never
495
+ per-table/per-column; replaces the old N x GET_TABLE_DIRECT_UPSTREAMS loop).
496
+
497
+ Runs Kahn's algorithm over the contracted adjacency. Returns (ordered_tables,
498
+ had_cycle). On a cycle the acyclic prefix is emitted in topological order
499
+ and the remaining cyclic tables are appended in input order — no exception
500
+ is raised.
501
+
502
+ Args:
503
+ affected_tables: Noise-filtered, synthetic-excluded table set to order.
504
+ db: Graph backend.
505
+ closure_col_ids: Full column-id set of the closure (start columns plus
506
+ their transitive downstream columns) — both endpoints of every
507
+ producer -> consumer hop (including synthetic bridges) must be
508
+ present here for the adjacency query to find the connecting path.
433
509
  """
434
510
  table_set = set(affected_tables)
435
511
  successors: dict[str, set[str]] = {t: set() for t in affected_tables}
436
512
  indegree: dict[str, int] = {t: 0 for t in affected_tables}
437
513
 
438
- for table in affected_tables:
514
+ if closure_col_ids:
439
515
  rows = db.run_read(
440
- GET_TABLE_DIRECT_UPSTREAMS_QUERY,
441
- {"table_qualified": table, "table_qualified2": table},
516
+ GET_TABLE_ADJACENCY_FOR_COLUMNS_QUERY,
517
+ {"col_ids": closure_col_ids, "col_ids2": closure_col_ids},
442
518
  )
519
+ # Raw rolled-up adjacency, including synthetic (cte/derived) endpoints.
520
+ raw_successors: dict[str, set[str]] = {}
521
+ synthetic_nodes: set[str] = set()
443
522
  for row in rows:
444
- src = row["upstream_table"]
445
- if src in table_set and src != table and table not in successors[src]:
446
- successors[src].add(table)
447
- indegree[table] += 1
523
+ src, dst = row["upstream_table"], row["downstream_table"]
524
+ if src == dst:
525
+ continue
526
+ raw_successors.setdefault(src, set()).add(dst)
527
+ if row["upstream_kind"] in _SYNTHETIC_TABLE_KINDS:
528
+ synthetic_nodes.add(src)
529
+ if row["downstream_kind"] in _SYNTHETIC_TABLE_KINDS:
530
+ synthetic_nodes.add(dst)
531
+
532
+ # Contract synthetic-node hops: from each real producer, BFS through
533
+ # any chain of synthetic nodes to find the real consumers it reaches —
534
+ # `real -> synthetic [-> synthetic ...] -> real` collapses to `real -> real`.
535
+ for src in table_set:
536
+ seen_chain: set[str] = set()
537
+ frontier = deque(raw_successors.get(src, ()))
538
+ while frontier:
539
+ node = frontier.popleft()
540
+ if node in seen_chain:
541
+ continue
542
+ seen_chain.add(node)
543
+ if node in table_set:
544
+ if node != src and node not in successors[src]:
545
+ successors[src].add(node)
546
+ indegree[node] += 1
547
+ elif node in synthetic_nodes:
548
+ frontier.extend(raw_successors.get(node, ()))
549
+ # Real nodes outside table_set (e.g. noise-filtered) are dead ends —
550
+ # do not traverse through them; only synthetic bridges are contracted.
448
551
 
449
552
  # Sort the zero-indegree frontier for deterministic output.
450
553
  ready: deque[str] = deque(sorted(t for t in affected_tables if indegree[t] == 0))
@@ -842,7 +945,10 @@ def find_definition(table_qualified: str) -> DefinitionResult:
842
945
  with _open_backend() as db:
843
946
  _assert_indexed(db)
844
947
 
845
- rows = db.run_read(FIND_DEFINITION_QUERY, {"table_qualified": table_qualified.lower()})
948
+ target = table_qualified.lower()
949
+ rows = db.run_read(FIND_DEFINITION_QUERY, {"table_qualified": target})
950
+ producer_rows = db.run_read(GET_PRODUCER_FILES_FOR_TABLE_QUERY, {"table_qualified": target})
951
+ producer_files = _dedup_preserve_order([r["file_path"] for r in producer_rows])
846
952
  noise_filter = NoiseFilter.from_config()
847
953
 
848
954
  duplicate_ddl = len(rows) > 1
@@ -874,6 +980,7 @@ def find_definition(table_qualified: str) -> DefinitionResult:
874
980
  table_qualified=table_qualified,
875
981
  definitions=definitions,
876
982
  duplicate_ddl=duplicate_ddl,
983
+ producer_files=producer_files,
877
984
  noise_excluded=noise_excluded,
878
985
  hint=hint,
879
986
  )
@@ -905,7 +1012,10 @@ def get_change_scope(table_qualified: str) -> ChangeScopeResult:
905
1012
  noise_filter = NoiseFilter.from_config()
906
1013
 
907
1014
  def_rows = db.run_read(GET_TABLE_DEFINING_FILES_QUERY, {"table_qualified": target})
908
- defining_files = _dedup_preserve_order([r["file_path"] for r in def_rows])
1015
+ producer_rows = db.run_read(GET_PRODUCER_FILES_FOR_TABLE_QUERY, {"table_qualified": target})
1016
+ defining_files = _dedup_preserve_order(
1017
+ [r["file_path"] for r in def_rows] + [r["file_path"] for r in producer_rows]
1018
+ )
909
1019
 
910
1020
  up_rows = db.run_read(
911
1021
  GET_TABLE_DIRECT_UPSTREAMS_QUERY,
@@ -921,7 +1031,13 @@ def get_change_scope(table_qualified: str) -> ChangeScopeResult:
921
1031
  affected_cols, _depth, truncated = _affected_columns_closure(db, start_cols, max_depth=None)
922
1032
 
923
1033
  affected_tables_all = [t for t in _rollup_to_tables(affected_cols) if t != target]
924
- affected_tables, noise_excluded = noise_filter.filter_nodes(affected_tables_all)
1034
+ affected_tables_noise_filtered, noise_excluded = noise_filter.filter_nodes(
1035
+ affected_tables_all
1036
+ )
1037
+ affected_tables, synthetic_excluded = _exclude_synthetic_tables(
1038
+ db, affected_tables_noise_filtered
1039
+ )
1040
+ noise_excluded = [*noise_excluded, *synthetic_excluded]
925
1041
  kept_tables = set(affected_tables)
926
1042
  affected_columns = [c for c in affected_cols if c.rsplit(".", 1)[0] in kept_tables]
927
1043
 
@@ -985,11 +1101,21 @@ def get_backfill_order(table_qualified: str) -> BackfillOrderResult:
985
1101
  affected_cols, _depth, truncated = _affected_columns_closure(db, start_cols, max_depth=None)
986
1102
 
987
1103
  affected_tables_all = [t for t in _rollup_to_tables(affected_cols) if t != target]
988
- affected_tables, noise_excluded = noise_filter.filter_nodes(affected_tables_all)
1104
+ affected_tables_noise_filtered, noise_excluded = noise_filter.filter_nodes(
1105
+ affected_tables_all
1106
+ )
1107
+ affected_tables, synthetic_excluded = _exclude_synthetic_tables(
1108
+ db, affected_tables_noise_filtered
1109
+ )
1110
+ noise_excluded = [*noise_excluded, *synthetic_excluded]
989
1111
  kept_tables = set(affected_tables)
990
1112
  affected_columns = [c for c in affected_cols if c.rsplit(".", 1)[0] in kept_tables]
991
1113
 
992
- order, had_cycle = _kahn_topological_sort(affected_tables, db)
1114
+ # Option A (issue #38): adjacency is derived from the same COLUMN_LINEAGE
1115
+ # closure as membership — pass the full id set (start + affected) so the
1116
+ # direct producer->consumer edge endpoints are both present.
1117
+ closure_col_ids = _dedup_preserve_order([*start_cols, *affected_cols])
1118
+ order, had_cycle = _kahn_topological_sort(affected_tables, db, closure_col_ids)
993
1119
 
994
1120
  hint = None
995
1121
  if had_cycle:
@@ -1044,19 +1170,26 @@ def diff_impact(changed_files: list[str]) -> DiffImpactResult:
1044
1170
  fp = Path(file_path)
1045
1171
  if not fp.is_absolute():
1046
1172
  fp = root / fp
1047
- rows = db.run_read(GET_TABLES_DEFINED_IN_FILE_QUERY, {"file_path": str(fp)})
1048
- for row in rows:
1049
- tq = row["table_qualified"]
1050
- if tq not in seen_changed:
1051
- seen_changed.add(tq)
1052
- changed_tables.append(tq)
1173
+ for query in (GET_TABLES_DEFINED_IN_FILE_QUERY, GET_TARGET_TABLES_FOR_FILE_QUERY):
1174
+ rows = db.run_read(query, {"file_path": str(fp)})
1175
+ for row in rows:
1176
+ tq = row["table_qualified"]
1177
+ if tq and tq not in seen_changed:
1178
+ seen_changed.add(tq)
1179
+ changed_tables.append(tq)
1053
1180
 
1054
1181
  all_affected_cols: list[str] = []
1055
1182
  affected_seen: set[str] = set()
1183
+ all_start_cols: list[str] = []
1184
+ start_seen: set[str] = set()
1056
1185
  truncated = False
1057
1186
  for tq in changed_tables:
1058
1187
  col_rows = db.run_read(GET_COLUMNS_FOR_TABLE_QUERY, {"table_qualified": tq})
1059
1188
  start_cols = [r["col_id"] for r in col_rows]
1189
+ for col in start_cols:
1190
+ if col not in start_seen:
1191
+ start_seen.add(col)
1192
+ all_start_cols.append(col)
1060
1193
  cols, _depth, tr = _affected_columns_closure(db, start_cols, max_depth=None)
1061
1194
  truncated = truncated or tr
1062
1195
  for col in cols:
@@ -1067,9 +1200,18 @@ def diff_impact(changed_files: list[str]) -> DiffImpactResult:
1067
1200
  affected_tables_all = [
1068
1201
  t for t in _rollup_to_tables(all_affected_cols) if t not in seen_changed
1069
1202
  ]
1070
- affected_tables, noise_excluded = noise_filter.filter_nodes(affected_tables_all)
1203
+ affected_tables_noise_filtered, noise_excluded = noise_filter.filter_nodes(
1204
+ affected_tables_all
1205
+ )
1206
+ affected_tables, synthetic_excluded = _exclude_synthetic_tables(
1207
+ db, affected_tables_noise_filtered
1208
+ )
1209
+ noise_excluded = [*noise_excluded, *synthetic_excluded]
1071
1210
  presentation_facing = [t for t in affected_tables if any(t.startswith(p) for p in prefixes)]
1072
- order, had_cycle = _kahn_topological_sort(affected_tables, db)
1211
+ # Option A (issue #38): same closure-derived adjacency as get_backfill_order —
1212
+ # union of every changed table's start columns plus the unioned affected set.
1213
+ closure_col_ids = _dedup_preserve_order([*all_start_cols, *all_affected_cols])
1214
+ order, had_cycle = _kahn_topological_sort(affected_tables, db, closure_col_ids)
1073
1215
 
1074
1216
  # Resolve external consumers for the blast radius in a single BATCH query (not per-table).
1075
1217
  external_consumers: list[str] = []