sql-code-graph 1.33.0__py3-none-any.whl → 1.34.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: sql-code-graph
3
- Version: 1.33.0
3
+ Version: 1.34.0
4
4
  Summary: SQL code graph analyzer and lineage tracer
5
5
  Project-URL: Homepage, https://github.com/Warhorze/sql-code-graph
6
6
  Project-URL: Repository, https://github.com/Warhorze/sql-code-graph
@@ -1,4 +1,4 @@
1
- sqlcg/__init__.py,sha256=eUMoB2vKl2ho6_w-fT0h0Atb36ONmYWYv3oayOjVeIs,116
1
+ sqlcg/__init__.py,sha256=2T8VTIp-T6SnT3Yg3veGg8ucJmETIyfJ2e1VhoKrDYU,116
2
2
  sqlcg/__main__.py,sha256=1YoFLcqEgTwYq1J3TbUwpkdG0zeeLIf2fJvwWI-CLFU,109
3
3
  sqlcg/cli/__init__.py,sha256=W8fD0LpMq2xm_5WKGNMvJh2WBL1ho5E8hUeAqXQYT1g,28
4
4
  sqlcg/cli/coverage.py,sha256=Xm9ITzZDHv2mJ70Q5jCacVuhDStVrE3gq12_-Ypvtd8,43823
@@ -19,21 +19,21 @@ sqlcg/cli/commands/uninstall.py,sha256=WrA1FnINxnd6mmE4-_QBK0aHBnibstJeAT8swnQKG
19
19
  sqlcg/cli/commands/watch.py,sha256=_OvUQ4m_Pp2lAzI_f9UX2Gwb56zbsaSC4Tj8h51uciA,2806
20
20
  sqlcg/core/__init__.py,sha256=dXvLWpbQ72f5CM6sKSvBDnEGqGuIZaN5MmHyD8Vf1aA,154
21
21
  sqlcg/core/config.py,sha256=QxhIiF_let_ablMa-WttGbcAlCCDIpT4lR4tUVymIEI,19091
22
- sqlcg/core/duckdb_backend.py,sha256=Yc1TBYvR7Yc3jaz4cZ6VseFEhTemm9rBe4jAGK0Fx0s,40112
22
+ sqlcg/core/duckdb_backend.py,sha256=zPsxiO2_qcE6axVZu8P3gUNHnJ1cgCCFv1XXy_t_LEw,42298
23
23
  sqlcg/core/freshness.py,sha256=Cm-w7jACPrt8TIstZpCzDJpYsHCiBo63ni5js8wCGbg,6248
24
- sqlcg/core/graph_db.py,sha256=QLkJk1bDA9s00QQlivBBfoUl6GSZiVSC6_9kY-IxsuY,10255
24
+ sqlcg/core/graph_db.py,sha256=3iZ1J32cEn7_cVhNA1tLiNuJHCyatN4F40HXLLfmwHY,10815
25
25
  sqlcg/core/jobs.py,sha256=Je-fCdSKRgiSsv1W8SgNAlp36a7t7-pJZ-qKPbka9OE,3298
26
26
  sqlcg/core/noise_match.py,sha256=8jxMBum0HB7wnVyS8hHaklO279QOSNKDmPtv-0RujD8,4781
27
27
  sqlcg/core/queries.cypher,sha256=cvPOVe5GUOzJN4bxUvDxNI--xIIP8gm42TR-gUnea4U,4685
28
- sqlcg/core/queries.py,sha256=LgJcxMfienHD-ug2tAKMfcpfFrhtJvVXCsxrco9jAb0,3513
29
- sqlcg/core/queries.sql,sha256=EDSnJKz6gnWWYTlXBmyGmNYNhuX5V11CTP9IWVs0YlU,13695
28
+ sqlcg/core/queries.py,sha256=mxpG_zBENG7kcoz3YolMvHv7s_HagWluakTUw3nYSts,3631
29
+ sqlcg/core/queries.sql,sha256=gT8vl7CgOUZXjElpMZrHvuE_bs2ZJ4FLFA83EKaeRWM,15601
30
30
  sqlcg/core/schema.cypher,sha256=StaJqIzOLwF2X9HYvfHSdGiYGFb_YkiHubPrTu-Qpmg,2898
31
- sqlcg/core/schema.py,sha256=6KrMzbfceM4k0p2i7hPz8eNBco5_1eGXwdGTd3VUaok,1477
31
+ sqlcg/core/schema.py,sha256=pVGzf__czAddmz-fsXgXqq2DTxN9EKom5RvIHLctIPU,1519
32
32
  sqlcg/indexer/__init__.py,sha256=Wh20Unz2OHs1oIyWLrpurPAasF0BET2g4iXtNk7mh2U,56
33
33
  sqlcg/indexer/dbt_adapter.py,sha256=EB5x1WU5Z9d-I97ADDj88S_hG1C4z4nbrv8JUCzXfy8,686
34
34
  sqlcg/indexer/error_classify.py,sha256=-sp8cRmuOBHu_CxnCtaXf34YxHFYwIFNjIrn4LaEv6M,7142
35
35
  sqlcg/indexer/git_delta.py,sha256=zYdH5q-jV7w_ne8Oxdywsy0N3rwUjpd5RjEDurlrMSA,5026
36
- sqlcg/indexer/indexer.py,sha256=qUTMkK3P3eCghjGzmM9QwzfUL23FOJ5siQ5HS_1l4nk,98037
36
+ sqlcg/indexer/indexer.py,sha256=de-3aYaAP_EdG-bwtFwp4e2NGKw6kpDpxb_wsOsQhdU,101712
37
37
  sqlcg/indexer/pool.py,sha256=iMmCQtpDRKBTQBep2_EUq9THcsE18Zgk0hdaFB_CwiA,19006
38
38
  sqlcg/indexer/walker.py,sha256=Cft6JiJtdBFy0HR6L9pJdr5Fg0eRR3XBW1OMtM2apto,1947
39
39
  sqlcg/indexer/watcher.py,sha256=mJQq1LASRLKKwhz0WhCUWPLLqyPR2_-FD_8efYU6gE8,8442
@@ -43,8 +43,8 @@ sqlcg/lineage/schema_resolver.py,sha256=iXt6LYF6UVWsGUpcfbmjmGn9wCgXl721lTGf_8Aa
43
43
  sqlcg/metrics/__init__.py,sha256=hLJ6wm4St8qqYwKh3o9QG7lcEt1BEYM31ccqO9tGpIg,133
44
44
  sqlcg/metrics/store.py,sha256=KuDtxvyAgug9_KtiSCpvgKM2VZM7VSaI3D11uMLjJJk,10604
45
45
  sqlcg/parsers/__init__.py,sha256=AamA8wBbDZV9_zEtZCI4Hyen5UAVKHmBwjTghTt2PZE,785
46
- sqlcg/parsers/ansi_parser.py,sha256=KLbNapfPU20eWK-pullEFvhP78aroU9qlg2aj8ZcESo,37926
47
- sqlcg/parsers/base.py,sha256=D9FwirsitJEGoYNUvclWVbeMgvkePr4Frqbs3G29XEg,92871
46
+ sqlcg/parsers/ansi_parser.py,sha256=Iayv93KDOs3qKrQlrPcpQRd4_K8dEsGHOaTY076e-GY,38032
47
+ sqlcg/parsers/base.py,sha256=QDBDqF55vf4tmlfq_oyB9ZivHvhAuhMXU6sH3mQ5k5Q,97743
48
48
  sqlcg/parsers/bigquery_parser.py,sha256=mOnWTfXB_Dp4JwFE1PVYOB6CDPf5nYE0Dea8kJCl9uQ,2827
49
49
  sqlcg/parsers/postgres_parser.py,sha256=lYfUpQY6j4Qm7ndXBtXbgPoGzYqYddWt5YeFnWKdA6I,946
50
50
  sqlcg/parsers/registry.py,sha256=LXy1F6rqQI6VdxpRvZg_tNpoEucW3mXZHYBMlMONbX4,1496
@@ -59,13 +59,13 @@ sqlcg/server/read_client.py,sha256=ReeDHCff1G-WQzCH3Ow_hAUSEULRGq1lBr2AIOqur5c,1
59
59
  sqlcg/server/selfheal.py,sha256=eL6D81N4tz-Feuw1uh3VzrUkW5qfbhLuLtyMqIiib3A,8299
60
60
  sqlcg/server/server.py,sha256=GUo6vVR2z80zH3geCanm1Eg-B4miUXmv78oLVBD4dlA,37407
61
61
  sqlcg/server/skill.py,sha256=TMCSn3kBYS7GzGz3L50ZwvuBUCD4otS0FSsPGK8juWs,14434
62
- sqlcg/server/tools.py,sha256=PhOvHotGJnTlcRWy_w8IMpIrJ4DSNLhi1vRzQW27aSg,121092
62
+ sqlcg/server/tools.py,sha256=kAig7GqzBRmlKsoqwXnCyB0YRXybgPi5e3sP47m9L1g,121685
63
63
  sqlcg/server/writer.py,sha256=wXaXoL0adyDqrYxmnWb8mWXOmUQJxtLcPImeDPyxVes,18284
64
64
  sqlcg/utils/__init__.py,sha256=--iqt5ThTXmT8Wz7da8hs3n0zDfYPl8P-z5OgRJ_77E,154
65
65
  sqlcg/utils/hashing.py,sha256=H25-sYfxHKb3_IERFnHyAIYNiXN470Oqo5sJT_D3YOA,438
66
66
  sqlcg/utils/ignore.py,sha256=6__DfiYgynI2b8nH52O2BzI1EZcZ_LRazhqEOJVGGcE,1955
67
67
  sqlcg/utils/logging.py,sha256=u0fCmYsLj9o81vawm3xZTHaw68GQYVm7JxG-gP81u8A,840
68
- sql_code_graph-1.33.0.dist-info/METADATA,sha256=zEAtNat6hBOJ-K6FUzyTtc_qrR9waQWGdV8d_1meoQo,17791
69
- sql_code_graph-1.33.0.dist-info/WHEEL,sha256=mffPy8wBnZQn2VnJUU5jE99KsxaSfiyMHV9Yt0aLVxs,87
70
- sql_code_graph-1.33.0.dist-info/entry_points.txt,sha256=Wfe49sVzV9p4eVFGo5RxcV-frr3HOP0yzzst8JBxQLQ,46
71
- sql_code_graph-1.33.0.dist-info/RECORD,,
68
+ sql_code_graph-1.34.0.dist-info/METADATA,sha256=8Upovle3wZwH5UCvVxmYTOMtZDXUjPBeeDLixMzd5wQ,17791
69
+ sql_code_graph-1.34.0.dist-info/WHEEL,sha256=mffPy8wBnZQn2VnJUU5jE99KsxaSfiyMHV9Yt0aLVxs,87
70
+ sql_code_graph-1.34.0.dist-info/entry_points.txt,sha256=Wfe49sVzV9p4eVFGo5RxcV-frr3HOP0yzzst8JBxQLQ,46
71
+ sql_code_graph-1.34.0.dist-info/RECORD,,
sqlcg/__init__.py CHANGED
@@ -1,5 +1,5 @@
1
1
  """SQL Code Graph - SQL lineage and dependency analysis tool."""
2
2
 
3
- __version__ = "1.33.0"
3
+ __version__ = "1.34.0"
4
4
 
5
5
  __all__ = ["__version__"]
@@ -203,6 +203,14 @@ _EDGE_DDLS = [
203
203
  PRIMARY KEY (src_key, dst_key)
204
204
  )
205
205
  """,
206
+ """
207
+ CREATE TABLE IF NOT EXISTS "JOIN_COL_RESOLVE" (
208
+ src_key VARCHAR NOT NULL,
209
+ dst_key VARCHAR NOT NULL,
210
+ bare_col VARCHAR,
211
+ PRIMARY KEY (src_key, dst_key)
212
+ )
213
+ """,
206
214
  ]
207
215
 
208
216
  _INDEX_DDLS = [
@@ -230,6 +238,8 @@ _INDEX_DDLS = [
230
238
  'CREATE INDEX IF NOT EXISTS idx_STAR_SOURCE_dst ON "STAR_SOURCE" (dst_key)',
231
239
  'CREATE INDEX IF NOT EXISTS idx_CONSUMED_BY_src ON "CONSUMED_BY" (src_key)',
232
240
  'CREATE INDEX IF NOT EXISTS idx_CONSUMED_BY_dst ON "CONSUMED_BY" (dst_key)',
241
+ 'CREATE INDEX IF NOT EXISTS idx_JOIN_COL_RESOLVE_src ON "JOIN_COL_RESOLVE" (src_key)',
242
+ 'CREATE INDEX IF NOT EXISTS idx_JOIN_COL_RESOLVE_dst ON "JOIN_COL_RESOLVE" (dst_key)',
233
243
  ]
234
244
 
235
245
  # Node label → column list (all columns of that table, in order)
@@ -304,6 +314,7 @@ _EDGE_EXTRA_COLUMNS: dict[str, list[str]] = {
304
314
  RelType.DECLARES: [],
305
315
  RelType.STAR_SOURCE: ["qualifier", "target_table", "confidence"],
306
316
  RelType.CONSUMED_BY: [],
317
+ RelType.JOIN_COL_RESOLVE: ["bare_col"],
307
318
  }
308
319
 
309
320
 
@@ -752,6 +763,7 @@ class DuckDBBackend(GraphBackend):
752
763
  "UPDATES",
753
764
  "DECLARES",
754
765
  "STAR_SOURCE",
766
+ "JOIN_COL_RESOLVE",
755
767
  ):
756
768
  self._conn.execute(
757
769
  f'DELETE FROM "{edge_table}" WHERE src_key IN ({placeholders})',
@@ -891,6 +903,39 @@ class DuckDBBackend(GraphBackend):
891
903
  row = result.fetchone()
892
904
  return int(row[0]) if row else 0
893
905
 
906
+ def resolve_join_columns(self) -> int:
907
+ """Resolve JOIN_COL_RESOLVE markers into concrete COLUMN_LINEAGE edges.
908
+
909
+ Runs AFTER ingestion AND after the catalog has been applied, so that
910
+ information_schema ``HAS_COLUMN`` rows are present. For each parse-time
911
+ marker (a bare unqualified column in a >=2-table join projection), it
912
+ finds which of the query's source tables owns a column with that bare
913
+ name (any HAS_COLUMN source), normalising casing on both sides:
914
+
915
+ - exactly one owner -> one high-confidence (0.9) edge (the dominant
916
+ live case: the second table is visible only via information_schema);
917
+ - more than one owner -> one low-confidence (0.5) edge per owner
918
+ (genuine ambiguity; over-attribute, never silently mis-pick);
919
+ - zero owners -> emit nothing (honest empty; XML-DDL gap).
920
+
921
+ Mirrors :meth:`expand_star_sources` (post-index SQL pass). The
922
+ suppressed sqlglot mis-bind edge is never emitted at parse time, so this
923
+ pass is the SOLE producer of these columns' edges — it MUST run in every
924
+ path that re-creates the markers (full index + both incremental paths).
925
+
926
+ Returns:
927
+ Number of COLUMN_LINEAGE JOIN_COL_RESOLVED edges present afterward.
928
+ """
929
+ from sqlcg.core.queries import RESOLVE_JOIN_COLUMNS_QUERY
930
+
931
+ self._conn.execute(RESOLVE_JOIN_COLUMNS_QUERY)
932
+ result = self._conn.execute(
933
+ 'SELECT count(*) AS n FROM "COLUMN_LINEAGE" WHERE transform = ?',
934
+ ["JOIN_COL_RESOLVED"],
935
+ )
936
+ row = result.fetchone()
937
+ return int(row[0]) if row else 0
938
+
894
939
  # ------------------------------------------------------------------
895
940
  # Full-rebuild helpers
896
941
  # ------------------------------------------------------------------
@@ -927,6 +972,7 @@ class DuckDBBackend(GraphBackend):
927
972
  "DECLARES",
928
973
  "STAR_SOURCE",
929
974
  "CONSUMED_BY",
975
+ "JOIN_COL_RESOLVE",
930
976
  ]
931
977
  for tbl in edge_tables + node_tables:
932
978
  self._conn.execute(f'DELETE FROM "{tbl}"')
sqlcg/core/graph_db.py CHANGED
@@ -273,6 +273,16 @@ class GraphBackend(ABC):
273
273
  """
274
274
  raise NotImplementedError(f"{type(self).__name__} does not support expand_star_sources")
275
275
 
276
+ def resolve_join_columns(self) -> int:
277
+ """Resolve JOIN_COL_RESOLVE markers into per-column COLUMN_LINEAGE edges.
278
+
279
+ Runs once per index after ingestion AND catalog apply, resolving bare
280
+ unqualified join columns against the full post-index HAS_COLUMN catalog
281
+ (including information_schema rows unavailable at parse time). Concrete
282
+ backends must override this; returns the total JOIN_COL_RESOLVED edge count.
283
+ """
284
+ raise NotImplementedError(f"{type(self).__name__} does not support resolve_join_columns")
285
+
276
286
 
277
287
  def indexed_repo_root(db: "GraphBackend") -> Path | None:
278
288
  """Return the indexed root path stored on the first Repo node, or None.
sqlcg/core/queries.py CHANGED
@@ -51,6 +51,8 @@ LIST_DIALECTS_AND_REPOS_QUERY = _Q["LIST_DIALECTS_AND_REPOS"]
51
51
  EXPAND_STAR_SOURCES_QUERY = _Q["EXPAND_STAR_SOURCES"]
52
52
  EXPAND_STAR_SOURCES_HAS_COLUMN_QUERY = _Q["EXPAND_STAR_SOURCES_HAS_COLUMN"]
53
53
  EXPAND_STAR_SOURCES_LINEAGE_QUERY = _Q["EXPAND_STAR_SOURCES_LINEAGE"]
54
+ RESOLVE_JOIN_COLUMNS_QUERY = _Q["RESOLVE_JOIN_COLUMNS"]
55
+ COUNT_JOIN_COL_RESOLVED_QUERY = _Q["COUNT_JOIN_COL_RESOLVED"]
54
56
  COUNT_STAR_SOURCES_QUERY = _Q["COUNT_STAR_SOURCES"]
55
57
  COUNT_STAR_EXPANSIONS_QUERY = _Q["COUNT_STAR_EXPANSIONS"]
56
58
  FIND_DEFINITION_QUERY = _Q["FIND_DEFINITION"]
sqlcg/core/queries.sql CHANGED
@@ -148,6 +148,53 @@ JOIN "SqlTable" tgt ON tgt.qualified = q.target_table
148
148
  WHERE q.target_table <> ''
149
149
  AND q.target_table <> t.qualified
150
150
 
151
+ -- RESOLVE_JOIN_COLUMNS
152
+ -- Resolve JOIN_COL_RESOLVE markers (bare unqualified join columns whose owning
153
+ -- table was only knowable post-index, e.g. catalogued via information_schema)
154
+ -- into concrete COLUMN_LINEAGE edges.
155
+ --
156
+ -- For each marker (src_key=query_id, dst_key=<dst_table>.<dst_col>, bare_col):
157
+ -- find which of the query's SELECTS_FROM source tables OWNS a column named
158
+ -- bare_col (any HAS_COLUMN source: ddl | information_schema | star_expansion |
159
+ -- usage). Casing is normalised on BOTH sides of the owner-match join because
160
+ -- information_schema stores col_name as exported (e.g. STATUS) while the parsed
161
+ -- bare name is lowercased (status).
162
+ --
163
+ -- Confidence contract:
164
+ -- - exactly one owning table -> HIGH confidence (0.9), the resolved edge
165
+ -- - more than one owning table -> one LOW confidence (0.5) edge per owner
166
+ -- (genuine ambiguity: over-attribute, never silently mis-pick)
167
+ -- - zero owning tables -> emit nothing (honest empty; XML-DDL gap)
168
+ -- A window COUNT over the owning tables per marker selects the confidence in one
169
+ -- pass.
170
+ -- params: []
171
+ INSERT OR REPLACE INTO "COLUMN_LINEAGE"
172
+ (src_key, dst_key, transform, confidence, query_id, inferred_from_source_name)
173
+ SELECT
174
+ src_key,
175
+ dst_key,
176
+ 'JOIN_COL_RESOLVED' AS transform,
177
+ CASE WHEN n_owners = 1 THEN 0.9 ELSE 0.5 END AS confidence,
178
+ query_id,
179
+ FALSE AS inferred_from_source_name
180
+ FROM (
181
+ SELECT
182
+ c.id AS src_key,
183
+ m.dst_key AS dst_key,
184
+ m.src_key AS query_id,
185
+ count(*) OVER (PARTITION BY m.src_key, m.dst_key) AS n_owners
186
+ FROM "JOIN_COL_RESOLVE" m
187
+ JOIN "SELECTS_FROM" sf ON sf.src_key = m.src_key
188
+ JOIN "HAS_COLUMN" hc ON hc.src_key = sf.dst_key
189
+ JOIN "SqlColumn" c
190
+ ON c.id = hc.dst_key
191
+ AND lower(c.col_name) = lower(m.bare_col)
192
+ ) owners
193
+
194
+ -- COUNT_JOIN_COL_RESOLVED
195
+ -- params: []
196
+ SELECT count(*) AS n FROM "COLUMN_LINEAGE" WHERE transform = 'JOIN_COL_RESOLVED'
197
+
151
198
  -- COUNT_STAR_SOURCES
152
199
  -- params: []
153
200
  SELECT count(*) AS n FROM "STAR_SOURCE"
sqlcg/core/schema.py CHANGED
@@ -3,7 +3,7 @@
3
3
  from enum import StrEnum
4
4
  from importlib.resources import files
5
5
 
6
- SCHEMA_VERSION = "10"
6
+ SCHEMA_VERSION = "11"
7
7
 
8
8
 
9
9
  class NodeLabel(StrEnum):
@@ -29,6 +29,7 @@ class RelType(StrEnum):
29
29
  DECLARES = "DECLARES"
30
30
  STAR_SOURCE = "STAR_SOURCE"
31
31
  CONSUMED_BY = "CONSUMED_BY"
32
+ JOIN_COL_RESOLVE = "JOIN_COL_RESOLVE"
32
33
 
33
34
 
34
35
  # Backward-compatible aliases
sqlcg/indexer/indexer.py CHANGED
@@ -47,6 +47,7 @@ class FileRowSet:
47
47
  selects_from_edges: list[dict] = field(default_factory=list)
48
48
  column_lineage_edges: list[dict] = field(default_factory=list)
49
49
  star_source_edges: list[dict] = field(default_factory=list)
50
+ join_col_resolve_edges: list[dict] = field(default_factory=list)
50
51
  counts: dict = field(
51
52
  default_factory=lambda: {"tables": 0, "edges": 0, "columns_defined": 0, "star_sources": 0}
52
53
  )
@@ -71,6 +72,7 @@ class BatchRowBuffer:
71
72
  selects_from_edges: list[dict] = field(default_factory=list)
72
73
  column_lineage_edges: list[dict] = field(default_factory=list)
73
74
  star_source_edges: list[dict] = field(default_factory=list)
75
+ join_col_resolve_edges: list[dict] = field(default_factory=list)
74
76
 
75
77
  def extend(self, rows: FileRowSet) -> None:
76
78
  """Accumulate one file's row sets into this buffer."""
@@ -84,6 +86,7 @@ class BatchRowBuffer:
84
86
  self.selects_from_edges.extend(rows.selects_from_edges)
85
87
  self.column_lineage_edges.extend(rows.column_lineage_edges)
86
88
  self.star_source_edges.extend(rows.star_source_edges)
89
+ self.join_col_resolve_edges.extend(rows.join_col_resolve_edges)
87
90
 
88
91
  @classmethod
89
92
  def from_single(cls, rows: FileRowSet) -> "BatchRowBuffer":
@@ -292,6 +295,9 @@ def _flush_row_batch(
292
295
  star_source_edges = list(
293
296
  {(r["src_key"], r["dst_key"]): r for r in buf.star_source_edges}.values()
294
297
  )
298
+ join_col_resolve_edges = list(
299
+ {(r["src_key"], r["dst_key"]): r for r in buf.join_col_resolve_edges}.values()
300
+ )
295
301
 
296
302
  # --- Phase B2: usage-derived catalog harvest ---
297
303
  # Post-dedup: for each COLUMN_LINEAGE src endpoint whose table is a physical
@@ -345,6 +351,11 @@ def _flush_row_batch(
345
351
  star_source_edges = [
346
352
  r for r in star_source_edges if r["dst_key"] not in excluded_tables
347
353
  ]
354
+ join_col_resolve_edges = [
355
+ r
356
+ for r in join_col_resolve_edges
357
+ if _table_of_column_id(r["dst_key"]) not in excluded_tables
358
+ ]
348
359
  column_lineage_edges = [
349
360
  r
350
361
  for r in column_lineage_edges
@@ -372,6 +383,9 @@ def _flush_row_batch(
372
383
  NodeLabel.COLUMN, NodeLabel.COLUMN, RelType.COLUMN_LINEAGE, column_lineage_edges
373
384
  )
374
385
  db.upsert_edges_bulk(NodeLabel.QUERY, NodeLabel.TABLE, RelType.STAR_SOURCE, star_source_edges)
386
+ db.upsert_edges_bulk(
387
+ NodeLabel.QUERY, NodeLabel.COLUMN, RelType.JOIN_COL_RESOLVE, join_col_resolve_edges
388
+ )
375
389
 
376
390
 
377
391
  def _subprocess_parse_worker(parser_cls, dialect, path, sql, q):
@@ -882,6 +896,12 @@ class Indexer:
882
896
  if profile:
883
897
  _t_catalog_end = time.perf_counter()
884
898
 
899
+ # Post-catalog: resolve JOIN_COL_RESOLVE markers against the full HAS_COLUMN
900
+ # catalog (Bug #5, PR-5 option b). MUST run after the catalog is applied so
901
+ # information_schema columns are present; the suppressed sqlglot mis-bind
902
+ # edge is never emitted, so this is the sole producer of the resolved edges.
903
+ join_cols_resolved = self._resolve_join_columns(db)
904
+
885
905
  # Classify all errors into buckets for measurement and reporting
886
906
  error_summary: dict[str, int] = {
887
907
  "E1": 0,
@@ -931,6 +951,7 @@ class Indexer:
931
951
  "columns_defined": nonlocal_counts["columns_defined"],
932
952
  "star_sources": nonlocal_counts["star_sources"],
933
953
  "star_edges_expanded": star_edges_expanded,
954
+ "join_cols_resolved": join_cols_resolved,
934
955
  "quality": nonlocal_counts["quality"],
935
956
  "error_summary": error_summary,
936
957
  "degraded_files": degraded_files,
@@ -1329,6 +1350,14 @@ class Indexer:
1329
1350
  # ---- Step 8: Single star expansion (same as index_repo) ------------------
1330
1351
  self._expand_star_sources(db)
1331
1352
 
1353
+ # ---- Step 8b: Resolve JOIN_COL_RESOLVE markers (Bug #5, PR-5 option b) ----
1354
+ # The incremental path does NOT reapply the catalog — it resolves against the
1355
+ # information_schema HAS_COLUMN rows persisted by the prior full index. Without
1356
+ # this, a delta touching a join file would re-create the markers but never
1357
+ # resolve them, silently dropping the join-column edges until the next full
1358
+ # index (plan-review BLOCKER).
1359
+ self._resolve_join_columns(db)
1360
+
1332
1361
  # ---- Step 9: Persist new SHA on success ----------------------------------
1333
1362
  db.set_indexed_sha(new_sha)
1334
1363
 
@@ -1382,6 +1411,13 @@ class Indexer:
1382
1411
  # Re-run star expansion after re-indexing
1383
1412
  self._expand_star_sources(db)
1384
1413
 
1414
+ # Resolve JOIN_COL_RESOLVE markers (Bug #5, PR-5 option b) against the
1415
+ # information_schema HAS_COLUMN rows persisted by the prior full index. The
1416
+ # single-file path does NOT reapply the catalog; without re-resolving here a
1417
+ # reindex of a join file would re-create the markers but drop the resolved
1418
+ # join-column edges until the next full index (plan-review BLOCKER).
1419
+ self._resolve_join_columns(db)
1420
+
1385
1421
  def _index_single_file(self, parser, path: Path, sql: str, timeout: int) -> ParsedFile:
1386
1422
  """Parse one file, with optional timeout via subprocess isolation.
1387
1423
 
@@ -1706,6 +1742,21 @@ class Indexer:
1706
1742
  )
1707
1743
  rows.counts["star_sources"] += 1
1708
1744
 
1745
+ # JOIN_COL_RESOLVE markers (Bug #5, PR-5 option b): bare unqualified
1746
+ # join-projection columns whose owning source table is only knowable
1747
+ # post-index (e.g. catalogued via information_schema). The sqlglot
1748
+ # mis-bind edge was suppressed at parse time; the post-index resolver
1749
+ # (DuckDBBackend.resolve_join_columns) is the SOLE producer of these
1750
+ # columns' COLUMN_LINEAGE edges. src_key=query_id, dst_key=<dst>.<col>.
1751
+ for jcr in stmt.join_col_resolves:
1752
+ rows.join_col_resolve_edges.append(
1753
+ {
1754
+ "src_key": query_id,
1755
+ "dst_key": jcr.dst.full_id,
1756
+ "bare_col": jcr.bare_col,
1757
+ }
1758
+ )
1759
+
1709
1760
  # Upsert target table node (if not already a defined_table)
1710
1761
  # so that star expansion can create destination columns.
1711
1762
  # #44: when a canonical_by_bare index is available, attempt to resolve
@@ -1930,6 +1981,22 @@ class Indexer:
1930
1981
  """
1931
1982
  return db.expand_star_sources()
1932
1983
 
1984
+ def _resolve_join_columns(self, db: GraphBackend) -> int:
1985
+ """Run the post-index join-column resolution (Bug #5, PR-5 option b).
1986
+
1987
+ Calls DuckDBBackend.resolve_join_columns(), which resolves JOIN_COL_RESOLVE
1988
+ markers (bare unqualified join columns) against the full post-index
1989
+ HAS_COLUMN catalog — including information_schema rows unavailable at parse
1990
+ time. MUST run AFTER the catalog is applied (full-index path) and AFTER star
1991
+ expansion in every path that re-creates the markers; the suppressed sqlglot
1992
+ mis-bind edge is never emitted, so this pass is the SOLE producer of these
1993
+ columns' edges.
1994
+
1995
+ Returns:
1996
+ Number of COLUMN_LINEAGE JOIN_COL_RESOLVED edges after resolution.
1997
+ """
1998
+ return db.resolve_join_columns()
1999
+
1933
2000
  def _ingest_external_consumers(self, db: GraphBackend, path: Path) -> dict:
1934
2001
  """Ingest declared external downstream consumers from .sqlcg.toml.
1935
2002
 
@@ -622,6 +622,7 @@ class AnsiParser(SqlParser):
622
622
  )
623
623
  column_lineage = extraction.edges
624
624
  star_sources = extraction.star_sources
625
+ join_col_resolves = extraction.join_col_resolves
625
626
  stmt_qualify_failed = extraction.qualify_failed
626
627
 
627
628
  # Extract defined columns for CREATE TABLE statements
@@ -666,6 +667,7 @@ class AnsiParser(SqlParser):
666
667
  parsing_mode="sqlglot",
667
668
  defined_columns=defined_columns,
668
669
  star_sources=star_sources,
670
+ join_col_resolves=join_col_resolves,
669
671
  defined_body=defined_body,
670
672
  clone_source=clone_source,
671
673
  )
sqlcg/parsers/base.py CHANGED
@@ -204,6 +204,32 @@ class StarSource:
204
204
  qualifier: str | None = None
205
205
 
206
206
 
207
+ @dataclass(frozen=True)
208
+ class JoinColResolve:
209
+ """A deferred-resolution marker for a bare unqualified join-projection column.
210
+
211
+ Emitted (instead of the sqlglot mis-bind edge) when a top-level INSERT/CREATE
212
+ SELECT projection contains a bare column (``exp.Column`` with no ``.table``)
213
+ AND the statement has >=2 source tables. The owning source table is only
214
+ knowable AFTER indexing (e.g. catalogued via information_schema), so the
215
+ binding is deferred to ``DuckDBBackend.resolve_join_columns`` — the post-index
216
+ pass is the SOLE producer of the resolved COLUMN_LINEAGE edge for this column.
217
+
218
+ Mirrors the over-attribution-instead-of-mis-bind discipline of the
219
+ ``CTE_PROJECTION_AMBIGUOUS`` precedent (see PR-5 in
220
+ plan/sprints/bugfix_lineage_correctness_validation.md).
221
+
222
+ Attributes:
223
+ dst: The destination ColumnRef the projection would have produced
224
+ (``<dst_table>.<dst_col_name>``); its full_id is the marker's dst_key.
225
+ bare_col: The bare (unqualified) column name to resolve against source
226
+ tables' HAS_COLUMN catalog.
227
+ """
228
+
229
+ dst: ColumnRef
230
+ bare_col: str = ""
231
+
232
+
207
233
  @dataclass(frozen=True)
208
234
  class LineageExtraction:
209
235
  """Result of column lineage extraction, including both edges and star markers.
@@ -211,6 +237,8 @@ class LineageExtraction:
211
237
  Attributes:
212
238
  edges: List of LineageEdge objects extracted from the statement
213
239
  star_sources: List of StarSource markers for SELECT * projections
240
+ join_col_resolves: List of JoinColResolve markers for bare unqualified
241
+ >=2-table join projections, resolved post-index against HAS_COLUMN.
214
242
  qualify_failed: True when the qualify() step raised an exception for
215
243
  this statement (schema-free retry may still yield a partial scope).
216
244
  Propagated to QueryNode.qualify_failed for per-statement storage.
@@ -218,6 +246,7 @@ class LineageExtraction:
218
246
 
219
247
  edges: list[LineageEdge]
220
248
  star_sources: list[StarSource]
249
+ join_col_resolves: list[JoinColResolve] = field(default_factory=list)
221
250
  qualify_failed: bool = False
222
251
 
223
252
 
@@ -268,6 +297,7 @@ class QueryNode:
268
297
  confidence: float = 1.0
269
298
  parsing_mode: str = "sqlglot"
270
299
  star_sources: list[StarSource] = field(default_factory=list)
300
+ join_col_resolves: list[JoinColResolve] = field(default_factory=list)
271
301
  defined_columns: list[str] = field(default_factory=list)
272
302
  defined_body: Any | None = None
273
303
  start_line: int = 0 # 1-based start line of statement in file; 0 = unknown
@@ -413,6 +443,22 @@ def normalize_keys(parsed: "ParsedFile", aliases: dict[str, str]) -> None:
413
443
  if stmt.clone_source is not None:
414
444
  stmt.clone_source = _norm_ref(stmt.clone_source)
415
445
 
446
+ # join_col_resolves (JoinColResolve.dst is a ColumnRef over a TableRef).
447
+ # The dst table is the INSERT/CREATE target, which is aliased above; keep
448
+ # the marker's dst_key in lockstep so the post-index resolver writes a
449
+ # COLUMN_LINEAGE edge with the same dst key as the rest of the graph.
450
+ new_join_col_resolves: list[JoinColResolve] = []
451
+ for jcr in stmt.join_col_resolves:
452
+ new_dst_table = _norm_ref(jcr.dst.table)
453
+ if new_dst_table is None:
454
+ # Empty-identity target — drop the marker (skip already logged).
455
+ continue
456
+ if new_dst_table is not jcr.dst.table:
457
+ new_dst = dataclasses.replace(jcr.dst, table=new_dst_table)
458
+ jcr = dataclasses.replace(jcr, dst=new_dst)
459
+ new_join_col_resolves.append(jcr)
460
+ stmt.join_col_resolves = new_join_col_resolves
461
+
416
462
  # column_lineage — LineageEdge and ColumnRef are frozen; rewrite via replace
417
463
  new_edges: list[LineageEdge] = []
418
464
  for edge in stmt.column_lineage:
@@ -1067,6 +1113,7 @@ class SqlParser(ABC):
1067
1113
 
1068
1114
  edges: list[LineageEdge] = []
1069
1115
  star_sources: list[StarSource] = []
1116
+ join_col_resolves: list[JoinColResolve] = []
1070
1117
  _qualify_failed: bool = False
1071
1118
 
1072
1119
  # NEW (T-07-06): Record MERGE statements explicitly as deferred.
@@ -1417,6 +1464,23 @@ class SqlParser(ABC):
1417
1464
  except Exception:
1418
1465
  pass
1419
1466
 
1467
+ # Bug #5 (PR-5, option b): count distinct PHYSICAL source tables ONCE per
1468
+ # statement (never per column — CLAUDE.md hot-loop invariant). When a
1469
+ # top-level INSERT/CREATE-SELECT join has >=2 physical sources, bare
1470
+ # unqualified projection columns cannot be reliably bound at parse time
1471
+ # (the owning table may be catalogued only via information_schema, which
1472
+ # is unavailable to qualify()). For those columns we SUPPRESS the sqlglot
1473
+ # mis-bind edge and record a JoinColResolve marker instead — resolved
1474
+ # post-index against the full HAS_COLUMN catalog by
1475
+ # DuckDBBackend.resolve_join_columns(). This mirrors the
1476
+ # CTE_PROJECTION_AMBIGUOUS over-attribution discipline above, but defers
1477
+ # resolution rather than over-attributing immediately. No new
1478
+ # qualify/build_scope/exp.expand/sg_lineage call is added.
1479
+ _physical_source_count = sum(
1480
+ 1 for _s in (query_sources or []) if getattr(_s, "role", "table") == "table"
1481
+ )
1482
+ _join_resolve_active = dst_table is not None and _physical_source_count >= 2
1483
+
1420
1484
  # Extract output columns — skip positions handled by the positional INSERT block
1421
1485
  for loop_idx, col_expr in enumerate(col_expressions):
1422
1486
  if loop_idx in positional_col_names:
@@ -1490,6 +1554,26 @@ class SqlParser(ABC):
1490
1554
  expr_type = type(col_expr).__name__
1491
1555
  out.errors.append(f"col_lineage_skip:func_fallback:{expr_type}")
1492
1556
  continue
1557
+
1558
+ # Bug #5 (PR-5, option b): bare unqualified column in a >=2-physical-
1559
+ # table join. SUPPRESS the sqlglot mis-bind edge (continue past
1560
+ # sg_lineage) and record a JoinColResolve marker; the post-index
1561
+ # resolver is the SOLE producer of this column's edge(s). A qualified
1562
+ # reference (o.amount) has c.table set and fails the bare test, so it
1563
+ # takes the existing sg_lineage path unchanged. This is the same
1564
+ # cardinality as the existing edge append — no new hot-loop op.
1565
+ if _join_resolve_active and dst_table is not None:
1566
+ _bare_cols = [c for c in col_expr.find_all(exp.Column) if not c.table]
1567
+ if _bare_cols:
1568
+ _bare_name = _bare_cols[0].name or col_name
1569
+ join_col_resolves.append(
1570
+ JoinColResolve(
1571
+ dst=ColumnRef(dst_table, col_name),
1572
+ bare_col=_bare_name,
1573
+ )
1574
+ )
1575
+ continue # skip sg_lineage for this projection
1576
+
1493
1577
  # Schema validation: if schema is loaded and column isn't in it,
1494
1578
  # emit a reduced-confidence edge rather than a full-confidence one.
1495
1579
  # Skip when the expression has an explicit alias — the alias is the
@@ -1789,7 +1873,10 @@ class SqlParser(ABC):
1789
1873
  out.errors.append(f"col_lineage:statement:{exc}")
1790
1874
 
1791
1875
  return LineageExtraction(
1792
- edges=edges, star_sources=star_sources, qualify_failed=_qualify_failed
1876
+ edges=edges,
1877
+ star_sources=star_sources,
1878
+ join_col_resolves=join_col_resolves,
1879
+ qualify_failed=_qualify_failed,
1793
1880
  )
1794
1881
 
1795
1882
  def _resolve_star_source(
sqlcg/server/tools.py CHANGED
@@ -2532,12 +2532,16 @@ def _diff_lost_producers(
2532
2532
  lost_raw = before - after
2533
2533
  new_raw = after - before
2534
2534
 
2535
- # Exclude synthetic from both sides.
2536
- if lost_raw:
2537
- lost_cleaned, _ = _exclude_synthetic_tables(db, sorted(lost_raw))
2538
- lost_set = set(lost_cleaned)
2539
- else:
2540
- lost_set = set()
2535
+ # Lost-producer side: do NOT apply _exclude_synthetic_tables.
2536
+ # The lost set comes exclusively from GET_PRODUCER_TABLES (SqlQuery.target_table),
2537
+ # which is only ever a real top-level write target — a CTE/derived intermediate alias
2538
+ # is never a SqlQuery.target_table. An INSERT target with no surviving consumers
2539
+ # keeps kind='derived' (the indexer only upgrades 'derived'→'table' when a SELECT
2540
+ # reads the table elsewhere), so applying _exclude_synthetic_tables here would drop
2541
+ # genuine write targets that happen to be unconsumed (bug #3 false-safety signal).
2542
+ # The synthetic filter is still applied to the NEW-producer side below (for rename
2543
+ # classification), where genuine intermediate nodes can appear via new CTEs/views.
2544
+ lost_set = set(lost_raw)
2541
2545
 
2542
2546
  if new_raw:
2543
2547
  new_cleaned, _ = _exclude_synthetic_tables(db, sorted(new_raw))