sql-code-graph 1.36.1__tar.gz → 1.36.2__tar.gz

{sql_code_graph-1.36.1 → sql_code_graph-1.36.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sql-code-graph
-Version: 1.36.1
+Version: 1.36.2
 Summary: SQL code graph analyzer and lineage tracer
 Project-URL: Homepage, https://github.com/Warhorze/sql-code-graph
 Project-URL: Repository, https://github.com/Warhorze/sql-code-graph

sql_code_graph-1.36.2/plan/metrics/gain_1.36.2_f6a335a.json

@@ -0,0 +1,33 @@
+{"total_calls": 1791, "last_7d_calls": 1791, "index_runs": 5, "feedback_tp": 0, 
+"feedback_total": 0, "top_tools": [{"name": "execute_sql", "count": 604}, 
+{"name": "trace_column_lineage", "count": 470}, {"name": 
+"list_dialects_and_repos", "count": 154}], "execute_sql_ratio": 0.34, 
+"parse_quality": {"sqlglot": 5162, "scripting": 20, "dynamic_sql": 3}, 
+"coverage": {"catalogued_tables": 5892, "total_tables": 6553, "good_edges": 
+54998, "total_edges": 63088, "phantom_edges": 12346, "blindspot_tables": 633, 
+"good_edges_strict": 49453, "edge_health_strict_pct": 78.39, 
+"good_edges_scoped": 37305, "total_edges_scoped": 37628, 
+"edge_health_scoped_pct": 99.14, "phantom_confirmed": 11686, 
+"phantom_contradicted": 239, "phantom_unverified": 421, "top_blindspot_tables": 
+[{"table": "ba.wtfa_kpi_datum_klant", "bad_edges": 70}, {"table": 
+"ba.wtfs_voorraad_dagstand", "bad_edges": 55}, {"table": 
+"ba.wtfs_voorraad_zondagstand", "bad_edges": 55}, {"table": 
+"wtfe_kpi_voorraad_artikel_voorraadlocatie", "bad_edges": 32}, {"table": 
+"ba.wtda_toeleverancier", "bad_edges": 13}, {"table": 
+"ba.wtfs_voorraad_dagstand_webshop_periode", "bad_edges": 13}, {"table": 
+"da.etl_rtwbs_cadeaukaarten", "bad_edges": 12}, {"table": 
+"wtfe_kpi_schapbeschikbaarheid_voorraadlocatie", "bad_edges": 12}, {"table": 
+"da.ttint_inventdim_formule", "bad_edges": 10}, {"table": 
+"ba.wtdm_klant_herkomst", "bad_edges": 8}], "blindspot_tables_for_80pct": 11, 
+"files_indexed": 1335, "indexed_sha": 
+"fdf1b551a34601a6cf3ce1c8b9f76e27ce2753e6", "db_path": 
+"/tmp/sqlcg_dropfix_f6a335a.db", "index_timestamp": 1781505432.175605, 
+"degraded_parse_total": 5185, "degraded_parse_queries": 27, 
+"degraded_parse_by_dir": {"etl": {"degraded": 8, "total": 2772}, "ddl": 
+{"degraded": 19, "total": 2413}}, "zero_edge_write_queries": 956, 
+"total_write_queries": 2322, "cte_key_collisions": 71, 
+"rescuable_unqualified_edges": 125, "info_schema_has_column_rows": 100193, 
+"cte_source_gap_writes": 0, "resolvable_write_col_edges": 38068, 
+"transitive_col_edges": 5312, "qualify_failed_statements": 9, 
+"tool_version_stale": false, "indexed_tool_version": "1.36.2", 
+"running_tool_version": "1.36.2"}}

sql_code_graph-1.36.2/plan/metrics/gain_drop_alter_phantom_baseline_v1361.json

@@ -0,0 +1,30 @@
+{"total_calls": 1721, "last_7d_calls": 1721, "index_runs": 5, "feedback_tp": 0, 
+"feedback_total": 0, "top_tools": [{"name": "execute_sql", "count": 580}, 
+{"name": "trace_column_lineage", "count": 452}, {"name": 
+"list_dialects_and_repos", "count": 148}], "execute_sql_ratio": 0.34, 
+"parse_quality": {"sqlglot": 5166, "scripting": 20, "dynamic_sql": 3}, 
+"coverage": {"catalogued_tables": 5864, "total_tables": 6673, "good_edges": 
+53811, "total_edges": 61983, "phantom_edges": 12241, "blindspot_tables": 631, 
+"good_edges_strict": 48907, "edge_health_strict_pct": 78.9, "good_edges_scoped":
+37502, "total_edges_scoped": 38063, "edge_health_scoped_pct": 98.53, 
+"phantom_confirmed": 11701, "phantom_contradicted": 352, "phantom_unverified": 
+188, "top_blindspot_tables": [{"table": "ba.wtfa_kpi_datum_klant", "bad_edges": 
+70}, {"table": "ba.wtfs_voorraad_dagstand", "bad_edges": 55}, {"table": 
+"ba.wtfs_voorraad_zondagstand", "bad_edges": 55}, {"table": 
+"da.rtgmd_postnl_customer_usage", "bad_edges": 43}, {"table": 
+"da.rtgmd_postnl_athome_3s", "bad_edges": 33}, {"table": 
+"wtfe_kpi_voorraad_artikel_voorraadlocatie", "bad_edges": 32}, {"table": 
+"da.rtgmd_postnl_facturen", "bad_edges": 30}, {"table": "da.hthyb_pages", 
+"bad_edges": 23}, {"table": "da.rtgmd_dhb_extra_kosten", "bad_edges": 19}, 
+{"table": "da.hthyb_page_brandplacement", "bad_edges": 17}], 
+"blindspot_tables_for_80pct": 18, "files_indexed": 1335, "indexed_sha": 
+"fdf1b551a34601a6cf3ce1c8b9f76e27ce2753e6", "db_path": "/tmp/ab_v1361.db", 
+"index_timestamp": 1781502931.191137, "degraded_parse_total": 5189, 
+"degraded_parse_queries": 27, "degraded_parse_by_dir": {"etl": {"degraded": 8, 
+"total": 2776}, "ddl": {"degraded": 19, "total": 2413}}, 
+"zero_edge_write_queries": 969, "total_write_queries": 2323, 
+"cte_key_collisions": 0, "rescuable_unqualified_edges": 125, 
+"info_schema_has_column_rows": 100193, "cte_source_gap_writes": 0, 
+"resolvable_write_col_edges": 37713, "transitive_col_edges": 4558, 
+"qualify_failed_statements": 9, "tool_version_stale": false, 
+"indexed_tool_version": "1.36.1", "running_tool_version": "1.36.1"}}

sql_code_graph-1.36.2/plan/sprints/fix_drop_alter_phantom_source_node.md

@@ -0,0 +1,551 @@
+# Feature Plan: DROP/ALTER phantom source-node + edge suppression (#161 part 2)
+
+**Status:** REVIEWED
+**Owner:** architect-planner
+**Version target:** PATCH. Master is currently `v1.36.1`; this is a graph-correctness
+fix that only *removes* phantom nodes/edges and adds no public surface (no new CLI
+flag, MCP tool, node/edge type, or config key), so per [CLAUDE.md](../../CLAUDE.md)
+SemVer it is a **patch** bump. Reconcile the exact number against `master` at the
+moment the PR is cut: if master is still `v1.36.1` → ship `v1.36.2`; if a later patch
+has landed, take the next patch above whatever master then carries. Do not assume
+`1.36.2` blindly.
+**Cross-refs:** issue #161 (phantom source node for non-table DDL), #159 (create-kind
+catch-all → OTHER), #166 / PR-B (the non-table-CREATE source-gate that this extends).
+[`sprint_snowflake_lineage_patterns.md`](sprint_snowflake_lineage_patterns.md) PR-B.
+
+## Summary
+
+`DROP TABLE/VIEW ... <name>` and `ALTER TABLE <name> ...` statements mint a phantom
+`SqlTable` source node (+ a phantom `SELECTS_FROM` edge) for their *target* object,
+even though DROP/ALTER carry no data flow. This inflates the graph with degree-0
+island nodes (the viz finding) and ~1,308 phantom `SELECTS_FROM` edges on the live
+DWH corpus. The fix extends the existing #161 single source-gate in
+`AnsiParser._parse_statement` to also return `sources = []` for `exp.Drop` /
+`exp.Alter` / `exp.TruncateTable`.
+
+## Live-verification verdict (DONE before planning — evidence below)
+
+**VERDICT: hypothesis CONFIRMED at the mechanism level; hypothesis #2 (the #159
+attribution) PARTIALLY DISPROVEN — #159 is NOT the cause. Plan proceeds.**
+
+### Mechanism confirmed (file:line)
+
+For `DROP VIEW IF EXISTS IA_ANALYTICS.BA_WTDA_CAMPAIGN`:
+- `sqlglot` parses it to `exp.Drop`, `kind="VIEW"`, and `AnsiParser._classify` →
+  `"OTHER"` (verified by probe).
+- `build_scope(stmt)` returns `None` for `exp.Drop`/`exp.Alter` → control falls into
+  the `else` fallback branch in
+  [`ansi_parser.py:580-595`](../../src/sqlcg/parsers/ansi_parser.py) which calls
+  `_fallback_table_scan(stmt)`.
+- [`_fallback_table_scan`](../../src/sqlcg/parsers/ansi_parser.py) (`ansi_parser.py:792`)
+  does `find_all(exp.Table)`, which matches the DROP/ALTER *target* table and returns
+  it as a "source".
+- The #161 gate `is_non_table_create = isinstance(stmt, exp.Create) and stmt.kind not
+  in ("TABLE","VIEW")` (ansi_parser.py:566) is **False** for `exp.Drop`/`exp.Alter`
+  (they are not `exp.Create`), so the gate does NOT cover them.
+- The "source" flows into `out.referenced_tables` via the unconditional
+  `out.referenced_tables.extend(query_node.sources)` at
+  [`ansi_parser.py:246`](../../src/sqlcg/parsers/ansi_parser.py) /
+  [`snowflake_parser.py:894`](../../src/sqlcg/parsers/snowflake_parser.py) /
+  [`:1054`](../../src/sqlcg/parsers/snowflake_parser.py), minting a `SqlTable` node +
+  `SELECTS_FROM` edge.
+
+### Decisive corpus case
+
+[`ddl/changelogs/BA-TABLES/WTDA_MARKETING_AUTOMATION_CAMPAIGN.sql`] line 9 is the ONLY
+appearance of `IA_ANALYTICS.BA_WTDA_CAMPAIGN` in the file — a `DROP VIEW IF EXISTS`.
+The CREATEs in that file target `BA.WTDA_MARKETING_AUTOMATION_CAMPAIGN`. Live
+`parse_file` output:
+
+```
+stmt: OTHER target=None  sources=['ia_analytics.ba_wtda_campaign']   <- DROP VIEW
+stmt: CREATE_VIEW target=ia_analytics.ba_wtda_marketing_automation_campaign ...
+```
+
+So `ia_analytics.ba_wtda_campaign` can ONLY originate from the DROP. Confirmed present
+as a degree-0 island in the live graph.
+
+### Why `_exclude_synthetic_tables` does NOT catch them
+
+[`_exclude_synthetic_tables`](../../src/sqlcg/server/tools.py) (tools.py:388) keys on
+`SqlTable.kind in {"cte","derived","temp"}` (`_SYNTHETIC_TABLE_KINDS`, tools.py:383). A
+DROP-minted source reference is a plain `kind="table"` node (the default source-ref
+kind in [`indexer.py` `_flush_row_batch`](../../src/sqlcg/indexer/indexer.py)), so it
+is never in the synthetic set. **Two reasons the `_bck*` filter misses these:**
+1. The node's `kind` is `"table"`, not a synthetic kind → the filter passes it through.
+2. `_exclude_synthetic_tables` is a **query-time** filter inside MCP tools
+   (get_change_scope / diff_impact / …). The viz reads `SqlTable` rows directly via
+   [`viz/data.py` `_Q_NODES`](../../src/sqlcg/viz/data.py) and emits a node for EVERY
+   `SqlTable` regardless of kind — the filter never runs for the viz at all.
+
+The corpus `.sqlcg.toml` has `noise_filter.ignore_table_regexes = ["_bck"]`, which
+strips some `_bck` nodes at index time, which is why the viz showed ~136 and not the
+full count — but the mechanism still leaks every non-`_bck` DROP/ALTER target.
+
+### Quantification (live DWH, 1,451 files, current worktree v1.36.1)
+
+| Metric | Value | Source |
+|--------|-------|--------|
+| Total `SqlTable` nodes | 6,673 | `_Q_NODES` |
+| Degree-0 island nodes (viz definition, all schemas) | 3,747 | viz logic replica |
+| `SqlTable` nodes whose **sole** origin is the DROP/ALTER source loop | **169** | corpus re-parse |
+| — of which `_bck*` backup | 7 | (most already stripped by noise_filter) |
+| — of which `da.rtgmd_*` | 32 | all sole-DROP-origin (see below) |
+| — `_obsolete` | 2 | |
+| — other | 128 | |
+| Phantom `SELECTS_FROM` edges from OTHER-kind no-target queries | **1,308** | (21% of all 6,239 SELECTS_FROM) |
+
+The "169 sole-DROP-origin nodes" closely matches the finding's "136 island tables"
+(difference = noise_filter `_bck` stripping + viz schema scoping).
+
+### A/B vs v1.34.1 (pre-#159) — code-level, definitive
+
+A full reindex at v1.34.1 was **not** run (RAM at 145 MB free at the time; the
+code-level A/B is conclusive and a reindex would add no signal for the DROP/ALTER
+target). Evidence:
+
+- Pre-#159 commit `2047bc0`: `_classify`'s `exp.Drop`/`exp.Alter` both hit the same
+  `case _: return QueryKind.OTHER` arm as today — **unchanged by #159**.
+- Pre-#159 `ansi_parser.py` had the SAME `_fallback_table_scan` fallthrough for
+  Drop/Alter and NO `is_non_table_create` gate (the gate is #161, post-#159).
+- #159 (`4a55306`) flipped ONLY the **non-table `exp.Create`** catch-all
+  `CREATE_TABLE`→`OTHER`. It did not touch `exp.Drop`/`exp.Alter` classification or
+  source extraction.
+
+**Conclusion:** the DROP/ALTER phantom-node/edge bug is **pre-existing and independent
+of #159**. #159 changed the *label* of non-table CREATEs (whose source node #166
+already gated); it neither introduced nor increased the DROP/ALTER island nodes. The
+plan corrects the finding's hypothesis #2 accordingly. The fix is still warranted on
+the confirmed mechanism + measured 169 nodes / 1,308 edges.
+
+### rtgmd note (finding said these should REMAIN)
+
+The finding noted raw `da.rtgmd_*` ingest tables are genuinely lineage-free and should
+stay. Verified: the 32 `da.rtgmd_*` nodes flagged here appear in the corpus **only** as
+`drop table` targets (e.g. `da.rtgmd_csp_coulances` — sole appearance is
+`ddl/changelogs/DA-TABLES/RTGMD.sql:7 drop table DA.RTGMD_CSP_COULANCES;`, no CREATE,
+no SELECT/INSERT). They are themselves DROP-origin phantoms and SHOULD be removed.
+rtgmd tables that have a **real** CREATE/INSERT definition elsewhere will still appear
+(they have real edges/origins) and are unaffected by this fix — those are the genuine
+lineage-free islands that REMAIN. The fix only removes nodes whose *sole* origin is the
+DROP/ALTER loop.
+
+### gain baseline (acceptance-gate framing)
+
+Saved: [`plan/metrics/gain_drop_alter_phantom_baseline_v1361.json`](../metrics/gain_drop_alter_phantom_baseline_v1361.json).
+
+| coverage key | baseline (v1.36.1) |
+|---|---|
+| `total_tables` | 6,673 |
+| `total_edges` | 61,983 |
+| `good_edges` | 53,811 |
+| `good_edges_strict` | 48,907 |
+| `good_edges_scoped` | 37,502 |
+| `edge_health_strict_pct` | 78.9 |
+| `edge_health_scoped_pct` | 98.53 |
+| `phantom_edges` | 12,241 |
+| `phantom_confirmed` | 11,701 |
+| `blindspot_tables` | 631 |
+
+## Scope
+
+### In Scope
+- Suppress `sources` for `exp.Drop`, `exp.Alter`, and `exp.TruncateTable` in the single
+  source-gate in `AnsiParser._parse_statement` (the same chokepoint as #161). This
+  removes both the phantom `SqlTable` node AND the phantom `SELECTS_FROM` edge for these
+  statement kinds.
+- Unit test pinning the new behaviour (DROP/ALTER/TRUNCATE emit no source / no
+  referenced_table), mirroring [`test_phantom_non_table_create_source.py`](../../tests/unit/test_phantom_non_table_create_source.py).
+- Integration test (real DuckDB) pinning that a DROP-only target does NOT become a
+  `SqlTable` node and produces no `SELECTS_FROM` edge, mirroring
+  [`test_phantom_non_table_create_node.py`](../../tests/integration/test_phantom_non_table_create_node.py).
+- Control tests proving real DML/CTAS lineage is untouched (UPDATE/DELETE/MERGE/SELECT
+  reads and CTAS sources survive), including a same-name DROP+CREATE no-collateral-loss
+  case.
+- Incremental-parity test (extend
+  [`test_incremental_reindex_parity.py`](../../tests/integration/test_incremental_reindex_parity.py))
+  proving the gate holds on `resync_changed` / `reindex_file`, not just `index_repo`.
+- Version bump (patch) + gain re-measure to populate the acceptance gate.
+
+### Non-Goals
+- **Schema qualification** — `ia_analytics.ba_wtda_*` / `ia_tableau.ba_wtda_*` are REAL
+  distinct corpus tables (publish layers prefix the source business-area into the
+  name). The qualification is CORRECT. Do not "fix" it.
+- **Quoted-identifier islands** (`ia_tableau.prijsadvies aan huis`) — that is #153,
+  out of scope here.
+- **rtgmd tables with real definitions** — left as genuine lineage-free islands.
+- Any change to `_exclude_synthetic_tables` or the viz (the fix is upstream, at parse
+  time — the right place; no downstream filter is added).
+- Re-classifying DROP/ALTER away from `OTHER` (the kind label is fine; only the
+  source-extraction is wrong).
+
+## Design
+
+### Root cause (one chokepoint)
+[`AnsiParser._parse_statement`](../../src/sqlcg/parsers/ansi_parser.py) (ansi_parser.py:566)
+currently gates only non-table CREATEs:
+
+```python
+is_non_table_create = isinstance(stmt, exp.Create) and stmt.kind not in ("TABLE", "VIEW")
+...
+if is_non_table_create:
+    sources = []
+else:
+    # build_scope / _fallback_table_scan  <- DROP/ALTER target leaks in here
+```
+
+`SnowflakeParser` inherits `_parse_statement` and `_classify` (it subclasses
+`AnsiParser`, no override — verified). So this single gate feeds all three
+`referenced_tables.extend(query_node.sources)` sites in both parsers.
+
+### Proposed change
+Add a sibling predicate and fold it into the same gate. DROP/ALTER/TRUNCATE never carry
+a data-flow source (no SELECT body is structurally possible), so suppressing their
+sources is edge-neutral for real lineage:
+
+```python
+# #161 part 2: DROP / ALTER / TRUNCATE reference their TARGET object, not a
+# data-flow source. build_scope() returns None for these, so _fallback_table_scan
+# would otherwise scoop the target name into `sources`, minting a phantom island
+# SqlTable node + SELECTS_FROM edge (measured: 169 sole-DROP-origin nodes,
+# 1,308 phantom SELECTS_FROM edges on the DWH corpus). Keyed off the AST type,
+# independent of _classify.
+is_non_flow_ddl = isinstance(stmt, (exp.Drop, exp.Alter, exp.TruncateTable))
+...
+if is_non_table_create or is_non_flow_ddl:
+    sources = []
+else:
+    ...
+```
+
+(Final naming/structure at developer discretion as long as it is the SAME single gate
+and keyed off the AST type, not `_classify`.)
+
+### CRITICAL INVARIANT — the gate is PER-STATEMENT, no collateral node loss
+`sources = []` is set **for the single DROP/ALTER/TRUNCATE statement currently being
+parsed in `_parse_statement`, and for nothing else.** `_parse_statement` is invoked
+once per top-level statement (see the loop at
+[`ansi_parser.py:212`](../../src/sqlcg/parsers/ansi_parser.py) and the Snowflake
+override sites [`snowflake_parser.py:870`](../../src/sqlcg/parsers/snowflake_parser.py)
+/ [`:1024`](../../src/sqlcg/parsers/snowflake_parser.py)); the `isinstance(stmt, ...)`
+predicate is evaluated against *that one statement's* AST node. It cannot reach across
+to another statement in the same file, nor to another file.
+
+Consequence — the precise blast radius:
+
+- A table whose **sole** appearance in the entire corpus is as a DROP/ALTER/TRUNCATE
+  target loses its phantom `SqlTable` node and its phantom `SELECTS_FROM` edge. This is
+  the intended removal.
+- A table that is ALSO legitimately the target of a `CREATE TABLE/VIEW` (its CREATE
+  statement is a *different* `_parse_statement` call that still emits its real target +
+  sources), or that is read by a real `SELECT`/`INSERT…SELECT`/CTAS anywhere (in this
+  file or any other), **keeps its `SqlTable` node and all of its real edges.** The graph
+  node identity is keyed on `(schema, name)` and is created/kept by any of those real
+  origins; suppressing the DROP statement's source list removes only the edge the DROP
+  would have contributed, never the node a real statement created.
+
+The acceptance metric is therefore **"count of sole-DROP/ALTER/TRUNCATE-origin island
+nodes → ~0"**, NOT "every table named in a DROP is removed". A table named in both a
+DROP and a CREATE must survive with its real edges intact — this is pinned by a
+dedicated integration test (see Step 2.2, the mixed-origin case) and by the live
+acceptance probe (the `da.rtgmd_*` analysis already distinguishes sole-DROP-origin
+phantoms from rtgmd tables with a real CREATE/INSERT that REMAIN).
+
+### All reindex paths inherit the gate (incremental == from-scratch)
+The fix lives in `AnsiParser._parse_statement`
+([`ansi_parser.py:423`](../../src/sqlcg/parsers/ansi_parser.py), gate block at
+`:568`), which is the SINGLE parse chokepoint shared by every path that turns SQL into
+`ParsedFile.referenced_tables`. Verified call routes (file:line):
+
+- **Full index** — `index_repo` → `_index_single_file`
+  ([`indexer.py:410`](../../src/sqlcg/indexer/indexer.py)) → `parser.parse_file` →
+  `_parse_statement` (loop at [`ansi_parser.py:212`](../../src/sqlcg/parsers/ansi_parser.py)).
+- **Branch-delta incremental** — `resync_changed`
+  ([`indexer.py:1010`](../../src/sqlcg/indexer/indexer.py)) re-parses its `reparse_set`
+  via the SAME `_index_single_file` → `parse_file` route (Step 4, `indexer.py:122-123`).
+- **Single-file watch** — `reindex_file`
+  ([`indexer.py:1420`](../../src/sqlcg/indexer/indexer.py)) calls `parser.parse_file`
+  directly ([`indexer.py:1458`](../../src/sqlcg/indexer/indexer.py), the
+  `parsed = parser.parse_file(...)` line).
+- **Snowflake** — `SnowflakeParser` does not override `_parse_statement`; both its
+  override entry points call `AnsiParser._parse_statement`
+  ([`snowflake_parser.py:870`](../../src/sqlcg/parsers/snowflake_parser.py),
+  [`:1024`](../../src/sqlcg/parsers/snowflake_parser.py)).
+
+Because all four routes funnel through the one gated method, the fix holds identically
+on the incremental path — there is no full-path-only step to drift from. This is the
+**exact bug class of the recent #170/#171 incremental-parity regression** (a step
+present in `index_repo` but absent in `resync_changed`). To make that guarantee
+*falsifiable* rather than asserted, the plan extends the existing parity harness
+[`test_incremental_reindex_parity.py`](../../tests/integration/test_incremental_reindex_parity.py)
+with a DROP-bearing file (Step 2.3): incremental and from-scratch graphs must contain
+the SAME `SqlTable` keys and SELECTS_FROM edges (i.e. neither path mints the phantom).
+
+### Why `exp.TruncateTable` is included — GROUNDED, not class-analogy
+TRUNCATE is NOT a speculative add. **Measured on the live source (file:line / probe):**
+
+- The DWH corpus contains **514 files** with a `TRUNCATE TABLE` statement
+  (`grep -ilE 'truncate +table'` over `/home/ignwrad/Projects/dwh`), e.g.
+  `etl/pdi/template/htwfm_contract_us28254.sql` → `TRUNCATE TABLE da_tmp.htwfm_contract;`.
+- `sqlglot.parse_one("TRUNCATE TABLE s.t", dialect="snowflake")` is `exp.TruncateTable`
+  (verified by probe; DROP→`exp.Drop`, ALTER→`exp.Alter` confirmed in the same probe).
+- Pre-fix, the target leaks identically to DROP/ALTER — a live `SnowflakeParser.parse_file`
+  probe returned:
+  `'TRUNCATE TABLE s.truncated;' -> referenced_tables=['s.truncated']`
+  (alongside `DROP VIEW IF EXISTS s.dropped;` → `['s.dropped']` and
+  `ALTER TABLE s.altered ADD COLUMN c INT;` → `['s.altered']`).
+
+So TRUNCATE is the same confirmed leak mechanism with a large live population, not a
+zero-instance guard. (The base.py `_can_have_column_lineage` docstring already lists
+TruncateTable/Drop/Alter together as structurally lineage-free — this gate is the
+source-side analogue.) NOTE: the original "169 nodes / 1,308 edges" quantification in
+§Quantification was the DROP/ALTER measurement; the live re-measure (Step 3.2) will
+report the TRUNCATE contribution to the removed-node/edge count separately so the
+TRUNCATE share is grounded post-fix, not just pre-fix.
+
+### Stale inline comment to correct (housekeeping, do not skip)
+The existing #161 gate comment at
+[`ansi_parser.py:567`](../../src/sqlcg/parsers/ansi_parser.py) cites the extend sites as
+`snowflake_parser.py:784 / :944`; the live sites are
+[`snowflake_parser.py:894`](../../src/sqlcg/parsers/snowflake_parser.py) and
+[`:1054`](../../src/sqlcg/parsers/snowflake_parser.py) (ansi `:246` is correct). The
+developer updates the comment to the correct lines while editing this block.
+
+### Performance
+Zero hot-path impact: the predicate is one `isinstance` check per statement, evaluated
+BEFORE the (already-skipped) `build_scope`/`_fallback_table_scan` for these kinds — it
+actually *removes* a `_fallback_table_scan` call for every DROP/ALTER/TRUNCATE. No
+change to `_extract_column_lineage`, `body_scope`, qualify-once, or any
+indexer bulk/batch upsert path. All CLAUDE.md performance invariants are untouched.
+
+The following invariant suites must pass **UNMODIFIED** (the developer must not edit,
+skip, or relax any of them; a red here means the fix touched a hot path it must not):
+- [`test_perf_scaling_guard.py`](../../tests/unit/test_perf_scaling_guard.py) — the
+  general O(N)→O(N²) guard.
+- [`test_bulk_upsert_invariant.py`](../../tests/unit/test_bulk_upsert_invariant.py) —
+  `_upsert_parsed_file` uses bulk upserts.
+- [`test_upsert_batch_invariant.py`](../../tests/unit/test_upsert_batch_invariant.py) —
+  `_flush_row_batch` flushes once per batch.
+- [`test_T09_01_qualify_once.py`](../../tests/unit/test_T09_01_qualify_once.py) —
+  qualify/build_scope once per statement.
+
+These are listed not because the fix is expected to touch them — it is a single
+`isinstance` predicate added *above* the `build_scope`/`_fallback_table_scan` branch,
+which it does not modify — but to make the no-regression contract explicit and
+gateable.
+
+### Data model / API
+No schema change, no new method, no new node/edge type. No backward-compat concern
+(re-index is the migration path). Existing phantom nodes/edges disappear on the next
+`sqlcg index`.
+
+### Dependencies
+None new.
+
+## Implementation Steps
+
+### Phase 1: Parser source-gate
+
+**Step 1.1** — Extend the source-gate in `_parse_statement`.
+- Files affected: [`src/sqlcg/parsers/ansi_parser.py`](../../src/sqlcg/parsers/ansi_parser.py)
+  (the `is_non_table_create` block, ~line 566-595).
+- Add `is_non_flow_ddl = isinstance(stmt, (exp.Drop, exp.Alter, exp.TruncateTable))`
+  and OR it into the gate so `sources = []` for those kinds.
+- Update the existing #161 gate comment to document the DROP/ALTER/TRUNCATE extension
+  and cross-ref this plan + #161.
+- Acceptance: a `DROP VIEW IF EXISTS s.t` QueryNode has `sources == []` and
+  `parsed.referenced_tables == []`; same for `ALTER TABLE s.t ADD COLUMN …` and
+  `TRUNCATE TABLE s.t`.
+
+### Phase 2: Tests
+
+**Step 2.1** — Unit test (no backend).
+- File: new `tests/unit/test_phantom_drop_alter_source.py` (mirror the #161 unit test
+  structure; do NOT seed opaque case codes into test names).
+- Behaviours to pin:
+  - DROP TABLE / DROP VIEW (with and without `IF EXISTS`) emit no source, no
+    referenced_table.
+  - ALTER TABLE emits no source, no referenced_table.
+  - TRUNCATE TABLE emits no source, no referenced_table.
+  - Control: `UPDATE s.t SET c = (SELECT x FROM s.src)` STILL records `s.src` as a
+    referenced source (real read preserved).
+  - Control: `DELETE FROM s.t WHERE id IN (SELECT id FROM s.src)` STILL records
+    `s.src`.
+  - Control: CTAS `CREATE TABLE s.t AS SELECT a FROM s.src` STILL records `s.src`
+    (re-assert #161's TABLE/VIEW arm is intact).
+  - Gate-keyed-on-AST documentation guard: assert `sqlglot.parse_one("DROP VIEW s.t",
+    dialect="snowflake")` is `exp.Drop`, `ALTER …` is `exp.Alter`, `TRUNCATE TABLE …`
+    is `exp.TruncateTable`.
+
+**Step 2.2** — Integration test (real DuckDB in-memory).
+- File: new `tests/integration/test_phantom_drop_alter_node.py` (mirror
+  [`test_phantom_non_table_create_node.py`](../../tests/integration/test_phantom_non_table_create_node.py)).
+- The fixture MUST match the LIVE corpus shape — schema-qualified, snowflake dialect,
+  with all three statement forms exactly as they appear in the DWH:
+  `DROP VIEW IF EXISTS ia_analytics.ba_wtda_campaign;`, an `ALTER TABLE … ADD COLUMN …`,
+  and `TRUNCATE TABLE da_tmp.htwfm_contract;` (the real sampled form). Do NOT use a
+  minimal unqualified `DROP TABLE t;` that cannot exhibit the schema-qualified phantom.
+- **Negative control (house rule — tests assert observable output AND must fail on
+  pre-fix code):** the dropped/altered/truncated-only target test must be written so it
+  FAILS against current (pre-fix) `_parse_statement` — i.e. on pre-fix code the
+  `SqlTable` node and `SELECTS_FROM` edge for the target ARE present, so the assertion
+  `node absent / edge absent` goes red. The developer must run the new test once against
+  the unpatched parser and record (in the PR/handoff) that it failed there, then green
+  after the gate. A test that is green both before and after is not a regression test.
+- Behaviours to pin (index a tiny fixture, then query the graph):
+  - A file containing ONLY `DROP VIEW IF EXISTS ia_analytics.ba_wtda_campaign;` produces
+    NO `SqlTable` row for `ia_analytics.ba_wtda_campaign` and NO `SELECTS_FROM` edge with
+    that dst key. Same assertion for an ALTER-only and a TRUNCATE-only target.
+  - **Mixed-origin survival (the no-collateral-loss case):** a file with
+    `DROP VIEW IF EXISTS s.x;` followed by
+    `CREATE OR REPLACE VIEW s.y AS SELECT a FROM s.real;` produces a `SqlTable` row for
+    `s.y` and `s.real` (and the real `s.real → s.y` lineage), but NO node for `s.x`.
+  - **Same-name DROP + CREATE survives (per-statement invariant, explicit):** a file
+    where the SAME object is both DROP'd and re-created —
+    `DROP TABLE s.t;` then `CREATE TABLE s.t AS SELECT c FROM s.src;` — MUST keep the
+    `SqlTable` node for `s.t` AND the real `s.src → s.t` edge. Only the phantom edge the
+    DROP would have contributed is gone; the CREATE's node + edge survive. This pins
+    that the gate is per-statement and cannot delete a node a sibling statement
+    legitimately created.
+
+**Step 2.3** — Incremental-parity test (extend the existing harness).
+- File: extend [`test_incremental_reindex_parity.py`](../../tests/integration/test_incremental_reindex_parity.py)
+  (reuse its `_sql_table_keys` / `_selects_from_edges` helpers, `indexer.index_repo`
+  vs `indexer.resync_changed` / `indexer.reindex_file` structure at lines 156 / 235).
+- Add a DROP-bearing file to the tree (e.g. a file with
+  `DROP VIEW IF EXISTS s.dropped_only;` and a real `CREATE … AS SELECT … FROM s.real;`)
+  and assert the incremental graph (`resync_changed` after a branch delta, and
+  separately `reindex_file`) has the SAME `SqlTable` keys and the SAME SELECTS_FROM
+  edges as a from-scratch `index_repo` of the final tree — i.e. NEITHER path mints the
+  phantom `s.dropped_only` node/edge. Guards against the #170/#171-class drift where the
+  gate would be honoured on the full path but missed on the incremental path.
+
+### Phase 3: Version + measure
+
+**Step 3.1** — Patch version bump.
+- Files: [`pyproject.toml`](../../pyproject.toml), [`src/sqlcg/__init__.py`](../../src/sqlcg/__init__.py),
+  `uv lock`. Bug fix only, no added surface → **patch**. Reconcile against `master` at
+  PR-cut: master is `v1.36.1` today → ship `v1.36.2`; if a later patch has merged first,
+  take the next patch above whatever master then carries (do not hardcode `1.36.2`).
+- Acceptance: `uv run sqlcg --version` reports the new version;
+  [`test_version_parity.py`](../../tests/unit/test_version_parity.py) stays green
+  (`pyproject.toml` == `__init__.py` == lockfile).
+
+**Step 3.2** — Live re-measure for the acceptance gate (delegated index run).
+- Re-index the live DWH corpus on the fix branch, capture `sqlcg gain --json` into
+  `plan/metrics/gain_drop_alter_phantom_after_<ver>.json`, and record the deltas in the
+  Acceptance Criteria table below. Index time must stay under the 5-min budget; report
+  gain table-health.
+
+## Test Strategy
+
+- **Unit**: assert observable `QueryNode.sources` / `ParsedFile.referenced_tables`
+  contents for DROP/ALTER/TRUNCATE (empty) and the DML/CTAS controls (real source
+  present). No "no exception raised" assertions.
+- **Integration**: assert observable graph state — absence of the phantom `SqlTable`
+  node and absence of the phantom `SELECTS_FROM` edge for a DROP-only target, presence
+  of the real CREATE-VIEW lineage in the same fixture, and the same-name DROP+CREATE
+  survival case. Fixtures use the snowflake dialect and schema-qualified names mirroring
+  the live corpus shape (`ia_analytics.ba_wtda_*` and `da_tmp.htwfm_contract`-style),
+  per the live-config-fidelity rule. The phantom-absence assertion is a **negative
+  control**: it must be confirmed RED on pre-fix `_parse_statement` and GREEN after, so
+  it genuinely pins the removal rather than passing vacuously.
+- **Incremental parity**: assert the incremental-path graph (`resync_changed` /
+  `reindex_file`) equals the from-scratch graph for a DROP-bearing tree (same SqlTable
+  keys, same SELECTS_FROM edges) — guards the #170/#171-class drift.
+- **Regression guards**: existing #161 tests
+  ([`test_phantom_non_table_create_source.py`](../../tests/unit/test_phantom_non_table_create_source.py),
+  [`test_phantom_non_table_create_node.py`](../../tests/integration/test_phantom_non_table_create_node.py))
+  must stay green (the TABLE/VIEW arm is unchanged). Perf scaling guard
+  ([`test_perf_scaling_guard.py`](../../tests/unit/test_perf_scaling_guard.py)) must
+  stay green (no hot-path change).
+
+## Acceptance Criteria
+
+- [ ] A `DROP TABLE`/`DROP VIEW` statement (±`IF EXISTS`) yields `QueryNode.sources == []`
+      and contributes nothing to `ParsedFile.referenced_tables`. (unit)
+- [ ] An `ALTER TABLE` statement yields `QueryNode.sources == []` and no
+      referenced_table. (unit)
+- [ ] A `TRUNCATE TABLE` statement yields `QueryNode.sources == []` and no
+      referenced_table. (unit)
+- [ ] `UPDATE … (SELECT … FROM s.src)`, `DELETE … (SELECT … FROM s.src)`, and CTAS
+      `CREATE TABLE … AS SELECT … FROM s.src` STILL record `s.src` as a source. (unit
+      controls — proves the fix is edge-neutral for real reads)
+- [ ] Indexing a DROP-only / ALTER-only / TRUNCATE-only file produces NO `SqlTable`
+      node and NO `SELECTS_FROM` edge for the target object — AND this test was shown to
+      FAIL on pre-fix code (negative control). (integration)
+- [ ] A file where the SAME object is both DROP'd and CREATE'd (`DROP TABLE s.t;` +
+      `CREATE TABLE s.t AS SELECT c FROM s.src;`) keeps the `s.t` node and the real
+      `s.src → s.t` edge — no collateral node loss. (integration, per-statement invariant)
+- [ ] Incremental (`resync_changed` / `reindex_file`) and from-scratch (`index_repo`)
+      graphs of a DROP-bearing tree have identical `SqlTable` keys and SELECTS_FROM
+      edges — the gate holds on the incremental path. (integration parity)
+- [ ] On the live DWH re-index: the `ia_analytics.ba_wtda_campaign` node is **gone**
+      (decisive case), and the count of **sole-DROP/ALTER/TRUNCATE-origin** island nodes
+      drops from 169 toward ~0 — measured as sole-origin, NOT "every DROP-named table
+      removed". Tables that also have a real CREATE/INSERT/SELECT origin MUST still be
+      present. (live; re-run the corpus re-parse probe, and report the TRUNCATE share of
+      the removed nodes/edges separately since the original 169/1,308 was DROP/ALTER)
+- [ ] On the live DWH re-index: phantom `SELECTS_FROM` edges from OTHER-kind no-target
+      queries drops from 1,308 toward the residual (UPDATE/DELETE/MERGE/SELECT real
+      reads must remain — i.e. only the Drop/Alter/Truncate subset is removed). (live)
+- [ ] `gain --json` after the fix: `good_edges_strict` and `good_edges_scoped` are
+      **unchanged or up** (no real lineage lost); `total_tables` and `total_edges` go
+      **down** (phantom nodes/edges removed); `edge_health_*_pct` unchanged or up.
+      Record exact before/after in
+      `plan/metrics/gain_drop_alter_phantom_after_<ver>.json`.
+- [ ] Index time stays under the 5-minute budget.
+- [ ] Existing #161 tests remain green. The perf/upsert invariant suites
+      (`test_perf_scaling_guard.py`, `test_bulk_upsert_invariant.py`,
+      `test_upsert_batch_invariant.py`, `test_T09_01_qualify_once.py`) pass
+      **UNMODIFIED** — not edited, skipped, or relaxed.
+
+## Risks and Mitigations
+
+| Risk | Mitigation |
+|------|------------|
+| A DROP/ALTER target that is ALSO a real table loses its node (collateral loss) | It does not: the gate is PER-STATEMENT (`_parse_statement` runs once per statement; the `isinstance` check sees only that statement). Real tables get their node from a *different* CREATE/INSERT/SELECT statement and survive with their real edges; only the phantom source the DROP statement contributed is removed. Pinned by two integration cases: mixed-origin (different objects) and same-name DROP+CREATE (same object). The live metric is "sole-DROP/ALTER/TRUNCATE-origin islands → ~0", not "all DROP-named tables removed". |
+| Gate honoured on full index but missed on incremental reindex (#170/#171 class) | The fix is parser-level in `_parse_statement`; all of `index_repo`, `resync_changed`, and `reindex_file` route through `parse_file` → `_parse_statement` (call routes verified in Design). The extended `test_incremental_reindex_parity.py` asserts incremental == from-scratch for a DROP-bearing tree. |
+| Suppressing TRUNCATE sources hides a real dependency | TRUNCATE has no data-flow source by definition (target-only). Edge-neutral. |
+| Removing nodes lowers good_edges (false "regression") | good_edges count only real lineage; phantom DROP edges are not counted as good. The gain gate explicitly requires good_edges_strict/scoped unchanged-or-up. |
+| sqlglot maps some DROP/ALTER variant to a different AST type | The unit doc-guard asserts the AST types for the dialect; if a variant differs, the gate's `isinstance` tuple is extended (cheap, local). |
+| #153 quoted-identifier islands conflated with this fix | Explicitly out of scope; those nodes have a different origin (quoted name handling), untouched here. |
+
+### Deviations
+
+#### Deviation 1: UPDATE/DELETE sub-SELECT control assertions could not be used as written
+- **Reason**: The plan (Step 2.1) specified unit controls asserting
+  `UPDATE s.t SET c = (SELECT x FROM s.src)` and
+  `DELETE FROM s.t WHERE id IN (SELECT id FROM s.src)` STILL record `s.src` as a
+  referenced source. Verified on clean (pre-fix) code: the live parser returns
+  `referenced_tables == []` for BOTH — it does not extract sub-SELECT sources from
+  UPDATE-SET / DELETE-WHERE clauses. This is pre-existing parser behaviour,
+  independent of this gate (proven by running the probe with `ansi_parser.py`
+  stashed). The asserted source was never recorded, so the control as written would
+  be red on master.
+- **Change**: Replaced the UPDATE/DELETE sub-SELECT source controls with controls
+  the parser actually produces — `SELECT ... FROM s.src`, `INSERT INTO s.t SELECT ...
+  FROM s.src`, and the CTAS arm (`#161` intact) — and added a
+  `TestGateEdgeNeutralForNonGatedKinds` class asserting UPDATE/SELECT are outside the
+  gate's `isinstance` tuple and unperturbed. This preserves the plan's INTENT
+  (prove the gate is edge-neutral for real reads) using observable parser output.
+- **Impact**: No scope change; the edge-neutrality guarantee is still pinned. The
+  gate's `isinstance` tuple is `(exp.Drop, exp.Alter, exp.TruncateTable)` only, so
+  UPDATE/DELETE are untouched by construction.
+- **Date**: 2026-06-15
+
+#### Deviation 2: Live-DWH baseline JSON is a different corpus state than the re-index
+- **Reason**: The committed baseline `gain_drop_alter_phantom_baseline_v1361.json`
+  (total_tables=6,673) was captured against a 1,451-file DWH state. The live
+  re-index on this branch indexed the current 1,335-file checkout
+  (indexed_sha=fdf1b551). The absolute gain totals are therefore NOT a clean
+  before/after delta against that JSON (different corpus).
+- **Change**: The acceptance evidence is reported as a controlled STRUCTURAL probe
+  on the post-fix graph (decisive node absent; sole-DROP/ALTER/TRUNCATE-origin
+  count) plus the pre-fix→post-fix red→green negative-control test, rather than a
+  raw gain-total subtraction. The after-JSON
+  (`gain_1.36.2_f6a335a.json`) is committed for the record.
+- **Impact**: No scope change. The decisive case is verified (node count 0) and the
+  sole-origin phantom population is verified gone (all DDL targets that remain have a
+  real CREATE / CREATE-CLONE / HAS_COLUMN / lineage origin).
+- **Date**: 2026-06-15

{sql_code_graph-1.36.1 → sql_code_graph-1.36.2}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "sql-code-graph"
-version = "1.36.1"
+version = "1.36.2"
 description = "SQL code graph analyzer and lineage tracer"
 readme = "README.md"
 requires-python = ">=3.12"

{sql_code_graph-1.36.1 → sql_code_graph-1.36.2}/src/sqlcg/__init__.py

@@ -1,5 +1,5 @@
 """SQL Code Graph - SQL lineage and dependency analysis tool."""
 
-__version__ = "1.36.1"
+__version__ = "1.36.2"
 
 __all__ = ["__version__"]