sql-code-graph 1.35.2__tar.gz → 1.35.3__tar.gz

{sql_code_graph-1.35.2 → sql_code_graph-1.35.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sql-code-graph
-Version: 1.35.2
+Version: 1.35.3
 Summary: SQL code graph analyzer and lineage tracer
 Project-URL: Homepage, https://github.com/Warhorze/sql-code-graph
 Project-URL: Repository, https://github.com/Warhorze/sql-code-graph

{sql_code_graph-1.35.2 → sql_code_graph-1.35.3}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "sql-code-graph"
-version = "1.35.2"
+version = "1.35.3"
 description = "SQL code graph analyzer and lineage tracer"
 readme = "README.md"
 requires-python = ">=3.12"

{sql_code_graph-1.35.2 → sql_code_graph-1.35.3}/src/sqlcg/__init__.py

@@ -1,5 +1,5 @@
 """SQL Code Graph - SQL lineage and dependency analysis tool."""
 
-__version__ = "1.35.2"
+__version__ = "1.35.3"
 
 __all__ = ["__version__"]

{sql_code_graph-1.35.2 → sql_code_graph-1.35.3}/src/sqlcg/indexer/indexer.py

@@ -388,7 +388,7 @@ def _flush_row_batch(
     )
 
 
-def _subprocess_parse_worker(parser_cls, dialect, path, sql, q):
+def _subprocess_parse_worker(parser_cls, dialect, path, sql, q, rel_path=None):
     """Parse a single file in a subprocess; queue the ParsedFile (or exception).
 
     parser_cls must be the *class* (pickleable), not an instance. The worker
@@ -398,10 +398,16 @@ def _subprocess_parse_worker(parser_cls, dialect, path, sql, q):
     T-09-04: Parser constructors require a SchemaResolver. The subprocess gets a
     fresh empty resolver; column resolution runs in infer-only mode, the same as
     small-repo mode.
+
+    #171: rel_path is the repo-relative posix path used for CTE/temp namespace
+    keying.  It MUST be forwarded so the incremental path produces the same keys
+    as index_repo (which threads rel_path through its task dict); without it the
+    namespace falls back to the absolute OS path, creating duplicate CTE/temp
+    nodes after an incremental reindex.
     """
     try:
         parser = parser_cls(SchemaResolver(dialect=str(dialect) if dialect else None))
-        out = parser.parse_file(path, sql)
+        out = parser.parse_file(path, sql, rel_path=rel_path)
         q.put(out)
     except BaseException as exc:
         # Send the exception back; parent will re-raise.
@@ -1129,6 +1135,22 @@ class Indexer:
         schema_resolver = SchemaResolver(dialect=dialect)
         parser = get_parser(dialect, schema_resolver)
 
+        # #170/#171: load schema_aliases and compute repo-relative posix paths the
+        # SAME way index_repo does, so the incremental path applies the same alias
+        # normalisation (#170) and CTE/temp namespace keys (#171).  Without these,
+        # a branch-switch resync produced phantom *_tmp.* nodes (aliases unapplied)
+        # and duplicate CTE/temp nodes (absolute-path keys).
+        from sqlcg.core.config import get_schema_aliases
+
+        schema_aliases = get_schema_aliases(root)
+        root_resolved = Path(root).resolve()
+
+        def _rel_posix(fp: Path) -> str:
+            try:
+                return fp.resolve().relative_to(root_resolved).as_posix()
+            except ValueError:
+                return fp.as_posix()
+
         pass1_results: list[ParsedFile] = []
         for file_path in reparse_set:
             try:
@@ -1141,7 +1163,9 @@ class Indexer:
                 pass1_results.append(placeholder)
                 continue
             try:
-                parsed = self._index_single_file(parser, file_path, sql, timeout_per_file)
+                parsed = self._index_single_file(
+                    parser, file_path, sql, timeout_per_file, rel_path=_rel_posix(file_path)
+                )
             except Exception as exc:
                 logger.warning("resync_changed: parse failed %s: %s", file_path, exc)
                 parsed = ParsedFile(path=file_path, dialect=dialect)
@@ -1287,7 +1311,11 @@ class Indexer:
                         def_path = Path(definer_fp)
                         def_sql = def_path.read_text(encoding="utf-8")
                         def_parsed = self._index_single_file(
-                            parser, def_path, def_sql, timeout_per_file
+                            parser,
+                            def_path,
+                            def_sql,
+                            timeout_per_file,
+                            rel_path=_rel_posix(def_path),
                         )
                         # Harvest only — register for cross_file_sources but do NOT upsert
                         aggregator.register_pass1(def_parsed)
@@ -1315,7 +1343,7 @@ class Indexer:
                 )
                 continue
             try:
-                cl_parsed = parser.parse_file(cl_path, cl_sql)
+                cl_parsed = parser.parse_file(cl_path, cl_sql, rel_path=_rel_posix(cl_path))
             except Exception as exc:
                 logger.warning("resync_changed: parse failed for closure file %s: %s", cl_path, exc)
                 cl_parsed = ParsedFile(path=cl_path, dialect=dialect)
@@ -1325,7 +1353,19 @@ class Indexer:
         # ---- Step 7: Batched bulk upsert (same _flush_batch path as index_repo) ----
         all_results = pass1_results + closure_results
 
-        # Build a registry for duplicate DDL detection
+        # #170: key-normalisation choke point — apply schema_aliases + empty-identity
+        # guard to EVERY parse result BEFORE the defined_table_registry is built and
+        # before _upsert_file_batch.  index_repo (line ~797) and reindex_file
+        # (line ~1421) both do this; resync_changed previously did not, so a
+        # branch-switch incremental reindex left staging-alias schemas (e.g. ba_tmp)
+        # un-normalised, producing phantom *_tmp.* nodes that a from-scratch index
+        # never creates.  O(edges) per file, once per resync — outside the hot loop.
+        from sqlcg.parsers.base import normalize_keys as _normalize_keys
+
+        for pf in all_results:
+            _normalize_keys(pf, schema_aliases)
+
+        # Build a registry for duplicate DDL detection (post-normalisation full_ids)
         defined_table_registry: dict[str, str] = {}
         for pf in all_results:
             for table in pf.defined_tables:
@@ -1432,7 +1472,14 @@ class Indexer:
         # join-column edges until the next full index (plan-review BLOCKER).
         self._resolve_join_columns(db)
 
-    def _index_single_file(self, parser, path: Path, sql: str, timeout: int) -> ParsedFile:
+    def _index_single_file(
+        self,
+        parser,
+        path: Path,
+        sql: str,
+        timeout: int,
+        rel_path: str | None = None,
+    ) -> ParsedFile:
         """Parse one file, with optional timeout via subprocess isolation.
 
         T-09-04: Subprocess isolation via multiprocessing.Process + spawn context.
@@ -1446,12 +1493,17 @@ class Indexer:
             path: Path to the file
             sql: SQL text
             timeout: Timeout in seconds (0 = no timeout)
+            rel_path: Repo-relative posix path for CTE/temp namespace keying
+                (#171).  Threaded through to parse_file (both the in-process and
+                subprocess branch) so the incremental path produces the same
+                namespace keys as index_repo.  Falls back to str(path) inside the
+                parser when None.
 
         Returns:
             ParsedFile with parse_failed flag set if timeout occurs
         """
         if timeout <= 0:
-            return parser.parse_file(path, sql)
+            return parser.parse_file(path, sql, rel_path=rel_path)
 
         ctx = mp.get_context("spawn")  # avoid fork-inherit pitfalls (KuzuDB connection FD etc.)
         # Unbounded queue: the child writes one large ParsedFile (192–552 KB pickled).
@@ -1462,7 +1514,7 @@ class Indexer:
         q: mp.Queue = ctx.Queue()
         proc = ctx.Process(
             target=_subprocess_parse_worker,
-            args=(parser.__class__, parser.DIALECT, path, sql, q),
+            args=(parser.__class__, parser.DIALECT, path, sql, q, rel_path),
             daemon=True,
         )
         proc.start()

{sql_code_graph-1.35.2 → sql_code_graph-1.35.3}/src/sqlcg/parsers/bigquery_parser.py

@@ -35,12 +35,16 @@ class BigQueryParser(AnsiParser):
         """
         super().__init__(schema_resolver, schema_aliases=schema_aliases)
 
-    def parse_file(self, path: Path, sql: str) -> ParsedFile:
+    def parse_file(self, path: Path, sql: str, rel_path: str | None = None) -> ParsedFile:
         """Parse BigQuery SQL file with scripting block detection.
 
         Args:
             path: Path to the source file
             sql: SQL text to parse
+            rel_path: Repo-relative posix path for CTE/temp namespace keying.
+                Accepted for signature parity with the other parsers (the pool and
+                resync paths always pass it); scripting-fallback BigQuery files do
+                not register CTE/temp nodes, so it is currently unused here.
 
         Returns:
             ParsedFile with parsed statements and metadata

sql_code_graph-1.35.3/tests/integration/test_incremental_reindex_parity.py

@@ -0,0 +1,278 @@
+"""Incremental-reindex == from-scratch parity guard (#170, #171).
+
+Two correctness bugs let the incremental ``resync_changed`` path drift from a
+from-scratch ``index_repo`` after a branch switch:
+
+  - #170: ``resync_changed`` upserted parse results WITHOUT first calling
+    ``normalize_keys`` (schema-alias normalisation), so a staging-alias schema
+    such as ``ba_tmp`` reached the graph un-normalised — phantom ``ba_tmp.*``
+    nodes a from-scratch index never creates.
+  - #171: ``_index_single_file`` (the incremental pass-1 parse) did not thread a
+    repo-relative ``rel_path`` into ``parse_file``, so CTE/temp namespace keys
+    used the absolute OS path instead of the repo-relative posix path
+    ``index_repo`` uses — duplicate CTE/temp nodes after an incremental reindex.
+
+This module builds the SAME final tree two ways and asserts the resulting graphs
+are EQUAL (same SqlTable nodes + kinds, same COLUMN_LINEAGE edges, same
+SELECTS_FROM edges):
+
+  (a) incremental — index the base tree, then apply a branch delta and run
+      ``resync_changed`` (and, separately, ``reindex_file``);
+  (b) from-scratch — ``index_repo`` of the final tree.
+
+The fixture deliberately exercises BOTH bugs: a ``*_tmp`` schema-alias table
+(catches #170) and an in-file CTE (catches #171).
+
+Guards the incremental-reindex parity fix
+([plan reference](https://github.com/Warhorze/sql-code-graph/issues/170)).
+A future incremental path that skips a normalisation or rel_path step goes red.
+"""
+
+import subprocess
+from pathlib import Path
+
+from sqlcg.core.duckdb_backend import DuckDBBackend
+from sqlcg.indexer.indexer import Indexer
+
+# ---------------------------------------------------------------------------
+# Fixture SQL — final state of each file
+# ---------------------------------------------------------------------------
+
+# .sqlcg.toml content: ba_tmp is a staging alias for ba (#170 trigger).
+SQLCG_TOML = """\
+[sqlcg.schema_aliases]
+ba_tmp = "ba"
+"""
+
+# Unchanged base file: defines a permanent table from a raw source.
+DIM_SQL = "CREATE TABLE ba.dim AS SELECT id, name FROM raw.source"
+
+# Modified-in-delta file: writes a *_tmp staging table (alias) and reads it back
+# via a CTE into a permanent fact table.  The CTE (#171) and the ba_tmp alias
+# (#170) both appear here.
+FACT_SQL_V1 = (
+    "CREATE TABLE ba_tmp.staging AS SELECT id, name FROM ba.dim;\n"
+    "CREATE TABLE ba.fact AS\n"
+    "WITH enriched AS (SELECT id, name FROM ba_tmp.staging)\n"
+    "SELECT id, name FROM enriched"
+)
+FACT_SQL_V2 = (
+    "CREATE TABLE ba_tmp.staging AS SELECT id, name FROM ba.dim;\n"
+    "CREATE TABLE ba.fact AS\n"
+    "WITH enriched AS (SELECT id, name AS renamed_name FROM ba_tmp.staging)\n"
+    "SELECT id, renamed_name FROM enriched"
+)
+
+# Added-in-delta file: a fresh aggregate reading the fact table through its own
+# CTE — exercises the rel_path keying on a brand-new file.
+AGG_SQL = "CREATE TABLE ba.agg AS\nWITH rolled AS (SELECT id FROM ba.fact)\nSELECT id FROM rolled"
+
+DIALECT = "snowflake"
+
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+
+def _git(repo: Path, *args: str) -> str:
+    result = subprocess.run(["git", *args], cwd=repo, check=True, capture_output=True, text=True)
+    return result.stdout.strip()
+
+
+def _init_repo(repo: Path) -> None:
+    _git(repo, "init", "--initial-branch=main")
+    _git(repo, "config", "user.email", "test@example.com")
+    _git(repo, "config", "user.name", "Test")
+
+
+def _commit_all(repo: Path, msg: str) -> str:
+    _git(repo, "add", "-A")
+    _git(repo, "commit", "-m", msg)
+    return _git(repo, "rev-parse", "HEAD")
+
+
+def _fresh_db() -> DuckDBBackend:
+    backend = DuckDBBackend(":memory:")
+    backend.init_schema()
+    return backend
+
+
+def _sql_table_keys(db: DuckDBBackend) -> set[tuple[str, str]]:
+    """All (qualified, kind) SqlTable identities in the graph."""
+    rows = db.run_read('SELECT qualified, kind FROM "SqlTable"', {})
+    return {(r["qualified"], r["kind"]) for r in rows}
+
+
+def _column_lineage_edges(db: DuckDBBackend) -> set[tuple[str, str]]:
+    rows = db.run_read('SELECT src_key AS s, dst_key AS d FROM "COLUMN_LINEAGE"', {})
+    return {(r["s"], r["d"]) for r in rows}
+
+
+def _selects_from_edges(db: DuckDBBackend, repo: Path) -> set[tuple[str, str]]:
+    """SELECTS_FROM edges with the absolute-repo-path prefix stripped from src.
+
+    The src_key is a query id of the form ``<abs file path>:<stmt idx>``.  The
+    File-node path is intentionally absolute and so differs between two repos
+    rooted at different temp dirs; strip the repo prefix so the comparison
+    reflects only the graph identity (which is what the bugs corrupt), not the
+    incidental temp-dir location.
+    """
+    prefix = str(repo.resolve())
+    rows = db.run_read('SELECT src_key AS s, dst_key AS d FROM "SELECTS_FROM"', {})
+    out: set[tuple[str, str]] = set()
+    for r in rows:
+        s = r["s"]
+        if s.startswith(prefix):
+            s = s[len(prefix) :].lstrip("/")
+        out.add((s, r["d"]))
+    return out
+
+
+def _write_base_tree(repo: Path) -> None:
+    (repo / ".sqlcg.toml").write_text(SQLCG_TOML)
+    (repo / "dim.sql").write_text(DIM_SQL)
+    (repo / "fact.sql").write_text(FACT_SQL_V1)
+
+
+def _apply_delta(repo: Path) -> None:
+    """Branch delta: modify fact.sql + add agg.sql (dim.sql untouched)."""
+    (repo / "fact.sql").write_text(FACT_SQL_V2)
+    (repo / "agg.sql").write_text(AGG_SQL)
+
+
+def _write_final_tree(repo: Path) -> None:
+    (repo / ".sqlcg.toml").write_text(SQLCG_TOML)
+    (repo / "dim.sql").write_text(DIM_SQL)
+    (repo / "fact.sql").write_text(FACT_SQL_V2)
+    (repo / "agg.sql").write_text(AGG_SQL)
+
+
+# ---------------------------------------------------------------------------
+# Tests
+# ---------------------------------------------------------------------------
+
+
+def test_resync_changed_graph_equals_from_scratch(tmp_path):
+    """resync_changed of a branch delta must yield the SAME graph as a full index.
+
+    Builds the final tree two ways and asserts SqlTable nodes, COLUMN_LINEAGE
+    edges, and SELECTS_FROM edges are identical.  Fails pre-fix because:
+      - #170 leaves phantom ``ba_tmp.staging`` nodes/edges (aliases unapplied);
+      - #171 keys the CTE node by the absolute path instead of the repo-relative
+        path, so the incremental and from-scratch CTE keys differ.
+    """
+    # ---- (a) incremental path -------------------------------------------------
+    inc_repo = tmp_path / "incremental"
+    inc_repo.mkdir()
+    _init_repo(inc_repo)
+    _write_base_tree(inc_repo)
+    old_sha = _commit_all(inc_repo, "base")
+
+    inc_db = _fresh_db()
+    indexer = Indexer()
+    indexer.index_repo(inc_repo, dialect=DIALECT, db=inc_db)
+
+    _apply_delta(inc_repo)
+    new_sha = _commit_all(inc_repo, "delta")
+
+    result = indexer.resync_changed(inc_repo, old_sha, new_sha, inc_db, dialect=DIALECT)
+    assert result["fell_back_to_full"] is False, (
+        "resync must take the incremental path, not fall back to a full reindex"
+    )
+
+    # ---- (b) from-scratch path ------------------------------------------------
+    scr_repo = tmp_path / "scratch"
+    scr_repo.mkdir()
+    _init_repo(scr_repo)
+    _write_final_tree(scr_repo)
+    _commit_all(scr_repo, "final")
+
+    scr_db = _fresh_db()
+    indexer.index_repo(scr_repo, dialect=DIALECT, db=scr_db)
+
+    # ---- Compare graphs (path-independent identities) -------------------------
+    inc_tables = _sql_table_keys(inc_db)
+    scr_tables = _sql_table_keys(scr_db)
+
+    # No phantom *_tmp.* nodes survived the incremental path (#170).
+    inc_tmp = {q for (q, _k) in inc_tables if ".staging" in q and "ba_tmp" in q}
+    assert inc_tmp == set(), (
+        f"#170: incremental path leaked un-aliased ba_tmp nodes: {sorted(inc_tmp)}"
+    )
+
+    assert inc_tables == scr_tables, (
+        "SqlTable identities differ between incremental and from-scratch.\n"
+        f"  only-incremental: {sorted(inc_tables - scr_tables)}\n"
+        f"  only-from-scratch: {sorted(scr_tables - inc_tables)}"
+    )
+
+    inc_edges = _column_lineage_edges(inc_db)
+    scr_edges = _column_lineage_edges(scr_db)
+    assert inc_edges == scr_edges, (
+        "COLUMN_LINEAGE edges differ between incremental and from-scratch.\n"
+        f"  only-incremental: {sorted(inc_edges - scr_edges)}\n"
+        f"  only-from-scratch: {sorted(scr_edges - inc_edges)}"
+    )
+
+    inc_sf = _selects_from_edges(inc_db, inc_repo)
+    scr_sf = _selects_from_edges(scr_db, scr_repo)
+    assert inc_sf == scr_sf, (
+        "SELECTS_FROM edges differ between incremental and from-scratch.\n"
+        f"  only-incremental: {sorted(inc_sf - scr_sf)}\n"
+        f"  only-from-scratch: {sorted(scr_sf - inc_sf)}"
+    )
+
+    # Observable-output floor: the from-scratch graph is non-trivial, so equality
+    # is not vacuously satisfied by two empty graphs.
+    assert len(scr_tables) >= 3, "fixture must register at least dim/fact/agg + CTE nodes"
+    assert len(scr_edges) >= 1, "fixture must produce column lineage"
+
+    inc_db.close()
+    scr_db.close()
+
+
+def test_reindex_file_cte_node_matches_from_scratch(tmp_path):
+    """reindex_file of a CTE+alias file must key CTE/temp nodes like index_repo.
+
+    Guards the single-file incremental path (``reindex_file``) for the same
+    rel_path/alias parity (#170/#171): the CTE node key and any ba_tmp alias
+    must match the from-scratch graph for the same file.
+    """
+    # from-scratch reference graph
+    scr_repo = tmp_path / "scratch"
+    scr_repo.mkdir()
+    _init_repo(scr_repo)
+    (scr_repo / ".sqlcg.toml").write_text(SQLCG_TOML)
+    (scr_repo / "fact.sql").write_text(FACT_SQL_V2)
+    _commit_all(scr_repo, "final")
+    scr_db = _fresh_db()
+    indexer = Indexer()
+    indexer.index_repo(scr_repo, dialect=DIALECT, db=scr_db)
+    scr_tables = _sql_table_keys(scr_db)
+
+    # incremental: index v1, then reindex the changed file to v2
+    inc_repo = tmp_path / "incremental"
+    inc_repo.mkdir()
+    _init_repo(inc_repo)
+    (inc_repo / ".sqlcg.toml").write_text(SQLCG_TOML)
+    (inc_repo / "fact.sql").write_text(FACT_SQL_V1)
+    _commit_all(inc_repo, "base")
+    inc_db = _fresh_db()
+    indexer.index_repo(inc_repo, dialect=DIALECT, db=inc_db)
+
+    (inc_repo / "fact.sql").write_text(FACT_SQL_V2)
+    indexer.reindex_file(str((inc_repo / "fact.sql").resolve()), inc_db, dialect=DIALECT)
+    inc_tables = _sql_table_keys(inc_db)
+
+    inc_tmp = {q for (q, _k) in inc_tables if ".staging" in q and "ba_tmp" in q}
+    assert inc_tmp == set(), f"#170: reindex_file leaked un-aliased ba_tmp nodes: {sorted(inc_tmp)}"
+    assert inc_tables == scr_tables, (
+        "reindex_file SqlTable identities differ from from-scratch.\n"
+        f"  only-incremental: {sorted(inc_tables - scr_tables)}\n"
+        f"  only-from-scratch: {sorted(scr_tables - inc_tables)}"
+    )
+    assert len(scr_tables) >= 3
+
+    inc_db.close()
+    scr_db.close()

{sql_code_graph-1.35.2 → sql_code_graph-1.35.3}/tests/unit/test_subprocess_isolate.py

@@ -31,7 +31,7 @@ class SlowParser:
     def __init__(self, schema_resolver=None):
         self.schema_resolver = schema_resolver
 
-    def parse_file(self, path, sql):
+    def parse_file(self, path, sql, rel_path=None):
         time.sleep(60)
         return ParsedFile(path=path, dialect=self.DIALECT)
 
@@ -44,7 +44,7 @@ class FailingParser:
     def __init__(self, schema_resolver=None):
         self.schema_resolver = schema_resolver
 
-    def parse_file(self, path, sql):
+    def parse_file(self, path, sql, rel_path=None):
         raise ValueError("bad sql")
 
 
@@ -56,7 +56,7 @@ class ExitParser:
     def __init__(self, schema_resolver=None):
         self.schema_resolver = schema_resolver
 
-    def parse_file(self, path, sql):
+    def parse_file(self, path, sql, rel_path=None):
         os._exit(0)
 
 

{sql_code_graph-1.35.2 → sql_code_graph-1.35.3}/tests/unit/test_timeout_cancel.py

@@ -18,7 +18,7 @@ class _FastParser:
     def __init__(self, schema_resolver=None):
         self.schema_resolver = schema_resolver
 
-    def parse_file(self, path, sql):
+    def parse_file(self, path, sql, rel_path=None):
         return ParsedFile(path=path, dialect=self.DIALECT)
 
 
@@ -28,7 +28,7 @@ class _SlowParser:
     def __init__(self, schema_resolver=None):
         self.schema_resolver = schema_resolver
 
-    def parse_file(self, path, sql):
+    def parse_file(self, path, sql, rel_path=None):
         time.sleep(10)
         return ParsedFile(path=path, dialect=self.DIALECT)
 

{sql_code_graph-1.35.2 → sql_code_graph-1.35.3}/uv.lock

@@ -2844,7 +2844,7 @@ wheels = [
 
 [[package]]
 name = "sql-code-graph"
-version = "1.35.2"
+version = "1.35.3"
 source = { editable = "." }
 dependencies = [
     { name = "duckdb" },