sql-code-graph 1.37.1__tar.gz → 1.38.0__tar.gz

{sql_code_graph-1.37.1 → sql_code_graph-1.38.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sql-code-graph
-Version: 1.37.1
+Version: 1.38.0
 Summary: SQL code graph analyzer and lineage tracer
 Project-URL: Homepage, https://github.com/Warhorze/sql-code-graph
 Project-URL: Repository, https://github.com/Warhorze/sql-code-graph

{sql_code_graph-1.37.1 → sql_code_graph-1.38.0}/docs/cli.md

@@ -473,11 +473,19 @@ and consumers that read the table only via COLUMN_LINEAGE / STAR_SOURCE
 one-hop consumer completeness ``find_table_usages`` already has. Stays
 one-hop (NOT transitive — that is ``downstream``'s job).
 
+With ``--through-temps`` the consumer set is augmented with the downstream
+**temp-collapsed** closure: physical tables reachable from this table only
+by threading through ``kind IN ('cte','derived','temp')`` intermediates,
+computed from the structural COLUMN_LINEAGE hops (read-only; it never
+mutates the graph). This recovers consumers — e.g. the star-into-temp
+final target — that the one-hop default and TEMP_INLINE do not surface.
+
 ### Options
 
 | Option | Type | Required | Repeatable | Default | Description |
 | --- | --- | --- | --- | --- | --- |
 | --raw | BOOLEAN | No | No | False | Disable noise filtering on results |
+| --through-temps | BOOLEAN | No | No | False | Also report consumers reachable only THROUGH a temp/cte/derived intermediate (table-grain temp-collapsed closure). Off by default — the default impact stays one-hop and does not thread temps. |
 
 ## `sqlcg analyze failures`
 

{sql_code_graph-1.37.1 → sql_code_graph-1.38.0}/plan/sprints/lineage_gold_regression_and_temp_collapse.md

@@ -382,6 +382,30 @@ fallback). Grep-confirm the call site lands in this PR.
 
 ---
 
+### Deviations
+
+#### Deviation 1: intermediate node = `kind IN ('cte','derived','temp')` AND namespaced (`::`)
+- **Reason**: The plan said thread *through* nodes whose `kind IN ('cte','derived','temp')`
+  and terminate at `kind IN ('table','external')`/unknown. Live indexing of the PR-1
+  synthetic fixtures showed the FINAL physical targets — the gold-set endpoints
+  `mart.dim_person`, `mart.f_orders`, `stgmart.f_compet_article` — are stamped
+  `kind='derived'` (they are INSERT/CTAS targets), NOT `'table'`. Threading through every
+  `derived` node would walk past the gold-set finals and never terminate (the recovered
+  edge `raw.src_emp_backup -> mart.dim_person` was returned EMPTY under the literal rule).
+- **Change**: A node is treated as a temp/cte/derived INTERMEDIATE to thread *through* only
+  when its `kind IN ('cte','derived','temp')` **AND** its key is namespaced
+  (`table_qualified LIKE '%::%'`). Namespacing (`<file>::<db>.<name>`, [`base.py:97-98`](../../src/sqlcg/parsers/base.py))
+  is the load-bearing discriminator: true CTE/temp intermediates are always namespaced,
+  whereas a `derived` INSERT/CTAS target is never namespaced and is a genuine endpoint.
+  Endpoints are everything else (non-namespaced, or kind not temp/cte/derived, or NULL).
+  This realises the plan's stated NODE-keyed intent against the actual stored data model.
+- **Impact**: No scope change; the headline acceptance (`raw.src_emp_backup -> mart.dim_person`
+  recovered, 4 gating absences not fabricated) holds and is asserted. The plan's
+  `_PHYSICAL_KIND_VALUES = ('table','external')` constant is therefore not used in the
+  traversal predicate (the namespacing test subsumes it); the temp-kind set
+  `('cte','derived','temp')` is still the authoritative constant, defined once in the module.
+- **Date**: 2026-06-15
+
 ## Versioning
 
 - **Base: master @ `1.37.0`.**

{sql_code_graph-1.37.1 → sql_code_graph-1.38.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "sql-code-graph"
-version = "1.37.1"
+version = "1.38.0"
 description = "SQL code graph analyzer and lineage tracer"
 readme = "README.md"
 requires-python = ">=3.12"

{sql_code_graph-1.37.1 → sql_code_graph-1.38.0}/src/sqlcg/__init__.py

@@ -1,5 +1,5 @@
 """SQL Code Graph - SQL lineage and dependency analysis tool."""
 
-__version__ = "1.37.1"
+__version__ = "1.38.0"
 
 __all__ = ["__version__"]

{sql_code_graph-1.37.1 → sql_code_graph-1.38.0}/src/sqlcg/cli/commands/analyze.py

@@ -314,6 +314,15 @@ def downstream(  # noqa: B008
 def impact(  # noqa: B008
     table: str = typer.Argument(..., help="Table name to analyze"),  # noqa: B008
     raw: bool = typer.Option(False, "--raw", help="Disable noise filtering on results"),  # noqa: B008
+    through_temps: bool = typer.Option(  # noqa: B008
+        False,
+        "--through-temps",
+        help=(
+            "Also report consumers reachable only THROUGH a temp/cte/derived "
+            "intermediate (table-grain temp-collapsed closure). Off by default — "
+            "the default impact stays one-hop and does not thread temps."
+        ),
+    ),
 ) -> None:
     """Show all queries impacted by a table.
 
@@ -323,6 +332,13 @@ def impact(  # noqa: B008
     ``ba.all_rows_in_selection``). Bug #6 fix: brings ``impact`` to the same
     one-hop consumer completeness ``find_table_usages`` already has. Stays
     one-hop (NOT transitive — that is ``downstream``'s job).
+
+    With ``--through-temps`` the consumer set is augmented with the downstream
+    **temp-collapsed** closure: physical tables reachable from this table only
+    by threading through ``kind IN ('cte','derived','temp')`` intermediates,
+    computed from the structural COLUMN_LINEAGE hops (read-only; it never
+    mutates the graph). This recovers consumers — e.g. the star-into-temp
+    final target — that the one-hop default and TEMP_INLINE do not surface.
     """
     direct = run_read_routed(
         "SELECT DISTINCT q.id AS id, q.kind AS kind, q.target_table AS target"
@@ -341,6 +357,32 @@ def impact(  # noqa: B008
     )
     seen = {r["id"] for r in direct}
     results = direct + [r for r in via if r["id"] not in seen]
+
+    # --through-temps: augment with the temp-collapsed downstream closure. These
+    # are physical final tables reached only by threading through a temp/cte/
+    # derived node (the one-hop default + the queries above miss them). Rendered
+    # as synthetic "[collapsed] <final>" rows so they are visibly distinct from
+    # the direct/via consumer-query rows above.
+    if through_temps:
+        from sqlcg.lineage.temp_collapse import collapse_through_temps
+
+        collapsed = collapse_through_temps(table.lower(), direction="downstream")
+        seen_finals = {r.get("target", "") for r in results}
+        for edge in collapsed:
+            if not edge.via_temp:
+                # reached by a single direct hop; already covered above
+                continue
+            if edge.final_table in seen_finals:
+                continue
+            seen_finals.add(edge.final_table)
+            results.append(
+                {
+                    "id": f"[collapsed] {edge.final_table}",
+                    "kind": f"via-temp (hops={edge.hop_count})",
+                    "target": edge.final_table,
+                }
+            )
+
     # Preserve the existing 100-row bound on the merged consumer set.
     results = results[:100]
     if not raw:

sql_code_graph-1.38.0/src/sqlcg/lineage/temp_collapse.py

@@ -0,0 +1,252 @@
+"""Non-destructive, read-only temp-collapse primitive (PR-2).
+
+Given a physical start table, :func:`collapse_through_temps` returns the
+**temp-collapsed transitive closure** — physical ``(src, final)`` table pairs
+threaded *through* intermediate nodes whose ``SqlTable.kind IN
+('cte','derived','temp')`` — computed from the stored **structural**
+COLUMN_LINEAGE hops (``transform IS DISTINCT FROM 'TEMP_INLINE'``), NOT from
+TEMP_INLINE presence.
+
+Why structural-only + node-keyed matters: the star-into-temp gap is precisely
+"TEMP_INLINE did not fire" — but the per-hop structural edges
+(``src -> <ns>::temp`` and ``<ns>::temp -> final``) always exist. Threading
+through the temp *node* (table_qualified, namespace-stripped to its table grain)
+recovers the collapsed ``src -> final`` edge that TEMP_INLINE does not emit.
+
+This module performs **read-only** traversal only: every query is a ``SELECT``
+issued through :func:`run_read_routed`. It never deletes, replaces, or mutates
+any stored ``COLUMN_LINEAGE`` row (the HARD non-destructive invariant — guarded
+behaviorally in the test suite). It lives outside ``base.py`` / ``indexer.py``
+so it adds nothing to the parse/qualify hot loop (CLAUDE.md perf invariants).
+
+Guards PR-2 of the lineage gold-set + temp-collapse plan
+([plan doc](../../../plan/sprints/lineage_gold_regression_and_temp_collapse.md)).
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+
+from sqlcg.server.read_client import run_read_routed
+
+# ---------------------------------------------------------------------------
+# Authoritative constant sets — mirror the existing read surface, do NOT invent.
+# ---------------------------------------------------------------------------
+
+#: Intermediate node kinds the collapse threads *through* (never an endpoint).
+#: Mirrors ``coverage.py``'s ``kind IN ('cte','derived','temp')`` usage and the
+#: cross-batch downgrade guard in ``duckdb_backend.py``. Define once here.
+_TEMP_KIND_VALUES: tuple[str, ...] = ("cte", "derived", "temp")
+
+#: COLUMN_LINEAGE.transform value emitted for inlined temp tables (E8
+#: dual-emission). Mirrors ``analyze.py:_TEMP_INLINE_TRANSFORM``. The collapse
+#: is computed from STRUCTURAL hops only, so these edges are filtered out.
+_TEMP_INLINE_TRANSFORM = "TEMP_INLINE"
+
+#: Default depth cap for the recursive traversal. Bounds runaway closures on a
+#: dense graph; the visited-path guard already breaks cycles. Mirrors the
+#: depth-bound convention in ``analyze.py`` (1..100).
+_DEFAULT_MAX_DEPTH = 50
+
+
+@dataclass(frozen=True)
+class CollapsedEdge:
+    """One temp-collapsed table-grain edge.
+
+    Attributes:
+        src_table: The physical (table/external/unknown) start endpoint.
+        final_table: The physical endpoint reached by threading through temps.
+        via_temp: True when the path passed through at least one
+            cte/derived/temp node (a genuine collapse), False for a direct
+            physical->physical structural hop reached at depth 1.
+        hop_count: Number of structural COLUMN_LINEAGE hops on the path.
+    """
+
+    src_table: str
+    final_table: str
+    via_temp: bool
+    hop_count: int
+
+
+def _temp_kind_in_list_sql() -> str:
+    """SQL literal list for the intermediate-kind IN (...) clause."""
+    return ", ".join(f"'{v}'" for v in _TEMP_KIND_VALUES)
+
+
+def _strip_namespace(key: str) -> str:
+    """Drop the ``<file>::`` namespace prefix from a temp/cte/derived key.
+
+    Temp/CTE/derived nodes are keyed ``<repo-rel-file>::<db>.<name>`` in the
+    graph (see ``base.py:97-98``). The collapse threads through them but reports
+    endpoints at their bare ``<db>.<name>`` table grain — the same convention
+    the PR-1 gold-set rollup uses.
+    """
+    return key.split("::", 1)[1] if "::" in key else key
+
+
+def _collapse_sql(*, direction: str, max_depth: int) -> str:
+    """Build the table-grain temp-collapse recursive-CTE SQL.
+
+    The traversal rolls COLUMN_LINEAGE to ``SqlColumn.table_qualified`` and
+    threads through nodes whose ``SqlTable.kind IN ('cte','derived','temp')``,
+    on STRUCTURAL hops only (``transform IS DISTINCT FROM 'TEMP_INLINE'``).
+    Every emitted row is a ``(start_physical, reached_physical)`` pair whose
+    intermediate hops are all temp/cte/derived and whose endpoint is physical
+    (``table``/``external``) or unknown (NULL kind).
+
+    Cycle-safe: the visited ``path`` of table-qualified node keys carries the
+    ``NOT ... = ANY(reach.path)`` guard (mirrors ``analyze.py:144/191``).
+    Depth-bounded: ``reach.depth < {max_depth}``.
+
+    Single ``?`` placeholder: the (lowercased) start table-qualified name.
+
+    Args:
+        direction: ``"downstream"`` (src -> final) or ``"upstream"``
+            (final -> src). ``analyze impact --through-temps`` uses downstream.
+        max_depth: Maximum number of structural hops to traverse.
+
+    Returns:
+        A parameterised SQL string with one ``?`` placeholder.
+    """
+    if direction == "downstream":
+        from_col, to_col = "src", "dst"
+    elif direction == "upstream":
+        from_col, to_col = "dst", "src"
+    else:  # pragma: no cover - guarded by the public function
+        raise ValueError(f"direction must be 'downstream' or 'upstream', got {direction!r}")
+
+    temp_kinds = _temp_kind_in_list_sql()
+    structural_base = f"cl.transform IS DISTINCT FROM '{_TEMP_INLINE_TRANSFORM}'"
+    structural_rec = f"cl2.transform IS DISTINCT FROM '{_TEMP_INLINE_TRANSFORM}'"
+
+    # What counts as a temp/cte/derived INTERMEDIATE node to thread *through*:
+    # a node whose ``SqlTable.kind IN ('cte','derived','temp')`` AND whose key is
+    # NAMESPACED (``<file>::<db>.<name>`` — see base.py:97-98). Namespacing is the
+    # load-bearing discriminator: CTE/temp nodes are always namespaced, whereas an
+    # INSERT/CTAS *target* table (e.g. ``mart.dim_person``) is also stamped
+    # ``kind='derived'`` but is NOT namespaced and is a genuine physical endpoint.
+    # Keying on kind alone would thread through final targets and never terminate;
+    # requiring the ``::`` namespace keeps endpoints (the gold-set finals) terminal.
+    # (See PR-2 Deviation 1 in the plan.)
+    def _is_intermediate(kind_alias: str, node_alias: str) -> str:
+        return f"({kind_alias}.kind IN ({temp_kinds}) AND {node_alias} LIKE '%::%')"
+
+    # ``reach`` walks the table-grain structural graph. Each row is a node
+    # reached from the start, carrying:
+    #   node    — the table_qualified of the reached endpoint
+    #   depth   — number of structural hops taken
+    #   via_temp— whether any intermediate node so far was a temp/cte/derived
+    #   path    — visited table_qualified keys (cycle guard)
+    # The recursive step only CONTINUES through an intermediate node; a physical
+    # endpoint (non-namespaced, or kind not temp/cte/derived) is terminal and is
+    # not re-expanded.
+    seed_via_temp = _is_intermediate("kt", "n.table_qualified")
+    rec_via_temp = _is_intermediate("kt2", "n2.table_qualified")
+    cur_is_intermediate = _is_intermediate("kcur", "reach.node")
+    return f"""
+WITH RECURSIVE reach(node, depth, via_temp, path) AS (
+  SELECT
+    n.table_qualified AS node,
+    1 AS depth,
+    {seed_via_temp} AS via_temp,
+    ARRAY[s.table_qualified, n.table_qualified] AS path
+  FROM "COLUMN_LINEAGE" cl
+  JOIN "SqlColumn" s ON s.id = cl.{from_col}_key
+  JOIN "SqlColumn" n ON n.id = cl.{to_col}_key
+  LEFT JOIN "SqlTable" kt ON kt.qualified = n.table_qualified
+  WHERE s.table_qualified = ?
+    AND {structural_base}
+    AND n.table_qualified IS DISTINCT FROM s.table_qualified
+  UNION ALL
+  SELECT
+    n2.table_qualified,
+    reach.depth + 1,
+    reach.via_temp OR {rec_via_temp},
+    array_append(reach.path, n2.table_qualified)
+  FROM reach
+  JOIN "SqlColumn" cur ON cur.table_qualified = reach.node
+  JOIN "COLUMN_LINEAGE" cl2 ON cl2.{from_col}_key = cur.id
+  JOIN "SqlColumn" n2 ON n2.id = cl2.{to_col}_key
+  LEFT JOIN "SqlTable" kcur ON kcur.qualified = reach.node
+  LEFT JOIN "SqlTable" kt2 ON kt2.qualified = n2.table_qualified
+  WHERE reach.depth < {max_depth}
+    AND {structural_rec}
+    -- only thread THROUGH a temp/cte/derived intermediate (namespaced) node
+    AND {cur_is_intermediate}
+    AND n2.table_qualified IS DISTINCT FROM reach.node
+    AND NOT n2.table_qualified = ANY(reach.path)
+)
+SELECT
+  reach.node AS final_node,
+  min(reach.depth) AS hop_count,
+  bool_or(reach.via_temp) AS via_temp
+FROM reach
+LEFT JOIN "SqlTable" fk ON fk.qualified = reach.node
+-- emit only endpoints, never a bare temp/cte intermediate node
+WHERE NOT {_is_intermediate("fk", "reach.node")}
+GROUP BY reach.node
+"""
+
+
+def collapse_through_temps(
+    start_table: str,
+    direction: str = "downstream",
+    max_depth: int = _DEFAULT_MAX_DEPTH,
+    db_path: Path | None = None,
+) -> list[CollapsedEdge]:
+    """Return the temp-collapsed transitive closure from ``start_table``.
+
+    Read-only, NODE-keyed, structural-hop-only table-grain BFS. Threads through
+    ``kind IN ('cte','derived','temp')`` intermediates and emits one
+    :class:`CollapsedEdge` per physical ``(start, reached)`` pair. Recovers the
+    star-into-temp ``src -> final`` edge that TEMP_INLINE does not emit (the
+    closure is built from the always-present structural hops, not from
+    TEMP_INLINE rows).
+
+    Non-destructive: issues only ``SELECT`` queries via
+    :func:`run_read_routed`. It performs no DELETE/INSERT/UPDATE/upsert and
+    never touches any write path.
+
+    Args:
+        start_table: The qualified start table name (case-insensitive — graph
+            keys are lowercased at index time, C2 normalization).
+        direction: ``"downstream"`` (default; consumers reached from the table)
+            or ``"upstream"`` (sources the table is derived from).
+        max_depth: Maximum number of structural hops (1..100). Bounds the
+            closure; the visited-path guard breaks cycles independently.
+        db_path: Explicit database path; defaults to the routed read target.
+
+    Returns:
+        A list of :class:`CollapsedEdge`, one per distinct physical endpoint
+        reached. ``src_table`` is the (namespace-stripped) start table;
+        ``final_table`` the (namespace-stripped) reached physical table.
+
+    Raises:
+        ValueError: ``direction`` is not ``"downstream"``/``"upstream"`` or
+            ``max_depth`` is outside 1..100.
+    """
+    if direction not in ("downstream", "upstream"):
+        raise ValueError(f"direction must be 'downstream' or 'upstream', got {direction!r}")
+    if max_depth < 1 or max_depth > 100:
+        raise ValueError(f"max_depth must be between 1 and 100, got {max_depth}")
+
+    start = start_table.lower()  # graph keys are lowercased at index time
+    sql = _collapse_sql(direction=direction, max_depth=max_depth)
+    rows = run_read_routed(sql, {"start": start}, db_path=db_path)
+
+    src_bare = _strip_namespace(start)
+    edges: list[CollapsedEdge] = []
+    for r in rows:
+        final_bare = _strip_namespace(r["final_node"])
+        if final_bare == src_bare:
+            continue  # never report the start as its own endpoint
+        edges.append(
+            CollapsedEdge(
+                src_table=src_bare,
+                final_table=final_bare,
+                via_temp=bool(r["via_temp"]),
+                hop_count=int(r["hop_count"]),
+            )
+        )
+    return edges