sql-dag-flow 0.4.7__tar.gz → 0.4.8__tar.gz

{sql_dag_flow-0.4.7/src/sql_dag_flow.egg-info → sql_dag_flow-0.4.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sql-dag-flow
-Version: 0.4.7
+Version: 0.4.8
 Summary: A sophisticated SQL lineage visualization tool for Medallion Architectures.
 Author-email: Flavio Sandoval <dsandovalflavio@gmail.com>
 License: MIT
@@ -89,12 +89,15 @@ Specially optimized for the **Medallion Architecture** (Bronze, Silver, Gold) an
 ### 📊 Discovery & Analysis Tools
 *   **Impact Analysis**: Visualize blast radius before making changes. Highlights downstream models, column usage, and risk levels.
 *   **Diff View on Refresh**: Automatically summarizes added, removed, and modified nodes/edges after code changes.
-*   **Column Usage Tracking**: Schema Preview shows which specific columns are used by downstream consumers.
+*   **Column Usage Tracking (Improved in v0.4.8 🔧)**: Schema Preview shows which specific columns are used by downstream consumers. Now tracks both qualified (`u.name`) and unqualified (`name`) column references for accurate counts.
 *   **Staleness Detection**: Automatically flags inactive models (`Last Modified > 90d` = Stale) to help clean up legacy pipelines.
 *   **Business Rule Extraction**: Automatically detects and displays WHERE filters, CASE logic, HAVING clauses, and aggregations from each SQL model.
 *   **Complexity Scoring**: Weighted metric per node (JOINs×3, CTEs×2, Subqueries×3, Filters×1, CASE×2, Aggregations×1, UNIONs×2) with color-coded badges (🟢 Low, 🟡 Medium, 🟠 High, 🔴 Very High). Toggleable via ⚡ button.
+*   **Node Comparison**: Select exactly 2 nodes and compare them side-by-side. Highlights differences across metadata, schema columns (shared vs unique), dependencies (Venn-style), complexity scores (with delta indicators), business rules, and SQL content (synced scroll).
 *   **Statistics Panel**: Centered popup with layer distribution bars, edge/source/sink/orphan counts, project/dataset tree, and architecture health validation (now detects **circular dependencies**).
-*   **Schema Preview**: Extracts columns from DDL, CTAS (`CREATE TABLE AS SELECT`), and `CREATE VIEW` statements.
+*   **Schema Extraction (Improved in v0.4.8 🔧)**: Backend AST-based extraction via `sqlglot` handles DDL, CTAS, `CREATE VIEW AS`, CTEs, window functions, CASE expressions, and `SELECT *`.
+*   **Batch Hide from Toolbar (New in v0.4.8 🆕)**: Select multiple nodes → click "Hide" in the selection toolbar to hide them all at once.
+*   **Discovery Mode Fix (v0.4.8 🔧)**: Ghost nodes from Discovery Mode are now hidden when their connected source nodes are hidden, preventing orphan ghost nodes.
 
 ### 🎨 Linear-Inspired UI (New in v0.4.6 ✨)
 *   **Design Token System**: ~80 CSS custom properties for consistent theming across all components.
@@ -139,7 +142,7 @@ Install easily via `pip`:
 pip install sql-dag-flow
 ```
 
-To update to the latest version (**v0.4.6**):
+To update to the latest version (**v0.4.8**):
 
 ```bash
 pip install --upgrade sql-dag-flow

{sql_dag_flow-0.4.7 → sql_dag_flow-0.4.8}/README.md

@@ -64,12 +64,15 @@ Specially optimized for the **Medallion Architecture** (Bronze, Silver, Gold) an
 ### 📊 Discovery & Analysis Tools
 *   **Impact Analysis**: Visualize blast radius before making changes. Highlights downstream models, column usage, and risk levels.
 *   **Diff View on Refresh**: Automatically summarizes added, removed, and modified nodes/edges after code changes.
-*   **Column Usage Tracking**: Schema Preview shows which specific columns are used by downstream consumers.
+*   **Column Usage Tracking (Improved in v0.4.8 🔧)**: Schema Preview shows which specific columns are used by downstream consumers. Now tracks both qualified (`u.name`) and unqualified (`name`) column references for accurate counts.
 *   **Staleness Detection**: Automatically flags inactive models (`Last Modified > 90d` = Stale) to help clean up legacy pipelines.
 *   **Business Rule Extraction**: Automatically detects and displays WHERE filters, CASE logic, HAVING clauses, and aggregations from each SQL model.
 *   **Complexity Scoring**: Weighted metric per node (JOINs×3, CTEs×2, Subqueries×3, Filters×1, CASE×2, Aggregations×1, UNIONs×2) with color-coded badges (🟢 Low, 🟡 Medium, 🟠 High, 🔴 Very High). Toggleable via ⚡ button.
+*   **Node Comparison**: Select exactly 2 nodes and compare them side-by-side. Highlights differences across metadata, schema columns (shared vs unique), dependencies (Venn-style), complexity scores (with delta indicators), business rules, and SQL content (synced scroll).
 *   **Statistics Panel**: Centered popup with layer distribution bars, edge/source/sink/orphan counts, project/dataset tree, and architecture health validation (now detects **circular dependencies**).
-*   **Schema Preview**: Extracts columns from DDL, CTAS (`CREATE TABLE AS SELECT`), and `CREATE VIEW` statements.
+*   **Schema Extraction (Improved in v0.4.8 🔧)**: Backend AST-based extraction via `sqlglot` handles DDL, CTAS, `CREATE VIEW AS`, CTEs, window functions, CASE expressions, and `SELECT *`.
+*   **Batch Hide from Toolbar (New in v0.4.8 🆕)**: Select multiple nodes → click "Hide" in the selection toolbar to hide them all at once.
+*   **Discovery Mode Fix (v0.4.8 🔧)**: Ghost nodes from Discovery Mode are now hidden when their connected source nodes are hidden, preventing orphan ghost nodes.
 
 ### 🎨 Linear-Inspired UI (New in v0.4.6 ✨)
 *   **Design Token System**: ~80 CSS custom properties for consistent theming across all components.
@@ -114,7 +117,7 @@ Install easily via `pip`:
 pip install sql-dag-flow
 ```
 
-To update to the latest version (**v0.4.6**):
+To update to the latest version (**v0.4.8**):
 
 ```bash
 pip install --upgrade sql-dag-flow

{sql_dag_flow-0.4.7 → sql_dag_flow-0.4.8}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "sql-dag-flow"
-version = "0.4.7"
+version = "0.4.8"
 description = "A sophisticated SQL lineage visualization tool for Medallion Architectures."
 readme = "README.md"
 requires-python = ">=3.8"

{sql_dag_flow-0.4.7 → sql_dag_flow-0.4.8}/src/sql_dag_flow/parser.py

@@ -5,6 +5,83 @@ import sqlglot
 from sqlglot import exp
 import networkx as nx
 
+
+def extract_output_columns(parsed, dialect="bigquery"):
+    """
+    Extract output column schema from a parsed SQL AST.
+    Returns a list of {"name": str, "type": str} dicts.
+    
+    Handles:
+    - DDL with explicit column definitions (CREATE TABLE t (col TYPE))
+    - CTAS / CREATE VIEW AS SELECT
+    - WITH...SELECT (final SELECT after CTEs)
+    - Standalone SELECT
+    - Window functions, nested functions, CASE expressions
+    - SELECT * (returns [{"name": "*", "type": "ALL"}])
+    """
+    columns = []
+
+    # Case 1: DDL with explicit column definitions
+    if isinstance(parsed, exp.Create):
+        schema_node = parsed.this
+        if isinstance(schema_node, exp.Schema):
+            for col_def in schema_node.expressions:
+                if isinstance(col_def, exp.ColumnDef):
+                    col_name = col_def.name
+                    col_type_node = col_def.args.get("kind")
+                    type_str = col_type_node.sql(dialect=dialect) if col_type_node else "UNKNOWN"
+                    columns.append({"name": col_name, "type": type_str})
+            if columns:
+                return columns
+
+    # Case 2: Find the final SELECT statement
+    # For CTAS, CREATE VIEW AS, WITH...SELECT, standalone SELECT
+    # We want the outermost SELECT that isn't inside a CTE or subquery
+    select_node = None
+    
+    if isinstance(parsed, exp.Create):
+        # For CTAS / CREATE VIEW AS: get the SELECT inside the CREATE
+        inner = parsed.expression
+        if inner:
+            if isinstance(inner, exp.Select):
+                select_node = inner
+            elif hasattr(inner, 'find'):
+                select_node = inner.find(exp.Select)
+    
+    if not select_node:
+        # For standalone queries: find WITH wrapper or direct SELECT
+        # Walk to find the top-level Select (not nested in subquery)
+        if isinstance(parsed, exp.Select):
+            select_node = parsed
+        else:
+            # Could be a WITH or UNION — find the first select
+            select_node = parsed.find(exp.Select)
+    
+    if select_node:
+        for expr in select_node.expressions:
+            try:
+                if isinstance(expr, exp.Star):
+                    columns.append({"name": "*", "type": "ALL"})
+                elif isinstance(expr, exp.Alias):
+                    alias_name = expr.alias
+                    inner_expr = expr.this
+                    type_str = inner_expr.sql(dialect=dialect, pretty=False)
+                    # Truncate very long expressions for readability
+                    if len(type_str) > 80:
+                        type_str = type_str[:77] + "..."
+                    columns.append({"name": alias_name, "type": type_str})
+                elif isinstance(expr, exp.Column):
+                    columns.append({"name": expr.name, "type": "column"})
+                else:
+                    # Computed expression without alias
+                    expr_sql = expr.sql(dialect=dialect, pretty=False)
+                    name = expr_sql[:40] + "..." if len(expr_sql) > 40 else expr_sql
+                    columns.append({"name": name, "type": "expression"})
+            except Exception:
+                continue
+
+    return columns
+
 def parse_sql_files(directory, allowed_subfolders=None, dialect="bigquery"):
     """
     Recursively scans a directory for .sql files and parses them.
@@ -328,6 +405,9 @@ def parse_sql_files(directory, allowed_subfolders=None, dialect="bigquery"):
                         alias_map[t_name] = t_full
                     
                     # Extract column references with their table qualifier
+                    # Also capture unqualified columns and assign them to source tables
+                    unqualified_columns = set()
+                    
                     for col in parsed.find_all(exp.Column):
                         col_name = col.name
                         col_table = col.table  # The table qualifier (alias or name)
@@ -336,6 +416,36 @@ def parse_sql_files(directory, allowed_subfolders=None, dialect="bigquery"):
                             if source not in column_references:
                                 column_references[source] = set()
                             column_references[source].add(col_name)
+                        elif not col_table:
+                            # Unqualified column — track separately
+                            unqualified_columns.add(col_name)
+                    
+                    # Assign unqualified columns to source tables
+                    # Filter out CTEs and the target table itself (both short and full name)
+                    source_tables = [
+                        v for k, v in alias_map.items() 
+                        if v != target_table_name 
+                        and v.split('.')[-1] != target_table_name
+                        and v not in defined_ctes
+                        and v.split('.')[-1] not in defined_ctes
+                    ]
+                    # Deduplicate (aliases may point to same table)
+                    source_tables = list(set(source_tables))
+                    
+                    if unqualified_columns and source_tables:
+                        if len(source_tables) == 1:
+                            # Single source: assign all unqualified columns to it
+                            src = source_tables[0]
+                            if src not in column_references:
+                                column_references[src] = set()
+                            column_references[src].update(unqualified_columns)
+                        else:
+                            # Multiple sources: assign to ALL sources (best effort)
+                            # The UI will show them as "used" which is better than missing
+                            for src in source_tables:
+                                if src not in column_references:
+                                    column_references[src] = set()
+                                column_references[src].update(unqualified_columns)
                     
                     # Convert sets to sorted lists for JSON serialization
                     column_references = {k: sorted(list(v)) for k, v in column_references.items()}
@@ -389,6 +499,7 @@ def parse_sql_files(directory, allowed_subfolders=None, dialect="bigquery"):
                         "content": sql_content,
                         "ctes": defined_ctes,
                         "cte_deps": cte_deps,
+                        "schema": extract_output_columns(parsed, dialect),
                         "business_rules": business_rules,
                         "complexity": complexity_breakdown,
                         "column_references": column_references,
@@ -637,6 +748,9 @@ def build_graph(tables, discovery_mode=False, expanded_nodes=None):
                 source_node_id = lookup.get(source_ref.split(".")[-1])
             
             if source_node_id and source_node_id in all_nodes_data:
+                # Skip self-references (a model shouldn't be its own consumer)
+                if source_node_id == consumer_id:
+                    continue
                 if source_node_id not in column_consumers:
                     column_consumers[source_node_id] = {}
                 for col in columns: