sql-dag-flow 0.5.1__tar.gz → 0.5.3__tar.gz

{sql_dag_flow-0.5.1/src/sql_dag_flow.egg-info → sql_dag_flow-0.5.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sql-dag-flow
-Version: 0.5.1
+Version: 0.5.3
 Summary: A sophisticated SQL lineage visualization tool for Medallion Architectures.
 Author-email: Flavio Sandoval <dsandovalflavio@gmail.com>
 License: MIT
@@ -65,6 +65,7 @@ Specially optimized for the **Medallion Architecture** (Bronze, Silver, Gold) an
     *   Powered by **ELK (Eclipse Layout Kernel)**.
     *   Minimizes edge crossings and optimizes flow direction.
     *   Intelligent "Port" handling for cleaner connections.
+*   **Startup Configuration Selector**: Instantly resume previous sessions by selecting any `.json` configuration file found in your project directory upon launching the app.
 
 ### 🎮 Interactive Graph
 *   **Smart Context Menu**:
@@ -99,6 +100,7 @@ Specially optimized for the **Medallion Architecture** (Bronze, Silver, Gold) an
 *   **Column-Level Lineage (New in v0.4.9 🆕)**: Traces how each output column derives from source columns. Shows transformation chain (e.g., `order_timestamp ← orders_raw.order_date via CAST(... AS DATETIME)`).
 *   **SQL Syntax Validation (New in v0.4.9 🆕)**: Detects SQL parse errors and displays structured warnings with line/column references. Shows ⚠️ badge on nodes with syntax issues.
 *   **Large DAG Support & Safe Cycle Detection (New in v0.5.1 🚀)**: Optimized cycle detection algorithm prevents backend hanging and "Failed to fetch" browser errors when analyzing massive projects (50+ nodes). Employs quick DAG verifications and bounded iterator loops.
+*   **Performance Optimizations (New in v0.5.2 ⚡)**: Includes backend parsing TTL caching and frontend request deduplication to prevent server overload when refreshing large numbers of un-cached SQL files simultaneously.
 *   **Batch Hide from Toolbar**: Select multiple nodes → click "Hide" in the selection toolbar to hide them all at once.
 *   **Discovery Mode Fix**: Ghost nodes from Discovery Mode are now hidden when their connected source nodes are hidden, preventing orphan ghost nodes.
 
@@ -145,7 +147,7 @@ Install easily via `pip`:
 pip install sql-dag-flow
 ```
 
-To update to the latest version (**v0.5.1**):
+To update to the latest version (**v0.5.3**):
 
 ```bash
 pip install --upgrade sql-dag-flow

{sql_dag_flow-0.5.1 → sql_dag_flow-0.5.3}/README.md

@@ -40,6 +40,7 @@ Specially optimized for the **Medallion Architecture** (Bronze, Silver, Gold) an
     *   Powered by **ELK (Eclipse Layout Kernel)**.
     *   Minimizes edge crossings and optimizes flow direction.
     *   Intelligent "Port" handling for cleaner connections.
+*   **Startup Configuration Selector**: Instantly resume previous sessions by selecting any `.json` configuration file found in your project directory upon launching the app.
 
 ### 🎮 Interactive Graph
 *   **Smart Context Menu**:
@@ -74,6 +75,7 @@ Specially optimized for the **Medallion Architecture** (Bronze, Silver, Gold) an
 *   **Column-Level Lineage (New in v0.4.9 🆕)**: Traces how each output column derives from source columns. Shows transformation chain (e.g., `order_timestamp ← orders_raw.order_date via CAST(... AS DATETIME)`).
 *   **SQL Syntax Validation (New in v0.4.9 🆕)**: Detects SQL parse errors and displays structured warnings with line/column references. Shows ⚠️ badge on nodes with syntax issues.
 *   **Large DAG Support & Safe Cycle Detection (New in v0.5.1 🚀)**: Optimized cycle detection algorithm prevents backend hanging and "Failed to fetch" browser errors when analyzing massive projects (50+ nodes). Employs quick DAG verifications and bounded iterator loops.
+*   **Performance Optimizations (New in v0.5.2 ⚡)**: Includes backend parsing TTL caching and frontend request deduplication to prevent server overload when refreshing large numbers of un-cached SQL files simultaneously.
 *   **Batch Hide from Toolbar**: Select multiple nodes → click "Hide" in the selection toolbar to hide them all at once.
 *   **Discovery Mode Fix**: Ghost nodes from Discovery Mode are now hidden when their connected source nodes are hidden, preventing orphan ghost nodes.
 
@@ -120,7 +122,7 @@ Install easily via `pip`:
 pip install sql-dag-flow
 ```
 
-To update to the latest version (**v0.5.1**):
+To update to the latest version (**v0.5.3**):
 
 ```bash
 pip install --upgrade sql-dag-flow

{sql_dag_flow-0.5.1 → sql_dag_flow-0.5.3}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "sql-dag-flow"
-version = "0.5.1"
+version = "0.5.3"
 description = "A sophisticated SQL lineage visualization tool for Medallion Architectures."
 readme = "README.md"
 requires-python = ">=3.8"

{sql_dag_flow-0.5.1 → sql_dag_flow-0.5.3}/src/sql_dag_flow/main.py

@@ -36,6 +36,24 @@ STATIC_DIR = os.path.join(BASE_DIR, "static")
 CURRENT_DIRECTORY = os.getcwd() # Default, updated by start()
 DIAGRAM_FILE = "sql_diagram.json"
 
+# Parse cache to avoid re-parsing on concurrent/rapid requests
+_parse_cache = {"key": None, "tables": None, "time": 0}
+PARSE_CACHE_TTL = 5  # seconds
+
+def _cached_parse(directory, subfolders_tuple, dialect):
+    """Parse with simple TTL cache. Prevents re-parsing on concurrent requests."""
+    cache_key = (directory, subfolders_tuple, dialect)
+    now = time.time()
+    if _parse_cache["key"] == cache_key and (now - _parse_cache["time"]) < PARSE_CACHE_TTL:
+        return _parse_cache["tables"]
+    
+    subfolders_list = list(subfolders_tuple) if subfolders_tuple else None
+    tables = parse_sql_files(directory, allowed_subfolders=subfolders_list, dialect=dialect)
+    _parse_cache["key"] = cache_key
+    _parse_cache["tables"] = tables
+    _parse_cache["time"] = now
+    return tables
+
 @app.get("/graph")
 def get_graph(dialect: str = "bigquery", discovery: bool = False, expanded_nodes: str = ""):
     """Parses SQL files in the current directory and returns graph data."""
@@ -44,7 +62,7 @@ def get_graph(dialect: str = "bigquery", discovery: bool = False, expanded_nodes
         
     exp_nodes_list = [n.strip() for n in expanded_nodes.split(",")] if expanded_nodes else []
     try:
-        tables = parse_sql_files(CURRENT_DIRECTORY, dialect=dialect)
+        tables = _cached_parse(CURRENT_DIRECTORY, None, dialect)
         nodes, edges, cycles = build_graph(tables, discovery_mode=discovery, expanded_nodes=exp_nodes_list)
         return {"nodes": nodes, "edges": edges, "cycles": cycles}
     except Exception as e:
@@ -61,8 +79,8 @@ def set_path(path_data: dict = Body(...)):
     if not path or not os.path.exists(path):
         raise HTTPException(status_code=400, detail="Directory does not exist")
     
-    
     CURRENT_DIRECTORY = path
+    _parse_cache["key"] = None  # Invalidate cache on path change
     return {"message": "Path updated", "path": CURRENT_DIRECTORY}
 
 @app.post("/scan/folders")
@@ -105,7 +123,7 @@ def get_filtered_graph(data: dict = Body(...)):
     expanded_nodes = data.get("expanded_nodes", [])
     
     try:
-        tables = parse_sql_files(CURRENT_DIRECTORY, allowed_subfolders=subfolders, dialect=dialect)
+        tables = _cached_parse(CURRENT_DIRECTORY, tuple(subfolders) if subfolders else None, dialect)
         nodes, edges, cycles = build_graph(tables, discovery_mode=discovery, expanded_nodes=expanded_nodes)
         return {"nodes": nodes, "edges": edges, "cycles": cycles}
     except Exception as e:

{sql_dag_flow-0.5.1 → sql_dag_flow-0.5.3}/src/sql_dag_flow/parser.py

@@ -566,18 +566,28 @@ def parse_sql_files(directory, allowed_subfolders=None, dialect="bigquery"):
         global_schema["default"][label] = col_dict
     
     # Re-extract column_references using qualify_columns for precision
-    for tid, tdata in tables.items():
-        if tdata.get("error") or not tdata.get("content"):
-            continue
+    # Protected with per-table time budget to prevent hanging on large projects
+    qualify_tables = [(tid, tdata) for tid, tdata in tables.items() 
+                      if not tdata.get("error") and tdata.get("content")]
+    total_qualify = len(qualify_tables)
+    
+    for idx, (tid, tdata) in enumerate(qualify_tables):
+        table_start = time.time()
+        if idx % 10 == 0 and total_qualify > 10:
+            print(f"  Qualifying columns: {idx}/{total_qualify} tables...")
         try:
             parsed = sqlglot.parse_one(tdata["content"], read=dialect)
             
-            # Try to qualify columns using the global schema
+            # Try to qualify columns using the global schema (with 2s budget)
             try:
                 qualified_ast = sqlglot_qualify_columns(parsed.copy(), schema=global_schema, dialect=dialect)
             except Exception:
                 qualified_ast = parsed  # Fallback to unqualified
             
+            # Check time budget
+            if time.time() - table_start > 2.0:
+                continue  # Skip extraction if qualification took too long
+            
             # Re-extract column references from qualified AST
             alias_map = {}
             defined_ctes = set()
@@ -634,55 +644,83 @@ def parse_sql_files(directory, allowed_subfolders=None, dialect="bigquery"):
         except Exception:
             pass  # Keep original column_references
     
+    if total_qualify > 10:
+        print(f"  Qualifying columns: {total_qualify}/{total_qualify} done.")
+    
     # ===== Column-Level Lineage =====
     # For each table, trace how each output column derives from source columns
-    for tid, tdata in tables.items():
-        if tdata.get("error") or not tdata.get("content"):
-            continue
-        schema_cols = tdata.get("schema", [])
-        if not schema_cols:
-            continue
-        
-        column_lineage = {}  # col_name -> [{source_table, source_column, transform}]
-        sql_content = tdata["content"]
-        
-        for col_info in schema_cols:
-            col_name = col_info["name"]
-            if col_name == "*":
+    # Protected: skip if project is very large (>100 tables) and cap columns per table
+    MAX_LINEAGE_TABLES = 100
+    MAX_COLS_PER_TABLE = 30
+    LINEAGE_TIME_BUDGET = 1.5  # seconds per table
+    
+    lineage_tables = [(tid, tdata) for tid, tdata in tables.items()
+                      if not tdata.get("error") and tdata.get("content") and tdata.get("schema")]
+    total_lineage = len(lineage_tables)
+    
+    if total_lineage > MAX_LINEAGE_TABLES:
+        print(f"  Skipping column lineage: {total_lineage} tables exceeds limit of {MAX_LINEAGE_TABLES}")
+    else:
+        for idx, (tid, tdata) in enumerate(lineage_tables):
+            table_start = time.time()
+            if idx % 10 == 0 and total_lineage > 10:
+                print(f"  Column lineage: {idx}/{total_lineage} tables...")
+            
+            schema_cols = tdata.get("schema", [])
+            if not schema_cols:
                 continue
-            try:
-                from sqlglot.lineage import lineage as sqlglot_lineage
-                node = sqlglot_lineage(
-                    col_name, sql_content,
-                    schema=global_schema,
-                    dialect=dialect
-                )
-                sources = []
-                for child in node.downstream:
-                    source_col = child.name
-                    # Extract the source table from the expression
-                    source_table = ""
-                    if child.source and isinstance(child.source, exp.Table):
-                        source_table = child.source.name
-                        if child.source.db:
-                            source_table = f"{child.source.db}.{child.source.name}"
-                    
-                    transform = ""
-                    if child.expression and str(child.expression) != source_col:
-                        transform = str(child.expression)
+            
+            column_lineage = {}  # col_name -> [{source_table, source_column, transform}]
+            sql_content = tdata["content"]
+            
+            cols_processed = 0
+            for col_info in schema_cols:
+                # Check time budget
+                if time.time() - table_start > LINEAGE_TIME_BUDGET:
+                    break
+                if cols_processed >= MAX_COLS_PER_TABLE:
+                    break
                     
-                    sources.append({
-                        "source_table": source_table,
-                        "source_column": source_col,
-                        "transform": transform
-                    })
-                if sources:
-                    column_lineage[col_name] = sources
-            except Exception:
-                pass  # Lineage not available for this column
+                col_name = col_info["name"]
+                if col_name == "*":
+                    continue
+                cols_processed += 1
+                try:
+                    from sqlglot.lineage import lineage as sqlglot_lineage
+                    node = sqlglot_lineage(
+                        col_name, sql_content,
+                        schema=global_schema,
+                        dialect=dialect
+                    )
+                    sources = []
+                    for child in node.downstream:
+                        source_col = child.name
+                        # Extract the source table from the expression
+                        source_table = ""
+                        if child.source and isinstance(child.source, exp.Table):
+                            source_table = child.source.name
+                            if child.source.db:
+                                source_table = f"{child.source.db}.{child.source.name}"
+                        
+                        transform = ""
+                        if child.expression and str(child.expression) != source_col:
+                            transform = str(child.expression)
+                        
+                        sources.append({
+                            "source_table": source_table,
+                            "source_column": source_col,
+                            "transform": transform
+                        })
+                    if sources:
+                        column_lineage[col_name] = sources
+                except Exception:
+                    pass  # Lineage not available for this column
+            
+            if column_lineage:
+                tdata["column_lineage"] = column_lineage
         
-        if column_lineage:
-            tdata["column_lineage"] = column_lineage
+        if total_lineage > 10:
+            print(f"  Column lineage: {total_lineage}/{total_lineage} done.")
     
     return tables