dvt-core 0.59.0a51__py3-none-any.whl

dbt/task/docs/api/lineage.py

@@ -0,0 +1,234 @@
+"""
+DVT Docs Serve - Lineage API
+
+v0.56.0: Lineage graph, node traversal, and cross-connection edges.
+"""
+
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+try:
+    from fastapi import APIRouter, HTTPException, Query
+except ImportError:
+    class APIRouter:
+        def __init__(self, *args, **kwargs):
+            pass
+        def get(self, *args, **kwargs):
+            def decorator(func):
+                return func
+            return decorator
+    class HTTPException(Exception):
+        def __init__(self, status_code, detail):
+            self.status_code = status_code
+            self.detail = detail
+    def Query(*args, **kwargs):
+        return None
+
+
+router = APIRouter(prefix="/api/lineage", tags=["lineage"])
+
+# Will be set by serve.py
+_project_root: Optional[Path] = None
+_store_initialized: bool = False
+
+
+def set_project_root(project_root: Path) -> None:
+    """Set project root for API access."""
+    global _project_root, _store_initialized
+    _project_root = project_root
+    _store_initialized = True  # Assume serve.py already initialized
+
+
+def _get_store():
+    """Get metadata store instance (lazy initialization)."""
+    global _store_initialized
+
+    if _project_root is None:
+        raise HTTPException(status_code=500, detail="Project root not configured")
+
+    try:
+        from dbt.compute.metadata import ProjectMetadataStore
+        store = ProjectMetadataStore(_project_root)
+        # Only initialize if not already done by serve.py
+        if not _store_initialized:
+            store.initialize()
+            _store_initialized = True
+        return store
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Could not open metadata store: {e}")
+
+
+@router.get("/graph")
+async def get_lineage_graph() -> Dict[str, Any]:
+    """
+    Get the full lineage graph.
+
+    Returns nodes and edges suitable for visualization.
+    """
+    store = _get_store()
+
+    try:
+        graph = store.get_lineage_graph()
+        return graph
+    finally:
+        store.close()
+
+
+@router.get("/node/{node_id}")
+async def get_node_lineage(
+    node_id: str,
+    depth: int = Query(1, ge=1, le=10, description="Traversal depth"),
+) -> Dict[str, Any]:
+    """
+    Get lineage for a specific node.
+
+    Returns upstream and downstream nodes up to specified depth using BFS traversal.
+    """
+    store = _get_store()
+
+    try:
+        # Get the node
+        node = store.get_catalog_node(node_id)
+        if not node:
+            raise HTTPException(status_code=404, detail=f"Node not found: {node_id}")
+
+        # BFS traversal for upstream nodes
+        upstream_nodes = _traverse_lineage(
+            store, node_id, depth, direction="upstream"
+        )
+
+        # BFS traversal for downstream nodes
+        downstream_nodes = _traverse_lineage(
+            store, node_id, depth, direction="downstream"
+        )
+
+        return {
+            "node": {
+                "unique_id": node.unique_id,
+                "name": node.name,
+                "resource_type": node.resource_type,
+                "connection": node.connection_name,
+                "icon": node.icon_type,
+                "color": node.color_hex,
+            },
+            "upstream": upstream_nodes,
+            "downstream": downstream_nodes,
+            "depth": depth,
+        }
+    finally:
+        store.close()
+
+
+def _traverse_lineage(
+    store, start_node_id: str, max_depth: int, direction: str
+) -> List[Dict[str, Any]]:
+    """
+    BFS traversal of lineage graph.
+
+    Args:
+        store: ProjectMetadataStore instance
+        start_node_id: Starting node ID
+        max_depth: Maximum traversal depth
+        direction: "upstream" or "downstream"
+
+    Returns:
+        List of nodes with depth information
+    """
+    from collections import deque
+
+    visited = set()
+    result = []
+
+    # Queue: (node_id, current_depth, edge_info)
+    queue = deque([(start_node_id, 0, None)])
+    visited.add(start_node_id)
+
+    while queue:
+        current_id, current_depth, edge_info = queue.popleft()
+
+        # Skip the start node itself
+        if current_depth > 0:
+            node = store.get_catalog_node(current_id)
+            if node:
+                result.append({
+                    "unique_id": node.unique_id,
+                    "name": node.name,
+                    "resource_type": node.resource_type,
+                    "connection": node.connection_name,
+                    "depth": current_depth,
+                    "edge_type": edge_info.edge_type if edge_info else None,
+                    "is_cross_connection": edge_info.is_cross_connection if edge_info else False,
+                })
+
+        # Stop if we've reached max depth
+        if current_depth >= max_depth:
+            continue
+
+        # Get edges based on direction
+        if direction == "upstream":
+            edges = store.get_upstream_edges(current_id)
+            next_nodes = [(e.source_node_id, e) for e in edges]
+        else:
+            edges = store.get_downstream_edges(current_id)
+            next_nodes = [(e.target_node_id, e) for e in edges]
+
+        # Add unvisited neighbors to queue
+        for next_id, edge in next_nodes:
+            if next_id not in visited:
+                visited.add(next_id)
+                queue.append((next_id, current_depth + 1, edge))
+
+    return result
+
+
+@router.get("/cross-connection")
+async def get_cross_connection_edges() -> List[Dict[str, Any]]:
+    """
+    Get all cross-connection edges.
+
+    Returns edges that span different database connections.
+    """
+    store = _get_store()
+
+    try:
+        edges = store.get_cross_connection_edges()
+        return [
+            {
+                "id": e.id,
+                "source_node_id": e.source_node_id,
+                "target_node_id": e.target_node_id,
+                "edge_type": e.edge_type,
+                "source_connection": e.source_connection,
+                "target_connection": e.target_connection,
+            }
+            for e in edges
+        ]
+    finally:
+        store.close()
+
+
+@router.get("/stats")
+async def get_lineage_stats() -> Dict[str, Any]:
+    """
+    Get lineage statistics.
+
+    Returns edge counts, cross-connection counts, etc.
+    """
+    store = _get_store()
+
+    try:
+        all_edges = store.get_all_lineage_edges()
+        cross_edges = store.get_cross_connection_edges()
+
+        # Count by edge type
+        type_counts: Dict[str, int] = {}
+        for edge in all_edges:
+            type_counts[edge.edge_type] = type_counts.get(edge.edge_type, 0) + 1
+
+        return {
+            "total_edges": len(all_edges),
+            "cross_connection_edges": len(cross_edges),
+            "by_type": type_counts,
+        }
+    finally:
+        store.close()

dbt/task/docs/api/profile.py

@@ -0,0 +1,204 @@
+"""
+DVT Docs Serve - Profile API
+
+v0.56.0: Profile results, alerts, and table profiles.
+"""
+
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+try:
+    from fastapi import APIRouter, HTTPException, Query
+except ImportError:
+    class APIRouter:
+        def __init__(self, *args, **kwargs):
+            pass
+        def get(self, *args, **kwargs):
+            def decorator(func):
+                return func
+            return decorator
+    class HTTPException(Exception):
+        def __init__(self, status_code, detail):
+            self.status_code = status_code
+            self.detail = detail
+    def Query(*args, **kwargs):
+        return None
+
+
+router = APIRouter(prefix="/api/profile", tags=["profile"])
+
+# Will be set by serve.py
+_project_root: Optional[Path] = None
+_store_initialized: bool = False
+
+
+def set_project_root(project_root: Path) -> None:
+    """Set project root for API access."""
+    global _project_root, _store_initialized
+    _project_root = project_root
+    _store_initialized = True  # Assume serve.py already initialized
+
+
+def _get_store():
+    """Get metadata store instance (lazy initialization)."""
+    global _store_initialized
+
+    if _project_root is None:
+        raise HTTPException(status_code=500, detail="Project root not configured")
+
+    try:
+        from dbt.compute.metadata import ProjectMetadataStore
+        store = ProjectMetadataStore(_project_root)
+        # Only initialize if not already done by serve.py
+        if not _store_initialized:
+            store.initialize()
+            _store_initialized = True
+        return store
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Could not open metadata store: {e}")
+
+
+@router.get("/tables")
+async def list_profiled_tables() -> List[Dict[str, Any]]:
+    """
+    List all profiled tables.
+
+    Returns table names with last profile timestamp and mode.
+    """
+    store = _get_store()
+
+    try:
+        tables = store.get_all_profiled_tables()
+        return [
+            {
+                "source_name": t[0],
+                "table_name": t[1],
+                "profile_mode": t[2],
+                "last_profiled": t[3].isoformat() if t[3] else None,
+            }
+            for t in tables
+        ]
+    finally:
+        store.close()
+
+
+@router.get("/tables/{source_name}/{table_name}")
+async def get_table_profile(
+    source_name: str,
+    table_name: str,
+    mode: Optional[str] = Query(None, description="Profile mode filter"),
+) -> Dict[str, Any]:
+    """
+    Get profile results for a specific table.
+
+    Returns all column profiles with metrics.
+    """
+    import json
+
+    store = _get_store()
+
+    try:
+        profiles = store.get_table_profile(source_name, table_name, mode)
+        if not profiles:
+            raise HTTPException(
+                status_code=404,
+                detail=f"No profile found for {source_name}.{table_name}"
+            )
+
+        # Get first profile for table-level stats
+        first = profiles[0]
+
+        return {
+            "source_name": source_name,
+            "table_name": table_name,
+            "profile_mode": first.profile_mode,
+            "row_count": first.row_count,
+            "column_count": len(profiles),
+            "profiled_at": first.profiled_at.isoformat() if first.profiled_at else None,
+            "columns": [
+                {
+                    "column_name": p.column_name,
+                    "null_count": p.null_count,
+                    "null_percent": p.null_percent,
+                    "distinct_count": p.distinct_count,
+                    "distinct_percent": p.distinct_percent,
+                    "min_value": p.min_value,
+                    "max_value": p.max_value,
+                    "mean_value": p.mean_value,
+                    "median_value": p.median_value,
+                    "stddev_value": p.stddev_value,
+                    "p25": p.p25,
+                    "p50": p.p50,
+                    "p75": p.p75,
+                    "min_length": p.min_length,
+                    "max_length": p.max_length,
+                    "avg_length": p.avg_length,
+                    "histogram": json.loads(p.histogram) if p.histogram else None,
+                    "top_values": json.loads(p.top_values) if p.top_values else None,
+                    "alerts": json.loads(p.alerts) if p.alerts else [],
+                    "duration_ms": p.duration_ms,
+                }
+                for p in profiles
+            ],
+        }
+    finally:
+        store.close()
+
+
+@router.get("/alerts")
+async def get_profile_alerts(
+    source_name: Optional[str] = Query(None, description="Filter by source"),
+    severity: Optional[str] = Query(None, description="Filter by severity: info, warning, error"),
+) -> List[Dict[str, Any]]:
+    """
+    Get all profile alerts.
+
+    Returns alerts from profile results, optionally filtered.
+    """
+    store = _get_store()
+
+    try:
+        alerts = store.get_profile_alerts(source_name)
+
+        # Filter by severity if provided
+        if severity:
+            alerts = [a for a in alerts if a.get("severity") == severity]
+
+        return alerts
+    finally:
+        store.close()
+
+
+@router.get("/stats")
+async def get_profile_stats() -> Dict[str, Any]:
+    """
+    Get profiling statistics.
+
+    Returns counts of profiled tables, columns, alerts.
+    """
+    store = _get_store()
+
+    try:
+        tables = store.get_all_profiled_tables()
+        alerts = store.get_profile_alerts()
+
+        # Count alerts by type
+        alert_types: Dict[str, int] = {}
+        for alert in alerts:
+            alert_type = alert.get("type", "unknown")
+            alert_types[alert_type] = alert_types.get(alert_type, 0) + 1
+
+        # Count alerts by severity
+        alert_severities: Dict[str, int] = {}
+        for alert in alerts:
+            severity = alert.get("severity", "info")
+            alert_severities[severity] = alert_severities.get(severity, 0) + 1
+
+        return {
+            "total_tables": len(tables),
+            "total_alerts": len(alerts),
+            "alerts_by_type": alert_types,
+            "alerts_by_severity": alert_severities,
+        }
+    finally:
+        store.close()

dbt/task/docs/api/spark.py

@@ -0,0 +1,186 @@
+"""
+DVT Docs Serve - Spark Status API
+
+v0.56.0: Basic Spark status for local Spark only.
+
+Note: For external clusters (EMR, Dataproc, Databricks),
+use the platform's native monitoring tools.
+"""
+
+from pathlib import Path
+from typing import Any, Dict, Optional
+
+try:
+    from fastapi import APIRouter, HTTPException
+except ImportError:
+    class APIRouter:
+        def __init__(self, *args, **kwargs):
+            pass
+        def get(self, *args, **kwargs):
+            def decorator(func):
+                return func
+            return decorator
+    class HTTPException(Exception):
+        def __init__(self, status_code, detail):
+            self.status_code = status_code
+            self.detail = detail
+
+
+router = APIRouter(prefix="/api/spark", tags=["spark"])
+
+# Will be set by serve.py
+_project_root: Optional[Path] = None
+_store_initialized: bool = False
+
+
+def set_project_root(project_root: Path) -> None:
+    """Set project root for API access."""
+    global _project_root, _store_initialized
+    _project_root = project_root
+    _store_initialized = True  # Assume serve.py already initialized
+
+
+@router.get("/status")
+async def get_spark_status() -> Dict[str, Any]:
+    """
+    Get current Spark status.
+
+    Returns basic status info for local Spark.
+    """
+    try:
+        # Try to detect if Spark is running
+        from dbt.config.compute import ComputeRegistry
+
+        registry = ComputeRegistry(_project_root) if _project_root else None
+        if not registry:
+            return {
+                "engine": "unknown",
+                "status": "not_configured",
+                "message": "Compute registry not available",
+            }
+
+        # Get default compute
+        default_compute = registry.target_compute
+        compute_cluster = registry.get(default_compute)
+
+        if not compute_cluster:
+            return {
+                "engine": "none",
+                "status": "not_configured",
+                "message": "No compute engine configured",
+            }
+
+        # Check if it's local Spark
+        platform = compute_cluster.detect_platform()
+        is_local = platform == "local"
+
+        # Try to get Spark session status
+        spark_ui_url = None
+        spark_running = False
+
+        if is_local:
+            try:
+                from pyspark.sql import SparkSession
+
+                # Check for existing session (don't create new one)
+                existing = SparkSession.getActiveSession()
+                if existing:
+                    spark_running = True
+                    spark_ui_url = existing.sparkContext.uiWebUrl
+            except Exception:
+                pass
+
+        return {
+            "engine": compute_cluster.name,
+            "platform": platform,
+            "status": "running" if spark_running else "ready",
+            "is_local": is_local,
+            "spark_ui_url": spark_ui_url,
+            "message": "Local Spark ready" if is_local else f"External platform: {platform}",
+            "config": {
+                "master": compute_cluster.config.get("master", "local[*]") if is_local else None,
+                "driver_memory": compute_cluster.config.get("spark.driver.memory"),
+                "executor_memory": compute_cluster.config.get("spark.executor.memory"),
+            },
+        }
+
+    except Exception as e:
+        return {
+            "engine": "unknown",
+            "status": "error",
+            "message": f"Could not get Spark status: {str(e)}",
+        }
+
+
+@router.get("/ui-url")
+async def get_spark_ui_url() -> Dict[str, Any]:
+    """
+    Get Spark UI URL if available.
+
+    Only works for local Spark when a session is active.
+    """
+    try:
+        from pyspark.sql import SparkSession
+
+        existing = SparkSession.getActiveSession()
+        if existing:
+            return {
+                "available": True,
+                "url": existing.sparkContext.uiWebUrl,
+            }
+        else:
+            return {
+                "available": False,
+                "url": None,
+                "message": "No active Spark session",
+            }
+    except ImportError:
+        return {
+            "available": False,
+            "url": None,
+            "message": "PySpark not installed",
+        }
+    except Exception as e:
+        return {
+            "available": False,
+            "url": None,
+            "message": str(e),
+        }
+
+
+@router.get("/config")
+async def get_spark_config() -> Dict[str, Any]:
+    """
+    Get Spark configuration from computes.yml.
+
+    Returns the configured Spark settings without starting a session.
+    """
+    try:
+        from dbt.config.compute import ComputeRegistry
+
+        registry = ComputeRegistry(_project_root) if _project_root else None
+        if not registry:
+            raise HTTPException(status_code=500, detail="Compute registry not available")
+
+        # Get all Spark computes
+        all_computes = registry.list()
+        spark_computes = [c for c in all_computes if c.type == "spark"]
+
+        return {
+            "default_compute": registry.target_compute,
+            "spark_computes": [
+                {
+                    "name": c.name,
+                    "platform": c.detect_platform(),
+                    "config": {
+                        "master": c.config.get("master"),
+                        "driver_memory": c.config.get("spark.driver.memory"),
+                        "executor_memory": c.config.get("spark.executor.memory"),
+                        "jars_packages": c.config.get("spark.jars.packages"),
+                    },
+                }
+                for c in spark_computes
+            ],
+        }
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))