odibi 2.5.0__py3-none-any.whl

odibi/transformers/validation.py

@@ -0,0 +1,165 @@
+"""Validation transformers."""
+
+import time
+from typing import Any, List
+
+from pydantic import BaseModel, Field
+
+from odibi.context import EngineContext
+from odibi.exceptions import ValidationError
+from odibi.registry import transform
+from odibi.utils.logging_context import get_logging_context
+
+
+class CrossCheckParams(BaseModel):
+    """
+    Configuration for cross-node validation checks.
+
+    Example (Row Count Mismatch):
+    ```yaml
+    transformer: "cross_check"
+    params:
+      type: "row_count_diff"
+      inputs: ["node_a", "node_b"]
+      threshold: 0.05  # Allow 5% difference
+    ```
+
+    Example (Schema Match):
+    ```yaml
+    transformer: "cross_check"
+    params:
+      type: "schema_match"
+      inputs: ["staging_orders", "prod_orders"]
+    ```
+    """
+
+    type: str = Field(description="Check type: 'row_count_diff', 'schema_match'")
+    inputs: List[str] = Field(description="List of node names to compare")
+    threshold: float = Field(default=0.0, description="Threshold for diff (0.0-1.0)")
+
+
+@transform("cross_check", param_model=CrossCheckParams)
+def cross_check(context: EngineContext, params: CrossCheckParams) -> Any:
+    """
+    Perform cross-node validation checks.
+
+    Does not return a DataFrame (returns None).
+    Raises ValidationError on failure.
+    """
+    ctx = get_logging_context()
+    start_time = time.time()
+
+    ctx.debug(
+        "CrossCheck starting",
+        check_type=params.type,
+        inputs=params.inputs,
+        threshold=params.threshold,
+    )
+
+    if len(params.inputs) < 2:
+        ctx.error(
+            "CrossCheck failed: insufficient inputs",
+            inputs_count=len(params.inputs),
+        )
+        raise ValueError(
+            f"Cross-check requires at least 2 inputs to compare, but got {len(params.inputs)}. "
+            f"Inputs provided: {params.inputs!r}. "
+            "Add another input dataset to the 'inputs' list."
+        )
+
+    dfs = {}
+    for name in params.inputs:
+        df = context.context.get(name)
+        if df is None:
+            ctx.error(
+                "CrossCheck failed: input not found",
+                missing_input=name,
+                available_inputs=(
+                    list(context.context._data.keys())
+                    if hasattr(context.context, "_data")
+                    else None
+                ),
+            )
+            raise ValueError(
+                f"Cross-check input '{name}' not found in context. "
+                f"Available inputs: {list(context.context._data.keys()) if hasattr(context.context, '_data') else 'unknown'}. "
+                f"Ensure '{name}' is listed in 'depends_on' for this node."
+            )
+        dfs[name] = df
+
+    if params.type == "row_count_diff":
+        counts = {name: context.engine.count_rows(df) for name, df in dfs.items()}
+        base_name = params.inputs[0]
+        base_count = counts[base_name]
+
+        ctx.debug(
+            "CrossCheck row counts",
+            counts=counts,
+        )
+
+        failures = []
+        for name, count in counts.items():
+            if name == base_name:
+                continue
+
+            if base_count == 0:
+                if count > 0:
+                    diff = 1.0
+                else:
+                    diff = 0.0
+            else:
+                diff = abs(count - base_count) / base_count
+
+            if diff > params.threshold:
+                failures.append(
+                    f"Row count mismatch: {name} ({count}) vs {base_name} ({base_count}). "
+                    f"Diff {diff:.1%} > {params.threshold:.1%}"
+                )
+
+        if failures:
+            ctx.warning(
+                "CrossCheck validation failed",
+                failures=failures,
+            )
+            raise ValidationError("cross_check", failures)
+
+    elif params.type == "schema_match":
+        base_name = params.inputs[0]
+        base_schema = context.engine.get_schema(dfs[base_name])
+
+        failures = []
+        for name, df in dfs.items():
+            if name == base_name:
+                continue
+
+            schema = context.engine.get_schema(df)
+            if base_schema != schema:
+                set_base = set(base_schema.items())
+                set_curr = set(schema.items())
+
+                missing = set_base - set_curr
+                extra = set_curr - set_base
+
+                msg = f"Schema mismatch: {name} vs {base_name}."
+                if missing:
+                    msg += f" Missing/Changed: {missing}"
+                if extra:
+                    msg += f" Extra/Changed: {extra}"
+                failures.append(msg)
+
+        if failures:
+            ctx.warning(
+                "CrossCheck validation failed",
+                failures=failures,
+            )
+            raise ValidationError("cross_check", failures)
+
+    elapsed_ms = (time.time() - start_time) * 1000
+    ctx.debug(
+        "CrossCheck completed",
+        check_type=params.type,
+        passed=True,
+        elapsed_ms=round(elapsed_ms, 2),
+    )
+
+    return None

odibi/ui/app.py

@@ -0,0 +1,195 @@
+import os
+from pathlib import Path
+
+from fastapi import FastAPI, Request
+from fastapi.responses import HTMLResponse
+from fastapi.staticfiles import StaticFiles
+from fastapi.templating import Jinja2Templates
+
+from odibi.state import StateManager
+
+app = FastAPI(title="Odibi UI")
+
+# Resolve paths
+BASE_DIR = Path(__file__).parent
+TEMPLATES_DIR = BASE_DIR / "templates"
+
+templates = Jinja2Templates(directory=str(TEMPLATES_DIR))
+
+
+@app.get("/", response_class=HTMLResponse)
+async def dashboard(request: Request):
+    # Load config to get state backend
+    config_path = os.getenv("ODIBI_CONFIG")
+    if not config_path:
+        # Fallback to defaults
+        for p in ["odibi.yaml", "odibi.yml", "project.yaml"]:
+            if os.path.exists(p):
+                config_path = p
+                break
+
+    if config_path:
+        from odibi.config import load_config_from_file
+        from odibi.state import create_state_backend
+
+        try:
+            config = load_config_from_file(config_path)
+            # Create backend connected to System Catalog
+            backend = create_state_backend(config, project_root=os.path.dirname(config_path))
+            state_mgr = StateManager(backend=backend)
+            state = state_mgr.backend.load_state()
+        except Exception as e:
+            print(f"Failed to load state backend: {e}")
+            state = {}
+    else:
+        state = {}
+
+    pipelines = []
+    if state and "pipelines" in state:
+        for name, p_data in state["pipelines"].items():
+            nodes = p_data.get("nodes", {})
+            total = len(nodes)
+            # Determine pipeline status based on nodes
+            # This is a heuristic as we don't store pipeline-level success explicitly in simple backend
+            # usually pipeline_data has it?
+            # LocalFileStateBackend saves: {"last_run": ..., "nodes": ...}
+            # We don't check individual nodes for pipeline level status in backend save.
+            # But we can infer.
+
+            failed_nodes = [n for n in nodes.values() if not n.get("success")]
+            status = "FAILED" if failed_nodes else "SUCCESS"
+            if total == 0:
+                status = "UNKNOWN"
+
+            pipelines.append(
+                {
+                    "name": name,
+                    "last_run": p_data.get("last_run"),
+                    "status": status,
+                    "nodes_total": total,
+                    "nodes_success": total - len(failed_nodes),
+                }
+            )
+
+    return templates.TemplateResponse(
+        "index.html", {"request": request, "pipelines": pipelines, "project": "Odibi Project"}
+    )
+
+
+@app.get("/stories", response_class=HTMLResponse)
+async def stories(request: Request):
+    # Determine stories root from config if available
+    stories_root = Path("stories")
+    config_path = os.getenv("ODIBI_CONFIG")
+    if config_path:
+        from odibi.config import load_config_from_file
+
+        try:
+            config = load_config_from_file(config_path)
+            # Resolve story path
+            # Connection: system -> base_path + config.story.path
+            conn_name = config.story.connection
+            conn_config = config.connections.get(conn_name)
+            if conn_config and conn_config.type == "local":
+                base = conn_config.base_path
+                if not os.path.isabs(base):
+                    base = os.path.join(os.path.dirname(config_path), base)
+                stories_root = Path(base) / config.story.path
+        except Exception:
+            pass
+
+    runs = []
+
+    if stories_root.exists():
+        # Traverse: pipeline/date/run.html
+        for p_dir in stories_root.iterdir():
+            if p_dir.is_dir():
+                for d_dir in p_dir.iterdir():
+                    if d_dir.is_dir():
+                        for f in d_dir.glob("*.html"):
+                            runs.append(
+                                {
+                                    "pipeline": p_dir.name,
+                                    "date": d_dir.name,
+                                    "name": f.name,
+                                    "path": f"/stories_static/{p_dir.name}/{d_dir.name}/{f.name}",
+                                }
+                            )
+
+    # Sort
+    runs.sort(key=lambda x: (x["date"], x["name"]), reverse=True)
+
+    return templates.TemplateResponse("stories.html", {"request": request, "runs": runs})
+
+
+@app.get("/config", response_class=HTMLResponse)
+async def config_view(request: Request):
+    config_path = os.getenv("ODIBI_CONFIG")
+    content = ""
+    error = None
+
+    if config_path:
+        try:
+            with open(config_path, "r") as f:
+                content = f.read()
+        except Exception as e:
+            error = str(e)
+    else:
+        # Try default locations
+        for p in ["odibi.yaml", "odibi.yml", "project.yaml"]:
+            if os.path.exists(p):
+                config_path = p
+                with open(p, "r") as f:
+                    content = f.read()
+                break
+        if not content:
+            error = "No configuration file found. Run with 'odibi ui config.yaml'"
+
+    return templates.TemplateResponse(
+        "config.html",
+        {"request": request, "config_path": config_path, "content": content, "error": error},
+    )
+
+
+# Mount static files for stories
+# We try to mount the configured stories path if possible, otherwise default
+# This is tricky because mounting happens at startup, but config might change per request?
+# Actually config is set via env var before startup in CLI.
+config_path_env = os.getenv("ODIBI_CONFIG")
+print(f"DEBUG: ODIBI_CONFIG Env Var: {config_path_env}")
+static_stories_dir = Path("stories")
+
+if config_path_env and os.path.exists(config_path_env):
+    # Resolve absolute path to avoid ambiguity
+    abs_config_path = Path(config_path_env).resolve()
+
+    from odibi.config import load_config_from_file
+
+    try:
+        # Use the official loader to get Pydantic defaults/validation
+        config = load_config_from_file(str(abs_config_path))
+
+        s_conn = config.story.connection
+        s_path = config.story.path
+        print(f"DEBUG: Story Conn: {s_conn}, Path: {s_path}")
+        print(f"DEBUG: Available Connections: {list(config.connections.keys())}")
+
+        if s_conn in config.connections:
+            c_conf = config.connections[s_conn]
+            if c_conf.type == "local":
+                base = c_conf.base_path
+                if not os.path.isabs(base):
+                    base = os.path.join(abs_config_path.parent, base)
+                static_stories_dir = Path(base) / s_path
+                print(f"DEBUG: Config Path: {abs_config_path}")
+                print(f"DEBUG: Calculated Base: {base}")
+                print(f"DEBUG: Calculated Stories Dir: {static_stories_dir}")
+                print(f"DEBUG: Exists? {static_stories_dir.exists()}")
+    except Exception as e:
+        print(f"DEBUG: Failed to resolve story path: {e}")
+
+if static_stories_dir.exists():
+    print(f"DEBUG: Mounting stories from {static_stories_dir}")
+    app.mount(
+        "/stories_static", StaticFiles(directory=str(static_stories_dir)), name="stories_static"
+    )

odibi/utils/__init__.py

@@ -0,0 +1,66 @@
+"""Utilities for ODIBI setup and configuration.
+
+Includes:
+- Configuration loading with env var substitution
+- Structured logging and context-aware logging
+- Key Vault and connection helpers
+- Rich console output utilities
+- Pipeline progress visualization
+"""
+
+from .config_loader import load_yaml_with_env
+from .console import (
+    error,
+    get_console,
+    info,
+    is_rich_available,
+    print_panel,
+    print_rule,
+    print_table,
+    success,
+    warning,
+)
+from .logging import StructuredLogger, configure_logging, logger
+from .logging_context import (
+    LoggingContext,
+    OperationMetrics,
+    OperationType,
+    create_logging_context,
+    get_logging_context,
+    set_logging_context,
+)
+from .progress import NodeStatus, PipelineProgress
+from .setup_helpers import (
+    configure_connections_parallel,
+    fetch_keyvault_secrets_parallel,
+    validate_databricks_environment,
+)
+
+__all__ = [
+    "fetch_keyvault_secrets_parallel",
+    "configure_connections_parallel",
+    "validate_databricks_environment",
+    "load_yaml_with_env",
+    "StructuredLogger",
+    "configure_logging",
+    "logger",
+    "LoggingContext",
+    "OperationMetrics",
+    "OperationType",
+    "create_logging_context",
+    "get_logging_context",
+    "set_logging_context",
+    # Console utilities
+    "is_rich_available",
+    "get_console",
+    "success",
+    "error",
+    "warning",
+    "info",
+    "print_table",
+    "print_panel",
+    "print_rule",
+    # Progress utilities
+    "NodeStatus",
+    "PipelineProgress",
+]