odibi 2.5.0__py3-none-any.whl

odibi/utils/config_loader.py

@@ -0,0 +1,343 @@
+import os
+import re
+from typing import Any, Dict
+
+import yaml
+
+from odibi.utils.logging import logger
+
+# Pattern to match ${VAR} or ${env:VAR}
+# Captures the variable name in group 1
+ENV_PATTERN = re.compile(r"\$\{(?:env:)?([A-Za-z0-9_]+)\}")
+
+
+def _normalize_pattern_to_transformer(data: Dict[str, Any]) -> None:
+    """Normalize 'pattern:' block to 'transformer:' + 'params:' fields.
+
+    The documentation uses `pattern: type: X` syntax, but the node executor
+    reads `transformer:` and `params:`. This function converts the user-friendly
+    syntax to the internal representation.
+
+    Example:
+        pattern:
+          type: dimension
+          params:
+            natural_key: customer_id
+
+        Becomes:
+          transformer: dimension
+          params:
+            natural_key: customer_id
+    """
+    pipelines = data.get("pipelines", [])
+    for pipeline in pipelines:
+        if isinstance(pipeline, dict):
+            nodes = pipeline.get("nodes", [])
+            for node in nodes:
+                if isinstance(node, dict) and "pattern" in node:
+                    pattern_block = node.pop("pattern")
+                    if isinstance(pattern_block, dict):
+                        pattern_type = pattern_block.get("type")
+                        pattern_params = pattern_block.get("params", {})
+                        if pattern_type:
+                            node["transformer"] = pattern_type
+                        if pattern_params:
+                            existing_params = node.get("params", {})
+                            node["params"] = {**existing_params, **pattern_params}
+
+
+def _tag_nodes_with_source(data: Dict[str, Any], source_path: str) -> None:
+    """Tag all nodes in pipelines with their source YAML file path.
+
+    This enables sql_file resolution to work correctly when pipelines are imported
+    from different directories.
+    """
+    pipelines = data.get("pipelines", [])
+    for pipeline in pipelines:
+        if isinstance(pipeline, dict):
+            nodes = pipeline.get("nodes", [])
+            for node in nodes:
+                if isinstance(node, dict) and "_source_yaml" not in node:
+                    node["_source_yaml"] = source_path
+
+
+def _merge_semantic_config(base: Dict[str, Any], override: Dict[str, Any]) -> Dict[str, Any]:
+    """Merge semantic layer configs by appending lists.
+
+    Semantic configs have list fields (metrics, dimensions, views, materializations)
+    that should be accumulated from multiple imports.
+    """
+    result = base.copy()
+    list_keys = ["metrics", "dimensions", "views", "materializations"]
+
+    for key, value in override.items():
+        if key in list_keys and isinstance(value, list):
+            if key in result and isinstance(result[key], list):
+                logger.debug(
+                    "Appending semantic list",
+                    key=key,
+                    existing_count=len(result[key]),
+                    new_count=len(value),
+                )
+                result[key] = result[key] + value
+            else:
+                result[key] = value
+        elif isinstance(value, dict) and key in result and isinstance(result[key], dict):
+            result[key] = _merge_semantic_config(result[key], value)
+        else:
+            result[key] = value
+
+    return result
+
+
+def _deep_merge(base: Dict[str, Any], override: Dict[str, Any]) -> Dict[str, Any]:
+    """Merge override dictionary into base dictionary.
+
+    Rules:
+    1. Dicts are merged recursively.
+    2. List 'pipelines' are appended.
+    3. Semantic config lists (metrics, dimensions, views, materializations) are appended.
+    4. Other types (and other lists) are overwritten by the override.
+    """
+    result = base.copy()
+    for key, value in override.items():
+        if key in result and isinstance(result[key], dict) and isinstance(value, dict):
+            if key == "semantic":
+                logger.debug(
+                    "Merging semantic config",
+                    base_keys=list(result[key].keys()) if result[key] else [],
+                    override_keys=list(value.keys()),
+                )
+                result[key] = _merge_semantic_config(result[key], value)
+            else:
+                logger.debug(
+                    "Deep merging nested dictionary",
+                    key=key,
+                    base_keys=list(result[key].keys()),
+                    override_keys=list(value.keys()),
+                )
+                result[key] = _deep_merge(result[key], value)
+        elif key == "pipelines" and isinstance(value, list) and isinstance(result.get(key), list):
+            logger.debug(
+                "Appending pipelines list",
+                existing_count=len(result[key]),
+                new_count=len(value),
+            )
+            result[key] = result[key] + value
+        else:
+            if key in result:
+                logger.debug("Overwriting key during merge", key=key)
+            result[key] = value
+    return result
+
+
+def load_yaml_with_env(path: str, env: str = None) -> Dict[str, Any]:
+    """Load YAML file with environment variable substitution and imports.
+
+    Supports:
+    - ${VAR_NAME} substitution
+    - 'imports' list of relative paths
+    - 'environments' overrides based on env param
+
+    Args:
+        path: Path to YAML file
+        env: Environment name (e.g., 'prod', 'dev') to apply overrides
+
+    Returns:
+        Parsed dictionary (merged with imports and env overrides)
+
+    Raises:
+        FileNotFoundError: If file does not exist
+        ValueError: If environment variable is missing
+        yaml.YAMLError: If YAML parsing fails
+    """
+    logger.debug("Loading YAML configuration", path=path, env=env)
+
+    if not os.path.exists(path):
+        logger.error("Configuration file not found", path=path)
+        raise FileNotFoundError(f"YAML file not found: {path}")
+
+    # Get absolute path for relative import resolution
+    abs_path = os.path.abspath(path)
+    base_dir = os.path.dirname(abs_path)
+
+    logger.debug("Reading configuration file", absolute_path=abs_path)
+
+    with open(abs_path, "r", encoding="utf-8") as f:
+        content = f.read()
+
+    # Debug: Log first 100 chars to detect encoding/BOM issues
+    logger.debug(
+        "File content loaded",
+        file_size=len(content),
+        first_100_repr=repr(content[:100]),
+    )
+
+    env_vars_found = []
+
+    def replace_env(match):
+        var_name = match.group(1)
+        env_vars_found.append(var_name)
+        value = os.environ.get(var_name)
+        if value is None:
+            logger.error(
+                "Missing required environment variable",
+                variable=var_name,
+                file=abs_path,
+            )
+            raise ValueError(f"Missing environment variable: {var_name}")
+        # Check for problematic characters that could break YAML
+        # Note: Colons are NOT checked because URLs (https://) are common and safe
+        # when the value is substituted into a quoted YAML string
+        if any(c in value for c in ["\n", "\r", "#"]):
+            logger.warning(
+                "Environment variable contains YAML-sensitive characters",
+                variable=var_name,
+                has_newline="\n" in value or "\r" in value,
+                has_hash="#" in value,
+            )
+        logger.debug("Environment variable substituted", variable=var_name, length=len(value))
+        return value
+
+    # Substitute variables
+    substituted_content = ENV_PATTERN.sub(replace_env, content)
+
+    if env_vars_found:
+        logger.debug(
+            "Environment variable substitution complete",
+            variables_substituted=env_vars_found,
+            count=len(env_vars_found),
+        )
+
+    # Parse YAML
+    logger.debug("Parsing YAML content", path=abs_path)
+    try:
+        data = yaml.safe_load(substituted_content) or {}
+    except yaml.YAMLError as e:
+        logger.error("YAML parsing failed", path=abs_path, error=str(e))
+        raise
+
+    logger.debug("YAML parsed successfully", top_level_keys=list(data.keys()))
+
+    # Normalize pattern: blocks to transformer: + params: (user-friendly -> internal)
+    _normalize_pattern_to_transformer(data)
+
+    # Tag all nodes in this file with their source YAML path (for sql_file resolution)
+    _tag_nodes_with_source(data, abs_path)
+
+    # Handle imports
+    imports = data.pop("imports", [])
+    if imports:
+        if isinstance(imports, str):
+            imports = [imports]
+
+        logger.debug("Processing imports", import_count=len(imports), imports=imports)
+
+        # Start with current data as the base
+        merged_data = data.copy()
+
+        for import_path in imports:
+            # Resolve relative to current file
+            if not os.path.isabs(import_path):
+                full_import_path = os.path.join(base_dir, import_path)
+            else:
+                full_import_path = import_path
+
+            logger.debug(
+                "Resolving import",
+                import_path=import_path,
+                resolved_path=full_import_path,
+            )
+
+            if not os.path.exists(full_import_path):
+                logger.error(
+                    "Imported configuration file not found",
+                    import_path=import_path,
+                    resolved_path=full_import_path,
+                    parent_file=abs_path,
+                )
+                raise FileNotFoundError(f"Imported YAML file not found: {full_import_path}")
+
+            # Recursive load
+            # Note: We pass env down to imported files too
+            logger.debug("Loading imported configuration", path=full_import_path)
+            try:
+                imported_data = load_yaml_with_env(full_import_path, env=env)
+            except Exception as e:
+                logger.error(
+                    "Failed to load imported configuration",
+                    import_path=import_path,
+                    resolved_path=full_import_path,
+                    parent_file=abs_path,
+                    error=str(e),
+                )
+                raise ValueError(
+                    f"Failed to load import '{import_path}' (resolved: {full_import_path}): {e}"
+                ) from e
+
+            # Merge imported data INTO the current data
+            # This way, the main file acts as the "master" that accumulates imports
+            logger.debug(
+                "Merging imported configuration",
+                import_path=full_import_path,
+                imported_keys=list(imported_data.keys()),
+            )
+            merged_data = _deep_merge(merged_data, imported_data)
+
+        data = merged_data
+        logger.debug("All imports processed and merged", import_count=len(imports))
+
+    # Apply Environment Overrides from "environments" block in main file
+    if env:
+        environments = data.get("environments", {})
+        if env in environments:
+            logger.debug(
+                "Applying environment overrides from environments block",
+                env=env,
+                override_keys=list(environments[env].keys()),
+            )
+            override = environments[env]
+            data = _deep_merge(data, override)
+        else:
+            logger.debug(
+                "No environment override found in environments block",
+                env=env,
+                available_environments=list(environments.keys()),
+            )
+
+    # Apply Environment Overrides from external env.{env}.yaml file
+    if env:
+        env_file_name = f"env.{env}.yaml"
+        env_file_path = os.path.join(base_dir, env_file_name)
+        if os.path.exists(env_file_path):
+            logger.debug(
+                "Loading external environment override file",
+                env=env,
+                env_file=env_file_path,
+            )
+            # Load the env file (recursively, so it can have imports too)
+            # We pass env=None to avoid infinite recursion if it somehow references itself,
+            # though strictly it shouldn't matter as we look for env.{env}.yaml based on the passed env.
+            # But logically, an env specific file shouldn't load other env specific files for the same env.
+            env_data = load_yaml_with_env(env_file_path, env=None)
+            logger.debug(
+                "Merging external environment overrides",
+                env_file=env_file_path,
+                override_keys=list(env_data.keys()),
+            )
+            data = _deep_merge(data, env_data)
+        else:
+            logger.debug(
+                "No external environment override file found",
+                env=env,
+                expected_path=env_file_path,
+            )
+
+    logger.debug(
+        "Configuration loading complete",
+        path=path,
+        env=env,
+        final_keys=list(data.keys()),
+    )
+
+    return data

odibi/utils/console.py

@@ -0,0 +1,231 @@
+"""Rich console utilities for polished terminal/notebook output.
+
+This module provides Rich-based console output with graceful fallback
+when Rich is not installed. Works in both CLI and Jupyter/Databricks notebooks.
+"""
+
+from typing import Any, Dict, List, Optional, Union
+
+_RICH_AVAILABLE: Optional[bool] = None
+
+
+def is_rich_available() -> bool:
+    """Check if Rich library is available.
+
+    Returns:
+        True if Rich is installed and importable, False otherwise.
+    """
+    global _RICH_AVAILABLE
+    if _RICH_AVAILABLE is None:
+        try:
+            import importlib.util
+
+            _RICH_AVAILABLE = importlib.util.find_spec("rich") is not None
+        except (ImportError, ModuleNotFoundError):
+            _RICH_AVAILABLE = False
+    return _RICH_AVAILABLE
+
+
+def get_console() -> Optional[Any]:
+    """Get a Rich Console instance.
+
+    Returns:
+        Rich Console instance if available, None otherwise.
+    """
+    if not is_rich_available():
+        return None
+
+    from rich.console import Console
+
+    return Console()
+
+
+def _is_notebook_environment() -> bool:
+    """Detect if running in Jupyter/Databricks notebook."""
+    try:
+        from IPython import get_ipython
+
+        shell = get_ipython()
+        if shell is None:
+            return False
+        shell_class = shell.__class__.__name__
+        return shell_class in ("ZMQInteractiveShell", "DatabricksShell", "Shell")
+    except (ImportError, NameError):
+        return False
+
+
+def success(message: str, prefix: str = "✓") -> None:
+    """Print a success message in green.
+
+    Args:
+        message: The message to display.
+        prefix: Prefix symbol (default: ✓).
+    """
+    if is_rich_available():
+        console = get_console()
+        console.print(f"[green]{prefix}[/green] {message}")
+    else:
+        print(f"{prefix} {message}")
+
+
+def error(message: str, prefix: str = "✗") -> None:
+    """Print an error message in red.
+
+    Args:
+        message: The message to display.
+        prefix: Prefix symbol (default: ✗).
+    """
+    if is_rich_available():
+        console = get_console()
+        console.print(f"[red]{prefix}[/red] {message}")
+    else:
+        print(f"{prefix} {message}")
+
+
+def warning(message: str, prefix: str = "⚠") -> None:
+    """Print a warning message in yellow.
+
+    Args:
+        message: The message to display.
+        prefix: Prefix symbol (default: ⚠).
+    """
+    if is_rich_available():
+        console = get_console()
+        console.print(f"[yellow]{prefix}[/yellow] {message}")
+    else:
+        print(f"{prefix} {message}")
+
+
+def info(message: str, prefix: str = "ℹ") -> None:
+    """Print an info message in blue.
+
+    Args:
+        message: The message to display.
+        prefix: Prefix symbol (default: ℹ).
+    """
+    if is_rich_available():
+        console = get_console()
+        console.print(f"[blue]{prefix}[/blue] {message}")
+    else:
+        print(f"{prefix} {message}")
+
+
+def print_table(
+    data: Union[List[Dict[str, Any]], Any],
+    title: Optional[str] = None,
+    columns: Optional[List[str]] = None,
+) -> None:
+    """Render data as a Rich table or plain text fallback.
+
+    Args:
+        data: List of dicts or a DataFrame to display.
+        title: Optional table title.
+        columns: Optional list of column names to include.
+    """
+    if not data:
+        print("(empty)")
+        return
+
+    rows: List[Dict[str, Any]] = []
+    if hasattr(data, "to_dict"):
+        rows = data.to_dict("records")
+    elif isinstance(data, list):
+        rows = data
+    else:
+        print(str(data))
+        return
+
+    if not rows:
+        print("(empty)")
+        return
+
+    col_names = columns or list(rows[0].keys())
+
+    if is_rich_available():
+        from rich.table import Table
+
+        console = get_console()
+        table = Table(title=title, show_header=True, header_style="bold cyan")
+
+        for col in col_names:
+            table.add_column(col)
+
+        for row in rows:
+            table.add_row(*[str(row.get(col, "")) for col in col_names])
+
+        console.print(table)
+    else:
+        if title:
+            print(f"\n{title}")
+            print("-" * len(title))
+
+        max_widths = {col: len(col) for col in col_names}
+        for row in rows:
+            for col in col_names:
+                max_widths[col] = max(max_widths[col], len(str(row.get(col, ""))))
+
+        header = " | ".join(col.ljust(max_widths[col]) for col in col_names)
+        print(header)
+        print("-" * len(header))
+
+        for row in rows:
+            line = " | ".join(str(row.get(col, "")).ljust(max_widths[col]) for col in col_names)
+            print(line)
+
+
+def print_panel(
+    content: str,
+    title: Optional[str] = None,
+    border_style: str = "blue",
+    padding: tuple = (0, 1),
+) -> None:
+    """Display content in a boxed panel.
+
+    Args:
+        content: The content to display.
+        title: Optional panel title.
+        border_style: Border color/style (default: blue).
+        padding: Tuple of (vertical, horizontal) padding.
+    """
+    if is_rich_available():
+        from rich.panel import Panel
+
+        console = get_console()
+        panel = Panel(content, title=title, border_style=border_style, padding=padding)
+        console.print(panel)
+    else:
+        width = max(len(line) for line in content.split("\n")) + 4
+        if title:
+            width = max(width, len(title) + 6)
+
+        border = "─" * width
+        print(f"┌{border}┐")
+        if title:
+            centered_title = f" {title} ".center(width)
+            print(f"│{centered_title}│")
+            print(f"├{border}┤")
+
+        for line in content.split("\n"):
+            padded = f"  {line}  ".ljust(width)
+            print(f"│{padded}│")
+
+        print(f"└{border}┘")
+
+
+def print_rule(title: Optional[str] = None, style: str = "blue") -> None:
+    """Print a horizontal rule/divider.
+
+    Args:
+        title: Optional centered title in the rule.
+        style: Line color/style.
+    """
+    if is_rich_available():
+        from rich.rule import Rule
+
+        console = get_console()
+        console.print(Rule(title, style=style))
+    else:
+        if title:
+            print(f"\n{'─' * 10} {title} {'─' * 10}\n")
+        else:
+            print("─" * 40)