odibi 2.5.0__py3-none-any.whl

odibi/references.py

@@ -0,0 +1,151 @@
+"""Cross-pipeline reference resolution for Odibi.
+
+This module handles the resolution of $pipeline.node references, enabling
+pipelines to read data from other pipelines' outputs (e.g., bronze -> silver).
+
+Example:
+    inputs:
+      events: $read_bronze.opsvisdata_ShiftDowntimeEventsview
+      calendar: $read_bronze.opsvisdata_vw_calender
+"""
+
+from typing import Any, Dict, Union
+
+from odibi.catalog import CatalogManager
+
+
+class ReferenceResolutionError(Exception):
+    """Raised when a cross-pipeline reference cannot be resolved."""
+
+    pass
+
+
+def resolve_input_reference(
+    ref: str,
+    catalog: CatalogManager,
+) -> Dict[str, Any]:
+    """
+    Resolves $pipeline.node to read configuration.
+
+    Args:
+        ref: Reference string like "$read_bronze.opsvisdata_vw_calender"
+        catalog: CatalogManager instance
+
+    Returns:
+        Dict with keys for engine.read():
+            - For external_table: connection, path, format
+            - For managed_table: table, format
+
+    Raises:
+        ValueError: If reference format is invalid
+        ReferenceResolutionError: If referenced node output is not found
+    """
+    if not ref.startswith("$"):
+        raise ValueError(f"Invalid reference: {ref}. Expected $pipeline.node format.")
+
+    parts = ref[1:].split(".", 1)  # Remove $ and split
+    if len(parts) != 2:
+        raise ValueError(
+            f"Invalid reference format: {ref}. Expected $pipeline.node (e.g., $read_bronze.my_node)"
+        )
+
+    pipeline_name, node_name = parts
+
+    output = catalog.get_node_output(pipeline_name, node_name)
+
+    if output is None:
+        raise ReferenceResolutionError(
+            f"No output found for {ref}. "
+            f"Ensure pipeline '{pipeline_name}' has run and node '{node_name}' has a write block."
+        )
+
+    if output.get("output_type") == "managed_table":
+        return {
+            "table": output.get("table_name"),
+            "format": output.get("format"),
+        }
+    else:  # external_table
+        return {
+            "connection": output.get("connection_name"),
+            "path": output.get("path"),
+            "format": output.get("format"),
+        }
+
+
+def is_pipeline_reference(value: Any) -> bool:
+    """
+    Check if a value is a cross-pipeline reference.
+
+    Args:
+        value: Value to check
+
+    Returns:
+        True if value is a string starting with $
+    """
+    return isinstance(value, str) and value.startswith("$")
+
+
+def resolve_inputs(
+    inputs: Dict[str, Union[str, Dict[str, Any]]],
+    catalog: CatalogManager,
+) -> Dict[str, Dict[str, Any]]:
+    """
+    Resolve all inputs, converting $pipeline.node references to read configs.
+
+    Args:
+        inputs: Dict mapping input name to either:
+            - A $pipeline.node reference string
+            - An explicit read config dict
+
+    Returns:
+        Dict mapping input name to resolved read configuration
+
+    Example:
+        inputs = {
+            "events": "$read_bronze.shift_events",
+            "calendar": {
+                "connection": "goat_prod",
+                "path": "bronze/OEE/vw_calender",
+                "format": "delta"
+            }
+        }
+        resolved = resolve_inputs(inputs, catalog)
+        # Returns:
+        # {
+        #     "events": {"connection": "goat_prod", "path": "bronze/OEE/shift_events", "format": "delta"},
+        #     "calendar": {"connection": "goat_prod", "path": "bronze/OEE/vw_calender", "format": "delta"}
+        # }
+    """
+    resolved = {}
+
+    for name, ref in inputs.items():
+        if is_pipeline_reference(ref):
+            resolved[name] = resolve_input_reference(ref, catalog)
+        elif isinstance(ref, dict):
+            resolved[name] = ref
+        else:
+            raise ValueError(
+                f"Invalid input format for '{name}': {ref}. "
+                "Expected $pipeline.node reference or read config dict."
+            )
+
+    return resolved
+
+
+def validate_references(
+    inputs: Dict[str, Union[str, Dict[str, Any]]],
+    catalog: CatalogManager,
+) -> None:
+    """
+    Validate all cross-pipeline references at pipeline load time (fail fast).
+
+    Args:
+        inputs: Dict of input configurations
+        catalog: CatalogManager instance
+
+    Raises:
+        ReferenceResolutionError: If any reference cannot be resolved
+    """
+    for name, ref in inputs.items():
+        if is_pipeline_reference(ref):
+            resolve_input_reference(ref, catalog)

odibi/registry.py

@@ -0,0 +1,246 @@
+"""Function registry for transform functions."""
+
+import inspect
+from functools import wraps
+from typing import Any, Callable, Dict, Optional, Union
+
+
+class FunctionRegistry:
+    """Global registry of transform functions with type validation."""
+
+    _functions: Dict[str, Callable] = {}
+    _signatures: Dict[str, inspect.Signature] = {}
+    _param_models: Dict[str, Any] = {}  # New: Store Pydantic models
+
+    @classmethod
+    def register(cls, func: Callable, name: str = None, param_model: Any = None) -> Callable:
+        """Register a transform function.
+
+        Args:
+            func: Function to register
+            name: Optional name override (default: func.__name__)
+            param_model: Optional Pydantic model for validation
+
+        Returns:
+            The original function
+        """
+        if name is None:
+            name = func.__name__
+
+        cls._functions[name] = func
+        cls._signatures[name] = inspect.signature(func)
+        if param_model:
+            cls._param_models[name] = param_model
+
+        return func
+
+    @classmethod
+    def get(cls, name: str) -> Callable:
+        """Retrieve a registered function.
+
+        Args:
+            name: Function name
+
+        Returns:
+            The registered function
+
+        Raises:
+            ValueError: If function not found
+        """
+        if name not in cls._functions:
+            available = ", ".join(cls._functions.keys()) if cls._functions else "none"
+            raise ValueError(
+                f"Transform function '{name}' not registered. Available functions: {available}"
+            )
+        return cls._functions[name]
+
+    @classmethod
+    def has_function(cls, name: str) -> bool:
+        """Check if a function is registered."""
+        return name in cls._functions
+
+    @classmethod
+    def get_function(cls, name: str) -> Optional[Callable]:
+        """Get a function without raising if not found."""
+        return cls._functions.get(name)
+
+    @classmethod
+    def get_param_model(cls, name: str) -> Optional[Any]:
+        """Get the Pydantic model for a function's parameters."""
+        return cls._param_models.get(name)
+
+    @classmethod
+    def validate_params(cls, name: str, params: Dict[str, Any]) -> None:
+        """Validate parameters against function signature or Pydantic model.
+
+        Args:
+            name: Function name
+            params: Parameters to validate
+
+        Raises:
+            ValueError: If parameters are invalid
+            TypeError: If parameter types don't match
+        """
+        if name not in cls._functions:
+            raise ValueError(f"Function '{name}' not registered")
+
+        # Priority: Check Pydantic Model
+        if name in cls._param_models:
+            model = cls._param_models[name]
+            try:
+                model(**params)
+                return  # Validated successfully
+            except Exception as e:
+                raise ValueError(f"Validation failed for '{name}': {e}")
+
+        # Fallback: Check function signature (Legacy)
+        sig = cls._signatures[name]
+
+        # Get function parameters (excluding 'context' and 'current' which are injected)
+        func_params = {k: v for k, v in sig.parameters.items() if k not in ["context", "current"]}
+
+        # Check for missing required parameters
+        missing = []
+        for param_name, param in func_params.items():
+            if param.default is inspect.Parameter.empty:
+                # Required parameter
+                if param_name not in params:
+                    missing.append(param_name)
+
+        if missing:
+            raise ValueError(
+                f"Missing required parameters for function '{name}': {', '.join(missing)}"
+            )
+
+        # Check for unexpected parameters
+        unexpected = set(params.keys()) - set(func_params.keys())
+        if unexpected:
+            raise ValueError(
+                f"Unexpected parameters for function '{name}': {', '.join(unexpected)}"
+            )
+
+    @classmethod
+    def list_functions(cls) -> list[str]:
+        """List all registered function names.
+
+        Returns:
+            List of function names
+        """
+        return list(cls._functions.keys())
+
+    @classmethod
+    def get_function_info(cls, name: str) -> Dict[str, Any]:
+        """Get detailed information about a registered function.
+
+        Args:
+            name: Function name
+
+        Returns:
+            Dictionary with function metadata
+        """
+        if name not in cls._functions:
+            raise ValueError(f"Function '{name}' not registered")
+
+        func = cls._functions[name]
+        sig = cls._signatures[name]
+
+        # Extract parameter info
+        params_info = {}
+        for param_name, param in sig.parameters.items():
+            if param_name == "context":
+                continue  # Skip context param
+
+            param_info = {
+                "required": param.default is inspect.Parameter.empty,
+                "default": None if param.default is inspect.Parameter.empty else param.default,
+                "annotation": (
+                    param.annotation if param.annotation != inspect.Parameter.empty else None
+                ),
+            }
+            params_info[param_name] = param_info
+
+        return {
+            "name": name,
+            "docstring": inspect.getdoc(func),
+            "parameters": params_info,
+            "return_annotation": (
+                sig.return_annotation if sig.return_annotation != inspect.Signature.empty else None
+            ),
+        }
+
+
+def transform(name_or_func: Union[str, Callable] = None, **kwargs) -> Callable:
+    """Decorator to register a transform function.
+
+    Usage:
+        @transform
+        def my_transform(...): ...
+
+        @transform("my_name")
+        def my_transform(...): ...
+
+        @transform(name="my_name", category="foo")
+        def my_transform(...): ...
+
+    Args:
+        name_or_func: Function (if used without args) or Name (if used with args)
+        **kwargs: Additional metadata (ignored for now)
+
+    Returns:
+        The decorated function
+    """
+
+    # If called with keyword args only (e.g. @transform(name="foo")), name_or_func might be None
+    if name_or_func is None and "name" in kwargs:
+        name_or_func = kwargs["name"]
+
+    def _register(func, name=None):
+        @wraps(func)
+        def wrapper(*args, **kwargs):
+            return func(*args, **kwargs)
+
+        # Register the function
+        # If name passed to decorator is None, use func.__name__
+        # But FunctionRegistry.register handles None name by using func.__name__
+        # However, we want to use the explicit name if provided.
+        reg_name = name or func.__name__
+
+        # Extract param_model from kwargs (captured from decorator args)
+        # Note: kwargs here are from the outer scope (transform arguments), NOT wrapper args
+        # Wait, _register closes over kwargs from transform(..., **kwargs)
+        param_model = kwargs.get("param_model")
+
+        FunctionRegistry.register(wrapper, name=reg_name, param_model=param_model)
+        return wrapper
+
+    if callable(name_or_func):
+        # Called as @transform
+        return _register(name_or_func)
+    else:
+        # Called as @transform("name") or @transform(name="name")
+        def decorator(func):
+            return _register(func, name=name_or_func)
+
+        return decorator
+
+
+def get_registered_function(name: str) -> Callable:
+    """Get a registered transform function.
+
+    Args:
+        name: Function name
+
+    Returns:
+        The registered function
+    """
+    return FunctionRegistry.get(name)
+
+
+def validate_function_params(name: str, params: Dict[str, Any]) -> None:
+    """Validate parameters for a registered function.
+
+    Args:
+        name: Function name
+        params: Parameters to validate
+    """
+    FunctionRegistry.validate_params(name, params)

odibi/semantics/__init__.py

@@ -0,0 +1,71 @@
+"""
+Semantic Layer Module
+=====================
+
+This module provides a semantic layer for defining and querying metrics.
+
+Features:
+- Define metrics in YAML (expressions, filters, source tables)
+- Query interface: "revenue BY region, month"
+- Materialize metrics on schedule
+- Dimension hierarchies and drill-down
+
+Core Components:
+- MetricDefinition: Pydantic models for metric/dimension definitions
+- SemanticQuery: Parse and execute "metric BY dimensions" queries
+- Materialize: Execute and persist materialized aggregations
+
+Example Config (in odibi.yaml):
+    metrics:
+      - name: revenue
+        description: "Total revenue from completed orders"
+        expr: "SUM(total_amount)"
+        source: fact_orders
+        filters:
+          - "status = 'completed'"
+
+    dimensions:
+      - name: order_date
+        source: dim_date
+        hierarchy: [year, quarter, month, full_date]
+
+    materializations:
+      - name: monthly_revenue_by_region
+        metrics: [revenue, order_count]
+        dimensions: [region, month]
+        schedule: "0 2 1 * *"
+        output: gold/agg_monthly_revenue
+"""
+
+from odibi.semantics.materialize import Materializer
+from odibi.semantics.metrics import (
+    DimensionDefinition,
+    MaterializationConfig,
+    MetricDefinition,
+    SemanticLayerConfig,
+    ViewConfig,
+    parse_semantic_config,
+)
+from odibi.semantics.query import SemanticQuery
+from odibi.semantics.runner import SemanticLayerRunner, run_semantic_layer
+from odibi.semantics.story import SemanticStoryGenerator, SemanticStoryMetadata
+from odibi.semantics.views import ViewExecutionResult, ViewGenerator, ViewResult
+
+__all__ = [
+    "MetricDefinition",
+    "DimensionDefinition",
+    "MaterializationConfig",
+    "SemanticLayerConfig",
+    "ViewConfig",
+    "parse_semantic_config",
+    "SemanticQuery",
+    "Materializer",
+    "ViewGenerator",
+    "ViewResult",
+    "ViewExecutionResult",
+    "SemanticStoryGenerator",
+    "SemanticStoryMetadata",
+    "SemanticLayerRunner",
+    "run_semantic_layer",
+]
+__version__ = "1.1.0"