odibi 2.5.0__py3-none-any.whl

odibi/diagnostics/manager.py

@@ -0,0 +1,171 @@
+"""
+Diagnostics Manager
+===================
+
+Handles loading and managing run history for diagnostics.
+"""
+
+import json
+from datetime import datetime
+from pathlib import Path
+from typing import Dict, List, Optional
+
+from odibi.story.metadata import DeltaWriteInfo, NodeExecutionMetadata, PipelineStoryMetadata
+
+
+class HistoryManager:
+    """Manages access to pipeline run history."""
+
+    def __init__(self, history_path: str = "stories/"):
+        """
+        Initialize history manager.
+
+        Args:
+            history_path: Path where stories are stored
+        """
+        self.history_path = Path(history_path)
+        self.is_remote = "://" in history_path
+
+    def list_runs(self, pipeline_name: str) -> List[Dict[str, str]]:
+        """
+        List available runs for a pipeline.
+
+        Returns:
+            List of dicts with keys: run_id, timestamp, path
+        """
+        runs = []
+
+        if self.is_remote:
+            # Remote listing not implemented yet
+            return []
+
+        if not self.history_path.exists():
+            return []
+
+        # Look for .json files
+        # Pattern: {pipeline_name}_{timestamp}.json
+        pattern = f"{pipeline_name}_*.json"
+
+        for path in self.history_path.glob(pattern):
+            try:
+                # Parse timestamp from filename
+                # Filename: name_YYYYMMDD_HHMMSS.json
+                parts = path.stem.split("_")
+                if len(parts) >= 3:
+                    ts_str = f"{parts[-2]}_{parts[-1]}"
+                    # Validate format
+                    datetime.strptime(ts_str, "%Y%m%d_%H%M%S")
+
+                    runs.append({"run_id": ts_str, "timestamp": ts_str, "path": str(path)})
+            except (ValueError, IndexError):
+                continue
+
+        # Sort by timestamp descending (newest first)
+        runs.sort(key=lambda x: x["timestamp"], reverse=True)
+        return runs
+
+    def get_latest_run(self, pipeline_name: str) -> Optional[PipelineStoryMetadata]:
+        """Get the most recent run metadata."""
+        runs = self.list_runs(pipeline_name)
+        if not runs:
+            return None
+
+        return self.load_run(runs[0]["path"])
+
+    def get_run_by_id(self, pipeline_name: str, run_id: str) -> Optional[PipelineStoryMetadata]:
+        """Get specific run metadata."""
+        runs = self.list_runs(pipeline_name)
+        for run in runs:
+            if run["run_id"] == run_id:
+                return self.load_run(run["path"])
+        return None
+
+    def get_previous_run(
+        self, pipeline_name: str, current_run_id: str
+    ) -> Optional[PipelineStoryMetadata]:
+        """Get the run immediately preceding the specified one."""
+        runs = self.list_runs(pipeline_name)
+
+        found_current = False
+        for run in runs:
+            if found_current:
+                return self.load_run(run["path"])
+
+            if run["run_id"] == current_run_id:
+                found_current = True
+
+        return None
+
+    def load_run(self, path: str) -> PipelineStoryMetadata:
+        """Load run metadata from JSON file."""
+        if self.is_remote:
+            raise NotImplementedError("Remote history loading not supported yet")
+
+        with open(path, "r", encoding="utf-8") as f:
+            data = json.load(f)
+
+        return self._dict_to_metadata(data)
+
+    def _dict_to_metadata(self, data: Dict) -> PipelineStoryMetadata:
+        """Convert dictionary to PipelineStoryMetadata object."""
+        nodes = []
+        for n in data.get("nodes", []):
+            # Reconstruct Delta Info
+            delta_info = None
+            if n.get("delta_info"):
+                d = n["delta_info"]
+                delta_info = DeltaWriteInfo(
+                    version=d.get("version"),
+                    timestamp=(
+                        datetime.fromisoformat(d.get("timestamp")) if d.get("timestamp") else None
+                    ),
+                    operation=d.get("operation"),
+                    operation_metrics=d.get("operation_metrics"),
+                    read_version=d.get("read_version"),
+                )
+
+            node = NodeExecutionMetadata(
+                node_name=n["node_name"],
+                operation=n.get("operation", "unknown"),
+                status=n.get("status", "unknown"),
+                duration=n.get("duration", 0.0),
+                rows_in=n.get("rows_in"),
+                rows_out=n.get("rows_out"),
+                rows_change=n.get("rows_change"),
+                rows_change_pct=n.get("rows_change_pct"),
+                sample_data=n.get("sample_data"),
+                schema_in=n.get("schema_in"),
+                schema_out=n.get("schema_out"),
+                columns_added=n.get("columns_added", []),
+                columns_removed=n.get("columns_removed", []),
+                columns_renamed=n.get("columns_renamed", []),
+                executed_sql=n.get("executed_sql", []),
+                sql_hash=n.get("sql_hash"),
+                transformation_stack=n.get("transformation_stack", []),
+                config_snapshot=n.get("config_snapshot"),
+                delta_info=delta_info,
+                data_diff=n.get("data_diff"),
+                error_message=n.get("error_message"),
+                error_type=n.get("error_type"),
+                started_at=n.get("started_at"),
+                completed_at=n.get("completed_at"),
+            )
+            nodes.append(node)
+
+        return PipelineStoryMetadata(
+            pipeline_name=data["pipeline_name"],
+            pipeline_layer=data.get("pipeline_layer"),
+            started_at=data.get("started_at"),
+            completed_at=data.get("completed_at"),
+            duration=data.get("duration", 0.0),
+            total_nodes=data.get("total_nodes", 0),
+            completed_nodes=data.get("completed_nodes", 0),
+            failed_nodes=data.get("failed_nodes", 0),
+            skipped_nodes=data.get("skipped_nodes", 0),
+            nodes=nodes,
+            project=data.get("project"),
+            plant=data.get("plant"),
+            asset=data.get("asset"),
+            business_unit=data.get("business_unit"),
+            theme=data.get("theme", "default"),
+        )

odibi/engine/__init__.py

@@ -0,0 +1,20 @@
+"""Engine implementations for ODIBI."""
+
+from odibi.engine.base import Engine
+from odibi.engine.pandas_engine import PandasEngine
+
+# Try to import SparkEngine (optional dependency)
+try:
+    from odibi.engine.spark_engine import SparkEngine
+
+    __all__ = ["Engine", "PandasEngine", "SparkEngine"]
+except ImportError:
+    # PySpark not available
+    __all__ = ["Engine", "PandasEngine"]
+
+
+# Lazy import helper for Spark (backward compatibility)
+def get_spark_engine():
+    from .spark_engine import SparkEngine
+
+    return SparkEngine

odibi/engine/base.py

@@ -0,0 +1,334 @@
+"""Base engine interface."""
+
+from abc import ABC, abstractmethod
+from typing import Any, Dict, List, Optional
+
+from odibi.context import Context
+
+
+class Engine(ABC):
+    """Abstract base class for execution engines."""
+
+    # Custom format registry
+    _custom_readers: Dict[str, Any] = {}
+    _custom_writers: Dict[str, Any] = {}
+
+    @classmethod
+    def register_format(cls, fmt: str, reader: Optional[Any] = None, writer: Optional[Any] = None):
+        """Register custom format reader/writer.
+
+        Args:
+            fmt: Format name (e.g. 'netcdf')
+            reader: Function(path, **options) -> DataFrame
+            writer: Function(df, path, **options) -> None
+        """
+        if reader:
+            cls._custom_readers[fmt] = reader
+        if writer:
+            cls._custom_writers[fmt] = writer
+
+    @abstractmethod
+    def read(
+        self,
+        connection: Any,
+        format: str,
+        table: Optional[str] = None,
+        path: Optional[str] = None,
+        options: Optional[Dict[str, Any]] = None,
+    ) -> Any:
+        """Read data from source.
+
+        Args:
+            connection: Connection object
+            format: Data format (csv, parquet, delta, etc.)
+            table: Table name (for SQL/Delta)
+            path: File path (for file-based sources)
+            options: Format-specific options
+
+        Returns:
+            DataFrame (engine-specific type)
+        """
+        pass
+
+    def materialize(self, df: Any) -> Any:
+        """Materialize lazy dataset into memory (DataFrame).
+
+        Args:
+            df: DataFrame or LazyDataset
+
+        Returns:
+            Materialized DataFrame
+        """
+        return df
+
+    @abstractmethod
+    def write(
+        self,
+        df: Any,
+        connection: Any,
+        format: str,
+        table: Optional[str] = None,
+        path: Optional[str] = None,
+        mode: str = "overwrite",
+        options: Optional[Dict[str, Any]] = None,
+        streaming_config: Optional[Any] = None,
+    ) -> None:
+        """Write data to destination.
+
+        Args:
+            df: DataFrame to write
+            connection: Connection object
+            format: Output format
+            table: Table name (for SQL/Delta)
+            path: File path (for file-based outputs)
+            mode: Write mode (overwrite/append)
+            options: Format-specific options
+        """
+        pass
+
+    @abstractmethod
+    def execute_sql(self, sql: str, context: Context) -> Any:
+        """Execute SQL query.
+
+        Args:
+            sql: SQL query string
+            context: Execution context with registered DataFrames
+
+        Returns:
+            Result DataFrame
+        """
+        pass
+
+    @abstractmethod
+    def execute_operation(self, operation: str, params: Dict[str, Any], df: Any) -> Any:
+        """Execute built-in operation (pivot, etc.).
+
+        Args:
+            operation: Operation name
+            params: Operation parameters
+            df: Input DataFrame
+
+        Returns:
+            Result DataFrame
+        """
+        pass
+
+    @abstractmethod
+    def get_schema(self, df: Any) -> Any:
+        """Get DataFrame schema.
+
+        Args:
+            df: DataFrame
+
+        Returns:
+            Dict[str, str] mapping column names to types, or List[str] of names (deprecated)
+        """
+        pass
+
+    @abstractmethod
+    def get_shape(self, df: Any) -> tuple:
+        """Get DataFrame shape.
+
+        Args:
+            df: DataFrame
+
+        Returns:
+            (rows, columns)
+        """
+        pass
+
+    @abstractmethod
+    def count_rows(self, df: Any) -> int:
+        """Count rows in DataFrame.
+
+        Args:
+            df: DataFrame
+
+        Returns:
+            Row count
+        """
+        pass
+
+    @abstractmethod
+    def count_nulls(self, df: Any, columns: List[str]) -> Dict[str, int]:
+        """Count nulls in specified columns.
+
+        Args:
+            df: DataFrame
+            columns: Columns to check
+
+        Returns:
+            Dictionary of column -> null count
+        """
+        pass
+
+    @abstractmethod
+    def validate_schema(self, df: Any, schema_rules: Dict[str, Any]) -> List[str]:
+        """Validate DataFrame schema.
+
+        Args:
+            df: DataFrame
+            schema_rules: Validation rules
+
+        Returns:
+            List of validation failures (empty if valid)
+        """
+        pass
+
+    @abstractmethod
+    def validate_data(self, df: Any, validation_config: Any) -> List[str]:
+        """Validate data against rules.
+
+        Args:
+            df: DataFrame to validate
+            validation_config: ValidationConfig object
+
+        Returns:
+            List of validation failure messages (empty if valid)
+        """
+        pass
+
+    @abstractmethod
+    def get_sample(self, df: Any, n: int = 10) -> List[Dict[str, Any]]:
+        """Get sample rows as list of dictionaries.
+
+        Args:
+            df: DataFrame
+            n: Number of rows to return
+
+        Returns:
+            List of row dictionaries
+        """
+        pass
+
+    def get_source_files(self, df: Any) -> List[str]:
+        """Get list of source files that generated this DataFrame.
+
+        Args:
+            df: DataFrame
+
+        Returns:
+            List of file paths (or empty list if not applicable/supported)
+        """
+        return []
+
+    def profile_nulls(self, df: Any) -> Dict[str, float]:
+        """Calculate null percentage for each column.
+
+        Args:
+            df: DataFrame
+
+        Returns:
+            Dictionary of {column_name: null_percentage} (0.0 to 1.0)
+        """
+        return {}
+
+    @abstractmethod
+    def table_exists(
+        self, connection: Any, table: Optional[str] = None, path: Optional[str] = None
+    ) -> bool:
+        """Check if table or location exists.
+
+        Args:
+            connection: Connection object
+            table: Table name (for catalog tables)
+            path: File path (for path-based tables)
+
+        Returns:
+            True if table/location exists, False otherwise
+        """
+        pass
+
+    @abstractmethod
+    def harmonize_schema(self, df: Any, target_schema: Dict[str, str], policy: Any) -> Any:
+        """Harmonize DataFrame schema with target schema according to policy.
+
+        Args:
+            df: Input DataFrame
+            target_schema: Target schema (column name -> type)
+            policy: SchemaPolicyConfig object
+
+        Returns:
+            Harmonized DataFrame
+        """
+        pass
+
+    @abstractmethod
+    def anonymize(
+        self, df: Any, columns: List[str], method: str, salt: Optional[str] = None
+    ) -> Any:
+        """Anonymize specified columns.
+
+        Args:
+            df: DataFrame to anonymize
+            columns: List of columns to anonymize
+            method: Method ('hash', 'mask', 'redact')
+            salt: Optional salt for hashing
+
+        Returns:
+            Anonymized DataFrame
+        """
+        pass
+
+    def get_table_schema(
+        self,
+        connection: Any,
+        table: Optional[str] = None,
+        path: Optional[str] = None,
+        format: Optional[str] = None,
+    ) -> Optional[Dict[str, str]]:
+        """Get schema of an existing table/file.
+
+        Args:
+            connection: Connection object
+            table: Table name
+            path: File path
+            format: Data format (optional, helps with file-based sources)
+
+        Returns:
+            Schema dict or None if table doesn't exist or schema fetch fails.
+        """
+        return None
+
+    def maintain_table(
+        self,
+        connection: Any,
+        format: str,
+        table: Optional[str] = None,
+        path: Optional[str] = None,
+        config: Optional[Any] = None,
+    ) -> None:
+        """Run table maintenance operations (optimize, vacuum).
+
+        Args:
+            connection: Connection object
+            format: Table format
+            table: Table name
+            path: Table path
+            config: AutoOptimizeConfig object
+        """
+        pass
+
+    def add_write_metadata(
+        self,
+        df: Any,
+        metadata_config: Any,
+        source_connection: Optional[str] = None,
+        source_table: Optional[str] = None,
+        source_path: Optional[str] = None,
+        is_file_source: bool = False,
+    ) -> Any:
+        """Add metadata columns to DataFrame before writing (Bronze layer lineage).
+
+        Args:
+            df: DataFrame
+            metadata_config: WriteMetadataConfig or True (for all defaults)
+            source_connection: Name of the source connection
+            source_table: Name of the source table (SQL sources)
+            source_path: Path of the source file (file sources)
+            is_file_source: True if source is a file-based read
+
+        Returns:
+            DataFrame with metadata columns added (or unchanged if metadata_config is None/False)
+        """
+        return df  # Default: no-op