odibi 2.5.0__py3-none-any.whl

odibi/utils/content_hash.py

@@ -0,0 +1,202 @@
+"""Content hashing utilities for skip_if_unchanged feature.
+
+This module provides functions to compute deterministic hashes of DataFrames
+for change detection in snapshot ingestion patterns.
+"""
+
+import hashlib
+from typing import TYPE_CHECKING, List, Optional
+
+if TYPE_CHECKING:
+    import pandas as pd
+
+
+def compute_dataframe_hash(
+    df: "pd.DataFrame",
+    columns: Optional[List[str]] = None,
+    sort_columns: Optional[List[str]] = None,
+) -> str:
+    """Compute a deterministic SHA256 hash of a DataFrame's content.
+
+    Args:
+        df: Pandas DataFrame to hash
+        columns: Subset of columns to include in hash. If None, all columns.
+        sort_columns: Columns to sort by for deterministic ordering.
+            If None, DataFrame is not sorted (assumes consistent order).
+
+    Returns:
+        SHA256 hex digest string (64 characters)
+
+    Example:
+        >>> df = pd.DataFrame({"id": [1, 2], "value": ["a", "b"]})
+        >>> hash1 = compute_dataframe_hash(df, sort_columns=["id"])
+        >>> hash2 = compute_dataframe_hash(df, sort_columns=["id"])
+        >>> assert hash1 == hash2  # Same content = same hash
+    """
+    if df.empty:
+        return hashlib.sha256(b"EMPTY_DATAFRAME").hexdigest()
+
+    work_df = df
+
+    if columns:
+        missing = set(columns) - set(df.columns)
+        if missing:
+            raise ValueError(f"Hash columns not found in DataFrame: {missing}")
+        work_df = work_df[columns]
+
+    if sort_columns:
+        missing = set(sort_columns) - set(df.columns)
+        if missing:
+            raise ValueError(f"Sort columns not found in DataFrame: {missing}")
+        work_df = work_df.sort_values(sort_columns).reset_index(drop=True)
+
+    csv_bytes = work_df.to_csv(index=False).encode("utf-8")
+    return hashlib.sha256(csv_bytes).hexdigest()
+
+
+def compute_spark_dataframe_hash(
+    df,
+    columns: Optional[List[str]] = None,
+    sort_columns: Optional[List[str]] = None,
+    distributed: bool = True,
+) -> str:
+    """Compute a deterministic SHA256 hash of a Spark DataFrame's content.
+
+    Args:
+        df: Spark DataFrame to hash
+        columns: Subset of columns to include in hash. If None, all columns are used.
+        sort_columns: Columns to sort by for deterministic ordering.
+            (Only used in legacy mode when distributed=False)
+        distributed: If True (default), use distributed hash computation.
+            If False, use legacy collect-to-driver approach.
+
+    Returns:
+        SHA256 hex digest string (64 characters)
+
+    Note:
+        The distributed mode (default) computes hash without collecting data to driver,
+        making it safe for large datasets. The hash is computed as:
+        1. Per-row xxhash64 of all column values
+        2. Sum of all row hashes (order-independent)
+        3. Combined with row count for final SHA256
+
+        Since the sum is commutative, this produces consistent hashes regardless of
+        partition ordering, without requiring a full sort operation.
+    """
+    if df.isEmpty():
+        return hashlib.sha256(b"EMPTY_DATAFRAME").hexdigest()
+
+    work_df = df
+
+    if columns:
+        work_df = work_df.select(columns)
+
+    if distributed:
+        return _compute_spark_hash_distributed(work_df)
+    else:
+        return _compute_spark_hash_legacy(work_df, sort_columns)
+
+
+def _compute_spark_hash_distributed(df) -> str:
+    """Compute hash distributedly using Spark's xxhash64.
+
+    This approach:
+    - Never collects data to driver (except 2 scalar values)
+    - Uses xxhash64 for fast row-level hashing
+    - Uses commutative sum for order-independent aggregation
+    - Is safe for arbitrarily large DataFrames
+    """
+    from pyspark.sql import functions as F
+
+    hash_cols = [F.coalesce(F.col(c).cast("string"), F.lit("__NULL__")) for c in df.columns]
+    work_df = df.withColumn("_row_hash", F.xxhash64(*hash_cols))
+
+    result = work_df.agg(
+        F.count("*").alias("row_count"),
+        F.sum("_row_hash").alias("hash_sum"),
+    ).collect()[0]
+
+    row_count = result["row_count"] or 0
+    hash_sum = result["hash_sum"] or 0
+    combined = f"v2:{row_count}:{hash_sum}:{','.join(sorted(df.columns))}"
+    return hashlib.sha256(combined.encode()).hexdigest()
+
+
+def _compute_spark_hash_legacy(df, sort_columns: Optional[List[str]] = None) -> str:
+    """Legacy hash computation that collects to driver.
+
+    Warning: This can cause OOM on large datasets.
+    Use distributed=True for production workloads.
+    """
+    work_df = df
+
+    if sort_columns:
+        work_df = work_df.orderBy(sort_columns)
+
+    pandas_df = work_df.toPandas()
+    csv_bytes = pandas_df.to_csv(index=False).encode("utf-8")
+    return hashlib.sha256(csv_bytes).hexdigest()
+
+
+def make_content_hash_key(node_name: str, table_name: str) -> str:
+    """Generate a state key for content hash storage.
+
+    Args:
+        node_name: Pipeline node name
+        table_name: Target table name
+
+    Returns:
+        State key string
+    """
+    return f"content_hash:{node_name}:{table_name}"
+
+
+def get_content_hash_from_state(state_backend, node_name: str, table_name: str) -> Optional[str]:
+    """Retrieve stored content hash from state backend (catalog).
+
+    Args:
+        state_backend: CatalogStateBackend or compatible state backend
+        node_name: Pipeline node name
+        table_name: Target table name
+
+    Returns:
+        Previously stored hash string, or None if not found
+    """
+    if state_backend is None:
+        return None
+
+    try:
+        key = make_content_hash_key(node_name, table_name)
+        value = state_backend.get_hwm(key)
+        if isinstance(value, dict):
+            return value.get("hash")
+        return None
+    except Exception:
+        return None
+
+
+def set_content_hash_in_state(
+    state_backend,
+    node_name: str,
+    table_name: str,
+    content_hash: str,
+) -> None:
+    """Store content hash in state backend (catalog).
+
+    Args:
+        state_backend: CatalogStateBackend or compatible state backend
+        node_name: Pipeline node name
+        table_name: Target table name
+        content_hash: Hash string to store
+    """
+    if state_backend is None:
+        return
+
+    from datetime import datetime, timezone
+
+    key = make_content_hash_key(node_name, table_name)
+    value = {
+        "hash": content_hash,
+        "timestamp": datetime.now(timezone.utc).isoformat(),
+    }
+    state_backend.set_hwm(key, value)

odibi/utils/duration.py

@@ -0,0 +1,43 @@
+"""Duration parsing utilities."""
+
+from datetime import timedelta
+from typing import Optional
+
+
+def parse_duration(duration_str: str) -> Optional[timedelta]:
+    """Parse a duration string like '2h', '30m', '1d' into a timedelta.
+
+    Args:
+        duration_str: Duration string with suffix (h=hours, m=minutes, d=days, s=seconds)
+
+    Returns:
+        timedelta object or None if parsing fails
+
+    Examples:
+        >>> parse_duration("2h")
+        datetime.timedelta(seconds=7200)
+        >>> parse_duration("30m")
+        datetime.timedelta(seconds=1800)
+        >>> parse_duration("1d")
+        datetime.timedelta(days=1)
+    """
+    if not duration_str:
+        return None
+
+    duration_str = duration_str.strip().lower()
+
+    try:
+        if duration_str.endswith("h"):
+            return timedelta(hours=int(duration_str[:-1]))
+        elif duration_str.endswith("d"):
+            return timedelta(days=int(duration_str[:-1]))
+        elif duration_str.endswith("m"):
+            return timedelta(minutes=int(duration_str[:-1]))
+        elif duration_str.endswith("s"):
+            return timedelta(seconds=int(duration_str[:-1]))
+        elif duration_str.endswith("w"):
+            return timedelta(weeks=int(duration_str[:-1]))
+        else:
+            return None
+    except ValueError:
+        return None

odibi/utils/encoding.py

@@ -0,0 +1,102 @@
+"""Encoding detection utilities."""
+
+import logging
+from typing import Any, List, Optional
+
+logger = logging.getLogger(__name__)
+
+# Common encodings to try
+CANDIDATE_ENCODINGS = ["utf-8", "utf-8-sig", "latin1", "cp1252"]
+
+
+def detect_encoding(
+    connection: Any,
+    path: str,
+    sample_bytes: int = 65536,
+    candidates: Optional[List[str]] = None,
+) -> Optional[str]:
+    """Detect text encoding of a file.
+
+    Args:
+        connection: Connection object
+        path: File path (relative to connection base)
+        sample_bytes: Number of bytes to read for detection
+        candidates: List of encodings to try (default: common list)
+
+    Returns:
+        Detected encoding name or None if detection failed
+    """
+    full_path = connection.get_path(path)
+    candidates = candidates or CANDIDATE_ENCODINGS
+
+    # Read sample bytes
+    sample = _read_sample_bytes(connection, full_path, sample_bytes)
+    if not sample:
+        return None
+
+    # Try decoding
+    for encoding in candidates:
+        if _is_valid_encoding(sample, encoding):
+            return encoding
+
+    return None
+
+
+def _read_sample_bytes(connection: Any, path: str, size: int) -> Optional[bytes]:
+    """Read bytes from file using available methods."""
+    # 1. Try fsspec (supports local and remote)
+    try:
+        import fsspec
+
+        # Get storage options from connection if available
+        storage_options = {}
+        if hasattr(connection, "pandas_storage_options"):
+            storage_options = connection.pandas_storage_options()
+
+        with fsspec.open(path, "rb", **storage_options) as f:
+            return f.read(size)
+    except ImportError:
+        pass
+    except Exception as e:
+        logger.debug(f"fsspec read failed for {path}: {e}")
+
+    # 2. Try local open (if path is local)
+    # Handle 'file://' prefix or plain path
+    local_path = path
+    if path.startswith("file://"):
+        local_path = path[7:]
+    elif "://" in path:
+        # Remote path and no fsspec -> cannot read
+        return None
+
+    try:
+        with open(local_path, "rb") as f:
+            return f.read(size)
+    except Exception as e:
+        logger.debug(f"Local open failed for {local_path}: {e}")
+
+    return None
+
+
+def _is_valid_encoding(sample: bytes, encoding: str) -> bool:
+    """Check if bytes can be decoded with encoding and look reasonable."""
+    try:
+        sample.decode(encoding)
+
+        # Check for replacement characters (if decoder doesn't fail but inserts them)
+        # Note: strict errors would raise exception, but some encodings might be loose.
+        # We use strict check first.
+        sample.decode(encoding, errors="strict")
+
+        # Heuristics:
+        # 1. Check for excessive non-printable characters?
+        # For now, strict decoding is a strong signal.
+        # Latin1 accepts everything, so it always succeeds.
+        # So we prioritize UTF-8. If UTF-8 works, use it.
+        # If not, Latin1 will work but might show garbage.
+        # "Looks right" is hard.
+        # Maybe check for common delimiters if it's CSV?
+
+        return True
+    except UnicodeError:
+        return False

odibi/utils/extensions.py

@@ -0,0 +1,28 @@
+"""Extension loading utilities."""
+
+import importlib.util
+import sys
+from pathlib import Path
+
+from odibi.utils.logging import logger
+
+
+def load_extensions(path: Path):
+    """Load python extensions (transforms.py, plugins.py) from path."""
+    # Add path to sys.path to handle imports within the extensions
+    if str(path) not in sys.path:
+        sys.path.append(str(path))
+
+    for name in ["transforms.py", "plugins.py"]:
+        file_path = path / name
+        if file_path.exists():
+            try:
+                module_name = file_path.stem
+                spec = importlib.util.spec_from_file_location(module_name, file_path)
+                if spec and spec.loader:
+                    module = importlib.util.module_from_spec(spec)
+                    sys.modules[module_name] = module
+                    spec.loader.exec_module(module)
+                    logger.info(f"Loaded extension: {file_path}")
+            except Exception as e:
+                logger.warning(f"Failed to load {name}: {e}", exc_info=True)

odibi/utils/hashing.py

@@ -0,0 +1,61 @@
+"""Utilities for calculating configuration hashes."""
+
+import hashlib
+import json
+from typing import Any, Optional, Set
+
+
+DEFAULT_EXCLUDE_FIELDS: Set[str] = {"description", "tags", "log_level"}
+
+
+def calculate_config_hash(
+    config: Any,
+    exclude: Optional[Set[str]] = None,
+) -> str:
+    """
+    Calculate MD5 hash of a configuration object.
+
+    Args:
+        config: Pydantic model or object with model_dump/dict method
+        exclude: Set of field names to exclude from hash calculation
+
+    Returns:
+        MD5 hex digest of the config
+    """
+    exclude = exclude if exclude is not None else DEFAULT_EXCLUDE_FIELDS
+
+    if hasattr(config, "model_dump"):
+        dump = config.model_dump(mode="json", exclude=exclude)
+    elif hasattr(config, "dict"):
+        dump = config.model_dump(exclude=exclude)
+    else:
+        dump = config
+
+    dump_str = json.dumps(dump, sort_keys=True)
+    return hashlib.md5(dump_str.encode("utf-8")).hexdigest()
+
+
+def calculate_pipeline_hash(config: Any) -> str:
+    """
+    Calculate hash for a pipeline configuration.
+
+    Pipeline hashes include all fields (no exclusions).
+    """
+    if hasattr(config, "model_dump"):
+        dump = config.model_dump(mode="json")
+    elif hasattr(config, "dict"):
+        dump = config.model_dump()
+    else:
+        dump = config
+
+    dump_str = json.dumps(dump, sort_keys=True)
+    return hashlib.md5(dump_str.encode("utf-8")).hexdigest()
+
+
+def calculate_node_hash(config: Any) -> str:
+    """
+    Calculate hash for a node configuration.
+
+    Node hashes exclude description, tags, and log_level.
+    """
+    return calculate_config_hash(config, exclude=DEFAULT_EXCLUDE_FIELDS)

odibi/utils/logging.py

@@ -0,0 +1,203 @@
+"""Structured logging for Odibi framework.
+
+This module provides the core logging infrastructure:
+- StructuredLogger: Base logger with JSON/human-readable output
+- LoggingContext: Context-aware logging wrapper (imported from logging_context)
+- Secret redaction for sensitive data
+
+For enhanced observability features, use:
+    from odibi.utils.logging_context import LoggingContext, OperationType
+"""
+
+import codecs
+import json
+import logging
+import sys
+from datetime import datetime, timezone
+
+try:
+    from rich.console import Console
+    from rich.logging import RichHandler
+
+    RICH_AVAILABLE = True
+except ImportError:
+    RICH_AVAILABLE = False
+
+
+class StructuredLogger:
+    """Logger that supports both human-readable and JSON output with secret redaction.
+
+    This is the base logging class for Odibi. For context-aware logging with
+    automatic pipeline/node tracking, use LoggingContext instead.
+
+    Example:
+        >>> logger = StructuredLogger(structured=True, level="DEBUG")
+        >>> logger.info("Processing started", pipeline="daily_etl", rows=1000)
+    """
+
+    def __init__(self, structured: bool = False, level: str = "INFO"):
+        """Initialize structured logger.
+
+        Args:
+            structured: If True, output JSON logs; otherwise human-readable
+            level: Log level (DEBUG, INFO, WARNING, ERROR)
+        """
+        self.structured = structured
+        self.level = getattr(logging, level.upper(), logging.INFO)
+        self._secrets: set = set()
+        self._initialized = False
+
+        if (
+            sys.platform == "win32"
+            and sys.stdout
+            and sys.stdout.encoding
+            and sys.stdout.encoding.lower() != "utf-8"
+        ):
+            try:
+                sys.stdout.reconfigure(encoding="utf-8")
+            except AttributeError:
+                sys.stdout = codecs.getwriter("utf-8")(sys.stdout.detach())
+
+        self._setup_handlers()
+
+    def _setup_handlers(self) -> None:
+        """Set up logging handlers."""
+        if self._initialized:
+            return
+
+        if not self.structured and RICH_AVAILABLE:
+            logging.basicConfig(
+                level=self.level,
+                format="%(message)s",
+                datefmt="[%X]",
+                handlers=[
+                    RichHandler(
+                        rich_tracebacks=True,
+                        markup=True,
+                        show_path=False,
+                        console=(
+                            Console(force_terminal=True, legacy_windows=False)
+                            if sys.platform == "win32"
+                            else None
+                        ),
+                    )
+                ],
+            )
+        else:
+            logging.basicConfig(level=self.level, format="%(message)s", stream=sys.stdout)
+
+        self.logger = logging.getLogger("odibi")
+        self.logger.setLevel(self.level)
+
+        third_party_level = max(self.level, logging.WARNING)
+        for logger_name in [
+            "py4j",
+            "azure",
+            "azure.core.pipeline.policies.http_logging_policy",
+            "adlfs",
+            "urllib3",
+            "fsspec",
+        ]:
+            logging.getLogger(logger_name).setLevel(third_party_level)
+
+        self._initialized = True
+
+    def register_secret(self, secret: str) -> None:
+        """Register a secret string to be redacted from logs.
+
+        Args:
+            secret: Secret value to redact (passwords, keys, tokens)
+        """
+        if secret and isinstance(secret, str) and len(secret.strip()) > 0:
+            self._secrets.add(secret)
+
+    def _redact(self, text: str) -> str:
+        """Redact registered secrets from text.
+
+        Args:
+            text: Text to redact
+
+        Returns:
+            Text with secrets replaced by [REDACTED]
+        """
+        if not text or not self._secrets:
+            return text
+
+        for secret in self._secrets:
+            if secret in text:
+                text = text.replace(secret, "[REDACTED]")
+        return text
+
+    def info(self, message: str, **kwargs) -> None:
+        """Log info message."""
+        self._log("INFO", message, **kwargs)
+
+    def warning(self, message: str, **kwargs) -> None:
+        """Log warning message."""
+        self._log("WARNING", message, **kwargs)
+
+    def error(self, message: str, **kwargs) -> None:
+        """Log error message."""
+        self._log("ERROR", message, **kwargs)
+
+    def debug(self, message: str, **kwargs) -> None:
+        """Log debug message."""
+        self._log("DEBUG", message, **kwargs)
+
+    def _log(self, level: str, message: str, **kwargs) -> None:
+        """Internal log method with redaction and formatting.
+
+        Args:
+            level: Log level
+            message: Log message
+            **kwargs: Additional context to include
+        """
+        level_val = getattr(logging, level, logging.INFO)
+        if level_val < self.level:
+            return
+
+        message = self._redact(str(message))
+
+        redacted_kwargs = {}
+        for k, v in kwargs.items():
+            if isinstance(v, str):
+                redacted_kwargs[k] = self._redact(v)
+            elif v is None:
+                continue
+            else:
+                redacted_kwargs[k] = v
+
+        if self.structured:
+            log_entry = {
+                "timestamp": datetime.now(timezone.utc).isoformat(),
+                "level": level,
+                "message": message,
+                **redacted_kwargs,
+            }
+            print(json.dumps(log_entry, default=str))
+        else:
+            context_str = ""
+            if redacted_kwargs:
+                context_items = [f"{k}={v}" for k, v in redacted_kwargs.items()]
+                context_str = f" ({', '.join(context_items)})"
+
+            formatted_msg = f"{message}{context_str}"
+
+            if level == "INFO":
+                self.logger.info(formatted_msg)
+            elif level == "WARNING":
+                self.logger.warning(f"[WARN] {formatted_msg}")
+            elif level == "ERROR":
+                self.logger.error(f"[ERROR] {formatted_msg}")
+            elif level == "DEBUG":
+                self.logger.debug(f"[DEBUG] {formatted_msg}")
+
+
+# Global instance to be initialized
+logger = StructuredLogger()
+
+
+def configure_logging(structured: bool, level: str):
+    """Configure the global logger."""
+    global logger
+    logger = StructuredLogger(structured=structured, level=level)