odibi 2.5.0__py3-none-any.whl

odibi/connections/base.py

@@ -0,0 +1,28 @@
+"""Base connection interface."""
+
+from abc import ABC, abstractmethod
+
+
+class BaseConnection(ABC):
+    """Abstract base class for connections."""
+
+    @abstractmethod
+    def get_path(self, relative_path: str) -> str:
+        """Get full path for a relative path.
+
+        Args:
+            relative_path: Relative path or table name
+
+        Returns:
+            Full path to resource
+        """
+        pass
+
+    @abstractmethod
+    def validate(self) -> None:
+        """Validate connection configuration.
+
+        Raises:
+            ConnectionError: If validation fails
+        """
+        pass

odibi/connections/factory.py

@@ -0,0 +1,322 @@
+"""Connection factory for built-in connection types."""
+
+from typing import Any, Dict
+
+from odibi.plugins import register_connection_factory
+from odibi.utils.logging import logger
+from odibi.utils.logging_context import get_logging_context
+
+
+def create_local_connection(name: str, config: Dict[str, Any]) -> Any:
+    """Factory for LocalConnection."""
+    ctx = get_logging_context()
+    ctx.log_connection(connection_type="local", connection_name=name, action="create")
+
+    from odibi.connections.local import LocalConnection
+
+    base_path = config.get("base_path", "./data")
+    connection = LocalConnection(base_path=base_path)
+
+    ctx.log_connection(
+        connection_type="local", connection_name=name, action="created", base_path=base_path
+    )
+    return connection
+
+
+def create_http_connection(name: str, config: Dict[str, Any]) -> Any:
+    """Factory for HttpConnection."""
+    ctx = get_logging_context()
+    ctx.log_connection(connection_type="http", connection_name=name, action="create")
+
+    from odibi.connections.http import HttpConnection
+
+    base_url = config.get("base_url", "")
+    connection = HttpConnection(
+        base_url=base_url,
+        headers=config.get("headers"),
+        auth=config.get("auth"),
+    )
+
+    ctx.log_connection(
+        connection_type="http", connection_name=name, action="created", base_url=base_url
+    )
+    return connection
+
+
+def create_azure_blob_connection(name: str, config: Dict[str, Any]) -> Any:
+    """Factory for AzureADLS (Blob) Connection."""
+    ctx = get_logging_context()
+    ctx.log_connection(connection_type="azure_blob", connection_name=name, action="create")
+
+    try:
+        from odibi.connections.azure_adls import AzureADLS
+    except ImportError as e:
+        ctx.error(
+            f"Failed to import AzureADLS for connection '{name}'",
+            connection_name=name,
+            error=str(e),
+        )
+        raise ImportError(
+            "Azure ADLS support requires 'pip install odibi[azure]'. "
+            "See README.md for installation instructions."
+        )
+
+    # Handle config discrepancies
+    account = config.get("account_name") or config.get("account")
+    if not account:
+        ctx.error(
+            f"Connection '{name}' missing 'account_name'",
+            connection_name=name,
+            config_keys=list(config.keys()),
+        )
+        raise ValueError(
+            f"Connection '{name}' missing 'account_name'. "
+            f"Expected 'account_name' or 'account' in config, got keys: {list(config.keys())}"
+        )
+
+    auth_config = config.get("auth", {})
+
+    # Extract auth details
+    key_vault_name = auth_config.get("key_vault_name") or config.get("key_vault_name")
+    secret_name = auth_config.get("secret_name") or config.get("secret_name")
+    account_key = auth_config.get("account_key") or config.get("account_key")
+    sas_token = auth_config.get("sas_token") or config.get("sas_token")
+    tenant_id = auth_config.get("tenant_id") or config.get("tenant_id")
+    client_id = auth_config.get("client_id") or config.get("client_id")
+    client_secret = auth_config.get("client_secret") or config.get("client_secret")
+
+    auth_mode = auth_config.get("mode") or config.get("auth_mode", "key_vault")
+
+    # Auto-detect auth_mode if not explicitly set
+    if "auth_mode" not in config and "mode" not in auth_config:
+        if sas_token:
+            auth_mode = "sas_token"
+        elif key_vault_name and secret_name:
+            auth_mode = "key_vault"
+        elif account_key:
+            auth_mode = "direct_key"
+        elif tenant_id and client_id and client_secret:
+            auth_mode = "service_principal"
+        else:
+            auth_mode = "managed_identity"
+
+        ctx.debug(
+            f"Auto-detected auth_mode for connection '{name}'",
+            connection_name=name,
+            auth_mode=auth_mode,
+        )
+
+    validation_mode = config.get("validation_mode", "lazy")
+    validate = config.get("validate")
+    if validate is None:
+        validate = True if validation_mode == "eager" else False
+
+    # Register secrets (log that we're registering, not the values)
+    if account_key:
+        logger.register_secret(account_key)
+        ctx.debug(f"Registered account_key secret for connection '{name}'", connection_name=name)
+    if sas_token:
+        logger.register_secret(sas_token)
+        ctx.debug(f"Registered sas_token secret for connection '{name}'", connection_name=name)
+    if client_secret:
+        logger.register_secret(client_secret)
+        ctx.debug(f"Registered client_secret secret for connection '{name}'", connection_name=name)
+
+    try:
+        connection = AzureADLS(
+            account=account,
+            container=config["container"],
+            path_prefix=config.get("path_prefix", ""),
+            auth_mode=auth_mode,
+            key_vault_name=key_vault_name,
+            secret_name=secret_name,
+            account_key=account_key,
+            sas_token=sas_token,
+            tenant_id=tenant_id,
+            client_id=client_id,
+            client_secret=client_secret,
+            validate=validate,
+        )
+
+        ctx.log_connection(
+            connection_type="azure_blob",
+            connection_name=name,
+            action="created",
+            account=account,
+            container=config["container"],
+            auth_mode=auth_mode,
+            validation_mode=validation_mode,
+        )
+        return connection
+
+    except Exception as e:
+        ctx.error(
+            f"Failed to create Azure Blob connection '{name}'",
+            connection_name=name,
+            account=account,
+            container=config.get("container"),
+            auth_mode=auth_mode,
+            error=str(e),
+        )
+        raise
+
+
+def create_delta_connection(name: str, config: Dict[str, Any]) -> Any:
+    """Factory for Delta Connection."""
+    ctx = get_logging_context()
+    ctx.log_connection(connection_type="delta", connection_name=name, action="create")
+
+    # Local path-based Delta
+    if "path" in config:
+        from odibi.connections.local import LocalConnection
+
+        base_path = config.get("path") or config.get("base_path")
+        connection = LocalConnection(base_path=base_path)
+
+        ctx.log_connection(
+            connection_type="delta",
+            connection_name=name,
+            action="created",
+            mode="local_path",
+            base_path=base_path,
+        )
+        return connection
+
+    # Catalog based (Spark only)
+    from odibi.connections.base import BaseConnection
+
+    class DeltaCatalogConnection(BaseConnection):
+        def __init__(self, catalog, schema):
+            self.catalog = catalog
+            self.schema = schema
+
+        def get_path(self, table):
+            return f"{self.catalog}.{self.schema}.{table}"
+
+        def validate(self):
+            pass
+
+        def pandas_storage_options(self):
+            return {}
+
+    catalog = config.get("catalog")
+    schema = config.get("schema") or "default"
+    connection = DeltaCatalogConnection(catalog=catalog, schema=schema)
+
+    ctx.log_connection(
+        connection_type="delta",
+        connection_name=name,
+        action="created",
+        mode="catalog",
+        catalog=catalog,
+        schema=schema,
+    )
+    return connection
+
+
+def create_sql_server_connection(name: str, config: Dict[str, Any]) -> Any:
+    """Factory for SQL Server / Azure SQL Connection."""
+    ctx = get_logging_context()
+    ctx.log_connection(connection_type="sql_server", connection_name=name, action="create")
+
+    try:
+        from odibi.connections.azure_sql import AzureSQL
+    except ImportError as e:
+        ctx.error(
+            f"Failed to import AzureSQL for connection '{name}'",
+            connection_name=name,
+            error=str(e),
+        )
+        raise ImportError(
+            "Azure SQL support requires 'pip install odibi[azure]'. "
+            "See README.md for installation instructions."
+        )
+
+    server = config.get("host") or config.get("server")
+    if not server:
+        ctx.error(
+            f"Connection '{name}' missing 'host' or 'server'",
+            connection_name=name,
+            config_keys=list(config.keys()),
+        )
+        raise ValueError(
+            f"Connection '{name}' missing 'host' or 'server'. " f"Got keys: {list(config.keys())}"
+        )
+
+    auth_config = config.get("auth", {})
+    username = auth_config.get("username") or config.get("username")
+    password = auth_config.get("password") or config.get("password")
+    key_vault_name = auth_config.get("key_vault_name") or config.get("key_vault_name")
+    secret_name = auth_config.get("secret_name") or config.get("secret_name")
+
+    auth_mode = config.get("auth_mode")
+    if not auth_mode:
+        if username and password:
+            auth_mode = "sql"
+        elif key_vault_name and secret_name and username:
+            auth_mode = "key_vault"
+        else:
+            auth_mode = "aad_msi"
+
+        ctx.debug(
+            f"Auto-detected auth_mode for connection '{name}'",
+            connection_name=name,
+            auth_mode=auth_mode,
+        )
+
+    if password:
+        logger.register_secret(password)
+        ctx.debug(f"Registered password secret for connection '{name}'", connection_name=name)
+
+    try:
+        connection = AzureSQL(
+            server=server,
+            database=config["database"],
+            driver=config.get("driver", "ODBC Driver 18 for SQL Server"),
+            username=username,
+            password=password,
+            auth_mode=auth_mode,
+            key_vault_name=key_vault_name,
+            secret_name=secret_name,
+            port=config.get("port", 1433),
+            timeout=config.get("timeout", 30),
+        )
+
+        ctx.log_connection(
+            connection_type="sql_server",
+            connection_name=name,
+            action="created",
+            server=server,
+            database=config["database"],
+            auth_mode=auth_mode,
+            port=config.get("port", 1433),
+        )
+        return connection
+
+    except Exception as e:
+        ctx.error(
+            f"Failed to create SQL Server connection '{name}'",
+            connection_name=name,
+            server=server,
+            database=config.get("database"),
+            auth_mode=auth_mode,
+            error=str(e),
+        )
+        raise
+
+
+def register_builtins():
+    """Register all built-in connection factories."""
+    register_connection_factory("local", create_local_connection)
+    register_connection_factory("http", create_http_connection)
+
+    # Azure Blob / ADLS
+    register_connection_factory("azure_blob", create_azure_blob_connection)
+    register_connection_factory("azure_adls", create_azure_blob_connection)
+
+    # Delta
+    register_connection_factory("delta", create_delta_connection)
+
+    # SQL
+    register_connection_factory("sql_server", create_sql_server_connection)
+    register_connection_factory("azure_sql", create_sql_server_connection)

odibi/connections/http.py

@@ -0,0 +1,78 @@
+"""HTTP Connection implementation."""
+
+from typing import Any, Dict, Optional
+from urllib.parse import urljoin
+
+from odibi.connections.base import BaseConnection
+
+
+class HttpConnection(BaseConnection):
+    """Connection to HTTP/HTTPS APIs."""
+
+    def __init__(
+        self,
+        base_url: str,
+        headers: Optional[Dict[str, str]] = None,
+        auth: Optional[Dict[str, str]] = None,
+        validate: bool = True,
+    ):
+        """Initialize HTTP connection.
+
+        Args:
+            base_url: Base URL for API
+            headers: Default headers
+            auth: Authentication details
+            validate: Whether to validate connection (ping)
+        """
+        self.base_url = base_url.rstrip("/") + "/"
+        self.headers = headers or {}
+
+        if auth:
+            if "token" in auth:
+                self.headers["Authorization"] = f"Bearer {auth['token']}"
+            elif "username" in auth and "password" in auth:
+                import base64
+
+                creds = f"{auth['username']}:{auth['password']}"
+                b64_creds = base64.b64encode(creds.encode()).decode()
+                self.headers["Authorization"] = f"Basic {b64_creds}"
+            elif "api_key" in auth:
+                # Common pattern: X-API-Key header or similar
+                header_name = auth.get("header_name", "X-API-Key")
+                self.headers[header_name] = auth["api_key"]
+
+        if validate:
+            self.validate()
+
+    def validate(self) -> None:
+        """Validate connection configuration.
+
+        Raises:
+            ValueError: If validation fails
+        """
+        if not self.base_url:
+            raise ValueError("HTTP connection requires 'base_url'")
+
+    def get_path(self, path: str) -> str:
+        """Resolve endpoint path.
+
+        Args:
+            path: API endpoint (e.g., 'v1/users')
+
+        Returns:
+            Full URL
+        """
+        if path.startswith("http://") or path.startswith("https://"):
+            return path
+
+        # urljoin can be tricky if base_url doesn't end with /
+        return urljoin(self.base_url, path.lstrip("/"))
+
+    def pandas_storage_options(self) -> Dict[str, Any]:
+        """Get storage options for Pandas/fsspec.
+
+        Returns:
+            Dictionary with headers
+        """
+        # For HTTP(S) in Pandas (urllib), storage_options ARE the headers.
+        return self.headers

odibi/connections/local.py

@@ -0,0 +1,119 @@
+"""Local filesystem connection."""
+
+from pathlib import Path
+
+from odibi.connections.base import BaseConnection
+from odibi.utils.logging_context import get_logging_context
+
+
+class LocalConnection(BaseConnection):
+    """Connection to local filesystem or URI-based paths (e.g. dbfs:/, file://)."""
+
+    def __init__(self, base_path: str = "./data"):
+        """Initialize local connection.
+
+        Args:
+            base_path: Base directory for all paths (can be local path or URI)
+        """
+        ctx = get_logging_context()
+        ctx.log_connection(
+            connection_type="local",
+            connection_name="LocalConnection",
+            action="init",
+            base_path=base_path,
+        )
+
+        self.base_path_str = base_path
+        self.is_uri = "://" in base_path or ":/" in base_path
+
+        if not self.is_uri:
+            self.base_path = Path(base_path)
+            ctx.debug(
+                "LocalConnection initialized with filesystem path",
+                base_path=base_path,
+                is_uri=False,
+            )
+        else:
+            self.base_path = None  # Not used for URIs
+            ctx.debug(
+                "LocalConnection initialized with URI path",
+                base_path=base_path,
+                is_uri=True,
+            )
+
+    def get_path(self, relative_path: str) -> str:
+        """Get full path for a relative path.
+
+        Args:
+            relative_path: Relative path from base
+
+        Returns:
+            Full absolute path or URI
+        """
+        ctx = get_logging_context()
+
+        if self.is_uri:
+            # Use os.path for simple string joining, handling slashes manually for consistency
+            # Strip leading slash from relative to avoid root replacement
+            clean_rel = relative_path.lstrip("/").lstrip("\\")
+            # Handle cases where base_path might not have trailing slash
+            if self.base_path_str.endswith("/") or self.base_path_str.endswith("\\"):
+                full_path = f"{self.base_path_str}{clean_rel}"
+            else:
+                # Use forward slash for URIs
+                full_path = f"{self.base_path_str}/{clean_rel}"
+
+            ctx.debug(
+                "Resolved URI path",
+                relative_path=relative_path,
+                full_path=full_path,
+            )
+            return full_path
+        else:
+            # Standard local path logic
+            full_path = self.base_path / relative_path
+            resolved = str(full_path.absolute())
+
+            ctx.debug(
+                "Resolved local path",
+                relative_path=relative_path,
+                full_path=resolved,
+            )
+            return resolved
+
+    def validate(self) -> None:
+        """Validate that base path exists or can be created.
+
+        Raises:
+            ConnectionError: If validation fails
+        """
+        ctx = get_logging_context()
+        ctx.debug(
+            "Validating LocalConnection",
+            base_path=self.base_path_str,
+            is_uri=self.is_uri,
+        )
+
+        if self.is_uri:
+            # Cannot validate/create URIs with local os module
+            # Assume valid or handled by engine
+            ctx.debug(
+                "Skipping URI validation (handled by engine)",
+                base_path=self.base_path_str,
+            )
+        else:
+            # Create base directory if it doesn't exist
+            try:
+                self.base_path.mkdir(parents=True, exist_ok=True)
+                ctx.info(
+                    "LocalConnection validated successfully",
+                    base_path=str(self.base_path.absolute()),
+                    created=not self.base_path.exists(),
+                )
+            except Exception as e:
+                ctx.error(
+                    "LocalConnection validation failed",
+                    base_path=self.base_path_str,
+                    error=str(e),
+                )
+                raise

odibi/connections/local_dbfs.py

@@ -0,0 +1,61 @@
+"""Local DBFS mock for testing Databricks pipelines locally."""
+
+from pathlib import Path
+from typing import Union
+
+from .base import BaseConnection
+
+
+class LocalDBFS(BaseConnection):
+    """Mock DBFS connection for local development.
+
+    Maps dbfs:/ paths to local filesystem for testing.
+    Useful for developing Databricks pipelines locally.
+    """
+
+    def __init__(self, root: Union[str, Path] = ".dbfs"):
+        """Initialize local DBFS mock.
+
+        Args:
+            root: Local directory to use as DBFS root (default: .dbfs)
+        """
+        self.root = Path(root).resolve()
+
+    def resolve(self, path: str) -> str:
+        """Resolve dbfs:/ path to local filesystem path.
+
+        Args:
+            path: DBFS path (e.g., 'dbfs:/FileStore/data.csv')
+
+        Returns:
+            Absolute local filesystem path
+
+        Example:
+            >>> conn = LocalDBFS(root="/tmp/dbfs")
+            >>> conn.resolve("dbfs:/FileStore/data.csv")
+            '/tmp/dbfs/FileStore/data.csv'
+        """
+        # Remove dbfs:/ prefix
+        clean_path = path.replace("dbfs:/", "").lstrip("/")
+
+        # Join with root
+        local_path = self.root / clean_path
+
+        return str(local_path)
+
+    def ensure_dir(self, path: str) -> None:
+        """Create parent directories for given path.
+
+        Args:
+            path: DBFS path
+        """
+        local_path = Path(self.resolve(path))
+        local_path.parent.mkdir(parents=True, exist_ok=True)
+
+    def get_path(self, relative_path: str) -> str:
+        """Get local filesystem path for DBFS path."""
+        return self.resolve(relative_path)
+
+    def validate(self) -> None:
+        """Validate local DBFS configuration."""
+        pass  # No validation needed for local mock

odibi/constants.py

@@ -0,0 +1,17 @@
+"""Constants used throughout the Odibi framework."""
+
+# Delta table maintenance
+DEFAULT_VACUUM_RETENTION_HOURS = 168  # 7 days
+
+# SQL operations
+DEFAULT_SQL_CHUNK_SIZE = 1000
+
+# Connection timeouts
+DEFAULT_CONNECTION_TIMEOUT = 30  # seconds
+DEFAULT_KEY_VAULT_TIMEOUT = 30.0  # seconds
+
+# Story generation
+DEFAULT_MAX_SAMPLE_ROWS = 10
+
+# Delta history
+DEFAULT_HISTORY_LIMIT = 100