datacoolie 0.1.0__py3-none-any.whl

datacoolie/__init__.py

@@ -0,0 +1,242 @@
+"""DataCoolie — Metadata-driven, engine-unified, cloud-agnostic ETL framework."""
+
+__version__ = "0.1.0"
+
+__all__ = [
+    # Base classes
+    "PluginRegistry",
+    "BaseSecretResolver",
+    "BaseEngine",
+    "BasePlatform",
+    "BaseSourceReader",
+    "BaseDestinationWriter",
+    "BaseTransformer",
+    # Registries
+    "engine_registry",
+    "platform_registry",
+    "source_registry",
+    "destination_registry",
+    "transformer_registry",
+    "resolver_registry",
+    # Factory functions
+    "create_engine",
+    "create_platform",
+    "create_source",
+    "create_destination",
+    "create_transformer",
+    "create_resolver",
+]
+
+import logging as _logging
+
+from datacoolie.core.registry import PluginRegistry
+from datacoolie.core.secret_resolver import BaseSecretResolver
+from datacoolie.destinations.base import BaseDestinationWriter
+from datacoolie.engines.base import BaseEngine
+from datacoolie.platforms.base import BasePlatform
+from datacoolie.sources.base import BaseSourceReader
+from datacoolie.transformers.base import BaseTransformer
+
+_logger = _logging.getLogger(__name__)
+
+# ---------------------------------------------------------------------------
+# Global plugin registries
+# ---------------------------------------------------------------------------
+
+engine_registry: PluginRegistry[BaseEngine] = PluginRegistry("datacoolie.engines", BaseEngine)
+platform_registry: PluginRegistry[BasePlatform] = PluginRegistry("datacoolie.platforms", BasePlatform)
+source_registry: PluginRegistry[BaseSourceReader] = PluginRegistry("datacoolie.sources", BaseSourceReader)
+destination_registry: PluginRegistry[BaseDestinationWriter] = PluginRegistry("datacoolie.destinations", BaseDestinationWriter)
+transformer_registry: PluginRegistry[BaseTransformer] = PluginRegistry("datacoolie.transformers", BaseTransformer)
+resolver_registry: PluginRegistry[BaseSecretResolver] = PluginRegistry("datacoolie.resolvers", BaseSecretResolver)
+
+
+# ---------------------------------------------------------------------------
+# Factory functions
+# ---------------------------------------------------------------------------
+
+def create_engine(name: str, **kwargs: object) -> BaseEngine:
+    """Create an engine instance by name (e.g. ``"spark"``, ``"polars"``)."""
+    return engine_registry.get(name, **kwargs)
+
+
+def create_platform(name: str, **kwargs: object) -> BasePlatform:
+    """Create a platform instance by name (e.g. ``"local"``, ``"fabric"``)."""
+    return platform_registry.get(name, **kwargs)
+
+
+def create_source(name: str, **kwargs: object) -> BaseSourceReader:
+    """Create a source reader by name (e.g. ``"delta"``, ``"csv"``)."""
+    return source_registry.get(name, **kwargs)
+
+
+def create_destination(name: str, **kwargs: object) -> BaseDestinationWriter:
+    """Create a destination writer by name (e.g. ``"delta"``, ``"parquet"``)."""
+    return destination_registry.get(name, **kwargs)
+
+
+def create_transformer(name: str, **kwargs: object) -> BaseTransformer:
+    """Create a transformer by name (e.g. ``"schema_converter"``)."""
+    return transformer_registry.get(name, **kwargs)
+
+
+def create_resolver(name: str, **kwargs: object) -> BaseSecretResolver:
+    """Create a secret resolver by name (e.g. ``"env"``)."""
+    return resolver_registry.get(name, **kwargs)
+
+
+# ---------------------------------------------------------------------------
+# Built-in registrations (after all packages are initialised)
+# ---------------------------------------------------------------------------
+
+def _register_builtins() -> None:
+    """Register all built-in plugins.
+
+    Uses try/except for each registration so that missing optional
+    dependencies (e.g. PySpark, Polars) don't break the import.
+    """
+    # -- Engines --
+    try:
+        from datacoolie.engines.spark_engine import SparkEngine
+        engine_registry.register("spark", SparkEngine)
+    except Exception as exc:
+        _logger.debug("Skipping built-in 'spark' engine: %s", exc)
+
+    try:
+        from datacoolie.engines.polars_engine import PolarsEngine
+        engine_registry.register("polars", PolarsEngine)
+    except Exception as exc:
+        _logger.debug("Skipping built-in 'polars' engine: %s", exc)
+
+    # -- Platforms --
+    try:
+        from datacoolie.platforms.local_platform import LocalPlatform
+        platform_registry.register("local", LocalPlatform)
+    except Exception as exc:
+        _logger.debug("Skipping built-in 'local' platform: %s", exc)
+
+    try:
+        from datacoolie.platforms.fabric_platform import FabricPlatform
+        platform_registry.register("fabric", FabricPlatform)
+    except Exception as exc:
+        _logger.debug("Skipping built-in 'fabric' platform: %s", exc)
+
+    try:
+        from datacoolie.platforms.databricks_platform import DatabricksPlatform
+        platform_registry.register("databricks", DatabricksPlatform)
+    except Exception as exc:
+        _logger.debug("Skipping built-in 'databricks' platform: %s", exc)
+
+    try:
+        from datacoolie.platforms.aws_platform import AWSPlatform
+        platform_registry.register("aws", AWSPlatform)
+    except Exception as exc:
+        _logger.debug("Skipping built-in 'aws' platform: %s", exc)
+
+    # -- Sources --
+    try:
+        from datacoolie.sources.delta_reader import DeltaReader
+        source_registry.register("delta", DeltaReader)
+    except Exception as exc:
+        _logger.debug("Skipping built-in 'delta' source: %s", exc)
+
+    try:
+        from datacoolie.sources.file_reader import FileReader
+        source_registry.register("parquet", FileReader)
+        source_registry.register("csv", FileReader)
+        source_registry.register("json", FileReader)
+        source_registry.register("jsonl", FileReader)
+        source_registry.register("avro", FileReader)
+        source_registry.register("excel", FileReader)
+    except Exception as exc:
+        _logger.debug("Skipping built-in file source readers: %s", exc)
+
+    try:
+        from datacoolie.sources.python_function_reader import PythonFunctionReader
+        source_registry.register("function", PythonFunctionReader)
+    except Exception as exc:
+        _logger.debug("Skipping built-in 'function' source: %s", exc)
+
+    try:
+        from datacoolie.sources.database_reader import DatabaseReader
+        source_registry.register("sql", DatabaseReader)
+    except Exception as exc:
+        _logger.debug("Skipping built-in 'sql' source: %s", exc)
+
+    try:
+        from datacoolie.sources.iceberg_reader import IcebergReader
+        source_registry.register("iceberg", IcebergReader)
+    except Exception as exc:
+        _logger.debug("Skipping built-in 'iceberg' source: %s", exc)
+
+    try:
+        from datacoolie.sources.api_reader import APIReader
+        source_registry.register("api", APIReader)
+    except Exception as exc:
+        _logger.debug("Skipping built-in 'api' source: %s", exc)
+
+    # -- Destinations --
+    try:
+        from datacoolie.destinations.delta_writer import DeltaWriter
+        destination_registry.register("delta", DeltaWriter)
+    except Exception as exc:
+        _logger.debug("Skipping built-in 'delta' destination: %s", exc)
+
+    try:
+        from datacoolie.destinations.file_writer import FileWriter
+        destination_registry.register("parquet", FileWriter)
+        destination_registry.register("csv", FileWriter)
+        destination_registry.register("json", FileWriter)
+        destination_registry.register("jsonl", FileWriter)
+        destination_registry.register("avro", FileWriter)
+    except Exception as exc:
+        _logger.debug("Skipping built-in file destination writers: %s", exc)
+
+    try:
+        from datacoolie.destinations.iceberg_writer import IcebergWriter
+        destination_registry.register("iceberg", IcebergWriter)
+    except Exception as exc:
+        _logger.debug("Skipping built-in 'iceberg' destination: %s", exc)
+
+    # -- Transformers --
+    try:
+        from datacoolie.transformers.schema_converter import SchemaConverter
+        transformer_registry.register("schema_converter", SchemaConverter)
+    except Exception as exc:
+        _logger.debug("Skipping built-in 'schema_converter' transformer: %s", exc)
+
+    try:
+        from datacoolie.transformers.deduplicator import Deduplicator
+        transformer_registry.register("deduplicator", Deduplicator)
+    except Exception as exc:
+        _logger.debug("Skipping built-in 'deduplicator' transformer: %s", exc)
+
+    try:
+        from datacoolie.transformers.column_adder import ColumnAdder, SCD2ColumnAdder, SystemColumnAdder
+        transformer_registry.register("column_adder", ColumnAdder)
+        transformer_registry.register("scd2_column_adder", SCD2ColumnAdder)
+        transformer_registry.register("system_column_adder", SystemColumnAdder)
+    except Exception as exc:
+        _logger.debug("Skipping built-in column adder transformers: %s", exc)
+
+    try:
+        from datacoolie.transformers.partition_handler import PartitionHandler
+        transformer_registry.register("partition_handler", PartitionHandler)
+    except Exception as exc:
+        _logger.debug("Skipping built-in 'partition_handler' transformer: %s", exc)
+
+    try:
+        from datacoolie.transformers.column_name_sanitizer import ColumnNameSanitizer
+        transformer_registry.register("column_name_sanitizer", ColumnNameSanitizer)
+    except Exception as exc:
+        _logger.debug("Skipping built-in 'column_name_sanitizer' transformer: %s", exc)
+
+    # -- Secret Resolvers --
+    try:
+        from datacoolie.core.secret_resolver import EnvResolver
+        resolver_registry.register("env", EnvResolver)
+    except Exception as exc:
+        _logger.debug("Skipping built-in 'env' resolver: %s", exc)
+
+
+_register_builtins()

datacoolie/core/__init__.py

@@ -0,0 +1,81 @@
+"""DataCoolie core — domain models, constants, exceptions, and plugin registry."""
+
+from datacoolie.core.constants import (
+    ConnectionType,
+    DataFlowStatus,
+    ExecutionType,
+    DatabaseType,
+    Format,
+    LoadType,
+    LogPurpose,
+    LogType,
+    MaintenanceType,
+    ProcessingMode,
+    ColumnCaseMode,
+    SystemColumn,
+    FileInfoColumn,
+)
+from datacoolie.core.exceptions import (
+    ConfigurationError,
+    DataCoolieError,
+    DataFlowError,
+    DestinationError,
+    EngineError,
+    MetadataError,
+    PlatformError,
+    SourceError,
+    TransformError,
+    WatermarkError,
+)
+from datacoolie.core.models import (
+    SchemaHint,
+    PartitionColumn,
+    AdditionalColumn,
+    Connection,
+    Source,
+    Destination,
+    Transform,
+    DataFlow,
+    DataCoolieRunConfig,
+)
+from datacoolie.core.registry import PluginRegistry
+
+__all__ = [
+    # Enums
+    "ConnectionType",
+    "DataFlowStatus",
+    "ExecutionType",
+    "DatabaseType",
+    "Format",
+    "LoadType",
+    "LogPurpose",
+    "LogType",
+    "MaintenanceType",
+    "ProcessingMode",
+    "ColumnCaseMode",
+    "SystemColumn",
+    "FileInfoColumn",
+    # Exceptions
+    "ConfigurationError",
+    "DataCoolieError",
+    "DataFlowError",
+    "DestinationError",
+    "EngineError",
+    "MetadataError",
+    "PlatformError",
+    "SourceError",
+    "TransformError",
+    "WatermarkError",
+    # Models
+    "SchemaHint",
+    "PartitionColumn",
+    "AdditionalColumn",
+    "Connection",
+    "Source",
+    "Destination",
+    "Transform",
+    "DataFlow",
+    "DataCoolieRunConfig",
+    # Registry
+    "PluginRegistry",
+]

datacoolie/core/constants.py

@@ -0,0 +1,223 @@
+"""Framework-wide enums and constants for the DataCoolie framework.
+
+Every enum inherits from ``str`` so its ``.value`` is a plain string
+that serialises naturally to JSON/YAML and matches database values.
+"""
+
+from __future__ import annotations
+
+import os
+from enum import Enum
+from typing import Dict, List
+
+
+# ---------------------------------------------------------------------------
+# Enums
+# ---------------------------------------------------------------------------
+
+class LoadType(str, Enum):
+    """Supported load (write) strategies."""
+
+    FULL_LOAD = "full_load"
+    OVERWRITE = "overwrite"
+    APPEND = "append"
+    MERGE_UPSERT = "merge_upsert"
+    MERGE_OVERWRITE = "merge_overwrite"
+    SCD2 = "scd2"
+
+
+class Format(str, Enum):
+    """Supported data formats."""
+
+    DELTA = "delta"
+    ICEBERG = "iceberg"
+    PARQUET = "parquet"
+    CSV = "csv"
+    JSON = "json"
+    JSONL = "jsonl"
+    AVRO = "avro"
+    EXCEL = "excel"
+    SQL = "sql"
+    API = "api"
+    FUNCTION = "function"
+
+
+class ConnectionType(str, Enum):
+    """Connection endpoint categories."""
+
+    LAKEHOUSE = "lakehouse"
+    FILE = "file"
+    DATABASE = "database"
+    API = "api"
+    STREAMING = "streaming"
+    FUNCTION = "function"
+
+
+# ---------------------------------------------------------------------------
+# Connection type ↔ format mapping
+# ---------------------------------------------------------------------------
+
+CONNECTION_TYPE_FORMATS: Dict[str, frozenset] = {
+    ConnectionType.FILE.value: frozenset({
+        Format.PARQUET.value, 
+        Format.CSV.value, 
+        Format.JSON.value,
+        Format.JSONL.value, 
+        Format.AVRO.value,
+        Format.EXCEL.value,
+    }),
+    ConnectionType.LAKEHOUSE.value: frozenset({
+        Format.DELTA.value, 
+        Format.ICEBERG.value,
+    }),
+    ConnectionType.DATABASE.value: frozenset({
+        Format.SQL.value,
+    }),
+    ConnectionType.API.value: frozenset({
+        Format.API.value,
+    }),
+    ConnectionType.FUNCTION.value: frozenset({
+        Format.FUNCTION.value,
+    }),
+    ConnectionType.STREAMING.value: frozenset(
+
+    ),
+}
+
+
+class ProcessingMode(str, Enum):
+    """ETL processing modes."""
+
+    BATCH = "batch"
+    MICROBATCH = "microbatch"
+    STREAMING = "streaming"
+
+
+class DataFlowStatus(str, Enum):
+    """Dataflow execution statuses."""
+
+    PENDING = "pending"
+    RUNNING = "running"
+    SUCCEEDED = "succeeded"
+    FAILED = "failed"
+    SKIPPED = "skipped"
+
+
+class ExecutionType(str, Enum):
+    """ETL execution types."""
+
+    ETL = "etl"
+    MAINTENANCE = "maintenance"
+
+
+class DatabaseType(str, Enum):
+    """Supported database types for connection URL generation."""
+
+    POSTGRESQL = "postgresql"
+    MYSQL = "mysql"
+    MSSQL = "mssql"
+    ORACLE = "oracle"
+    SQLITE = "sqlite"
+
+
+class MaintenanceType(str, Enum):
+    """Maintenance operation types."""
+
+    COMPACT = "compact"
+    CLEANUP = "cleanup"
+
+
+class ColumnCaseMode(str, Enum):
+    """Column name case-conversion modes used by :class:`ColumnNameSanitizer`."""
+
+    LOWER = "lower"
+    SNAKE = "snake"
+
+
+class LogType(str, Enum):
+    """Log entry types for structured logging."""
+
+    JOB_RUN_LOG = "job_run_log"
+    DATAFLOW_RUN_LOG = "dataflow_run_log"
+
+
+class LogPurpose(str, Enum):
+    """Log output purposes / formats."""
+
+    DEBUG = "debug_json"
+    ANALYST = "analyst"
+
+
+# ---------------------------------------------------------------------------
+# System columns (auto-added to destination data)
+# ---------------------------------------------------------------------------
+
+class SystemColumn(str, Enum):
+    """Column names automatically added to every destination write."""
+
+    CREATED_AT = "__created_at"
+    UPDATED_AT = "__updated_at"
+    UPDATED_BY = "__updated_by"
+
+
+class FileInfoColumn(str, Enum):
+    """Column names injected for file-based sources."""
+
+    FILE_NAME = "__file_name"
+    FILE_PATH = "__file_path"
+    FILE_MODIFICATION_TIME = "__file_modification_time"
+
+
+class SCD2Column(str, Enum):
+    """Column names for Slowly Changing Dimension Type 2."""
+
+    VALID_FROM = "__valid_from"
+    VALID_TO = "__valid_to"
+    IS_CURRENT = "__is_current"
+
+
+# Canonical order for columns that should always appear at the end of a
+# DataFrame / table schema.  Shared by the transformer pipeline
+# (``ColumnNameSanitizer``) and the Iceberg adapter
+# (``_reorder_iceberg_trailing_columns``).
+TRAILING_COLUMNS: tuple[str, ...] = (
+    *(c.value for c in SCD2Column),
+    *(c.value for c in FileInfoColumn),
+    *(c.value for c in SystemColumn),
+)
+
+# ---------------------------------------------------------------------------
+# Watermark
+# ---------------------------------------------------------------------------
+
+WATERMARK_FILE_NAME: str = "watermark_value.json"
+DATETIME_PATTERN: str = "__datetime__"
+DATE_PATTERN: str = "__date__"
+TIME_PATTERN: str = "__time__"
+DATE_FOLDER_PARTITION_KEY: str = "__date_folder_partition__"
+
+# ---------------------------------------------------------------------------
+# Date folder partition placeholders
+# ---------------------------------------------------------------------------
+
+DATE_PLACEHOLDERS: Dict[str, tuple[str, str]] = {
+    "{year}": (r"(\d{4})", "year"),
+    "{month}": (r"(\d{2})", "month"),
+    "{day}": (r"(\d{2})", "day"),
+    "{hour}": (r"(\d{2})", "hour"),
+}
+
+# ---------------------------------------------------------------------------
+# Defaults
+# ---------------------------------------------------------------------------
+
+# Log partition pattern default is date-based, but can be overridden by config
+DEFAULT_PARTITION_PATTERN = "__run_date={year}-{month}-{day}"
+
+# Default values for various parameters across the framework
+DEFAULT_AUTHOR: str = "DataCoolie"
+DEFAULT_MAX_WORKERS: int = 8
+DEFAULT_RETRY_COUNT: int = 0
+DEFAULT_RETRY_DELAY: float = 5.0
+DEFAULT_RETENTION_HOURS: int = 168  # 7 days
+

datacoolie/core/exceptions.py

@@ -0,0 +1,114 @@
+"""Exception hierarchy for the DataCoolie framework.
+
+All framework exceptions inherit from :class:`DataCoolieError` which carries
+a human-readable ``message`` and an optional ``details`` dict for structured
+error context.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+
+class DataCoolieError(Exception):
+    """Base exception for all DataCoolie errors.
+
+    Attributes:
+        message: Human-readable error description.
+        details: Optional structured context for logging / debugging.
+    """
+
+    def __init__(self, message: str, details: dict[str, Any] | None = None) -> None:
+        self.message = message
+        self.details = details or {}
+        super().__init__(self.message)
+
+    def __str__(self) -> str:
+        if self.details:
+            return f"{self.message} | Details: {self.details}"
+        return self.message
+
+
+# ---------------------------------------------------------------------------
+# Component-specific errors
+# ---------------------------------------------------------------------------
+
+
+class ConfigurationError(DataCoolieError):
+    """Raised for invalid or missing configuration."""
+
+
+class MetadataError(DataCoolieError):
+    """Raised for metadata loading / parsing failures."""
+
+
+class SourceError(DataCoolieError):
+    """Raised when reading from a data source fails."""
+
+
+class TransformError(DataCoolieError):
+    """Raised when a transformation step fails."""
+
+
+class DestinationError(DataCoolieError):
+    """Raised when writing to a destination fails."""
+
+
+class WatermarkError(DataCoolieError):
+    """Raised for watermark read / write / parse failures."""
+
+
+class EngineError(DataCoolieError):
+    """Raised for engine-level failures (Spark, Polars)."""
+
+
+class PlatformError(DataCoolieError):
+    """Raised for platform-specific failures (Fabric, Databricks, AWS)."""
+
+
+class PipelineError(DataCoolieError):
+    """Raised when a pipeline step fails, carrying partial runtime results.
+
+    Attributes:
+        partial_result: The incomplete result tuple collected before the failure.
+    """
+
+    def __init__(
+        self,
+        message: str,
+        partial_result: Any = None,
+        details: dict[str, Any] | None = None,
+    ) -> None:
+        super().__init__(message, details)
+        self.partial_result = partial_result
+
+
+class DataFlowError(DataCoolieError):
+    """Raised for dataflow-level execution errors.
+
+    Attributes:
+        dataflow_id: Identifier of the failed dataflow.
+        stage: Pipeline stage where the error occurred.
+    """
+
+    def __init__(
+        self,
+        message: str,
+        dataflow_id: str | None = None,
+        stage: str | None = None,
+        details: dict[str, Any] | None = None,
+    ) -> None:
+        self.dataflow_id = dataflow_id
+        self.stage = stage
+        super().__init__(message, details)
+
+    def __str__(self) -> str:
+        parts = [self.message]
+        if self.dataflow_id:
+            parts.append(f"DataFlow: {self.dataflow_id}")
+        if self.stage:
+            parts.append(f"Stage: {self.stage}")
+        if self.details:
+            parts.append(f"Details: {self.details}")
+        return " | ".join(parts)
+