pycharter 0.0.24__py3-none-any.whl → 0.0.26__py3-none-any.whl

pycharter/etl_generator/config_models.py

@@ -0,0 +1,673 @@
+"""
+Pydantic models for ETL pipeline configuration.
+
+Replaces JSON Schema files with type-safe Python models for:
+- Extract configuration (HTTP, file, database, cloud storage)
+- Transform configuration (simple ops, JSONata, custom function)
+- Load configuration (Postgres, SQLite, file, cloud storage)
+- Settings (shared pipeline config)
+- Validation (source/target schema validation)
+- DLQ (dead letter queue)
+"""
+
+from enum import Enum
+from typing import Any, Dict, List, Literal, Optional, Union
+
+from pydantic import BaseModel, ConfigDict, Field
+
+
+# =============================================================================
+# ENUMS
+# =============================================================================
+
+
+class ExtractType(str, Enum):
+    """Extract source types."""
+
+    HTTP = "http"
+    FILE = "file"
+    DATABASE = "database"
+    CLOUD_STORAGE = "cloud_storage"
+
+
+class LoadType(str, Enum):
+    """Load destination types."""
+
+    POSTGRES = "postgres"
+    POSTGRESQL = "postgresql"
+    DATABASE = "database"
+    SQLITE = "sqlite"
+    FILE = "file"
+    CLOUD_STORAGE = "cloud_storage"
+
+
+class CloudProvider(str, Enum):
+    """Cloud storage providers."""
+
+    S3 = "s3"
+    GCS = "gcs"
+    AZURE = "azure"
+
+
+class WriteMethod(str, Enum):
+    """Database write methods."""
+
+    INSERT = "insert"
+    UPSERT = "upsert"
+    REPLACE = "replace"
+    UPDATE = "update"
+    DELETE = "delete"
+    TRUNCATE_AND_LOAD = "truncate_and_load"
+
+
+class FileFormat(str, Enum):
+    """File formats."""
+
+    JSON = "json"
+    JSONL = "jsonl"
+    CSV = "csv"
+    TSV = "tsv"
+    PARQUET = "parquet"
+    EXCEL = "excel"
+    XML = "xml"
+
+
+class WriteMode(str, Enum):
+    """File write modes."""
+
+    OVERWRITE = "overwrite"
+    APPEND = "append"
+
+
+class OnErrorAction(str, Enum):
+    """Actions on validation error."""
+
+    FAIL = "fail"
+    WARN = "warn"
+    SKIP = "skip"
+    QUARANTINE = "quarantine"
+
+
+class DLQBackend(str, Enum):
+    """DLQ storage backends."""
+
+    FILE = "file"
+    DATABASE = "database"
+    MEMORY = "memory"
+
+
+class MetadataStoreType(str, Enum):
+    """Metadata store types."""
+
+    POSTGRES = "postgres"
+    SQLITE = "sqlite"
+    MONGODB = "mongodb"
+    REDIS = "redis"
+    MEMORY = "memory"
+
+
+class FilterOperator(str, Enum):
+    """Filter comparison operators."""
+
+    EQ = "eq"
+    NE = "ne"
+    GT = "gt"
+    GTE = "gte"
+    LT = "lt"
+    LTE = "lte"
+    IN = "in"
+    NOT_IN = "not_in"
+    CONTAINS = "contains"
+    NOT_CONTAINS = "not_contains"
+    IS_NULL = "is_null"
+    IS_NOT_NULL = "is_not_null"
+
+
+class PaginationStrategy(str, Enum):
+    """HTTP pagination strategies."""
+
+    PAGE = "page"
+    OFFSET = "offset"
+    CURSOR = "cursor"
+    LINK = "link"
+
+
+class HTTPMethod(str, Enum):
+    """HTTP methods."""
+
+    GET = "GET"
+    POST = "POST"
+    PUT = "PUT"
+    PATCH = "PATCH"
+    DELETE = "DELETE"
+
+
+class ResponseFormat(str, Enum):
+    """HTTP response formats."""
+
+    JSON = "json"
+    TEXT = "text"
+    XML = "xml"
+
+
+class ConvertType(str, Enum):
+    """Type conversion targets."""
+
+    INT = "int"
+    INTEGER = "integer"
+    FLOAT = "float"
+    NUMBER = "number"
+    STR = "str"
+    STRING = "string"
+    BOOL = "bool"
+    BOOLEAN = "boolean"
+    DATETIME = "datetime"
+    DATE = "date"
+
+
+class JSONataMode(str, Enum):
+    """JSONata application mode."""
+
+    RECORD = "record"
+    BATCH = "batch"
+
+
+# =============================================================================
+# SHARED CONFIG MODELS
+# =============================================================================
+
+
+class SSHTunnelConfig(BaseModel):
+    """SSH tunnel configuration for database connections."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    host: str
+    port: int = 22
+    username: str
+    private_key_path: Optional[str] = None
+    remote_host: Optional[str] = None
+    remote_port: Optional[int] = None
+
+
+class DatabaseConfig(BaseModel):
+    """Database connection configuration."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    url: str
+
+
+class CSVOptions(BaseModel):
+    """CSV file options."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    delimiter: str = ","
+    quote_char: str = '"'
+    escape_char: Optional[str] = None
+    has_header: bool = True
+    include_header: bool = True  # for writing
+    encoding: str = "utf-8"
+
+
+class JSONOptions(BaseModel):
+    """JSON file options."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    encoding: str = "utf-8"
+    lines: bool = False  # for reading JSONL
+    indent: Optional[int] = None  # for writing
+    ensure_ascii: bool = False  # for writing
+
+
+class CloudStorageConfig(BaseModel):
+    """Cloud storage configuration."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    provider: CloudProvider
+    bucket: Optional[str] = None  # S3, GCS
+    container: Optional[str] = None  # Azure
+    path: Optional[str] = None
+    prefix: Optional[str] = None
+    credentials: Optional[Dict[str, Any]] = None
+
+
+# =============================================================================
+# DLQ CONFIG
+# =============================================================================
+
+
+class DLQConfig(BaseModel):
+    """Dead Letter Queue configuration."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    enabled: bool = True
+    backend: DLQBackend = DLQBackend.FILE
+    path: Optional[str] = None  # for file backend
+    connection_string: Optional[str] = None  # for database backend
+    schema_name: Optional[str] = Field(None, alias="schema")  # DB schema
+    table: Optional[str] = None  # DB table name
+
+
+# =============================================================================
+# VALIDATION CONFIG
+# =============================================================================
+
+
+class ContractRef(BaseModel):
+    """Reference to a data contract in database."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    type: Literal["database"] = "database"
+    name: str
+    version: Optional[str] = None
+
+
+class ExtractValidationConfig(BaseModel):
+    """Validation config for extract stage (source schema)."""
+
+    model_config = ConfigDict(extra="forbid", populate_by_name=True)
+
+    schema_path: Optional[str] = Field(None, alias="schema")
+    on_error: OnErrorAction = OnErrorAction.FAIL
+    dlq: Optional[Union[bool, DLQConfig]] = None
+
+
+class LoadValidationConfig(BaseModel):
+    """Validation config for load stage (target contract)."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    contract: Optional[Union[str, ContractRef]] = None  # file path or db ref
+    use_contract: bool = False  # use settings.yaml contract
+    on_error: OnErrorAction = OnErrorAction.FAIL
+    dlq: Optional[Union[bool, DLQConfig]] = None
+
+
+# =============================================================================
+# METADATA STORE CONFIG
+# =============================================================================
+
+
+class MetadataStoreConfig(BaseModel):
+    """Metadata store configuration for database-based contracts."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    type: MetadataStoreType
+    connection_string: str
+
+
+# =============================================================================
+# SETTINGS CONFIG
+# =============================================================================
+
+
+class SettingsConfig(BaseModel):
+    """Shared pipeline settings (settings.yaml)."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    name: Optional[str] = None
+    metadata_store: Optional[MetadataStoreConfig] = None
+    dlq: Optional[DLQConfig] = None
+    contract: Optional[str] = None  # default contract name
+
+
+# =============================================================================
+# EXTRACT CONFIGS
+# =============================================================================
+
+
+class PaginationConfig(BaseModel):
+    """HTTP pagination configuration."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    enabled: bool = False
+    strategy: Optional[PaginationStrategy] = None
+    page_param: Optional[str] = None
+    offset_param: Optional[str] = None
+    limit_param: Optional[str] = None
+    cursor_param: Optional[str] = None
+    cursor_path: Optional[str] = None
+    next_link_path: Optional[str] = None
+    page_size: Optional[int] = Field(None, ge=1)
+    max_pages: Optional[int] = Field(None, ge=1)
+
+
+class RetryConfig(BaseModel):
+    """HTTP retry configuration."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    max_attempts: int = Field(3, ge=1)
+    backoff_factor: float = Field(2.0, ge=0)
+    retry_on_status: List[int] = Field(
+        default_factory=lambda: [429, 500, 502, 503, 504]
+    )
+
+
+class TimeoutConfig(BaseModel):
+    """HTTP timeout configuration."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    connect: Optional[float] = Field(None, ge=0)
+    read: Optional[float] = Field(None, ge=0)
+    write: Optional[float] = Field(None, ge=0)
+
+
+class HTTPExtractConfig(BaseModel):
+    """HTTP extraction configuration."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    type: Literal["http"] = "http"
+    url: Optional[str] = None
+    base_url: Optional[str] = None
+    endpoint: Optional[str] = None
+    method: HTTPMethod = HTTPMethod.GET
+    headers: Optional[Dict[str, str]] = None
+    params: Optional[Dict[str, Any]] = None
+    body: Optional[Dict[str, Any]] = None
+    response_format: ResponseFormat = ResponseFormat.JSON
+    response_path: Optional[str] = None
+    pagination: Optional[PaginationConfig] = None
+    retry: Optional[RetryConfig] = None
+    timeout: Optional[TimeoutConfig] = None
+    rate_limit_delay: Optional[float] = Field(None, ge=0)
+    batch_size: int = Field(1000, ge=1)
+    validation: Optional[ExtractValidationConfig] = None
+
+
+class FileExtractConfig(BaseModel):
+    """File extraction configuration."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    type: Literal["file"] = "file"
+    path: str
+    format: Optional[FileFormat] = None
+    csv_options: Optional[CSVOptions] = None
+    json_options: Optional[JSONOptions] = None
+    batch_size: int = Field(1000, ge=1)
+    validation: Optional[ExtractValidationConfig] = None
+
+
+class DatabaseExtractConfig(BaseModel):
+    """Database extraction configuration."""
+
+    model_config = ConfigDict(extra="forbid", populate_by_name=True)
+
+    type: Literal["database"] = "database"
+    query: str
+    connection_string: Optional[str] = None
+    database: Optional[DatabaseConfig] = None
+    query_params: Optional[Dict[str, Any]] = Field(None, alias="params")
+    ssh_tunnel: Optional[SSHTunnelConfig] = None
+    batch_size: int = Field(1000, ge=1)
+    validation: Optional[ExtractValidationConfig] = None
+
+
+class CloudStorageExtractConfig(BaseModel):
+    """Cloud storage extraction configuration."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    type: Literal["cloud_storage"] = "cloud_storage"
+    storage: CloudStorageConfig
+    format: Optional[FileFormat] = None
+    batch_size: int = Field(1000, ge=1)
+    validation: Optional[ExtractValidationConfig] = None
+
+
+# Union type for extract config - use discriminator on 'type'
+ExtractConfig = Union[
+    HTTPExtractConfig,
+    FileExtractConfig,
+    DatabaseExtractConfig,
+    CloudStorageExtractConfig,
+]
+
+
+# =============================================================================
+# LOAD CONFIGS
+# =============================================================================
+
+
+class PostgresLoadConfig(BaseModel):
+    """PostgreSQL load configuration."""
+
+    model_config = ConfigDict(extra="forbid", populate_by_name=True)
+
+    type: Literal["postgres", "postgresql", "database"]
+    table: str
+    schema_name: Optional[str] = Field("public", alias="schema")
+    write_method: WriteMethod = WriteMethod.INSERT
+    primary_key: Optional[Union[str, List[str]]] = None
+    update_columns: Optional[List[str]] = None
+    connection_string: Optional[str] = None
+    database: Optional[DatabaseConfig] = None
+    ssh_tunnel: Optional[SSHTunnelConfig] = None
+    batch_size: int = Field(1000, ge=1)
+    validation: Optional[LoadValidationConfig] = None
+
+
+class SQLiteLoadConfig(BaseModel):
+    """SQLite load configuration."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    type: Literal["sqlite"] = "sqlite"
+    table: str
+    path: str
+    write_method: WriteMethod = WriteMethod.INSERT
+    primary_key: Optional[Union[str, List[str]]] = None
+    batch_size: int = Field(1000, ge=1)
+    validation: Optional[LoadValidationConfig] = None
+
+
+class FileLoadConfig(BaseModel):
+    """File load configuration."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    type: Literal["file"] = "file"
+    path: str
+    format: FileFormat = FileFormat.JSON
+    write_mode: WriteMode = WriteMode.OVERWRITE
+    csv_options: Optional[CSVOptions] = None
+    json_options: Optional[JSONOptions] = None
+    batch_size: int = Field(1000, ge=1)
+    validation: Optional[LoadValidationConfig] = None
+
+
+class CloudStorageLoadConfig(BaseModel):
+    """Cloud storage load configuration."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    type: Literal["cloud_storage"] = "cloud_storage"
+    storage: CloudStorageConfig
+    format: FileFormat = FileFormat.JSON
+    write_mode: WriteMode = WriteMode.OVERWRITE
+    partition_by: Optional[List[str]] = None
+    batch_size: int = Field(1000, ge=1)
+    validation: Optional[LoadValidationConfig] = None
+
+
+# Union type for load config
+LoadConfig = Union[
+    PostgresLoadConfig,
+    SQLiteLoadConfig,
+    FileLoadConfig,
+    CloudStorageLoadConfig,
+]
+
+
+# =============================================================================
+# TRANSFORM CONFIG
+# =============================================================================
+
+
+class FilterConfig(BaseModel):
+    """Filter operation configuration."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    expression: Optional[str] = None
+    field: Optional[str] = None
+    operator: Optional[FilterOperator] = None
+    value: Optional[Any] = None
+
+
+class MapConfig(BaseModel):
+    """Map (lookup) operation configuration."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    field: str
+    mapping: Dict[str, Any]
+    default: Optional[Any] = None
+
+
+class JSONataConfig(BaseModel):
+    """JSONata transformation configuration."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    expression: str
+    mode: JSONataMode = JSONataMode.RECORD
+
+
+class CustomFunctionConfig(BaseModel):
+    """Custom Python function configuration."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    module: str
+    function: str
+    kwargs: Optional[Dict[str, Any]] = None
+    mode: JSONataMode = JSONataMode.BATCH
+
+
+class SimpleOpsConfig(BaseModel):
+    """Simple transform operations (object format)."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    rename: Optional[Dict[str, str]] = None
+    convert: Optional[Dict[str, ConvertType]] = None
+    defaults: Optional[Dict[str, Any]] = None
+    add: Optional[Dict[str, Any]] = None
+    select: Optional[List[str]] = None
+    drop: Optional[List[str]] = None
+    filter: Optional[FilterConfig] = None
+
+
+class TransformConfig(BaseModel):
+    """Transform configuration (simple ops, JSONata, custom function)."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    # Simple ops: can be list (ordered) or dict (legacy)
+    transform: Optional[Union[SimpleOpsConfig, List[Dict[str, Any]]]] = None
+    # JSONata transformation
+    jsonata: Optional[JSONataConfig] = None
+    # Custom Python function
+    custom_function: Optional[CustomFunctionConfig] = None
+
+
+# =============================================================================
+# PIPELINE CONFIG (single-file format)
+# =============================================================================
+
+
+class PipelineConfig(BaseModel):
+    """Complete pipeline configuration (single-file format)."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    name: Optional[str] = None
+    version: Optional[str] = None
+    description: Optional[str] = None
+
+    # Core ETL steps (validated separately based on type)
+    extract: Dict[str, Any]
+    transform: Optional[Union[Dict[str, Any], List[Dict[str, Any]]]] = None
+    jsonata: Optional[JSONataConfig] = None
+    custom_function: Optional[CustomFunctionConfig] = None
+    load: Dict[str, Any]
+
+    # Inline settings (for single-file format)
+    metadata_store: Optional[MetadataStoreConfig] = None
+    dlq: Optional[DLQConfig] = None
+    contract: Optional[str] = None
+
+
+# =============================================================================
+# HELPER FUNCTIONS
+# =============================================================================
+
+
+def parse_extract_config(config: Dict[str, Any]) -> ExtractConfig:
+    """Parse extract config dict into typed model based on 'type' field."""
+    extract_type = config.get("type", "").lower()
+
+    if extract_type == "http":
+        return HTTPExtractConfig(**config)
+    elif extract_type == "file":
+        return FileExtractConfig(**config)
+    elif extract_type == "database":
+        return DatabaseExtractConfig(**config)
+    elif extract_type == "cloud_storage":
+        return CloudStorageExtractConfig(**config)
+    else:
+        raise ValueError(
+            f"Unknown extract type: '{extract_type}'. "
+            f"Supported types: http, file, database, cloud_storage"
+        )
+
+
+def parse_load_config(config: Dict[str, Any]) -> LoadConfig:
+    """Parse load config dict into typed model based on 'type' field."""
+    load_type = config.get("type", "").lower()
+
+    if load_type in ("postgres", "postgresql", "database"):
+        return PostgresLoadConfig(**config)
+    elif load_type == "sqlite":
+        return SQLiteLoadConfig(**config)
+    elif load_type == "file":
+        return FileLoadConfig(**config)
+    elif load_type == "cloud_storage":
+        return CloudStorageLoadConfig(**config)
+    else:
+        raise ValueError(
+            f"Unknown load type: '{load_type}'. "
+            f"Supported types: postgres, postgresql, database, sqlite, file, cloud_storage"
+        )
+
+
+def parse_transform_config(
+    config: Optional[Union[Dict[str, Any], List[Dict[str, Any]]]]
+) -> Optional[TransformConfig]:
+    """Parse transform config into typed model."""
+    if config is None:
+        return None
+
+    # Handle top-level transform that's just a list or simple ops dict
+    if isinstance(config, list):
+        return TransformConfig(transform=config)
+
+    return TransformConfig(**config)
+
+
+def parse_settings_config(config: Dict[str, Any]) -> SettingsConfig:
+    """Parse settings config into typed model."""
+    return SettingsConfig(**config)