pycharter 0.0.24__py3-none-any.whl → 0.0.26__py3-none-any.whl

pycharter/etl_generator/validation.py

@@ -0,0 +1,551 @@
+"""
+ETL Validation Module.
+
+Provides validation for ETL pipelines:
+- ETLValidator: Validates data against schemas/contracts with DLQ support
+- resolve_contract: Resolves contract references (file or database)
+- resolve_schema: Resolves schema file paths
+"""
+
+import logging
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Tuple, Union
+
+import yaml
+
+from pycharter.etl_generator.config_models import (
+    ContractRef,
+    DLQConfig,
+    DLQBackend,
+    ExtractValidationConfig,
+    LoadValidationConfig,
+    OnErrorAction,
+)
+from pycharter.etl_generator.dlq import DeadLetterQueue, DLQReason
+from pycharter.runtime_validator import Validator
+from pycharter.metadata_store import MetadataStoreClient
+
+logger = logging.getLogger(__name__)
+
+
+class ETLValidationError(Exception):
+    """Raised when validation fails with on_error=fail."""
+
+    def __init__(
+        self,
+        message: str,
+        invalid_count: int = 0,
+        errors: Optional[List[str]] = None,
+    ):
+        super().__init__(message)
+        self.invalid_count = invalid_count
+        self.errors = errors or []
+
+
+class ETLValidator:
+    """
+    Validates data in ETL pipelines with configurable error handling.
+
+    Supports:
+    - Source validation (after extract) against a schema file
+    - Target validation (before load) against a data contract
+    - Error handling modes: fail, warn, skip, quarantine
+    - DLQ integration for quarantining invalid records
+
+    Example:
+        >>> validator = ETLValidator(
+        ...     schema_or_contract={"type": "object", "properties": {...}},
+        ...     on_error=OnErrorAction.QUARANTINE,
+        ...     dlq=dlq_instance,
+        ...     pipeline_name="orders_pipeline",
+        ...     stage="extract",
+        ... )
+        >>> valid, invalid, error_count = await validator.validate(data)
+    """
+
+    def __init__(
+        self,
+        schema_or_contract: Dict[str, Any],
+        on_error: OnErrorAction = OnErrorAction.FAIL,
+        dlq: Optional[DeadLetterQueue] = None,
+        pipeline_name: str = "",
+        stage: str = "",
+    ):
+        """
+        Initialize the ETL validator.
+
+        Args:
+            schema_or_contract: JSON Schema dict or complete contract dict
+            on_error: Action on validation error (fail, warn, skip, quarantine)
+            dlq: Optional DeadLetterQueue instance for quarantine mode
+            pipeline_name: Name of the pipeline (for DLQ metadata)
+            stage: ETL stage (extract or load) for DLQ metadata
+        """
+        self.on_error = on_error
+        self.dlq = dlq
+        self.pipeline_name = pipeline_name
+        self.stage = stage
+
+        # Initialize the underlying Validator
+        # If it's a full contract (has 'schema' key), load it as contract_dict
+        # Otherwise treat it as a schema directly
+        if "schema" in schema_or_contract and isinstance(
+            schema_or_contract["schema"], dict
+        ):
+            schema = schema_or_contract["schema"]
+            # Ensure schema has version (required by Validator)
+            if "version" not in schema:
+                schema["version"] = "1.0.0"
+            self._validator = Validator(contract_dict=schema_or_contract)
+        else:
+            # It's a schema, wrap it in contract format
+            # Ensure schema has version (required by Validator)
+            if "version" not in schema_or_contract:
+                schema_or_contract = {**schema_or_contract, "version": "1.0.0"}
+            self._validator = Validator(contract_dict={"schema": schema_or_contract})
+
+    async def validate(
+        self,
+        data: List[Dict[str, Any]],
+        run_id: str = "",
+    ) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]], int]:
+        """
+        Validate a batch of data records.
+
+        Args:
+            data: List of data records to validate
+            run_id: Optional run identifier for DLQ metadata
+
+        Returns:
+            Tuple of (valid_records, quarantined_records, error_count)
+
+        Raises:
+            ETLValidationError: If on_error=fail and validation fails
+        """
+        if not data:
+            return [], [], 0
+
+        valid_records: List[Dict[str, Any]] = []
+        invalid_records: List[Dict[str, Any]] = []
+        all_errors: List[str] = []
+
+        # Validate each record
+        results = self._validator.validate_batch(data, strict=False)
+
+        for record, result in zip(data, results):
+            if result.is_valid:
+                # Return the original dict, not the Pydantic model
+                valid_records.append(record)
+            else:
+                invalid_records.append({
+                    "record": record,
+                    "errors": result.errors,
+                })
+                all_errors.extend(result.errors)
+
+        error_count = len(invalid_records)
+
+        # Handle based on on_error setting
+        if invalid_records:
+            if self.on_error == OnErrorAction.FAIL:
+                raise ETLValidationError(
+                    f"Validation failed: {error_count} record(s) invalid",
+                    invalid_count=error_count,
+                    errors=all_errors[:10],  # Limit errors in exception
+                )
+
+            elif self.on_error == OnErrorAction.WARN:
+                logger.warning(
+                    f"[{self.pipeline_name}] Validation warning: "
+                    f"{error_count} record(s) invalid in {self.stage} stage"
+                )
+                # Return all records (including invalid) for warn mode
+                return data, [], error_count
+
+            elif self.on_error == OnErrorAction.SKIP:
+                logger.info(
+                    f"[{self.pipeline_name}] Skipping {error_count} invalid record(s) "
+                    f"in {self.stage} stage"
+                )
+                # Return only valid records, no quarantine
+                return valid_records, [], error_count
+
+            elif self.on_error == OnErrorAction.QUARANTINE:
+                logger.info(
+                    f"[{self.pipeline_name}] Quarantining {error_count} invalid record(s) "
+                    f"in {self.stage} stage"
+                )
+                # Send to DLQ if configured
+                if self.dlq:
+                    await self._send_to_dlq(invalid_records, run_id)
+                # Return valid records, quarantined records
+                return (
+                    valid_records,
+                    [item["record"] for item in invalid_records],
+                    error_count,
+                )
+
+        return valid_records, [], 0
+
+    async def _send_to_dlq(
+        self,
+        invalid_records: List[Dict[str, Any]],
+        run_id: str = "",
+    ) -> None:
+        """Send invalid records to the Dead Letter Queue."""
+        reason = (
+            DLQReason.SCHEMA_MISMATCH
+            if self.stage == "extract"
+            else DLQReason.VALIDATION_ERROR
+        )
+
+        for item in invalid_records:
+            await self.dlq.add_record(
+                pipeline_name=self.pipeline_name,
+                record_data=item["record"],
+                reason=reason,
+                error_message="; ".join(item["errors"][:5]),  # Limit error message length
+                error_type="ValidationError",
+                stage=self.stage,
+                metadata={
+                    "run_id": run_id,
+                    "error_count": len(item["errors"]),
+                    "validation_errors": item["errors"][:10],  # Include detailed errors
+                },
+            )
+
+
+def resolve_schema(
+    schema_ref: str,
+    base_dir: Optional[Path] = None,
+) -> Dict[str, Any]:
+    """
+    Resolve a schema file path to a schema dict.
+
+    Args:
+        schema_ref: Path to schema file (absolute or relative)
+        base_dir: Base directory for relative paths
+
+    Returns:
+        Schema dict
+
+    Raises:
+        FileNotFoundError: If schema file not found
+        ValueError: If schema file is invalid
+    """
+    schema_path = Path(schema_ref)
+
+    # Resolve relative paths
+    if not schema_path.is_absolute() and base_dir:
+        schema_path = base_dir / schema_path
+
+    if not schema_path.exists():
+        raise FileNotFoundError(f"Schema file not found: {schema_path}")
+
+    with open(schema_path) as f:
+        schema = yaml.safe_load(f)
+
+    if not isinstance(schema, dict):
+        raise ValueError(f"Invalid schema file: {schema_path}")
+
+    return schema
+
+
+def resolve_contract(
+    contract_ref: Union[str, ContractRef, Dict[str, Any]],
+    metadata_store: Optional[MetadataStoreClient] = None,
+    base_dir: Optional[Path] = None,
+) -> Dict[str, Any]:
+    """
+    Resolve a contract reference to a complete contract dict.
+
+    Supports:
+    - String path: "contracts/orders/schema.yaml" (file path)
+    - ContractRef object: {"type": "database", "name": "orders", "version": "2.0"}
+    - Dict with 'name' key: {"name": "orders"} (uses metadata_store)
+    - Dict with 'type': file' and 'path' key
+
+    Args:
+        contract_ref: Contract reference (path, ContractRef, or dict)
+        metadata_store: Optional MetadataStoreClient for database contracts
+        base_dir: Base directory for relative file paths
+
+    Returns:
+        Complete contract dict with schema (and optionally coercion/validation rules)
+
+    Raises:
+        FileNotFoundError: If file-based contract not found
+        ValueError: If contract reference is invalid or store not provided
+    """
+    # Case 1: String path (file-based)
+    if isinstance(contract_ref, str):
+        return _load_contract_from_file(contract_ref, base_dir)
+
+    # Case 2: ContractRef Pydantic model
+    if isinstance(contract_ref, ContractRef):
+        if not metadata_store:
+            raise ValueError(
+                "metadata_store required for database contract reference"
+            )
+        return _load_contract_from_store(
+            metadata_store, contract_ref.name, contract_ref.version
+        )
+
+    # Case 3: Dict
+    if isinstance(contract_ref, dict):
+        ref_type = contract_ref.get("type", "file")
+
+        if ref_type == "file":
+            path = contract_ref.get("path")
+            if not path:
+                raise ValueError("Contract reference missing 'path' field")
+            return _load_contract_from_file(path, base_dir)
+
+        elif ref_type == "database":
+            name = contract_ref.get("name")
+            if not name:
+                raise ValueError("Contract reference missing 'name' field")
+            if not metadata_store:
+                raise ValueError(
+                    "metadata_store required for database contract reference"
+                )
+            return _load_contract_from_store(
+                metadata_store, name, contract_ref.get("version")
+            )
+
+        # Case 4: Dict with just 'name' (shorthand for database)
+        elif "name" in contract_ref:
+            if not metadata_store:
+                raise ValueError(
+                    "metadata_store required for contract name reference"
+                )
+            return _load_contract_from_store(
+                metadata_store,
+                contract_ref["name"],
+                contract_ref.get("version"),
+            )
+
+        else:
+            raise ValueError(f"Unknown contract reference type: {ref_type}")
+
+    raise ValueError(f"Invalid contract reference: {contract_ref}")
+
+
+def _load_contract_from_file(
+    path: str,
+    base_dir: Optional[Path] = None,
+) -> Dict[str, Any]:
+    """Load contract from file path."""
+    contract_path = Path(path)
+
+    # Resolve relative paths
+    if not contract_path.is_absolute() and base_dir:
+        contract_path = base_dir / contract_path
+
+    # Check if it's a directory (contract directory) or file
+    if contract_path.is_dir():
+        return _load_contract_from_directory(contract_path)
+    elif contract_path.exists():
+        return _load_contract_from_single_file(contract_path)
+    else:
+        # Try adding common extensions
+        for ext in (".yaml", ".yml", ".json"):
+            if contract_path.with_suffix(ext).exists():
+                return _load_contract_from_single_file(contract_path.with_suffix(ext))
+
+        raise FileNotFoundError(f"Contract not found: {contract_path}")
+
+
+def _load_contract_from_directory(directory: Path) -> Dict[str, Any]:
+    """Load contract from directory with separate files."""
+    contract: Dict[str, Any] = {}
+
+    # Load schema (required)
+    schema_path = directory / "schema.yaml"
+    if not schema_path.exists():
+        schema_path = directory / "schema.yml"
+    if not schema_path.exists():
+        raise FileNotFoundError(f"Schema not found in contract directory: {directory}")
+
+    with open(schema_path) as f:
+        contract["schema"] = yaml.safe_load(f)
+
+    # Load coercion rules (optional)
+    for name in ("coercion_rules.yaml", "coercion_rules.yml", "coercion.yaml"):
+        coercion_path = directory / name
+        if coercion_path.exists():
+            with open(coercion_path) as f:
+                contract["coercion_rules"] = yaml.safe_load(f)
+            break
+
+    # Load validation rules (optional)
+    for name in ("validation_rules.yaml", "validation_rules.yml", "validation.yaml"):
+        validation_path = directory / name
+        if validation_path.exists():
+            with open(validation_path) as f:
+                contract["validation_rules"] = yaml.safe_load(f)
+            break
+
+    return contract
+
+
+def _load_contract_from_single_file(path: Path) -> Dict[str, Any]:
+    """Load contract from single file."""
+    with open(path) as f:
+        content = yaml.safe_load(f)
+
+    if not isinstance(content, dict):
+        raise ValueError(f"Invalid contract file: {path}")
+
+    # If it has a 'schema' key, it's a complete contract
+    if "schema" in content:
+        return content
+
+    # Otherwise treat the whole file as a schema
+    return {"schema": content}
+
+
+def _load_contract_from_store(
+    store: MetadataStoreClient,
+    name: str,
+    version: Optional[str] = None,
+) -> Dict[str, Any]:
+    """Load contract from metadata store.
+    
+    Returns raw schema + separate rules. The Validator handles merging internally.
+    """
+    # Get raw schema (not merged)
+    schema = store.get_schema(name, version)
+
+    if not schema:
+        raise ValueError(f"Contract not found in store: {name} (version: {version})")
+
+    # Get coercion and validation rules separately
+    coercion_rules = store.get_coercion_rules(name, version)
+    validation_rules = store.get_validation_rules(name, version)
+
+    return {
+        "schema": schema,
+        "coercion_rules": coercion_rules or {},
+        "validation_rules": validation_rules or {},
+    }
+
+
+def create_dlq(
+    dlq_config: Optional[Union[bool, DLQConfig, Dict[str, Any]]],
+    default_config: Optional[DLQConfig] = None,
+    db_session: Optional[Any] = None,
+) -> Optional[DeadLetterQueue]:
+    """
+    Create a DeadLetterQueue from configuration.
+
+    Args:
+        dlq_config: DLQ configuration (bool, DLQConfig, or dict)
+        default_config: Default DLQ config from settings
+        db_session: Database session for database backend
+
+    Returns:
+        DeadLetterQueue instance or None if disabled
+    """
+    # Handle bool shorthand
+    if dlq_config is True:
+        # Use default config if available
+        if default_config:
+            dlq_config = default_config
+        else:
+            # Create minimal file-based DLQ
+            dlq_config = DLQConfig(enabled=True, backend=DLQBackend.FILE, path="./dlq")
+    elif dlq_config is False or dlq_config is None:
+        return None
+
+    # Convert dict to DLQConfig if needed
+    if isinstance(dlq_config, dict):
+        dlq_config = DLQConfig(**dlq_config)
+
+    if not dlq_config.enabled:
+        return None
+
+    # Create DLQ based on backend
+    return DeadLetterQueue(
+        db_session=db_session if dlq_config.backend == DLQBackend.DATABASE else None,
+        storage_backend=dlq_config.backend.value,
+        storage_path=dlq_config.path,
+        enabled=dlq_config.enabled,
+        schema_name=dlq_config.schema_name,
+    )
+
+
+def create_etl_validator(
+    validation_config: Union[ExtractValidationConfig, LoadValidationConfig, Dict[str, Any]],
+    pipeline_name: str,
+    stage: str,
+    metadata_store: Optional[MetadataStoreClient] = None,
+    default_contract: Optional[str] = None,
+    default_dlq_config: Optional[DLQConfig] = None,
+    base_dir: Optional[Path] = None,
+    db_session: Optional[Any] = None,
+) -> Optional[ETLValidator]:
+    """
+    Create an ETLValidator from validation configuration.
+
+    Args:
+        validation_config: Validation config (ExtractValidationConfig, LoadValidationConfig, or dict)
+        pipeline_name: Pipeline name for DLQ metadata
+        stage: ETL stage ('extract' or 'load')
+        metadata_store: Optional metadata store for database contracts
+        default_contract: Default contract name from settings
+        default_dlq_config: Default DLQ config from settings
+        base_dir: Base directory for relative file paths
+        db_session: Database session for DLQ database backend
+
+    Returns:
+        ETLValidator instance or None if validation not configured
+    """
+    if validation_config is None:
+        return None
+
+    # Convert dict to appropriate config type
+    if isinstance(validation_config, dict):
+        if stage == "extract":
+            validation_config = ExtractValidationConfig(**validation_config)
+        else:
+            validation_config = LoadValidationConfig(**validation_config)
+
+    # Resolve schema/contract
+    schema_or_contract: Optional[Dict[str, Any]] = None
+
+    if isinstance(validation_config, ExtractValidationConfig):
+        # Extract validation uses schema file
+        if validation_config.schema_path:
+            schema_or_contract = resolve_schema(validation_config.schema_path, base_dir)
+        else:
+            logger.warning(f"Extract validation config missing 'schema' field")
+            return None
+
+    elif isinstance(validation_config, LoadValidationConfig):
+        # Load validation uses contract
+        if validation_config.contract:
+            schema_or_contract = resolve_contract(
+                validation_config.contract, metadata_store, base_dir
+            )
+        elif validation_config.use_contract and default_contract:
+            schema_or_contract = resolve_contract(
+                {"name": default_contract}, metadata_store, base_dir
+            )
+        else:
+            logger.warning(f"Load validation config missing 'contract' field")
+            return None
+
+    if schema_or_contract is None:
+        return None
+
+    # Create DLQ if configured
+    dlq = create_dlq(validation_config.dlq, default_dlq_config, db_session)
+
+    return ETLValidator(
+        schema_or_contract=schema_or_contract,
+        on_error=validation_config.on_error,
+        dlq=dlq,
+        pipeline_name=pipeline_name,
+        stage=stage,
+    )