pycharter 0.0.22__py3-none-any.whl → 0.0.24__py3-none-any.whl

pycharter/etl_generator/orchestrator.py

@@ -1,1650 +0,0 @@
-"""
-ETL Orchestrator - Streaming ETL pipeline with simple operations, JSONata, and custom functions.
-
-Executes ETL pipelines: Extract → Transform (Simple Operations → JSONata → Custom Functions) → Load.
-
-Transformation Pipeline:
-1. Simple Operations: rename, convert, defaults, add, select, drop (declarative, easy to use)
-2. JSONata: Powerful query language for complex transformations (full JSONata support)
-3. Custom Functions: Import and run external Python modules/functions
-"""
-
-import asyncio
-import gc
-import importlib
-import logging
-import re
-import uuid
-import warnings
-from collections import Counter, defaultdict
-from datetime import datetime
-from pathlib import Path
-from typing import Any, AsyncIterator, Callable, Dict, List, Optional, Tuple
-
-import jsonata
-import yaml
-
-from pycharter.contract_parser import ContractMetadata, parse_contract_file
-from pycharter.etl_generator.checkpoint import CheckpointManager
-from pycharter.etl_generator.database import get_database_connection, load_data
-from pycharter.etl_generator.dlq import DeadLetterQueue, DLQReason
-from pycharter.etl_generator.extraction import extract_with_pagination_streaming
-from pycharter.etl_generator.progress import ETLProgress, ProgressTracker
-from pycharter.utils.value_injector import resolve_values
-
-logger = logging.getLogger(__name__)
-
-# Optional memory monitoring
-try:
-    import psutil
-    PSUTIL_AVAILABLE = True
-except ImportError:
-    PSUTIL_AVAILABLE = False
-
-DEFAULT_BATCH_SIZE = 1000
-
-
-class ETLOrchestrator:
-    """
-    Generic ETL Orchestrator that executes pipelines from contract artifacts and ETL configs.
-    
-    Processes data in streaming mode: Extract-Batch → Transform-Batch → Load-Batch.
-    This ensures constant memory usage regardless of dataset size.
-    
-    Example:
-        >>> from pycharter.etl_generator import ETLOrchestrator
-        >>> orchestrator = ETLOrchestrator(contract_dir="data/examples/my_contract")
-        >>> await orchestrator.run()
-    """
-    
-    def __init__(
-        self,
-        contract_dir: Optional[str] = None,
-        contract_file: Optional[str] = None,
-        contract_dict: Optional[Dict[str, Any]] = None,
-        contract_metadata: Optional[ContractMetadata] = None,
-        checkpoint_dir: Optional[str] = None,
-        progress_callback: Optional[Callable[[ETLProgress], None]] = None,
-        verbose: bool = True,
-        max_memory_mb: Optional[int] = None,
-        config_context: Optional[Dict[str, Any]] = None,
-        # ETL config options (alternative to loading from contract_dir)
-        extract_config: Optional[Dict[str, Any]] = None,
-        transform_config: Optional[Dict[str, Any]] = None,
-        load_config: Optional[Dict[str, Any]] = None,
-        extract_file: Optional[str] = None,
-        transform_file: Optional[str] = None,
-        load_file: Optional[str] = None,
-    ):
-        """
-        Initialize the ETL orchestrator with contract artifacts.
-        
-        Args:
-            contract_dir: Directory containing contract files and ETL configs
-            contract_file: Path to complete contract file (YAML/JSON)
-            contract_dict: Contract as dictionary
-            contract_metadata: ContractMetadata object (from parse_contract)
-            checkpoint_dir: Directory for checkpoint files (None = disabled)
-            progress_callback: Optional callback for progress updates
-            verbose: If True, print progress to stdout
-            max_memory_mb: Maximum memory usage in MB (None = no limit)
-            config_context: Optional context dictionary for value injection.
-                          Values in this dict have highest priority when resolving
-                          variables in config files (e.g., ${VAR}).
-                          Useful for injecting application-level settings.
-            extract_config: Optional extract configuration as dictionary.
-                           If provided, overrides extract.yaml from contract_dir.
-            transform_config: Optional transform configuration as dictionary.
-                            If provided, overrides transform.yaml from contract_dir.
-            load_config: Optional load configuration as dictionary.
-                        If provided, overrides load.yaml from contract_dir.
-            extract_file: Optional path to extract.yaml file.
-                         If provided, overrides extract.yaml from contract_dir.
-            transform_file: Optional path to transform.yaml file.
-                           If provided, overrides transform.yaml from contract_dir.
-            load_file: Optional path to load.yaml file.
-                      If provided, overrides load.yaml from contract_dir.
-        
-        Note:
-            ETL config priority: direct dict > file path > contract_dir
-            If contract_dir is not provided, you must provide extract_config/transform_config/load_config
-            or extract_file/transform_file/load_file.
-        
-        Note:
-            Tables must be created manually or via migrations (e.g., Alembic).
-            PyCharter no longer creates tables from schema.json.
-        """
-        self.contract_dir: Optional[Path] = None
-        self.schema: Optional[Dict[str, Any]] = None
-        self.coercion_rules: Dict[str, Any] = {}
-        self.validation_rules: Dict[str, Any] = {}
-        self.input_params: Dict[str, Dict[str, Any]] = {}
-        
-        # Configuration context for value injection
-        self.config_context = config_context or {}
-        
-        # Store ETL config parameters for later loading
-        self._extract_config_param = extract_config
-        self._transform_config_param = transform_config
-        self._load_config_param = load_config
-        self._extract_file_param = extract_file
-        self._transform_file_param = transform_file
-        self._load_file_param = load_file
-        
-        # Enhanced features
-        self.checkpoint_manager = CheckpointManager(checkpoint_dir)
-        self.progress_tracker = ProgressTracker(progress_callback, verbose)
-        self.max_memory_mb = max_memory_mb
-        self.process = None
-        if PSUTIL_AVAILABLE:
-            self.process = psutil.Process()
-        
-        # Logging context
-        self.run_id: Optional[str] = None  # Correlation ID for this run
-        self._current_stage: Optional[str] = None  # Current pipeline stage
-        
-        # Load contract artifacts
-        if contract_metadata:
-            self._load_from_metadata(contract_metadata)
-        elif contract_dict:
-            self._load_from_dict(contract_dict)
-        elif contract_file:
-            file_path = Path(contract_file)
-            self.contract_dir = file_path.parent
-            self._load_from_file(file_path)
-        elif contract_dir:
-            self.contract_dir = Path(contract_dir)
-            self._load_from_directory(self.contract_dir)
-        else:
-            # If no contract source provided, we still need contract_dir for ETL configs
-            # unless all ETL configs are provided directly
-            if not (extract_config or extract_file) and not contract_dir:
-                raise ValueError(
-                    "Must provide one of: contract_dir, contract_file, contract_dict, "
-                    "contract_metadata, or extract_config/extract_file"
-                )
-            # Set contract_dir to None if not provided (ETL configs will be loaded from params)
-            self.contract_dir = None
-        
-        # Load ETL configurations (extract, transform, load)
-        # Priority: direct dict > file path > contract_dir
-        self._load_etl_configs()
-    
-    # ============================================================================
-    # INITIALIZATION AND CONFIGURATION LOADING
-    # ============================================================================
-    
-    def _load_from_metadata(self, metadata: ContractMetadata) -> None:
-        """Load contract from ContractMetadata object."""
-        self.schema = metadata.schema
-        self.coercion_rules = metadata.coercion_rules or {}
-        self.validation_rules = metadata.validation_rules or {}
-    
-    def _load_from_dict(self, contract: Dict[str, Any]) -> None:
-        """Load contract from dictionary."""
-        self.schema = contract.get("schema")
-        if not self.schema:
-            raise ValueError("Contract dictionary must contain 'schema'")
-        
-        self.coercion_rules = self._extract_rules(contract.get("coercion_rules", {}))
-        self.validation_rules = self._extract_rules(contract.get("validation_rules", {}))
-    
-    @staticmethod
-    def _extract_rules(rules_data: Any) -> Dict[str, Any]:
-        """Extract rules from various formats."""
-        if not isinstance(rules_data, dict):
-            return {}
-        
-        if "rules" in rules_data:
-            return rules_data["rules"]
-        elif not any(k in rules_data for k in ["version", "description", "title"]):
-            return rules_data
-        else:
-            return {}
-    
-    def _load_from_file(self, file_path: Path) -> None:
-        """Load contract from file."""
-        contract_metadata = parse_contract_file(str(file_path))
-        self._load_from_metadata(contract_metadata)
-    
-    def _load_from_directory(self, contract_dir: Path) -> None:
-        """Load contract components from directory."""
-        if not contract_dir.exists():
-            raise ValueError(f"Contract directory not found: {contract_dir}")
-        
-        # Load schema (required) - support both YAML and JSON
-        schema_path_yaml = contract_dir / "schema.yaml"
-        schema_path_json = contract_dir / "schema.json"
-        
-        schema_path = None
-        if schema_path_yaml.exists():
-            schema_path = schema_path_yaml
-        elif schema_path_json.exists():
-            schema_path = schema_path_json
-        else:
-            # Try to find JSON schema files with dataset name pattern
-            dataset_name = contract_dir.name
-            possible_json_schemas = [
-                contract_dir / f"{dataset_name}_schema.json",
-                contract_dir / f"{dataset_name}.schema.json",
-                contract_dir / "schema.json",
-            ]
-            for possible_path in possible_json_schemas:
-                if possible_path.exists():
-                    schema_path = possible_path
-                    break
-        
-        if schema_path and schema_path.exists():
-            if schema_path.suffix == '.json':
-                import json
-                with open(schema_path, 'r', encoding='utf-8') as f:
-                    self.schema = json.load(f)
-            else:
-                self.schema = self._load_yaml(schema_path)
-        else:
-            raise ValueError(
-                f"Schema file not found in {contract_dir}. "
-                f"Expected: schema.yaml, schema.json, or {contract_dir.name}_schema.json"
-            )
-        
-        # Load coercion rules (optional)
-        coercion_path = contract_dir / "coercion_rules.yaml"
-        if coercion_path.exists():
-            coercion_data = self._load_yaml(coercion_path)
-            self.coercion_rules = self._extract_rules(coercion_data)
-        
-        # Load validation rules (optional)
-        validation_path = contract_dir / "validation_rules.yaml"
-        if validation_path.exists():
-            validation_data = self._load_yaml(validation_path)
-            self.validation_rules = self._extract_rules(validation_data)
-    
-    def _load_etl_configs(self) -> None:
-        """
-        Load ETL configuration files (extract, transform, load).
-        
-        Priority order:
-        1. Direct dictionary parameters (extract_config, transform_config, load_config)
-        2. File path parameters (extract_file, transform_file, load_file)
-        3. Files in contract_dir (extract.yaml, transform.yaml, load.yaml)
-        """
-        # Load extract config (required)
-        self.extract_config = self._load_single_config(
-            config_param=self._extract_config_param,
-            file_param=self._extract_file_param,
-            default_filename="extract.yaml",
-            required=True,
-            config_name="Extract"
-        )
-        
-        # Load transform config (optional)
-        self.transform_config = self._load_single_config(
-            config_param=self._transform_config_param,
-            file_param=self._transform_file_param,
-            default_filename="transform.yaml",
-            required=False,
-            config_name="Transform"
-        )
-        
-        # Load load config (required)
-        self.load_config = self._load_single_config(
-            config_param=self._load_config_param,
-            file_param=self._load_file_param,
-            default_filename="load.yaml",
-            required=True,
-            config_name="Load"
-        )
-        
-        # Parse input parameters from extract config
-        self._parse_input_params()
-        
-        if not self.schema:
-            raise ValueError("Schema not loaded")
-        
-        # Initialize Dead Letter Queue (will be configured with session in run() method)
-        self.dlq: Optional[DeadLetterQueue] = None
-    
-    def _load_single_config(
-        self,
-        config_param: Optional[Dict[str, Any]],
-        file_param: Optional[str],
-        default_filename: str,
-        required: bool,
-        config_name: str,
-    ) -> Dict[str, Any]:
-        """
-        Load a single ETL config following priority order.
-        
-        Args:
-            config_param: Direct dictionary config (highest priority)
-            file_param: File path to config (medium priority)
-            default_filename: Default filename in contract_dir (lowest priority)
-            required: Whether this config is required
-            config_name: Name for error messages
-            
-        Returns:
-            Loaded config dictionary (empty dict if not required and not found)
-        """
-        # Priority 1: Direct dictionary
-        if config_param is not None:
-            return config_param
-        
-        # Priority 2: File path
-        if file_param:
-            config_path = Path(file_param)
-            if not config_path.exists():
-                raise ValueError(f"{config_name} config file not found: {config_path}")
-            config = self._load_yaml(config_path)
-            # Set contract_dir from file if not already set
-            if not self.contract_dir:
-                self.contract_dir = config_path.parent
-            return config
-        
-        # Priority 3: From contract_dir
-        if self.contract_dir and self.contract_dir.exists():
-            config_path = self.contract_dir / default_filename
-            if config_path.exists():
-                return self._load_yaml(config_path)
-        
-        # Handle missing config
-        if required:
-            raise ValueError(
-                f"{config_name} configuration not found. Provide one of: "
-                f"{config_name.lower()}_config (dict), {config_name.lower()}_file (path), "
-                f"or contract_dir with {default_filename}"
-            )
-        
-        return {}
-    
-    def _parse_input_params(self) -> None:
-        """Parse input parameters from extract config."""
-        input_params_config = self.extract_config.get('input_params', [])
-        if isinstance(input_params_config, list):
-            self.input_params = {name: {} for name in input_params_config}
-        elif isinstance(input_params_config, dict):
-            self.input_params = input_params_config
-        else:
-            self.input_params = {}
-    
-    def _load_yaml(self, file_path: Path) -> Dict[str, Any]:
-        """Load YAML file, return empty dict if not found."""
-        if not file_path.exists():
-            return {}
-        with open(file_path, 'r', encoding='utf-8') as f:
-            return yaml.safe_load(f) or {}
-    
-    def _prepare_params(self, **kwargs) -> Tuple[Dict[str, Any], Dict[str, Any]]:
-        """Prepare params and headers from config and kwargs."""
-        params = self.extract_config.get('params', {}).copy()
-        headers = self.extract_config.get('headers', {})
-        
-        # Get parameter mapping from extract config (maps input param names to API param names)
-        param_mapping = self.extract_config.get('param_mapping', {})
-        
-        # Merge input arguments
-        for param_name, param_value in kwargs.items():
-            if param_name in self.input_params:
-                # Check if there's a mapping for this parameter
-                api_param_name = param_mapping.get(param_name, param_name)
-                params[api_param_name] = param_value
-            else:
-                warnings.warn(
-                    f"Unknown input parameter '{param_name}'. "
-                    f"Available: {list(self.input_params.keys())}",
-                    UserWarning
-                )
-        
-        # Validate required input parameters and apply defaults for optional ones
-        for param_name, param_meta in self.input_params.items():
-            if param_meta.get('required', False):
-                # Check if input parameter was provided in kwargs
-                if param_name not in kwargs:
-                    raise ValueError(
-                        f"Required input parameter '{param_name}' not provided. "
-                        f"Please provide: {param_name}=value"
-                    )
-            else:
-                # Apply default value for optional parameters if not provided
-                if param_name not in kwargs and 'default' in param_meta:
-                    default_value = param_meta.get('default')
-                    # Only add if default is not None (None means truly optional)
-                    if default_value is not None:
-                        api_param_name = param_mapping.get(param_name, param_name)
-                        params[api_param_name] = default_value
-        
-        # Resolve values with config context
-        source_file = str(self.contract_dir / "extract.yaml") if self.contract_dir else None
-        params = resolve_values(params, context=self.config_context, source_file=source_file)
-        headers = resolve_values(headers, context=self.config_context, source_file=source_file)
-        
-        return params, headers
-    
-    # ============================================================================
-    # EXTRACTION
-    # ============================================================================
-    
-    async def extract(
-        self,
-        batch_size: Optional[int] = None,
-        max_records: Optional[int] = None,
-        **kwargs
-    ) -> AsyncIterator[List[Dict[str, Any]]]:
-        """
-        Extract data in batches using async generator.
-        
-        Yields batches of records for memory-efficient processing.
-        
-        Args:
-            batch_size: Number of records per batch (defaults to extract.yaml config)
-            max_records: Maximum total records to extract (None = all)
-            **kwargs: Input parameters defined in extract.yaml's input_params section
-        
-        Yields:
-            Batches of extracted records (lists of dictionaries)
-        
-        Example:
-            >>> async for batch in orchestrator.extract(symbol="AAPL"):
-            ...     print(f"Extracted {len(batch)} records")
-        """
-        self._current_stage = 'extract'
-        if batch_size is None:
-            batch_size = self.extract_config.get('batch_size', DEFAULT_BATCH_SIZE)
-        
-        params, headers = self._prepare_params(**kwargs)
-        
-        async for batch in extract_with_pagination_streaming(
-            self.extract_config, params, headers, self.contract_dir, batch_size, max_records, config_context=self.config_context
-        ):
-            yield batch
-    
-    # ============================================================================
-    # TRANSFORMATION (Simple Operations → JSONata → Custom Functions)
-    # ============================================================================
-    
-    def transform(self, raw_data: List[Dict[str, Any]], **kwargs) -> List[Dict[str, Any]]:
-        """
-        Transform data using simple operations, JSONata expressions, and/or custom Python functions.
-        
-        Pipeline order (applied sequentially):
-        1. Simple operations (rename, select, drop, convert, defaults, add)
-        2. JSONata transformation (if configured)
-        3. Custom function execution (if configured)
-        
-        Args:
-            raw_data: Raw data from extraction
-            **kwargs: Additional parameters (passed to custom functions)
-        
-        Returns:
-            Transformed data
-        
-        Example - Simple operations:
-            transform_config:
-              rename:
-                oldName: new_name
-                camelCase: snake_case
-              select:
-                - field1
-                - field2
-              convert:
-                price: float
-                quantity: integer
-              defaults:
-                status: "pending"
-        
-        Example - JSONata (advanced):
-            transform_config:
-              jsonata:
-                expression: |
-                  $.{
-                    "ticker": symbol,
-                    "avg_price": $average(prices)
-                  }
-        
-        Example - Custom function:
-            transform_config:
-              custom_function:
-                module: "myproject.transforms"
-                function: "optimize_data"
-                mode: "batch"
-        """
-        if not self.transform_config:
-            return raw_data
-        
-        data = raw_data
-        
-        # Step 1: Apply simple operations (in order)
-        # Support both new 'transform' key and legacy top-level keys for backward compatibility
-        simple_ops = {}
-        
-        # New format: transform: { rename: {...}, select: [...] }
-        if 'transform' in self.transform_config:
-            simple_ops = self.transform_config.get('transform', {})
-        
-        # Legacy format: rename: {...} at top level (for backward compatibility)
-        if 'rename' in self.transform_config and 'transform' not in self.transform_config:
-            simple_ops['rename'] = self.transform_config.get('rename')
-        if 'select' in self.transform_config and 'transform' not in self.transform_config:
-            simple_ops['select'] = self.transform_config.get('select')
-        if 'drop' in self.transform_config and 'transform' not in self.transform_config:
-            simple_ops['drop'] = self.transform_config.get('drop')
-        if 'convert' in self.transform_config and 'transform' not in self.transform_config:
-            simple_ops['convert'] = self.transform_config.get('convert')
-        if 'defaults' in self.transform_config and 'transform' not in self.transform_config:
-            simple_ops['defaults'] = self.transform_config.get('defaults')
-        if 'add' in self.transform_config and 'transform' not in self.transform_config:
-            simple_ops['add'] = self.transform_config.get('add')
-        
-        if simple_ops:
-            data = self._apply_simple_operations(data, simple_ops)
-        
-        # Step 2: Apply JSONata transformation (if configured)
-        jsonata_config = self.transform_config.get('jsonata')
-        if jsonata_config:
-            data = self._apply_jsonata(data, jsonata_config)
-        
-        # Step 3: Apply custom function (if configured)
-        custom_func_config = self.transform_config.get('custom_function')
-        if custom_func_config:
-            data = self._apply_custom_function(data, custom_func_config, **kwargs)
-        
-        return data
-    
-    def _apply_simple_operations(
-        self,
-        data: List[Dict[str, Any]],
-        config: Dict[str, Any]
-    ) -> List[Dict[str, Any]]:
-        """
-        Apply simple declarative transformation operations.
-        
-        Operations are applied in this order:
-        1. rename - Rename fields (old_name: new_name)
-        2. convert - Convert field types (field: type)
-        3. defaults - Set default values for missing fields
-        4. add - Add computed fields with expressions
-        5. select - Keep only specified fields
-        6. drop - Remove specified fields
-        
-        Args:
-            data: Input data (list of records)
-            config: Simple operations configuration
-        
-        Returns:
-            Transformed data
-        
-        Example config:
-            transform:
-              rename:
-                oldName: new_name
-                camelCase: snake_case
-              convert:
-                price: float
-                quantity: integer
-                active: boolean
-              defaults:
-                status: "pending"
-                priority: 0
-              add:
-                full_name: "${first_name} ${last_name}"
-                created_at: "now()"
-                record_id: "uuid()"
-              select:
-                - field1
-                - field2
-              drop:
-                - internal_id
-                - debug_info
-        """
-        if not data:
-            return data
-        
-        result = []
-        
-        # Get available fields from first record for validation
-        available_fields = set(data[0].keys()) if data else set()
-        
-        # Step 1: Rename fields
-        rename_map = config.get('rename', {})
-        if rename_map:
-            # Validate rename mappings
-            missing_fields = [old for old in rename_map.keys() if old not in available_fields]
-            if missing_fields:
-                logger.warning(
-                    f"Rename operation: Fields not found in data: {missing_fields}. "
-                    f"Available fields: {sorted(available_fields)}"
-                )
-        
-        # Step 2: Convert types
-        convert_map = config.get('convert', {})
-        
-        # Step 3: Defaults
-        defaults_map = config.get('defaults', {})
-        
-        # Step 4: Add computed fields
-        add_map = config.get('add', {})
-        
-        # Step 5: Select fields (keep only these)
-        select_fields = config.get('select')
-        
-        # Step 6: Drop fields (remove these)
-        drop_fields = set(config.get('drop', []))
-        
-        for record in data:
-            transformed = dict(record)
-            
-            # 1. Rename
-            if rename_map:
-                for old_name, new_name in rename_map.items():
-                    if old_name in transformed:
-                        transformed[new_name] = transformed.pop(old_name)
-            
-            # 2. Convert types
-            if convert_map:
-                for field_name, target_type in convert_map.items():
-                    if field_name in transformed:
-                        try:
-                            transformed[field_name] = self._convert_type(
-                                transformed[field_name], target_type
-                            )
-                        except (ValueError, TypeError) as e:
-                            logger.warning(
-                                f"Failed to convert field '{field_name}' to {target_type}: {e}. "
-                                f"Keeping original value."
-                            )
-            
-            # 3. Apply defaults
-            if defaults_map:
-                for field_name, default_value in defaults_map.items():
-                    if field_name not in transformed or transformed[field_name] is None:
-                        transformed[field_name] = default_value
-            
-            # 4. Add computed fields
-            if add_map:
-                for field_name, expression in add_map.items():
-                    try:
-                        transformed[field_name] = self._evaluate_expression(
-                            expression, transformed
-                        )
-                    except Exception as e:
-                        logger.warning(
-                            f"Failed to compute field '{field_name}': {e}. "
-                            f"Skipping this field."
-                        )
-            
-            # 5. Select (keep only specified fields)
-            if select_fields:
-                transformed = {
-                    k: v for k, v in transformed.items()
-                    if k in select_fields
-                }
-            
-            # 6. Drop (remove specified fields)
-            if drop_fields:
-                transformed = {
-                    k: v for k, v in transformed.items()
-                    if k not in drop_fields
-                }
-            
-            result.append(transformed)
-        
-        return result
-    
-    def _convert_type(self, value: Any, target_type: str) -> Any:
-        """
-        Convert a value to the specified type.
-        
-        Args:
-            value: Value to convert
-            target_type: Target type (string, integer, float, boolean, datetime, date)
-        
-        Returns:
-            Converted value
-        """
-        if value is None:
-            return None
-        
-        target_type_lower = target_type.lower().strip()
-        
-        if target_type_lower in ('str', 'string'):
-            return str(value)
-        elif target_type_lower in ('int', 'integer'):
-            if isinstance(value, str):
-                # Try to parse as float first (handles "1.0" -> 1)
-                try:
-                    return int(float(value))
-                except ValueError:
-                    return int(value)
-            return int(value)
-        elif target_type_lower in ('float', 'number', 'numeric'):
-            if isinstance(value, str):
-                return float(value)
-            return float(value)
-        elif target_type_lower in ('bool', 'boolean'):
-            if isinstance(value, str):
-                return value.lower() in ('true', '1', 'yes', 'on')
-            return bool(value)
-        elif target_type_lower == 'datetime':
-            from datetime import datetime
-            if isinstance(value, str):
-                # Try common datetime formats
-                for fmt in [
-                    '%Y-%m-%dT%H:%M:%S',
-                    '%Y-%m-%dT%H:%M:%S.%f',
-                    '%Y-%m-%dT%H:%M:%SZ',
-                    '%Y-%m-%dT%H:%M:%S.%fZ',
-                    '%Y-%m-%d %H:%M:%S',
-                    '%Y-%m-%d %H:%M:%S.%f',
-                ]:
-                    try:
-                        return datetime.strptime(value, fmt)
-                    except ValueError:
-                        continue
-                raise ValueError(f"Cannot parse datetime: {value}")
-            return value
-        elif target_type_lower == 'date':
-            from datetime import date, datetime
-            if isinstance(value, str):
-                # Try common date formats
-                for fmt in ['%Y-%m-%d', '%Y/%m/%d', '%m/%d/%Y']:
-                    try:
-                        dt = datetime.strptime(value, fmt)
-                        return dt.date()
-                    except ValueError:
-                        continue
-                raise ValueError(f"Cannot parse date: {value}")
-            elif isinstance(value, datetime):
-                return value.date()
-            return value
-        else:
-            raise ValueError(f"Unsupported target type: {target_type}")
-    
-    def _evaluate_expression(self, expression: str, record: Dict[str, Any]) -> Any:
-        """
-        Evaluate a simple expression in the context of a record.
-        
-        Supports:
-        - Field references: "${field_name}"
-        - String concatenation: "${field1} ${field2}"
-        - Simple functions: "now()", "uuid()"
-        - Literal values (if no placeholders)
-        
-        Args:
-            expression: Expression string
-            record: Record dictionary for context
-        
-        Returns:
-            Evaluated result
-        
-        Examples:
-            "${first_name} ${last_name}" -> "John Doe"
-            "now()" -> "2024-01-01T12:00:00"
-            "uuid()" -> "123e4567-e89b-12d3-a456-426614174000"
-            "static_value" -> "static_value"
-        """
-        if not isinstance(expression, str):
-            return expression
-        
-        expression = expression.strip()
-        
-        # Handle special functions
-        if expression == 'now()':
-            return datetime.now().isoformat()
-        elif expression == 'uuid()':
-            return str(uuid.uuid4())
-        
-        # Handle field references and string interpolation
-        try:
-            # Simple string interpolation: "${field1} ${field2}"
-            result = expression
-            placeholders_found = False
-            
-            # Find all ${...} placeholders
-            placeholder_pattern = r'\$\{([^}]+)\}'
-            matches = re.findall(placeholder_pattern, expression)
-            
-            if matches:
-                placeholders_found = True
-                for field_name in matches:
-                    if field_name in record:
-                        value = record[field_name]
-                        placeholder = f"${{{field_name}}}"
-                        result = result.replace(placeholder, str(value) if value is not None else '')
-                    else:
-                        logger.warning(
-                            f"Expression '{expression}': Field '{field_name}' not found in record. "
-                            f"Available fields: {sorted(record.keys())}"
-                        )
-                        # Replace with empty string if field not found
-                        placeholder = f"${{{field_name}}}"
-                        result = result.replace(placeholder, '')
-            
-            # If no placeholders were found and it's not a function, return as literal
-            if not placeholders_found and not expression.endswith('()'):
-                return expression
-            
-            return result
-        except Exception as e:
-            raise ValueError(f"Failed to evaluate expression '{expression}': {e}") from e
-    
-    def _apply_jsonata(
-        self,
-        data: List[Dict[str, Any]],
-        config: Dict[str, Any]
-    ) -> List[Dict[str, Any]]:
-        """
-        Apply JSONata expression to transform data.
-        
-        Args:
-            data: Input data (list of records)
-            config: JSONata configuration with 'expression' and optional 'mode'
-        
-        Returns:
-            Transformed data
-        
-        Example config:
-            jsonata:
-              expression: |
-                $.{
-                  "ticker": symbol,
-                  "avg_price": $average(prices),
-                  "total_volume": $sum(volumes)
-                }
-              mode: "batch"  # or "record"
-        """
-        expression_str = config.get('expression')
-        if not expression_str:
-            return data
-        
-        mode = config.get('mode', 'batch')
-        
-        try:
-            expr = jsonata.Jsonata(expression_str)
-            
-            if mode == 'batch':
-                # Apply expression to entire dataset
-                result = expr.evaluate(data)
-                if result is None:
-                    return []
-                return result if isinstance(result, list) else [result]
-            else:
-                # Apply expression to each record individually
-                return [expr.evaluate(record) for record in data if expr.evaluate(record) is not None]
-                
-        except Exception as e:
-            logger.error(f"JSONata transformation failed: {e}")
-            raise ValueError(f"JSONata transformation error: {e}") from e
-    
-    def _apply_custom_function(
-        self,
-        data: List[Dict[str, Any]],
-        config: Dict[str, Any],
-        **kwargs
-    ) -> List[Dict[str, Any]]:
-        """
-        Execute a custom Python function for transformation.
-        
-        Args:
-            data: Input data
-            config: Custom function configuration
-            **kwargs: Additional parameters passed to the function
-        
-        Returns:
-            Transformed data
-        
-        Example config:
-            custom_function:
-              module: "pyoptima"
-              function: "optimize_from_etl_inputs"
-              mode: "batch"
-              kwargs:
-                method: "min_volatility"
-                solver: "ipopt"
-        
-        Alternative config (using callable path):
-            custom_function:
-              callable: "myproject.transforms.optimize_portfolio"
-              mode: "batch"
-        """
-        # Get module and function
-        callable_path = config.get('callable')
-        module_path = config.get('module')
-        func_name = config.get('function')
-        
-        if callable_path:
-            # Parse "module.submodule.function" format
-            parts = callable_path.rsplit('.', 1)
-            if len(parts) != 2:
-                raise ValueError(f"Invalid callable path: {callable_path}. Use 'module.function' format.")
-            module_path, func_name = parts
-        
-        if not module_path or not func_name:
-            raise ValueError("custom_function requires either 'callable' or 'module' + 'function'")
-        
-        # Dynamic import
-        try:
-            module = importlib.import_module(module_path)
-            func = getattr(module, func_name)
-        except ImportError as e:
-            raise ValueError(f"Cannot import module '{module_path}': {e}") from e
-        except AttributeError as e:
-            raise ValueError(f"Function '{func_name}' not found in module '{module_path}'") from e
-        
-        # Handle class-based methods (e.g., pyoptima optimization methods)
-        if isinstance(func, type):
-            instance = func()
-            if hasattr(instance, 'optimize'):
-                func = instance.optimize
-            elif hasattr(instance, 'run'):
-                func = instance.run
-            elif hasattr(instance, '__call__'):
-                func = instance
-            else:
-                raise ValueError(f"Class '{func_name}' has no 'optimize', 'run', or '__call__' method")
-        
-        # Get mode and kwargs
-        mode = config.get('mode', 'batch')
-        func_kwargs = config.get('kwargs', {})
-        
-        # Merge with runtime kwargs
-        merged_kwargs = {**func_kwargs, **kwargs}
-        
-        try:
-            if mode == 'batch':
-                result = func(data, **merged_kwargs)
-                if result is None:
-                    return []
-                return result if isinstance(result, list) else [result]
-            else:
-                # Record mode
-                results = []
-                for record in data:
-                    record_result = func(record, **merged_kwargs)
-                    if record_result is not None:
-                        if isinstance(record_result, list):
-                            results.extend(record_result)
-                        else:
-                            results.append(record_result)
-                return results
-                
-        except Exception as e:
-            logger.error(f"Custom function '{func_name}' failed: {e}")
-            raise ValueError(f"Custom function error: {e}") from e
-    
-    # ============================================================================
-    # LOADING
-    # ============================================================================
-    
-    async def load(
-        self,
-        transformed_data: List[Dict[str, Any]],
-        session: Any = None,
-        **kwargs,
-    ) -> Dict[str, Any]:
-        """Load transformed data into the database."""
-        target_table = self.load_config.get('target_table')
-        schema_name = self.load_config.get('schema_name')
-        if not schema_name:
-            raise ValueError(
-                "schema_name must be specified in load.yaml. "
-                "Example: schema_name: public"
-            )
-        write_method = self.load_config.get('write_method', 'upsert')
-        primary_key = self.load_config.get('primary_key')
-        unique_constraints = self.load_config.get('unique_constraints', [])
-        # Keep primary_key as-is (can be string or list for composite keys)
-        # The load functions now handle both single and composite primary keys
-        batch_size = self.load_config.get('batch_size', 1000)
-        
-        # If primary_key is 'id' and not in the data, use unique constraints for conflict detection
-        # This allows using UUID primary keys while upserting on natural keys
-        conflict_key = primary_key
-        if write_method == 'upsert' and transformed_data:
-            incoming_columns = set(transformed_data[0].keys())
-            # Check if primary_key is 'id' (string) or contains 'id' (list)
-            pk_is_id = (isinstance(primary_key, str) and primary_key == 'id') or \
-                       (isinstance(primary_key, list) and len(primary_key) == 1 and primary_key[0] == 'id')
-            
-            if pk_is_id and 'id' not in incoming_columns:
-                # Use first unique constraint for conflict detection
-                if unique_constraints:
-                    # unique_constraints can be a list of lists or a list of strings
-                    if isinstance(unique_constraints[0], list):
-                        conflict_key = unique_constraints[0]  # First constraint (can be composite)
-                    else:
-                        conflict_key = unique_constraints[0] if isinstance(unique_constraints[0], str) else unique_constraints
-                else:
-                    # Fallback: if no unique constraints, can't do upsert
-                    raise ValueError(
-                        f"Cannot perform upsert: primary_key is 'id' (auto-generated) but no unique_constraints "
-                        f"specified in load.yaml for conflict detection. Please specify unique_constraints."
-                    )
-        
-        if not target_table:
-            raise ValueError("target_table not specified in load configuration")
-        
-        tunnel = None
-        if session is None:
-            try:
-                engine, db_session, db_type, tunnel = get_database_connection(
-                    self.load_config, self.contract_dir, config_context=self.config_context
-                )
-                try:
-                    result = load_data(
-                        transformed_data,
-                        db_session,
-                        schema_name,
-                        target_table,
-                        write_method,
-                        conflict_key,  # Use conflict_key (may be unique constraint instead of PK)
-                        batch_size,
-                        db_type,
-                    )
-                    return result
-                finally:
-                    db_session.close()
-                    if tunnel:
-                        tunnel.stop()
-            except Exception as e:
-                if tunnel:
-                    try:
-                        tunnel.stop()
-                    except Exception:
-                        pass
-                raise
-        else:
-            from pycharter.etl_generator.database import detect_database_type
-            from sqlalchemy.ext.asyncio import AsyncSession
-            
-            # Detect database type
-            db_type = "postgresql"
-            if hasattr(session, 'bind') and hasattr(session.bind, 'url'):
-                db_url = str(session.bind.url)
-                db_type = detect_database_type(db_url)
-            
-            # load_data is now async and expects AsyncSession
-            if not isinstance(session, AsyncSession):
-                raise ValueError(
-                    f"load_data requires an AsyncSession, but got {type(session)}. "
-                    "Please use an AsyncSession for database operations."
-                )
-            
-            return await load_data(
-                transformed_data,
-                session,
-                schema_name,
-                target_table,
-                write_method,
-                conflict_key,  # Use conflict_key (may be unique constraint instead of PK)
-                batch_size,
-                db_type,
-            )
-    
-    # ============================================================================
-    # MEMORY MANAGEMENT
-    # ============================================================================
-    
-    def _check_memory(self) -> Optional[float]:
-        """Get current memory usage in MB, or None if psutil not available."""
-        if not PSUTIL_AVAILABLE or not self.process:
-            return None
-        return self.process.memory_info().rss / 1024 / 1024
-    
-    def _enforce_memory_limit(self):
-        """Check and enforce memory limits."""
-        if self.max_memory_mb:
-            current = self._check_memory()
-            if current and current > self.max_memory_mb:
-                gc.collect()
-                current = self._check_memory()
-                
-                if current and current > self.max_memory_mb:
-                    raise MemoryError(
-                        f"Memory limit exceeded: {current:.1f}MB > {self.max_memory_mb}MB. "
-                        f"Consider increasing batch_size."
-                    )
-    
-    # ============================================================================
-    # PIPELINE EXECUTION
-    # ============================================================================
-    
-    def _log_error(self, message: str, error: Exception, **context) -> None:
-        """Log error with context and traceback."""
-        extra = {
-            'run_id': self.run_id,
-            'pipeline': self.contract_dir.name if self.contract_dir else 'unknown',
-            'stage': self._current_stage,
-            'error_type': type(error).__name__,
-            **context
-        }
-        logger.error(message, extra=extra, exc_info=True)
-    
-    def _log_warning(self, message: str, **context) -> None:
-        """Log warning with context."""
-        extra = {
-            'run_id': self.run_id,
-            'pipeline': self.contract_dir.name if self.contract_dir else 'unknown',
-            'stage': self._current_stage,
-            **context
-        }
-        logger.warning(message, extra=extra)
-    
-    def _log_info(self, message: str, **context) -> None:
-        """Log info with context."""
-        extra = {
-            'run_id': self.run_id,
-            'pipeline': self.contract_dir.name if self.contract_dir else 'unknown',
-            'stage': self._current_stage,
-            **context
-        }
-        logger.info(message, extra=extra)
-    
-    def _summarize_errors(self, failed_batches: List[Dict[str, Any]]) -> str:
-        """
-        Summarize errors from failed batches.
-        
-        Groups errors by type and shows the most common errors first.
-        
-        Args:
-            failed_batches: List of failed batch dictionaries with 'error', 'error_type', 'batch_num', 'records'
-            
-        Returns:
-            Formatted error summary string
-        """
-        if not failed_batches:
-            return "No error details available."
-        
-        # Group errors by type
-        error_type_counts = Counter(batch['error_type'] for batch in failed_batches)
-        errors_by_type = defaultdict(list)
-        for batch in failed_batches:
-            errors_by_type[batch['error_type']].append(batch)
-        
-        # Build summary
-        lines = []
-        
-        # Summary statistics
-        total_failed = len(failed_batches)
-        total_records_failed = sum(batch.get('records', 0) for batch in failed_batches)
-        lines.append(f"Total failed batches: {total_failed}")
-        lines.append(f"Total records in failed batches: {total_records_failed}")
-        lines.append("")
-        
-        # Group by error type (most common first)
-        lines.append("Errors by type:")
-        for error_type, count in error_type_counts.most_common():
-            batches_of_type = errors_by_type[error_type]
-            lines.append(f"  {error_type}: {count} occurrence(s)")
-            
-            # Show sample error messages (up to 3 unique ones)
-            unique_errors = {}
-            for batch in batches_of_type:
-                error_msg = batch.get('error', 'Unknown error')
-                # Truncate very long error messages
-                if len(error_msg) > 200:
-                    error_msg = error_msg[:200] + "..."
-                if error_msg not in unique_errors:
-                    unique_errors[error_msg] = batch.get('batch_num', '?')
-                    if len(unique_errors) >= 3:
-                        break
-            
-            for error_msg, batch_num in unique_errors.items():
-                lines.append(f"    - Batch {batch_num}: {error_msg}")
-            
-            if len(unique_errors) < len(batches_of_type):
-                remaining = len(batches_of_type) - len(unique_errors)
-                lines.append(f"    ... and {remaining} more occurrence(s) of this error type")
-            lines.append("")
-        
-        return "\n".join(lines)
-    
-    async def run(
-        self,
-        dry_run: bool = False,
-        session: Any = None,
-        checkpoint_id: Optional[str] = None,
-        resume: bool = False,
-        batch_size: Optional[int] = None,
-        max_retries: int = 3,
-        error_threshold: float = 0.1,
-        **kwargs,
-    ) -> Dict[str, Any]:
-        """
-        Run the complete ETL pipeline in streaming mode.
-        
-        Processes data incrementally: Extract-Batch → Transform-Batch → Load-Batch.
-        This ensures constant memory usage regardless of dataset size.
-        
-        Args:
-            dry_run: If True, skip database operations
-            session: Optional database session
-            checkpoint_id: Optional checkpoint ID for resume capability
-            resume: If True, resume from checkpoint
-            batch_size: Batch size for processing (defaults to extract.yaml config)
-            max_retries: Maximum retries for failed batches
-            error_threshold: Error rate threshold (0.0-1.0) before aborting
-            **kwargs: Additional parameters passed to extract()
-            
-        Returns:
-            Pipeline execution results dictionary
-        """
-        # Generate correlation ID for this run
-        self.run_id = str(uuid.uuid4())[:8]
-        
-        if batch_size is None:
-            batch_size = self.extract_config.get('batch_size', DEFAULT_BATCH_SIZE)
-        
-        # Note: Tables must be created manually or via migrations.
-        # PyCharter no longer creates tables from schema.json.
-        
-        # Initialize Dead Letter Queue
-        dlq_config = self.load_config.get('dead_letter_queue', {})
-        dlq_enabled = dlq_config.get('enabled', True)
-        dlq_backend = dlq_config.get('backend', 'database')
-        dlq_storage_path = dlq_config.get('storage_path')
-        dlq_schema = dlq_config.get('schema_name')  # Optional schema name
-        
-        # Get pipeline name for DLQ
-        pipeline_name = (
-            self.contract_dir.name if self.contract_dir else 
-            self.extract_config.get('title', 'unknown_pipeline')
-        )
-        
-        self.dlq = DeadLetterQueue(
-            db_session=session if not dry_run else None,
-            storage_backend=dlq_backend,
-            storage_path=dlq_storage_path,
-            enabled=dlq_enabled,
-            schema_name=dlq_schema,  # Pass schema name if provided
-        )
-        
-        self._log_info(
-            "Starting ETL pipeline",
-            batch_size=batch_size,
-            dry_run=dry_run,
-            checkpoint_id=checkpoint_id,
-            resume=resume,
-            input_params=kwargs,
-            dlq_enabled=dlq_enabled,
-        )
-        
-        results = {
-            'extraction': {'batches_processed': 0, 'total_records': 0},
-            'transformation': {'batches_processed': 0, 'total_records': 0},
-            'loading': {'batches_processed': 0, 'total_records': 0, 'inserted': 0, 'updated': 0},
-            'success': False,
-            'failed_batches': [],
-            'dlq_records': 0,
-        }
-        
-        # Load checkpoint if resuming
-        start_batch = 0
-        if resume and checkpoint_id:
-            checkpoint_state = self.checkpoint_manager.load(checkpoint_id)
-            if checkpoint_state:
-                kwargs.update(checkpoint_state.last_processed_params)
-                start_batch = checkpoint_state.batch_num
-        
-        self.progress_tracker.start()
-        batch_num = 0
-        total_records = 0
-        failed_batches = []
-        
-        try:
-            async for batch in self.extract(batch_size=batch_size, **kwargs):
-                batch_num += 1
-                
-                # Skip batches if resuming
-                if batch_num <= start_batch:
-                    continue
-                
-                batch_start_time = datetime.now()
-                
-                try:
-                    self._enforce_memory_limit()
-                    
-                    # Transform batch
-                    self._current_stage = 'transform'
-                    transformed_batch = self.transform(batch, **kwargs)
-                    
-                    # Load batch
-                    self._current_stage = 'load'
-                    if not dry_run:
-                        load_result = await self.load(transformed_batch, session=session, **kwargs)
-                        results['loading']['inserted'] += load_result.get('inserted', 0)
-                        results['loading']['updated'] += load_result.get('updated', 0)
-                        results['loading']['total_records'] += load_result.get('total', 0)
-                    
-                    # Update counters
-                    total_records += len(batch)
-                    results['extraction']['total_records'] += len(batch)
-                    results['extraction']['batches_processed'] = batch_num
-                    results['transformation']['total_records'] += len(transformed_batch)
-                    results['transformation']['batches_processed'] = batch_num
-                    results['loading']['batches_processed'] = batch_num
-                    
-                    # Report progress
-                    memory_usage = self._check_memory()
-                    batch_time = (datetime.now() - batch_start_time).total_seconds()
-                    self.progress_tracker.record_batch_time(batch_time)
-                    self.progress_tracker.report(
-                        'extract',
-                        batch_num,
-                        total_records,
-                        memory_usage_mb=memory_usage,
-                    )
-                    
-                    # Save checkpoint
-                    if checkpoint_id:
-                        self.checkpoint_manager.save(
-                            checkpoint_id,
-                            'extract',
-                            batch_num,
-                            total_records,
-                            kwargs,
-                        )
-                    
-                    # Cleanup
-                    del batch, transformed_batch
-                    gc.collect()
-                    
-                except Exception as e:
-                    batch_duration = (datetime.now() - batch_start_time).total_seconds()
-                    error_msg = str(e)
-                    error_type = type(e).__name__
-                    
-                    # Check if this is a connection-related error
-                    is_connection_error = (
-                        'connection' in error_msg.lower() or 
-                        'closed' in error_msg.lower() or
-                        'ConnectionDoesNotExistError' in error_type or
-                        'ConnectionError' in error_type or
-                        'InterfaceError' in error_type or
-                        'DBAPIError' in error_type and ('connection' in error_msg.lower() or 'closed' in error_msg.lower())
-                    )
-                    
-                    self._log_error(
-                        "Batch processing failed",
-                        e,
-                        batch_num=batch_num,
-                        batch_size=len(batch) if batch else 0,
-                        total_records=total_records,
-                        is_connection_error=is_connection_error,
-                    )
-                    
-                    # For connection errors, retry before checking error rate
-                    # This prevents aborting on transient connection issues
-                    if is_connection_error and len(failed_batches) < max_retries:
-                        wait_time = min(2 ** len(failed_batches), 5.0)  # Exponential backoff, max 5s
-                        self._log_warning(
-                            f"Connection error in batch {batch_num}, retrying (attempt {len(failed_batches) + 1}/{max_retries})",
-                            batch_num=batch_num,
-                            retry_attempt=len(failed_batches) + 1,
-                            wait_seconds=wait_time,
-                        )
-                        await asyncio.sleep(wait_time)
-                        continue  # Retry the batch without adding to failed_batches
-                    
-                    # Not a connection error, or retries exhausted - add to failed batches
-                    failed_batches.append({
-                        'batch_num': batch_num,
-                        'error': error_msg,
-                        'error_type': error_type,
-                        'records': len(batch) if batch else 0,
-                    })
-                    
-                    # Add failed batch to Dead Letter Queue
-                    if batch and self.dlq:
-                        # Determine DLQ reason
-                        if is_connection_error:
-                            dlq_reason = DLQReason.CONNECTION_ERROR
-                        elif self._current_stage == 'extract':
-                            dlq_reason = DLQReason.EXTRACTION_ERROR
-                        elif self._current_stage == 'transform':
-                            dlq_reason = DLQReason.TRANSFORMATION_ERROR
-                        elif self._current_stage == 'load':
-                            dlq_reason = DLQReason.LOAD_ERROR
-                        else:
-                            dlq_reason = DLQReason.UNKNOWN
-                        
-                        # Add batch to DLQ
-                        dlq_records = await self.dlq.add_batch(
-                            pipeline_name=pipeline_name,
-                            batch=batch,
-                            reason=dlq_reason,
-                            error_message=error_msg,
-                            error_type=error_type,
-                            stage=self._current_stage or 'unknown',
-                            metadata={
-                                'batch_num': batch_num,
-                                'total_records': total_records,
-                                'run_id': self.run_id,
-                                'is_connection_error': is_connection_error,
-                            },
-                        )
-                        results['dlq_records'] += len(dlq_records)
-                    
-                    # Check error rate (only after connection retries are exhausted)
-                    # Also be more lenient for small batch counts (don't abort on first failure)
-                    min_batches_for_error_check = 3  # Need at least 3 batches before checking error rate
-                    if batch_num >= min_batches_for_error_check:
-                        error_rate = len(failed_batches) / batch_num if batch_num > 0 else 1.0
-                        if error_rate > error_threshold:
-                            # Summarize errors before raising
-                            error_summary = self._summarize_errors(failed_batches)
-                            
-                            self._log_error(
-                                "Error rate threshold exceeded",
-                                RuntimeError("Error rate threshold exceeded"),
-                                error_rate=error_rate,
-                                threshold=error_threshold,
-                                failed_batches=len(failed_batches),
-                                total_batches=batch_num,
-                                error_summary=error_summary,
-                            )
-                            
-                            error_msg = (
-                                f"Error rate too high: {error_rate:.1%} > {error_threshold:.1%}. "
-                                f"Aborting pipeline.\n\n"
-                                f"Error Summary ({len(failed_batches)} failed batches out of {batch_num} total):\n"
-                                f"{error_summary}"
-                            )
-                            raise RuntimeError(error_msg)
-                    
-                    # Retry logic for non-connection errors
-                    if len(failed_batches) <= max_retries:
-                        wait_time = 2 ** len(failed_batches)
-                        self._log_warning(
-                            f"Retrying batch {batch_num}",
-                            batch_num=batch_num,
-                            retry_attempt=len(failed_batches),
-                            wait_seconds=wait_time,
-                        )
-                        await asyncio.sleep(wait_time)
-                        continue
-                    else:
-                        self.progress_tracker.report(
-                            'extract',
-                            batch_num,
-                            total_records,
-                            error_count=len(failed_batches),
-                        )
-            
-            results['failed_batches'] = failed_batches
-            results['success'] = len(failed_batches) < batch_num * error_threshold
-            
-            # Add DLQ statistics to results
-            if self.dlq:
-                try:
-                    dlq_stats = self.dlq.get_statistics(pipeline_name=pipeline_name)
-                    results['dlq_statistics'] = dlq_stats
-                except Exception as e:
-                    logger.warning(f"Failed to get DLQ statistics: {e}")
-            
-            self._log_info(
-                "ETL pipeline completed",
-                batches=batch_num,
-                records=total_records,
-                failed_batches=len(failed_batches),
-                inserted=results['loading'].get('inserted', 0),
-                updated=results['loading'].get('updated', 0),
-                dlq_records=results.get('dlq_records', 0),
-            )
-            
-            # Delete checkpoint on success
-            if checkpoint_id and results['success']:
-                self.checkpoint_manager.delete(checkpoint_id)
-            
-        except Exception as e:
-            self._log_error(
-                "ETL pipeline failed",
-                e,
-                batches_processed=batch_num,
-                records_processed=total_records,
-                failed_batches=len(failed_batches),
-            )
-            
-            if checkpoint_id:
-                self.checkpoint_manager.save(
-                    checkpoint_id,
-                    'error',
-                    batch_num,
-                    total_records,
-                    kwargs,
-                    error=str(e),
-                )
-            results['error'] = str(e)
-            results['success'] = False
-            raise
-        
-        return results
-    
-    async def run_multiple(
-        self,
-        param_name: Optional[str] = None,
-        param_values: Optional[List[Any]] = None,
-        param_sets: Optional[List[Dict[str, Any]]] = None,
-        batch_size: int = 5,
-        delay_between_runs: float = 1.0,
-        dry_run: bool = False,
-        session: Any = None,
-        **kwargs,
-    ) -> List[Dict[str, Any]]:
-        """
-        Run ETL pipeline multiple times with different parameter sets.
-        
-        This method allows you to efficiently run the same ETL pipeline multiple times
-        with varying parameters. You can either:
-        1. Provide a single parameter name and list of values (simple case)
-        2. Provide a list of parameter dictionaries (complex case with multiple varying params)
-        
-        Args:
-            param_name: Name of the parameter to vary (e.g., 'symbol', 'ticker', 'date')
-                       Required if using param_values.
-            param_values: List of values for the specified parameter.
-                         Each value will be passed as {param_name: value} to run().
-            param_sets: List of parameter dictionaries. Each dict will be unpacked
-                       and passed to run() as **params. Use this when multiple
-                       parameters vary between runs.
-            batch_size: Number of runs to process before a brief pause (for rate limiting)
-            delay_between_runs: Delay in seconds between individual runs (for rate limiting)
-            dry_run: If True, skip database operations
-            session: Optional database session
-            **kwargs: Additional parameters passed to each run() call (common to all runs)
-        
-        Returns:
-            List of result dictionaries, each containing:
-            - 'params': The parameters used for this run
-            - 'success': Whether the run succeeded
-            - 'records': Number of records processed (if successful)
-            - 'result': Full result dictionary from run() (if successful)
-            - 'error': Error message (if failed)
-        
-        Examples:
-            # Simple case: vary a single parameter
-            >>> results = await orchestrator.run_multiple(
-            ...     param_name='symbol',
-            ...     param_values=['AAPL', 'MSFT', 'GOOGL'],
-            ...     batch_size=5,
-            ...     delay_between_runs=1.0
-            ... )
-            
-            # Complex case: vary multiple parameters
-            >>> results = await orchestrator.run_multiple(
-            ...     param_sets=[
-            ...         {'symbol': 'AAPL', 'date': '2024-01-01'},
-            ...         {'symbol': 'MSFT', 'date': '2024-01-02'},
-            ...     ],
-            ...     batch_size=3,
-            ...     delay_between_runs=0.5
-            ... )
-        """
-        # Validate inputs
-        if param_sets is not None:
-            if param_name is not None or param_values is not None:
-                raise ValueError(
-                    "Cannot use both param_sets and param_name/param_values. "
-                    "Use either param_sets OR param_name+param_values."
-                )
-            if not isinstance(param_sets, list) or len(param_sets) == 0:
-                raise ValueError("param_sets must be a non-empty list of dictionaries")
-            # Convert param_sets to list of dicts
-            runs = [dict(params) for params in param_sets]
-        elif param_name is not None and param_values is not None:
-            if not isinstance(param_values, list) or len(param_values) == 0:
-                raise ValueError("param_values must be a non-empty list")
-            # Convert param_name + param_values to list of dicts
-            runs = [{param_name: value} for value in param_values]
-        else:
-            raise ValueError(
-                "Must provide either (param_name + param_values) OR param_sets"
-            )
-        
-        results = []
-        
-        for i in range(0, len(runs), batch_size):
-            run_batch = runs[i:i + batch_size]
-            
-            for run_params in run_batch:
-                try:
-                    # Merge run_params with common kwargs
-                    merged_params = {**kwargs, **run_params}
-                    result = await self.run(
-                        dry_run=dry_run,
-                        session=session,
-                        **merged_params
-                    )
-                    results.append({
-                        'params': run_params,
-                        'success': result['success'],
-                        'records': result.get('loading', {}).get('total_records', 0),
-                        'result': result,
-                    })
-                except Exception as e:
-                    results.append({
-                        'params': run_params,
-                        'success': False,
-                        'error': str(e),
-                    })
-                
-                # Rate limiting
-                if i + batch_size < len(runs) or run_params != run_batch[-1]:
-                    await asyncio.sleep(delay_between_runs)
-        
-        return results
-
-
-def create_orchestrator(
-    contract_dir: Optional[str] = None,
-    **kwargs,
-) -> ETLOrchestrator:
-    """
-    Create an ETL orchestrator instance.
-    
-    Args:
-        contract_dir: Directory containing contract files and ETL configs
-        **kwargs: Additional arguments passed to ETLOrchestrator
-    
-    Returns:
-        ETLOrchestrator instance
-    """
-    return ETLOrchestrator(contract_dir=contract_dir, **kwargs)