pycharter 0.0.25__py3-none-any.whl → 0.0.26__py3-none-any.whl

pycharter/etl_generator/pipeline.py

@@ -2,28 +2,25 @@
 Pipeline class with | operator for chaining.
 
 Supports both config-driven and programmatic pipeline construction.
+Includes optional validation at extract (source) and load (target) stages.
 """
 
 import logging
 import os
-import re
 import uuid
 from datetime import datetime, timezone
 from pathlib import Path
-from typing import Any, Callable, Dict, List, Optional, Union
+from typing import Any, Dict, List, Optional, Union
 
 import yaml
 
-from pycharter.etl_generator.context import PipelineContext
+from pycharter.etl_generator.context import PipelineContext, VARIABLE_PATTERN
 from pycharter.etl_generator.protocols import Extractor, Transformer, Loader
 from pycharter.etl_generator.result import PipelineResult, BatchResult
 from pycharter.shared.errors import ErrorContext, ErrorMode, get_error_context
 
 logger = logging.getLogger(__name__)
 
-# Variable pattern: ${VAR} or ${VAR:-default} or ${VAR:?error}
-VARIABLE_PATTERN = re.compile(r'\$\{([^}:]+)(?::([?-])([^}]*))?\}')
-
 
 class Pipeline:
     """
@@ -68,12 +65,31 @@ class Pipeline:
         loader: Optional[Loader] = None,
         context: Optional[PipelineContext] = None,
         name: Optional[str] = None,
+        transform_config: Optional[Union[Dict[str, Any], List[Dict[str, Any]]]] = None,
+        # Validation configs
+        extract_validation_config: Optional[Dict[str, Any]] = None,
+        load_validation_config: Optional[Dict[str, Any]] = None,
+        # Shared settings
+        settings: Optional[Dict[str, Any]] = None,
+        # Base directory for resolving relative paths
+        base_dir: Optional[Path] = None,
     ):
         self.extractor = extractor
         self._transformers: List[Transformer] = list(transformers) if transformers else []
         self.loader = loader
         self.context = context or PipelineContext()
         self.name = name
+        # For config-driven runs, store raw transform config and use apply_transforms
+        self._transform_config = transform_config
+        # Validation configs
+        self._extract_validation_config = extract_validation_config
+        self._load_validation_config = load_validation_config
+        # Shared settings from settings.yaml
+        self._settings = settings or {}
+        # Base directory for resolving file paths
+        self._base_dir = base_dir
+        # Run ID (set during run())
+        self._run_id: Optional[str] = None
     
     def __or__(self, other: Union[Transformer, Loader]) -> "Pipeline":
         """Chain transformer or set loader using | operator."""
@@ -84,6 +100,10 @@ class Pipeline:
                 loader=other,
                 context=self.context,
                 name=self.name,
+                extract_validation_config=self._extract_validation_config,
+                load_validation_config=self._load_validation_config,
+                settings=self._settings,
+                base_dir=self._base_dir,
             )
         else:
             new_transformers = self._transformers.copy()
@@ -94,6 +114,10 @@ class Pipeline:
                 loader=self.loader,
                 context=self.context,
                 name=self.name,
+                extract_validation_config=self._extract_validation_config,
+                load_validation_config=self._load_validation_config,
+                settings=self._settings,
+                base_dir=self._base_dir,
             )
     
     async def run(
@@ -117,6 +141,7 @@ class Pipeline:
             PipelineResult with counts and any errors.
         """
         run_id = str(uuid.uuid4())[:8]
+        self._run_id = run_id
         start_time = datetime.now(timezone.utc)
         ctx = error_context or get_error_context()
         
@@ -133,14 +158,58 @@ class Pipeline:
         
         logger.info(f"[{run_id}] Starting pipeline: {self.name or 'unnamed'}")
         
+        # Create validators if configured
+        extract_validator = self._create_extract_validator()
+        load_validator = self._create_load_validator()
+        
         try:
             batch_index = 0
             async for batch in self.extractor.extract(**params):
                 batch_result = BatchResult(batch_index=batch_index, rows_in=len(batch))
+                result.rows_extracted += len(batch)
+                
+                # Source validation (after extract)
+                if extract_validator:
+                    try:
+                        batch, quarantined, error_count = await extract_validator.validate(
+                            batch, run_id=run_id
+                        )
+                        result.rows_quarantined_extract += len(quarantined)
+                        if error_count > 0:
+                            batch_result.errors.append(
+                                f"Extract validation: {error_count} record(s) invalid"
+                            )
+                    except Exception as e:
+                        # Validation error with on_error=fail
+                        ctx.handle_error(str(e), e, category="extract_validation")
+                        batch_result.errors.append(f"Extract validation failed: {e}")
+                        result.success = False
+                        result.errors.append(str(e))
+                        break
                 
                 # Transform
                 transformed = self._apply_transforms(batch)
                 batch_result.rows_out = len(transformed)
+                result.rows_transformed += len(transformed)
+                
+                # Target validation (before load)
+                if load_validator and transformed:
+                    try:
+                        transformed, quarantined, error_count = await load_validator.validate(
+                            transformed, run_id=run_id
+                        )
+                        result.rows_quarantined_load += len(quarantined)
+                        if error_count > 0:
+                            batch_result.errors.append(
+                                f"Load validation: {error_count} record(s) invalid"
+                            )
+                    except Exception as e:
+                        # Validation error with on_error=fail
+                        ctx.handle_error(str(e), e, category="load_validation")
+                        batch_result.errors.append(f"Load validation failed: {e}")
+                        result.success = False
+                        result.errors.append(str(e))
+                        break
                 
                 # Load
                 if not dry_run and self.loader and transformed:
@@ -160,8 +229,6 @@ class Pipeline:
                 elif dry_run:
                     result.rows_loaded += len(transformed)
                 
-                result.rows_extracted += len(batch)
-                result.rows_transformed += len(transformed)
                 result.batches_processed += 1
                 result.batch_results.append(batch_result)
                 batch_index += 1
@@ -179,11 +246,115 @@ class Pipeline:
         if result.errors:
             result.success = False
         
-        logger.info(f"[{run_id}] Complete: extracted={result.rows_extracted}, loaded={result.rows_loaded}")
+        logger.info(
+            f"[{run_id}] Complete: extracted={result.rows_extracted}, "
+            f"loaded={result.rows_loaded}, "
+            f"quarantined_extract={result.rows_quarantined_extract}, "
+            f"quarantined_load={result.rows_quarantined_load}"
+        )
         return result
     
+    def _create_extract_validator(self):
+        """Create extract validator if configured."""
+        if not self._extract_validation_config:
+            return None
+        
+        from pycharter.etl_generator.validation import create_etl_validator
+        from pycharter.etl_generator.config_models import DLQConfig
+        
+        # Get default DLQ config from settings
+        default_dlq = None
+        if self._settings.get("dlq"):
+            default_dlq = DLQConfig(**self._settings["dlq"])
+        
+        return create_etl_validator(
+            validation_config=self._extract_validation_config,
+            pipeline_name=self.name or "unnamed",
+            stage="extract",
+            metadata_store=None,  # Extract uses local schema, not store
+            default_contract=None,
+            default_dlq_config=default_dlq,
+            base_dir=self._base_dir,
+        )
+    
+    def _create_load_validator(self):
+        """Create load validator if configured."""
+        if not self._load_validation_config:
+            return None
+        
+        from pycharter.etl_generator.validation import create_etl_validator
+        from pycharter.etl_generator.config_models import DLQConfig
+        from pycharter.metadata_store import MetadataStoreClient
+        
+        # Get default DLQ config from settings
+        default_dlq = None
+        if self._settings.get("dlq"):
+            default_dlq = DLQConfig(**self._settings["dlq"])
+        
+        # Get default contract from settings
+        default_contract = self._settings.get("contract")
+        
+        # Create metadata store if configured
+        metadata_store = None
+        if self._settings.get("metadata_store"):
+            metadata_store = self._create_metadata_store()
+        
+        return create_etl_validator(
+            validation_config=self._load_validation_config,
+            pipeline_name=self.name or "unnamed",
+            stage="load",
+            metadata_store=metadata_store,
+            default_contract=default_contract,
+            default_dlq_config=default_dlq,
+            base_dir=self._base_dir,
+        )
+    
+    def _create_metadata_store(self) -> Optional["MetadataStoreClient"]:
+        """Create metadata store from settings."""
+        store_config = self._settings.get("metadata_store")
+        if not store_config:
+            return None
+        
+        store_type = store_config.get("type", "").lower()
+        connection_string = store_config.get("connection_string")
+        
+        if not connection_string:
+            logger.warning("metadata_store missing connection_string")
+            return None
+        
+        if store_type == "postgres":
+            from pycharter.metadata_store import PostgresMetadataStore
+            return PostgresMetadataStore(connection_string)
+        elif store_type == "sqlite":
+            from pycharter.metadata_store import SQLiteMetadataStore
+            return SQLiteMetadataStore(connection_string)
+        elif store_type == "mongodb":
+            from pycharter.metadata_store import MongoDBMetadataStore
+            return MongoDBMetadataStore(connection_string)
+        elif store_type == "redis":
+            from pycharter.metadata_store import RedisMetadataStore
+            return RedisMetadataStore(connection_string)
+        elif store_type == "memory":
+            from pycharter.metadata_store import InMemoryMetadataStore
+            return InMemoryMetadataStore()
+        else:
+            logger.warning(f"Unknown metadata_store type: {store_type}")
+            return None
+    
     def _apply_transforms(self, data: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
-        """Apply all transformers to data."""
+        """Apply all transformers to data.
+        
+        For config-driven pipelines (when _transform_config is set), uses the unified
+        apply_transforms() which supports simple_ops, jsonata, and custom_function.
+        For programmatic pipelines (when _transformers is set), loops through the
+        transformer chain.
+        """
+        # Config-driven path: use unified apply_transforms (supports jsonata, etc.)
+        if self._transform_config:
+            from pycharter.etl_generator.transformers import apply_transforms
+            return apply_transforms(data, self._transform_config)
+        
+        # Programmatic path: loop through transformer instances
         result = data
         for transformer in self._transformers:
             result = transformer.transform(result)
@@ -263,6 +434,7 @@ class Pipeline:
         - extract.yaml (required)
         - transform.yaml (optional)
         - load.yaml (required)
+        - settings.yaml (optional, for shared config like DLQ, metadata store)
         
         Args:
             directory: Path to directory containing config files
@@ -289,6 +461,7 @@ class Pipeline:
         extract_file = directory / "extract.yaml"
         load_file = directory / "load.yaml"
         transform_file = directory / "transform.yaml"
+        settings_file = directory / "settings.yaml"
         
         if not extract_file.exists():
             raise FileNotFoundError(f"Required file not found: {extract_file}")
@@ -299,6 +472,11 @@ class Pipeline:
         extract_config = _load_config_input(extract_file, variables)
         load_config = _load_config_input(load_file, variables)
         transform_config = _load_config_input(transform_file, variables) if transform_file.exists() else {}
+        settings_config = _load_config_input(settings_file, variables) if settings_file.exists() else {}
+        
+        # Extract validation configs from extract/load
+        extract_validation_config = extract_config.pop("validation", None) if isinstance(extract_config, dict) else None
+        load_validation_config = load_config.pop("validation", None) if isinstance(load_config, dict) else None
         
         return cls._build_from_configs(
             extract_config=extract_config,
@@ -307,6 +485,10 @@ class Pipeline:
             variables=variables,
             validate=validate,
             name=name or directory.name,
+            extract_validation_config=extract_validation_config,
+            load_validation_config=load_validation_config,
+            settings=settings_config if isinstance(settings_config, dict) else {},
+            base_dir=directory.resolve(),
         )
     
     @classmethod
@@ -325,11 +507,25 @@ class Pipeline:
             extract:
               type: http
               url: https://api.example.com
+              validation:  # optional
+                schema: ./source_schema.yaml
+                on_error: quarantine
             transform:
               - rename: {old: new}
             load:
               type: file
               path: output.json
+              validation:  # optional
+                contract: ./contracts/orders
+                on_error: fail
+            # Optional shared settings
+            metadata_store:
+              type: postgres
+              connection_string: ...
+            dlq:
+              enabled: true
+              backend: file
+              path: ./dlq
         
         Args:
             path: Path to pipeline config file (YAML)
@@ -361,13 +557,32 @@ class Pipeline:
         if "load" not in config:
             raise ValueError(f"Config file missing 'load' section: {path}")
         
+        # Extract validation configs
+        extract_config = config["extract"]
+        load_config = config["load"]
+        extract_validation_config = extract_config.pop("validation", None) if isinstance(extract_config, dict) else None
+        load_validation_config = load_config.pop("validation", None) if isinstance(load_config, dict) else None
+        
+        # Build inline settings from top-level keys
+        settings: Dict[str, Any] = {}
+        if config.get("metadata_store"):
+            settings["metadata_store"] = config["metadata_store"]
+        if config.get("dlq"):
+            settings["dlq"] = config["dlq"]
+        if config.get("contract"):
+            settings["contract"] = config["contract"]
+        
         return cls._build_from_configs(
-            extract_config=config["extract"],
+            extract_config=extract_config,
             transform_config=config.get("transform", {}),
-            load_config=config["load"],
+            load_config=load_config,
             variables=variables,
             validate=validate,
             name=config.get("name"),
+            extract_validation_config=extract_validation_config,
+            load_validation_config=load_validation_config,
+            settings=settings,
+            base_dir=path.parent.resolve(),
         )
     
     @classmethod
@@ -376,6 +591,7 @@ class Pipeline:
         config: Dict[str, Any],
         variables: Optional[Dict[str, str]] = None,
         validate: bool = True,
+        base_dir: Optional[Path] = None,
     ) -> "Pipeline":
         """
         Create pipeline from a configuration dictionary.
@@ -384,6 +600,7 @@ class Pipeline:
             config: Dict with 'extract', 'transform' (optional), 'load' sections
             variables: Variables for ${VAR} substitution
             validate: If True, validate config against schema
+            base_dir: Base directory for resolving relative file paths
         
         Returns:
             Configured Pipeline instance
@@ -406,15 +623,40 @@ class Pipeline:
         
         # Resolve variables in config
         extract_config = context.resolve_dict(config["extract"])
+        load_config = context.resolve_dict(config["load"])
+        
+        # Extract validation configs
+        extract_validation_config = extract_config.pop("validation", None) if isinstance(extract_config, dict) else None
+        load_validation_config = load_config.pop("validation", None) if isinstance(load_config, dict) else None
+        
+        # Build transform config - include transform, jsonata, and custom_function
+        # This allows top-level jsonata/custom_function in config (common in yaml files)
+        transform_config: Dict[str, Any] = {}
         raw_transform = config.get("transform", {})
         if isinstance(raw_transform, list):
-            transform_config = [
+            transform_config["transform"] = [
                 context.resolve_dict(item) if isinstance(item, dict) else item
                 for item in raw_transform
             ]
-        else:
-            transform_config = context.resolve_dict(raw_transform)
-        load_config = context.resolve_dict(config["load"])
+        elif raw_transform:
+            transform_config["transform"] = context.resolve_dict(raw_transform)
+        
+        # Include top-level jsonata if present
+        if config.get("jsonata"):
+            transform_config["jsonata"] = context.resolve_dict(config["jsonata"])
+        
+        # Include top-level custom_function if present
+        if config.get("custom_function"):
+            transform_config["custom_function"] = context.resolve_dict(config["custom_function"])
+        
+        # Build inline settings from top-level keys
+        settings: Dict[str, Any] = {}
+        if config.get("metadata_store"):
+            settings["metadata_store"] = context.resolve_dict(config["metadata_store"])
+        if config.get("dlq"):
+            settings["dlq"] = context.resolve_dict(config["dlq"])
+        if config.get("contract"):
+            settings["contract"] = config["contract"]
         
         return cls._build_from_configs(
             extract_config=extract_config,
@@ -423,6 +665,10 @@ class Pipeline:
             variables=variables,
             validate=validate,
             name=config.get("name"),
+            extract_validation_config=extract_validation_config,
+            load_validation_config=load_validation_config,
+            settings=settings,
+            base_dir=base_dir,
         )
     
     @classmethod
@@ -434,8 +680,21 @@ class Pipeline:
         variables: Dict[str, str],
         validate: bool,
         name: Optional[str],
+        extract_validation_config: Optional[Dict[str, Any]] = None,
+        load_validation_config: Optional[Dict[str, Any]] = None,
+        settings: Optional[Dict[str, Any]] = None,
+        base_dir: Optional[Path] = None,
     ) -> "Pipeline":
-        """Internal method to build pipeline from resolved configs."""
+        """Internal method to build pipeline from resolved configs.
+        
+        Uses the unified apply_transforms path for transformations, which supports:
+        - Simple ops: rename, convert, defaults, add, select, drop, filter
+        - JSONata expressions
+        - Custom functions
+        
+        Also supports validation at extract and load stages via extract_validation_config
+        and load_validation_config parameters.
+        """
         from pycharter.etl_generator.config_validator import ConfigValidator
         
         # Validate if enabled
@@ -455,15 +714,20 @@ class Pipeline:
         
         # Create components
         extractor = _create_extractor(extract_config)
-        transformers = _create_transformers(transform_config)
         loader_instance = _create_loader(load_config)
         
+        # Pass raw transform config - will be used by apply_transforms in _apply_transforms
         return cls(
             extractor=extractor,
-            transformers=transformers,
+            transformers=None,  # No transformer objects for config-driven runs
             loader=loader_instance,
             context=context,
             name=name,
+            transform_config=transform_config if transform_config else None,
+            extract_validation_config=extract_validation_config,
+            load_validation_config=load_validation_config,
+            settings=settings or {},
+            base_dir=base_dir,
         )
 
 
@@ -557,151 +821,6 @@ def _create_extractor(config: Dict[str, Any]) -> Optional[Extractor]:
     return extractor_class.from_config(config)
 
 
-def _create_transformers(config: Union[Dict[str, Any], List[Dict[str, Any]]]) -> List[Transformer]:
-    """Create transformer chain from config."""
-    if not config:
-        return []
-    
-    # Handle nested 'transform' key
-    if isinstance(config, dict) and "transform" in config:
-        config = config["transform"]
-    
-    # List format - ordered transforms
-    if isinstance(config, list):
-        return _create_transformers_from_list(config)
-    
-    # Dict format - fixed order
-    return _create_transformers_from_dict(config)
-
-
-def _create_transformers_from_list(config: List[Dict[str, Any]]) -> List[Transformer]:
-    """Create transformers from list format (user-specified order)."""
-    transformers = []
-    
-    for step in config:
-        if not isinstance(step, dict):
-            logger.warning(f"Invalid transform step (expected dict): {step}")
-            continue
-        
-        for op_name, op_config in step.items():
-            transformer = _create_single_transformer(op_name, op_config)
-            if transformer:
-                if isinstance(transformer, list):
-                    transformers.extend(transformer)
-                else:
-                    transformers.append(transformer)
-    
-    return transformers
-
-
-def _create_transformers_from_dict(config: Dict[str, Any]) -> List[Transformer]:
-    """Create transformers from dict format (fixed order)."""
-    transformers = []
-    ordered_ops = ["rename", "convert", "defaults", "add", "select", "drop", "filter", "custom_function"]
-    
-    for op_name in ordered_ops:
-        if op_name in config:
-            transformer = _create_single_transformer(op_name, config[op_name])
-            if transformer:
-                if isinstance(transformer, list):
-                    transformers.extend(transformer)
-                else:
-                    transformers.append(transformer)
-    
-    return transformers
-
-
-def _create_single_transformer(op_name: str, op_config: Any) -> Optional[Union[Transformer, List[Transformer]]]:
-    """Create a single transformer from operation name and config."""
-    from pycharter.etl_generator.transformers import (
-        Rename, AddField, Drop, Select, Filter, Convert, Default, CustomFunction,
-    )
-    from pycharter.etl_generator.transformers.simple_operations import convert_type
-    
-    op_name = op_name.lower()
-    
-    if op_name == "rename":
-        if isinstance(op_config, dict):
-            return Rename(op_config)
-    
-    elif op_name == "add":
-        if isinstance(op_config, dict):
-            return [AddField(field, value) for field, value in op_config.items()]
-    
-    elif op_name == "drop":
-        if isinstance(op_config, list):
-            return Drop(op_config)
-    
-    elif op_name == "select":
-        if isinstance(op_config, list):
-            return Select(op_config)
-    
-    elif op_name == "convert":
-        if isinstance(op_config, dict):
-            type_map = {
-                "int": int, "integer": int,
-                "float": float, "number": float, "numeric": float,
-                "str": str, "string": str,
-                "bool": bool, "boolean": bool,
-            }
-            conversions = {}
-            for field, target_type in op_config.items():
-                target_lower = target_type.lower() if isinstance(target_type, str) else str(target_type)
-                if target_lower in type_map:
-                    conversions[field] = type_map[target_lower]
-                elif target_lower in ("datetime", "date"):
-                    conversions[field] = lambda v, t=target_lower: convert_type(v, t)
-                else:
-                    conversions[field] = str
-            return Convert(conversions)
-    
-    elif op_name == "defaults":
-        if isinstance(op_config, dict):
-            return Default(op_config)
-    
-    elif op_name == "filter":
-        if isinstance(op_config, dict):
-            field = op_config.get("field")
-            operator = op_config.get("operator", "eq")
-            value = op_config.get("value")
-            if field and operator:
-                predicate = _create_filter_predicate(field, operator, value)
-                if predicate:
-                    return Filter(predicate)
-    
-    elif op_name == "custom_function":
-        if isinstance(op_config, dict):
-            return CustomFunction(
-                module=op_config.get("module"),
-                function=op_config.get("function"),
-                kwargs=op_config.get("kwargs", {}),
-            )
-    
-    else:
-        logger.warning(f"Unknown transform operation: {op_name}")
-    
-    return None
-
-
-def _create_filter_predicate(field: str, operator: str, value: Any) -> Optional[Callable]:
-    """Create a filter predicate function from operator and value."""
-    operators = {
-        "eq": lambda r: r.get(field) == value,
-        "ne": lambda r: r.get(field) != value,
-        "gt": lambda r: r.get(field) is not None and r.get(field) > value,
-        "gte": lambda r: r.get(field) is not None and r.get(field) >= value,
-        "lt": lambda r: r.get(field) is not None and r.get(field) < value,
-        "lte": lambda r: r.get(field) is not None and r.get(field) <= value,
-        "in": lambda r: r.get(field) in (value if isinstance(value, (list, tuple, set)) else [value]),
-        "not_in": lambda r: r.get(field) not in (value if isinstance(value, (list, tuple, set)) else [value]),
-        "contains": lambda r: value in str(r.get(field, "")),
-        "not_contains": lambda r: value not in str(r.get(field, "")),
-        "is_null": lambda r: r.get(field) is None,
-        "is_not_null": lambda r: r.get(field) is not None,
-    }
-    return operators.get(operator)
-
-
 def _create_loader(config: Dict[str, Any]) -> Optional[Loader]:
     """Create loader from config using explicit type field."""
     if not config:

pycharter/etl_generator/result.py

@@ -39,6 +39,12 @@ class PipelineResult:
     rows_transformed: int = 0
     rows_loaded: int = 0
     rows_failed: int = 0
+    # Validation tracking
+    rows_quarantined_extract: int = 0  # Records quarantined at extract stage
+    rows_quarantined_load: int = 0  # Records quarantined at load stage
+    validation_errors_extract: List[str] = field(default_factory=list)
+    validation_errors_load: List[str] = field(default_factory=list)
+    # Timing
     start_time: Optional[datetime] = None
     end_time: Optional[datetime] = None
     duration_seconds: Optional[float] = None
@@ -48,6 +54,11 @@ class PipelineResult:
     pipeline_name: Optional[str] = None
     run_id: Optional[str] = None
     
+    @property
+    def total_quarantined(self) -> int:
+        """Total records quarantined at both extract and load stages."""
+        return self.rows_quarantined_extract + self.rows_quarantined_load
+    
     def to_dict(self) -> Dict[str, Any]:
         return {
             "success": self.success,
@@ -55,6 +66,11 @@ class PipelineResult:
             "rows_transformed": self.rows_transformed,
             "rows_loaded": self.rows_loaded,
             "rows_failed": self.rows_failed,
+            "rows_quarantined_extract": self.rows_quarantined_extract,
+            "rows_quarantined_load": self.rows_quarantined_load,
+            "total_quarantined": self.total_quarantined,
+            "validation_errors_extract": self.validation_errors_extract,
+            "validation_errors_load": self.validation_errors_load,
             "duration_seconds": self.duration_seconds,
             "batches_processed": self.batches_processed,
             "errors": self.errors,