pycharter 0.0.24__py3-none-any.whl → 0.0.26__py3-none-any.whl

pycharter/etl_generator/config_validator.py

@@ -2,33 +2,27 @@
 Configuration validation for ETL pipelines.
 
 Provides validation of extract, transform, and load configurations
-with clear, actionable error messages.
+using Pydantic models with clear, actionable error messages.
 """
 
-import json
-from pathlib import Path
-from typing import Any, Dict, List, Optional, Tuple, Union
+from typing import Any, Dict, List, Optional, Tuple
 
-try:
-    import jsonschema
-    from jsonschema import Draft7Validator, ValidationError as JsonschemaValidationError
-    HAS_JSONSCHEMA = True
-except ImportError:
-    HAS_JSONSCHEMA = False
-    JsonschemaValidationError = Exception  # type: ignore[misc, assignment]
+from pydantic import ValidationError as PydanticValidationError
 
 from pycharter.shared.errors import ConfigValidationError
-from pycharter.etl_generator.schemas import (
-    get_extract_schema,
-    get_transform_schema,
-    get_load_schema,
-    get_pipeline_schema,
+from pycharter.etl_generator.config_models import (
+    parse_extract_config,
+    parse_load_config,
+    parse_transform_config,
+    parse_settings_config,
+    SettingsConfig,
+    TransformConfig,
 )
 
 
 class ConfigValidator:
     """
-    Validates ETL pipeline configurations against JSON schemas.
+    Validates ETL pipeline configurations using Pydantic models.
     
     Provides clear error messages to help users fix configuration issues.
     
@@ -56,146 +50,80 @@ class ConfigValidator:
                    If False, return errors without raising.
         """
         self.strict = strict
-        self._extract_schema = None
-        self._transform_schema = None
-        self._load_schema = None
-        self._pipeline_schema = None
     
-    @property
-    def extract_schema(self) -> Dict[str, Any]:
-        if self._extract_schema is None:
-            self._extract_schema = get_extract_schema()
-        return self._extract_schema
-    
-    @property
-    def transform_schema(self) -> Dict[str, Any]:
-        if self._transform_schema is None:
-            self._transform_schema = get_transform_schema()
-        return self._transform_schema
-    
-    @property
-    def load_schema(self) -> Dict[str, Any]:
-        if self._load_schema is None:
-            self._load_schema = get_load_schema()
-        return self._load_schema
-    
-    @property
-    def pipeline_schema(self) -> Dict[str, Any]:
-        if self._pipeline_schema is None:
-            self._pipeline_schema = get_pipeline_schema()
-        return self._pipeline_schema
-    
-    def _validate(
-        self,
-        config: Dict[str, Any],
-        schema: Dict[str, Any],
-        config_type: str,
-        config_path: Optional[str] = None,
-    ) -> Tuple[bool, List[Dict[str, Any]]]:
-        """
-        Validate config against schema.
-        
-        Returns:
-            Tuple of (is_valid, list of error dicts)
-        """
-        if not HAS_JSONSCHEMA:
-            # If jsonschema not installed, do basic validation
-            return self._basic_validate(config, config_type)
-        
-        errors = []
-        validator = Draft7Validator(schema)
-        
-        for error in sorted(validator.iter_errors(config), key=lambda e: str(e.path)):
-            error_dict = {
-                "path": ".".join(str(p) for p in error.absolute_path) or "(root)",
-                "message": self._format_error_message(error, config_type),
-                "validator": error.validator,
-                "schema_path": list(error.schema_path),
-            }
-            errors.append(error_dict)
-        
-        return len(errors) == 0, errors
-    
-    def _basic_validate(
+    def _format_pydantic_errors(
         self,
-        config: Dict[str, Any],
+        error: PydanticValidationError,
         config_type: str,
-    ) -> Tuple[bool, List[Dict[str, Any]]]:
-        """Basic validation without jsonschema library."""
+    ) -> List[Dict[str, Any]]:
+        """Convert Pydantic validation errors to our format."""
         errors = []
-        
-        if config_type == "extract":
-            if "type" not in config:
-                errors.append({
-                    "path": "type",
-                    "message": "Missing required field 'type'. Must be one of: http, file, database, cloud_storage",
-                })
-            elif config["type"] not in ("http", "file", "database", "cloud_storage"):
-                errors.append({
-                    "path": "type",
-                    "message": f"Invalid type '{config['type']}'. Must be one of: http, file, database, cloud_storage",
-                })
-        
-        elif config_type == "load":
-            if "type" not in config:
-                errors.append({
-                    "path": "type",
-                    "message": "Missing required field 'type'. Must be one of: postgres, sqlite, file, cloud_storage",
-                })
-            elif config["type"] not in ("postgres", "postgresql", "sqlite", "database", "file", "cloud_storage"):
-                errors.append({
-                    "path": "type",
-                    "message": f"Invalid type '{config['type']}'. Must be one of: postgres, sqlite, file, cloud_storage",
-                })
-        
-        elif config_type == "pipeline":
-            if "extract" not in config:
-                errors.append({
-                    "path": "extract",
-                    "message": "Missing required section 'extract'",
-                })
-            if "load" not in config:
-                errors.append({
-                    "path": "load",
-                    "message": "Missing required section 'load'",
-                })
-        
-        return len(errors) == 0, errors
+        for err in error.errors():
+            path = ".".join(str(p) for p in err["loc"]) or "(root)"
+            errors.append({
+                "path": path,
+                "message": self._format_error_message(err, config_type),
+                "type": err["type"],
+            })
+        return errors
     
-    def _format_error_message(self, error: JsonschemaValidationError, config_type: str) -> str:
-        """Format a validation error into a user-friendly message."""
-        msg = error.message
+    def _format_error_message(self, error: Dict[str, Any], config_type: str) -> str:
+        """Format a Pydantic error into a user-friendly message."""
+        error_type = error["type"]
+        msg = error["msg"]
         
         # Improve common error messages
-        if error.validator == "required":
-            missing = list(error.validator_value)
-            if "type" in missing:
+        if error_type == "missing":
+            field = error["loc"][-1] if error["loc"] else "unknown"
+            if field == "type":
                 valid_types = {
                     "extract": "http, file, database, cloud_storage",
-                    "load": "postgres, sqlite, file, cloud_storage",
+                    "load": "postgres, postgresql, sqlite, file, cloud_storage",
                 }
                 types = valid_types.get(config_type, "")
                 if types:
                     return f"Missing required 'type' field. Must be one of: {types}"
-            return f"Missing required field(s): {', '.join(missing)}"
+            return f"Missing required field: {field}"
         
-        elif error.validator == "enum":
-            allowed = error.validator_value
-            return f"Invalid value. Allowed values: {', '.join(str(v) for v in allowed)}"
+        elif error_type == "enum":
+            # Extract allowed values from the input
+            ctx = error.get("ctx", {})
+            expected = ctx.get("expected", "")
+            return f"Invalid value. Allowed values: {expected}"
         
-        elif error.validator == "type":
-            expected = error.validator_value
-            actual = type(error.instance).__name__
-            return f"Expected {expected}, got {actual}"
+        elif error_type in ("string_type", "int_type", "float_type", "bool_type"):
+            expected = error_type.replace("_type", "")
+            return f"Expected {expected}"
         
-        elif error.validator == "anyOf":
-            return "Config doesn't match any of the allowed patterns"
+        elif error_type == "extra_forbidden":
+            field = error["loc"][-1] if error["loc"] else "unknown"
+            return f"Unknown field: '{field}'"
         
-        elif error.validator == "oneOf":
-            return "Config must match exactly one of the allowed patterns"
+        elif error_type == "value_error":
+            return str(msg)
         
         return msg
     
+    def _validate_with_pydantic(
+        self,
+        config: Dict[str, Any],
+        config_type: str,
+        parse_func,
+    ) -> Tuple[bool, List[Dict[str, Any]]]:
+        """Validate config using Pydantic model."""
+        try:
+            parse_func(config)
+            return True, []
+        except PydanticValidationError as e:
+            return False, self._format_pydantic_errors(e, config_type)
+        except ValueError as e:
+            # Handle parse_* function ValueErrors (e.g., unknown type)
+            return False, [{
+                "path": "(root)",
+                "message": str(e),
+                "type": "value_error",
+            }]
+    
     def validate_extract(
         self,
         config: Dict[str, Any],
@@ -211,11 +139,13 @@ class ConfigValidator:
         Raises:
             ConfigValidationError: If validation fails and strict=True
         """
-        is_valid, errors = self._validate(config, self.extract_schema, "extract", config_path)
+        is_valid, errors = self._validate_with_pydantic(
+            config, "extract", parse_extract_config
+        )
         
         if not is_valid and self.strict:
             raise ConfigValidationError(
-                f"Invalid extract configuration",
+                "Invalid extract configuration",
                 errors=errors,
                 config_type="extract",
                 config_path=config_path,
@@ -240,11 +170,13 @@ class ConfigValidator:
         if not config:
             return
         
-        is_valid, errors = self._validate(config, self.transform_schema, "transform", config_path)
+        is_valid, errors = self._validate_with_pydantic(
+            config, "transform", lambda c: TransformConfig(**c)
+        )
         
         if not is_valid and self.strict:
             raise ConfigValidationError(
-                f"Invalid transform configuration",
+                "Invalid transform configuration",
                 errors=errors,
                 config_type="transform",
                 config_path=config_path,
@@ -265,16 +197,45 @@ class ConfigValidator:
         Raises:
             ConfigValidationError: If validation fails and strict=True
         """
-        is_valid, errors = self._validate(config, self.load_schema, "load", config_path)
+        is_valid, errors = self._validate_with_pydantic(
+            config, "load", parse_load_config
+        )
         
         if not is_valid and self.strict:
             raise ConfigValidationError(
-                f"Invalid load configuration",
+                "Invalid load configuration",
                 errors=errors,
                 config_type="load",
                 config_path=config_path,
             )
     
+    def validate_settings(
+        self,
+        config: Dict[str, Any],
+        config_path: Optional[str] = None,
+    ) -> None:
+        """
+        Validate settings configuration.
+        
+        Args:
+            config: Settings configuration dict
+            config_path: Optional path to config file (for error messages)
+            
+        Raises:
+            ConfigValidationError: If validation fails and strict=True
+        """
+        is_valid, errors = self._validate_with_pydantic(
+            config, "settings", parse_settings_config
+        )
+        
+        if not is_valid and self.strict:
+            raise ConfigValidationError(
+                "Invalid settings configuration",
+                errors=errors,
+                config_type="settings",
+                config_path=config_path,
+            )
+    
     def validate_pipeline(
         self,
         config: Dict[str, Any],
@@ -290,47 +251,50 @@ class ConfigValidator:
         Raises:
             ConfigValidationError: If validation fails and strict=True
         """
-        # Basic structure validation
         errors = []
         
+        # Validate extract (required)
         if "extract" not in config:
             errors.append({
                 "path": "extract",
                 "message": "Missing required 'extract' section",
+                "type": "missing",
             })
         else:
-            is_valid, extract_errors = self._validate(
-                config["extract"], self.extract_schema, "extract"
+            is_valid, extract_errors = self._validate_with_pydantic(
+                config["extract"], "extract", parse_extract_config
             )
             for e in extract_errors:
                 e["path"] = f"extract.{e['path']}" if e["path"] != "(root)" else "extract"
             errors.extend(extract_errors)
         
+        # Validate transform (optional)
         if "transform" in config and config["transform"]:
-            # Validate transform if present
             transform_config = config["transform"]
-            # If transform is directly the operations (list or dict), wrap it
+            # Wrap if it's just simple ops
             if isinstance(transform_config, (list, dict)):
                 if isinstance(transform_config, list) or not any(
                     k in transform_config for k in ("transform", "jsonata", "custom_function")
                 ):
                     transform_config = {"transform": transform_config}
             
-            is_valid, transform_errors = self._validate(
-                transform_config, self.transform_schema, "transform"
+            is_valid, transform_errors = self._validate_with_pydantic(
+                transform_config, "transform", lambda c: TransformConfig(**c)
             )
             for e in transform_errors:
                 e["path"] = f"transform.{e['path']}" if e["path"] != "(root)" else "transform"
             errors.extend(transform_errors)
         
+        # Validate load (required)
         if "load" not in config:
             errors.append({
                 "path": "load",
                 "message": "Missing required 'load' section",
+                "type": "missing",
             })
         else:
-            is_valid, load_errors = self._validate(
-                config["load"], self.load_schema, "load"
+            is_valid, load_errors = self._validate_with_pydantic(
+                config["load"], "load", parse_load_config
             )
             for e in load_errors:
                 e["path"] = f"load.{e['path']}" if e["path"] != "(root)" else "load"
@@ -338,7 +302,7 @@ class ConfigValidator:
         
         if errors and self.strict:
             raise ConfigValidationError(
-                f"Invalid pipeline configuration",
+                "Invalid pipeline configuration",
                 errors=errors,
                 config_type="pipeline",
                 config_path=config_path,
@@ -349,7 +313,7 @@ class ConfigValidator:
         config: Dict[str, Any],
     ) -> Tuple[bool, List[Dict[str, Any]]]:
         """Check extract config without raising. Returns (is_valid, errors)."""
-        return self._validate(config, self.extract_schema, "extract")
+        return self._validate_with_pydantic(config, "extract", parse_extract_config)
     
     def check_transform(
         self,
@@ -358,14 +322,23 @@ class ConfigValidator:
         """Check transform config without raising. Returns (is_valid, errors)."""
         if not config:
             return True, []
-        return self._validate(config, self.transform_schema, "transform")
+        return self._validate_with_pydantic(
+            config, "transform", lambda c: TransformConfig(**c)
+        )
     
     def check_load(
         self,
         config: Dict[str, Any],
     ) -> Tuple[bool, List[Dict[str, Any]]]:
         """Check load config without raising. Returns (is_valid, errors)."""
-        return self._validate(config, self.load_schema, "load")
+        return self._validate_with_pydantic(config, "load", parse_load_config)
+    
+    def check_settings(
+        self,
+        config: Dict[str, Any],
+    ) -> Tuple[bool, List[Dict[str, Any]]]:
+        """Check settings config without raising. Returns (is_valid, errors)."""
+        return self._validate_with_pydantic(config, "settings", parse_settings_config)
 
 
 def validate_config(
@@ -379,7 +352,7 @@ def validate_config(
     
     Args:
         config: Configuration dict
-        config_type: One of "extract", "transform", "load", "pipeline"
+        config_type: One of "extract", "transform", "load", "settings", "pipeline"
         config_path: Optional path to config file
         strict: If True, raise on validation failure
         
@@ -400,6 +373,9 @@ def validate_config(
     elif config_type == "load":
         validator.validate_load(config, config_path)
         return validator.check_load(config)
+    elif config_type == "settings":
+        validator.validate_settings(config, config_path)
+        return validator.check_settings(config)
     elif config_type == "pipeline":
         validator.validate_pipeline(config, config_path)
         # Return combined check

pycharter/etl_generator/context.py

@@ -10,8 +10,11 @@ from dataclasses import dataclass, field
 from typing import Any, Dict, Optional
 
 # Variable pattern: ${VAR_NAME} or ${VAR_NAME:-default} or ${VAR_NAME:?error}
+# This is the canonical definition - import from here in other modules
 VARIABLE_PATTERN = re.compile(r'\$\{([^}:]+)(?::([?-])([^}]*))?\}')
 
+__all__ = ["PipelineContext", "VARIABLE_PATTERN"]
+
 
 @dataclass
 class PipelineContext:

pycharter/etl_generator/database.py

@@ -39,6 +39,7 @@ DB_MYSQL = "mysql"
 DB_SQLITE = "sqlite"
 DB_MSSQL = "mssql"
 DB_ORACLE = "oracle"
+DB_MONGODB = "mongodb"
 
 # Default ports for SSH tunnel
 DEFAULT_POSTGRES_PORT = 5432
@@ -86,10 +87,13 @@ def detect_database_type(db_url: str) -> str:
     # Check for Oracle
     elif base_url.startswith("oracle://"):
         return DB_ORACLE
+    # Check for MongoDB
+    elif base_url.startswith(("mongodb://", "mongodb+srv://")):
+        return DB_MONGODB
     else:
         raise ValueError(
             f"Cannot detect database type from connection string. "
-            f"Supported formats: postgresql://, mysql://, sqlite://, mssql://, oracle:// "
+            f"Supported formats: postgresql://, mysql://, sqlite://, mssql://, oracle://, mongodb:// "
             f"(async variants like postgresql+asyncpg:// are also supported). "
             f"Got: {db_url[:50]}..."
         )

pycharter/etl_generator/extractors/__init__.py

@@ -5,6 +5,7 @@ This module provides a modular architecture for data extraction from various sou
 - HTTP/API extraction
 - File-based extraction (CSV, JSON, Parquet, Excel, TSV, XML)
 - Database extraction (PostgreSQL, MySQL, SQLite, MSSQL, Oracle)
+- MongoDB extraction (queries and aggregation pipelines)
 - Cloud storage extraction (S3, GCS, Azure Blob)
 
 Entry point for orchestration: extract_with_pagination_streaming().
@@ -14,17 +15,18 @@ from pycharter.etl_generator.extractors.base import BaseExtractor
 from pycharter.etl_generator.extractors.cloud_storage import CloudStorageExtractor
 from pycharter.etl_generator.extractors.database import DatabaseExtractor
 from pycharter.etl_generator.extractors.file import FileExtractor
-from pycharter.etl_generator.extractors.factory import ExtractorFactory, get_extractor
+from pycharter.etl_generator.extractors.factory import ExtractorFactory
 from pycharter.etl_generator.extractors.http import HTTPExtractor
+from pycharter.etl_generator.extractors.mongodb import MongoDBExtractor
 from pycharter.etl_generator.extractors.streaming import extract_with_pagination_streaming
 
 __all__ = [
     "BaseExtractor",
     "ExtractorFactory",
-    "get_extractor",
     "HTTPExtractor",
     "FileExtractor",
     "DatabaseExtractor",
+    "MongoDBExtractor",
     "CloudStorageExtractor",
     "extract_with_pagination_streaming",
 ]

pycharter/etl_generator/extractors/cloud_storage.py

@@ -125,10 +125,10 @@ class CloudStorageExtractor(BaseExtractor):
     
     def validate_config(self, extract_config: Dict[str, Any]) -> None:
         """Validate cloud storage extractor configuration."""
-        if 'source_type' in extract_config and extract_config['source_type'] != 'cloud_storage':
+        if 'type' in extract_config and extract_config['type'] != 'cloud_storage':
             raise ValueError(
-                f"CloudStorageExtractor requires source_type='cloud_storage', "
-                f"got '{extract_config.get('source_type')}'"
+                f"CloudStorageExtractor requires type='cloud_storage', "
+                f"got '{extract_config.get('type')}'"
             )
         
         storage_config = extract_config.get('storage', {})
@@ -266,7 +266,7 @@ class CloudStorageExtractor(BaseExtractor):
                             # Use FileExtractor to process the file
                             file_extractor = FileExtractor()
                             file_config = {
-                                'source_type': 'file',
+                                'type': 'file',
                                 'file_path': str(tmp_path),
                                 'format': file_format,
                             }
@@ -292,7 +292,7 @@ class CloudStorageExtractor(BaseExtractor):
                     # Use FileExtractor to process the file
                     file_extractor = FileExtractor()
                     file_config = {
-                        'source_type': 'file',
+                        'type': 'file',
                         'file_path': str(tmp_path),
                         'format': file_format,
                     }
@@ -369,7 +369,7 @@ class CloudStorageExtractor(BaseExtractor):
                         # Use FileExtractor
                         file_extractor = FileExtractor()
                         file_config = {
-                            'source_type': 'file',
+                            'type': 'file',
                             'file_path': str(tmp_path),
                             'format': file_format,
                         }
@@ -395,7 +395,7 @@ class CloudStorageExtractor(BaseExtractor):
                     # Use FileExtractor
                     file_extractor = FileExtractor()
                     file_config = {
-                        'source_type': 'file',
+                        'type': 'file',
                         'file_path': str(tmp_path),
                         'format': file_format,
                     }
@@ -473,7 +473,7 @@ class CloudStorageExtractor(BaseExtractor):
                         # Use FileExtractor
                         file_extractor = FileExtractor()
                         file_config = {
-                            'source_type': 'file',
+                            'type': 'file',
                             'file_path': str(tmp_path),
                             'format': file_format,
                         }
@@ -500,7 +500,7 @@ class CloudStorageExtractor(BaseExtractor):
                     # Use FileExtractor
                     file_extractor = FileExtractor()
                     file_config = {
-                        'source_type': 'file',
+                        'type': 'file',
                         'file_path': str(tmp_path),
                         'format': file_format,
                     }

pycharter/etl_generator/extractors/database.py

@@ -102,8 +102,8 @@ class DatabaseExtractor(BaseExtractor):
     
     def validate_config(self, extract_config: Dict[str, Any]) -> None:
         """Validate database extractor configuration."""
-        if 'source_type' in extract_config and extract_config['source_type'] != 'database':
-            raise ValueError(f"DatabaseExtractor requires source_type='database', got '{extract_config.get('source_type')}'")
+        if 'type' in extract_config and extract_config['type'] != 'database':
+            raise ValueError(f"DatabaseExtractor requires type='database', got '{extract_config.get('type')}'")
         
         db_config = extract_config.get('database', {})
         if not db_config.get('url'):