pycharter 0.0.24__py3-none-any.whl → 0.0.26__py3-none-any.whl

pycharter/etl_generator/extractors/factory.py

@@ -22,27 +22,27 @@ from pycharter.etl_generator.extractors.cloud_storage import CloudStorageExtract
 from pycharter.etl_generator.extractors.database import DatabaseExtractor
 from pycharter.etl_generator.extractors.file import FileExtractor
 from pycharter.etl_generator.extractors.http import HTTPExtractor
+from pycharter.etl_generator.extractors.mongodb import MongoDBExtractor
 
 logger = logging.getLogger(__name__)
 
 
 class ExtractorFactory:
     """
-    Factory for creating extractor instances based on source type.
+    Factory for creating extractor instances based on type.
     
     Supports:
     - Explicit 'type' field (recommended)
-    - Legacy 'source_type' field
-    - Auto-detection from config keys (for backward compatibility)
+    - Auto-detection from config keys
     
     Example:
         # With explicit type (recommended)
         config = {"type": "http", "url": "https://api.example.com/data"}
         extractor = ExtractorFactory.create(config)
         
-        # Auto-detected (legacy)
-        config = {"url": "https://api.example.com/data"}
-        extractor = ExtractorFactory.create(config)  # Detected as HTTP
+        # Auto-detected from config keys
+        config = {"file_path": "/path/to/data.csv"}
+        extractor = ExtractorFactory.create(config)  # Detected as file
     """
     
     # Registry of extractors by source type
@@ -51,6 +51,8 @@ class ExtractorFactory:
         "file": FileExtractor,
         "database": DatabaseExtractor,
         "cloud_storage": CloudStorageExtractor,
+        "mongodb": MongoDBExtractor,
+        "mongo": MongoDBExtractor,  # Alias
     }
     
     @classmethod
@@ -97,8 +99,8 @@ class ExtractorFactory:
         Raises:
             ValueError: If type cannot be determined or is not registered
         """
-        # Get type from config (check 'type' first, then 'source_type' for legacy)
-        extract_type = config.get("type") or config.get("source_type")
+        # Get type from config
+        extract_type = config.get("type")
         
         # Auto-detect if not specified
         if not extract_type:
@@ -140,6 +142,10 @@ class ExtractorFactory:
         
         This is for backward compatibility. New configs should use explicit 'type'.
         """
+        # MongoDB indicators (check first as it has specific 'mongodb' config key)
+        if "mongodb" in config:
+            return "mongodb"
+        
         # HTTP indicators
         if any(key in config for key in ("url", "base_url", "api_endpoint", "endpoint")):
             return "http"
@@ -148,7 +154,7 @@ class ExtractorFactory:
         if any(key in config for key in ("path", "file_path")) and "storage" not in config:
             return "file"
         
-        # Database indicators
+        # Database indicators (SQL databases)
         if "query" in config or (
             "database" in config and "connection_string" not in config.get("database", {}).get("url", "s3")
         ):
@@ -159,27 +165,3 @@ class ExtractorFactory:
             return "cloud_storage"
         
         return None
-    
-    # Legacy method names for backward compatibility
-    @classmethod
-    def register_extractor(cls, source_type: str, extractor_class: Type[BaseExtractor]) -> None:
-        """Legacy method. Use register() instead."""
-        cls.register(source_type, extractor_class)
-    
-    @classmethod
-    def get_extractor(cls, extract_config: Dict[str, Any]) -> BaseExtractor:
-        """Legacy method. Use create() instead."""
-        return cls.create(extract_config)
-
-
-def get_extractor(extract_config: Dict[str, Any]) -> BaseExtractor:
-    """
-    Convenience function to get extractor instance.
-    
-    Args:
-        extract_config: Extract configuration dictionary
-    
-    Returns:
-        Extractor instance
-    """
-    return ExtractorFactory.create(extract_config)

pycharter/etl_generator/extractors/file.py

@@ -99,8 +99,8 @@ class FileExtractor(BaseExtractor):
     
     def validate_config(self, extract_config: Dict[str, Any]) -> None:
         """Validate file extractor configuration."""
-        if 'source_type' in extract_config and extract_config['source_type'] != 'file':
-            raise ValueError(f"FileExtractor requires source_type='file', got '{extract_config.get('source_type')}'")
+        if 'type' in extract_config and extract_config['type'] != 'file':
+            raise ValueError(f"FileExtractor requires type='file', got '{extract_config.get('type')}'")
         
         file_path = extract_config.get('file_path')
         if not file_path:

pycharter/etl_generator/extractors/http.py

@@ -121,8 +121,8 @@ class HTTPExtractor(BaseExtractor):
     
     def validate_config(self, extract_config: Dict[str, Any]) -> None:
         """Validate HTTP extractor configuration."""
-        if 'source_type' in extract_config and extract_config['source_type'] != 'http':
-            raise ValueError(f"HTTPExtractor requires source_type='http', got '{extract_config.get('source_type')}'")
+        if 'type' in extract_config and extract_config['type'] != 'http':
+            raise ValueError(f"HTTPExtractor requires type='http', got '{extract_config.get('type')}'")
         
         # Check for required HTTP config fields
         if not extract_config.get('api_endpoint') and not extract_config.get('base_url'):

pycharter/etl_generator/extractors/mongodb.py

@@ -0,0 +1,393 @@
+"""
+MongoDB extractor for ETL orchestrator.
+
+Supports extracting data from MongoDB collections using:
+- Simple find() queries with filters
+- Aggregation pipelines for complex transformations
+
+Configuration example:
+    ```yaml
+    type: mongodb
+    mongodb:
+      url: mongodb://user:pass@localhost:27017
+      database: mydb
+      collection: users
+      ssh_tunnel:
+        enabled: false
+        host: bastion.example.com
+        username: tunnel_user
+        remote_host: mongo-internal.example.com
+        remote_port: 27017
+    
+    # Option 1: Simple query filter
+    query:
+      status: active
+      created_at:
+        $gte: "2024-01-01"
+    
+    # Option 2: Aggregation pipeline (takes precedence over query)
+    pipeline:
+      - $match:
+          status: active
+      - $project:
+          name: 1
+          email: 1
+          _id: 0
+    
+    # Optional: projection for simple queries
+    projection:
+      name: 1
+      email: 1
+      _id: 0
+    
+    # Optional: sorting
+    sort:
+      created_at: -1
+    
+    # Optional: limit results
+    limit: 1000
+    ```
+"""
+
+import logging
+from typing import Any, AsyncIterator, Dict, List, Optional
+
+from bson import ObjectId
+from pymongo import MongoClient
+
+from pycharter.etl_generator.extractors.base import BaseExtractor
+from pycharter.etl_generator.database import (
+    create_ssh_tunnel,
+    DEFAULT_TUNNEL_LOCAL_PORT,
+)
+from pycharter.utils.value_injector import resolve_values
+
+logger = logging.getLogger(__name__)
+
+# Default MongoDB port
+DEFAULT_MONGODB_PORT = 27017
+
+
+def _serialize_document(doc: Dict[str, Any]) -> Dict[str, Any]:
+    """
+    Serialize a MongoDB document for JSON compatibility.
+    
+    Converts:
+    - ObjectId to string
+    - Other BSON types as needed
+    
+    Args:
+        doc: MongoDB document
+        
+    Returns:
+        JSON-serializable dictionary
+    """
+    result = {}
+    for key, value in doc.items():
+        if isinstance(value, ObjectId):
+            result[key] = str(value)
+        elif isinstance(value, dict):
+            result[key] = _serialize_document(value)
+        elif isinstance(value, list):
+            result[key] = [
+                _serialize_document(item) if isinstance(item, dict)
+                else str(item) if isinstance(item, ObjectId)
+                else item
+                for item in value
+            ]
+        else:
+            result[key] = value
+    return result
+
+
+def _modify_mongodb_url_for_tunnel(url: str, local_port: int) -> str:
+    """
+    Modify MongoDB URL to use local tunnel port.
+    
+    Args:
+        url: Original MongoDB URL
+        local_port: Local tunnel port
+        
+    Returns:
+        Modified URL pointing to localhost tunnel
+    """
+    import re
+    # Match mongodb:// or mongodb+srv:// URLs
+    # Replace host:port with 127.0.0.1:local_port
+    # Handle URLs with or without port
+    pattern = r'mongodb(\+srv)?://([^@]+@)?([^/:]+)(:\d+)?'
+    
+    def replace_host(match):
+        protocol = match.group(1) or ''
+        auth = match.group(2) or ''
+        # For tunneled connections, we don't use SRV
+        return f'mongodb://{auth}127.0.0.1:{local_port}'
+    
+    return re.sub(pattern, replace_host, url)
+
+
+class MongoDBExtractor(BaseExtractor):
+    """
+    Extractor for MongoDB data sources.
+    
+    Supports two query modes:
+    1. Simple find() queries with optional projection and sorting
+    2. Aggregation pipelines for complex data transformations
+    
+    Example (programmatic API):
+        >>> extractor = MongoDBExtractor(
+        ...     connection_string="mongodb://localhost:27017",
+        ...     database="mydb",
+        ...     collection="users",
+        ...     query={"status": "active"},
+        ... )
+        >>> async for batch in extractor.extract():
+        ...     process(batch)
+    
+    Example (config-driven):
+        >>> extractor = MongoDBExtractor()
+        >>> async for batch in extractor.extract_streaming(config, params, headers):
+        ...     process(batch)
+    """
+    
+    def __init__(
+        self,
+        connection_string: Optional[str] = None,
+        database: Optional[str] = None,
+        collection: Optional[str] = None,
+        query: Optional[Dict[str, Any]] = None,
+        pipeline: Optional[List[Dict[str, Any]]] = None,
+        projection: Optional[Dict[str, Any]] = None,
+        sort: Optional[Dict[str, int]] = None,
+        batch_size: int = 1000,
+        max_records: Optional[int] = None,
+        ssh_tunnel: Optional[Dict[str, Any]] = None,
+    ):
+        """
+        Initialize MongoDB extractor.
+        
+        Args:
+            connection_string: MongoDB connection URL
+            database: Database name
+            collection: Collection name
+            query: MongoDB query filter (for find operations)
+            pipeline: Aggregation pipeline (takes precedence over query)
+            projection: Fields to include/exclude (for find operations)
+            sort: Sort specification (for find operations)
+            batch_size: Number of records per batch
+            max_records: Maximum records to extract (None = all)
+            ssh_tunnel: SSH tunnel configuration
+        """
+        self.connection_string = connection_string
+        self.database = database
+        self.collection = collection
+        self.query = query or {}
+        self.pipeline = pipeline
+        self.projection = projection
+        self.sort = sort
+        self.batch_size = batch_size
+        self.max_records = max_records
+        self.ssh_tunnel = ssh_tunnel
+    
+    @classmethod
+    def from_config(cls, config: Dict[str, Any]) -> "MongoDBExtractor":
+        """Create extractor from configuration dict."""
+        mongo_config = config.get("mongodb", {})
+        return cls(
+            connection_string=mongo_config.get("url") or config.get("connection_string"),
+            database=mongo_config.get("database") or config.get("database"),
+            collection=mongo_config.get("collection") or config.get("collection"),
+            query=config.get("query", {}),
+            pipeline=config.get("pipeline"),
+            projection=config.get("projection"),
+            sort=config.get("sort"),
+            batch_size=config.get("batch_size", 1000),
+            max_records=config.get("max_records"),
+            ssh_tunnel=mongo_config.get("ssh_tunnel"),
+        )
+    
+    async def extract(self, **params) -> AsyncIterator[List[Dict[str, Any]]]:
+        """
+        Extract data from MongoDB.
+        
+        Yields:
+            Batches of records as dictionaries
+        """
+        if not self.connection_string:
+            raise ValueError("MongoDB connection string is required")
+        if not self.database:
+            raise ValueError("MongoDB database name is required")
+        if not self.collection:
+            raise ValueError("MongoDB collection name is required")
+        
+        extract_config = {
+            "mongodb": {
+                "url": self.connection_string,
+                "database": self.database,
+                "collection": self.collection,
+                "ssh_tunnel": self.ssh_tunnel,
+            },
+            "query": self.query,
+            "pipeline": self.pipeline,
+            "projection": self.projection,
+            "sort": self.sort,
+        }
+        
+        async for batch in self.extract_streaming(
+            extract_config, params, {},
+            batch_size=self.batch_size,
+            max_records=self.max_records,
+        ):
+            yield batch
+    
+    def validate_config(self, extract_config: Dict[str, Any]) -> None:
+        """Validate MongoDB extractor configuration."""
+        source_type = extract_config.get('type') or extract_config.get('source_type')
+        if source_type and source_type not in ('mongodb', 'mongo'):
+            raise ValueError(
+                f"MongoDBExtractor requires type='mongodb', got '{source_type}'"
+            )
+        
+        mongo_config = extract_config.get('mongodb', {})
+        if not mongo_config.get('url'):
+            raise ValueError("MongoDB extractor requires 'mongodb.url' in extract_config")
+        if not mongo_config.get('database'):
+            raise ValueError("MongoDB extractor requires 'mongodb.database' in extract_config")
+        if not mongo_config.get('collection'):
+            raise ValueError("MongoDB extractor requires 'mongodb.collection' in extract_config")
+    
+    async def extract_streaming(
+        self,
+        extract_config: Dict[str, Any],
+        params: Dict[str, Any],
+        headers: Dict[str, Any],
+        contract_dir: Optional[Any] = None,
+        batch_size: int = 1000,
+        max_records: Optional[int] = None,
+        config_context: Optional[Dict[str, Any]] = None,
+    ) -> AsyncIterator[List[Dict[str, Any]]]:
+        """
+        Extract data from MongoDB using find() or aggregation pipeline.
+        
+        Yields batches of records as lists of dictionaries.
+        """
+        # Get MongoDB configuration
+        mongo_config = extract_config.get('mongodb', {})
+        
+        # Resolve variables
+        source_file = str(contract_dir / "extract.yaml") if contract_dir else None
+        mongo_url = resolve_values(
+            mongo_config.get('url'), 
+            context=config_context, 
+            source_file=source_file
+        )
+        db_name = resolve_values(
+            mongo_config.get('database'),
+            context=config_context,
+            source_file=source_file
+        )
+        collection_name = resolve_values(
+            mongo_config.get('collection'),
+            context=config_context,
+            source_file=source_file
+        )
+        
+        if not mongo_url:
+            raise ValueError("MongoDB URL is required")
+        if not db_name:
+            raise ValueError("MongoDB database name is required")
+        if not collection_name:
+            raise ValueError("MongoDB collection name is required")
+        
+        # Handle SSH tunnel if configured
+        ssh_config = mongo_config.get('ssh_tunnel', {})
+        tunnel = None
+        if ssh_config:
+            ssh_config = resolve_values(ssh_config, context=config_context, source_file=source_file)
+            enabled_value = ssh_config.get('enabled', False)
+            if isinstance(enabled_value, str):
+                enabled_lower = enabled_value.lower()
+                ssh_config['enabled'] = enabled_lower in ('true', '1', 'yes', 'on')
+            elif not isinstance(enabled_value, bool):
+                ssh_config['enabled'] = bool(enabled_value)
+            
+            if ssh_config.get('enabled', False):
+                # Set default remote port for MongoDB if not specified
+                if 'remote_port' not in ssh_config:
+                    ssh_config['remote_port'] = DEFAULT_MONGODB_PORT
+                
+                tunnel = create_ssh_tunnel(ssh_config)
+                if tunnel:
+                    local_port = int(ssh_config.get('local_port', DEFAULT_TUNNEL_LOCAL_PORT))
+                    mongo_url = _modify_mongodb_url_for_tunnel(mongo_url, local_port)
+        
+        # Get query/pipeline configuration
+        query = extract_config.get('query', {})
+        pipeline = extract_config.get('pipeline')
+        projection = extract_config.get('projection')
+        sort_spec = extract_config.get('sort')
+        limit = extract_config.get('limit')
+        
+        # Merge params into query for parameterized queries
+        if params:
+            query = {**query, **params}
+        
+        # Create MongoDB client
+        client = MongoClient(mongo_url)
+        db = client[db_name]
+        collection = db[collection_name]
+        
+        try:
+            logger.info(
+                f"Extracting from MongoDB: {db_name}.{collection_name} "
+                f"(pipeline: {bool(pipeline)}, query: {bool(query)})"
+            )
+            
+            # Choose extraction method
+            if pipeline:
+                # Use aggregation pipeline
+                cursor = collection.aggregate(pipeline, batchSize=batch_size)
+            else:
+                # Use find() with optional projection, sort, limit
+                cursor = collection.find(query, projection)
+                
+                if sort_spec:
+                    # Convert dict to list of tuples for pymongo
+                    sort_list = [(k, v) for k, v in sort_spec.items()]
+                    cursor = cursor.sort(sort_list)
+                
+                if limit:
+                    cursor = cursor.limit(limit)
+                
+                cursor = cursor.batch_size(batch_size)
+            
+            # Stream results in batches
+            current_batch = []
+            total_extracted = 0
+            
+            for doc in cursor:
+                if max_records and total_extracted >= max_records:
+                    break
+                
+                # Serialize document for JSON compatibility
+                record = _serialize_document(doc)
+                current_batch.append(record)
+                total_extracted += 1
+                
+                if len(current_batch) >= batch_size:
+                    yield current_batch
+                    current_batch = []
+            
+            # Yield remaining records
+            if current_batch:
+                yield current_batch
+            
+            logger.info(f"MongoDB extraction completed: {total_extracted} records extracted")
+            
+        except Exception as e:
+            logger.error(f"MongoDB extraction error: {e}", exc_info=True)
+            raise RuntimeError(f"MongoDB extraction failed: {e}") from e
+        finally:
+            client.close()
+            if tunnel:
+                tunnel.stop()

pycharter/etl_generator/extractors/streaming.py

@@ -28,7 +28,7 @@ async def extract_with_pagination_streaming(
     - Cloud storage (S3, GCS, Azure Blob)
 
     The source type is auto-detected from extract_config or can be explicitly set
-    via 'source_type' field.
+    via 'type' field.
 
     Yields batches as they are extracted, preventing memory exhaustion for large datasets.
 
@@ -44,7 +44,7 @@ async def extract_with_pagination_streaming(
     Yields:
         Batches of extracted records (lists of dictionaries)
     """
-    extractor = ExtractorFactory.get_extractor(extract_config)
+    extractor = ExtractorFactory.create(extract_config)
     async for batch in extractor.extract_streaming(
         extract_config,
         params,

pycharter/etl_generator/loaders/__init__.py

@@ -7,32 +7,38 @@ Two APIs:
 
 Supports:
 - Database (PostgreSQL, MySQL, SQLite, MSSQL)
+- MongoDB (insert, upsert, replace, update, delete)
 - File (local JSON, CSV, Parquet, JSONL)
 - Cloud storage (AWS S3, Google Cloud Storage, Azure Blob)
 """
 
-# Class-based loaders (new API)
+# Base class
 from pycharter.etl_generator.loaders.base import BaseLoader
+
+# Database loaders
 from pycharter.etl_generator.loaders.database import PostgresLoader, DatabaseLoader
-from pycharter.etl_generator.loaders.file import FileLoader
-from pycharter.etl_generator.loaders.cloud import CloudStorageLoader
 
-# Factory
-from pycharter.etl_generator.loaders.factory import LoaderFactory, get_loader
+# MongoDB loader
+from pycharter.etl_generator.loaders.mongodb import MongoDBLoader
+
+# File loaders (class and function APIs)
+from pycharter.etl_generator.loaders.file import FileLoader, load_to_file
 
-# Function-based loaders (config-driven)
-from pycharter.etl_generator.loaders.file_loader import load_to_file
-from pycharter.etl_generator.loaders.cloud_storage_loader import load_to_cloud_storage
+# Cloud storage loaders (class and function APIs)
+from pycharter.etl_generator.loaders.cloud_storage import CloudStorageLoader, load_to_cloud_storage
+
+# Factory
+from pycharter.etl_generator.loaders.factory import LoaderFactory
 
 __all__ = [
     # Base class
     "BaseLoader",
     # Factory
     "LoaderFactory",
-    "get_loader",
     # Class-based loaders
     "PostgresLoader",
     "DatabaseLoader",
+    "MongoDBLoader",
     "FileLoader",
     "CloudStorageLoader",
     # Function-based loaders