pycharter 0.0.24__py3-none-any.whl → 0.0.26__py3-none-any.whl

pycharter/etl_generator/loaders/{cloud_storage_loader.py → cloud_storage.py}

@@ -1,15 +1,19 @@
 """
-Cloud storage loader for ETL orchestrator.
+Cloud storage loader for ETL pipelines.
 
-Writes transformed data to AWS S3, Google Cloud Storage, or Azure Blob Storage.
+Provides both class-based (CloudStorageLoader) and function-based (load_to_cloud_storage) APIs.
+Supports AWS S3, Google Cloud Storage, and Azure Blob Storage.
 """
 
 import io
 import json
 import logging
+import time
 from pathlib import Path
 from typing import Any, Dict, List, Optional, Tuple
 
+from pycharter.etl_generator.loaders.base import BaseLoader
+from pycharter.etl_generator.result import LoadResult
 from pycharter.utils.value_injector import resolve_values
 
 logger = logging.getLogger(__name__)
@@ -43,6 +47,91 @@ except ImportError:
     BlobServiceClient = None
 
 
+# =============================================================================
+# CLASS-BASED API (for programmatic use)
+# =============================================================================
+
+class CloudStorageLoader(BaseLoader):
+    """
+    Loader for cloud storage (S3, GCS, Azure).
+    
+    Supports JSON, CSV, Parquet, and JSONL formats.
+    
+    Example:
+        >>> loader = CloudStorageLoader(
+        ...     provider="s3",
+        ...     bucket="my-bucket",
+        ...     path="output/data.json",
+        ...     format="json",
+        ... )
+        >>> result = await loader.load(data)
+    """
+    
+    def __init__(
+        self,
+        provider: str,
+        bucket: str,
+        path: str,
+        credentials: Optional[Dict[str, Any]] = None,
+        file_format: str = "json",
+    ):
+        self.provider = provider
+        self.bucket = bucket
+        self.path = path
+        self.credentials = credentials
+        self.file_format = file_format
+    
+    @classmethod
+    def from_config(cls, config: Dict[str, Any]) -> "CloudStorageLoader":
+        """Create loader from configuration dict."""
+        storage_config = config.get("storage", {})
+        return cls(
+            provider=storage_config.get("provider") or config.get("provider"),
+            bucket=storage_config.get("bucket") or config.get("bucket"),
+            path=storage_config.get("path") or config.get("path"),
+            credentials=storage_config.get("credentials") or config.get("credentials"),
+            file_format=config.get("format", "json"),
+        )
+    
+    async def load(self, data: List[Dict[str, Any]], **params) -> LoadResult:
+        """Load data to cloud storage."""
+        start_time = time.time()
+        
+        if not data:
+            return LoadResult(success=True, rows_loaded=0)
+        
+        try:
+            load_config = {
+                "storage": {
+                    "provider": self.provider,
+                    "bucket": self.bucket,
+                    "path": self.path,
+                    "credentials": self.credentials,
+                },
+                "format": self.file_format,
+            }
+            
+            result = load_to_cloud_storage(data, load_config)
+            
+            duration = time.time() - start_time
+            return LoadResult(
+                success=True,
+                rows_loaded=result.get("written", 0),
+                duration_seconds=duration,
+            )
+        
+        except Exception as e:
+            return LoadResult(
+                success=False,
+                error=str(e),
+                duration_seconds=time.time() - start_time,
+            )
+
+
+# =============================================================================
+# FUNCTION-BASED API (for config-driven use)
+# =============================================================================
+
 def load_to_cloud_storage(
     data: List[Dict[str, Any]],
     load_config: Dict[str, Any],
@@ -134,6 +223,10 @@ def load_to_cloud_storage(
     }
 
 
+# =============================================================================
+# INTERNAL HELPER FUNCTIONS
+# =============================================================================
+
 def _serialize_data(
     data: List[Dict[str, Any]], fmt: str
 ) -> Tuple[bytes, str]:

pycharter/etl_generator/loaders/factory.py

@@ -20,28 +20,28 @@ from typing import Any, Dict, List, Optional, Type
 from pycharter.etl_generator.loaders.base import BaseLoader
 from pycharter.etl_generator.loaders.database import PostgresLoader, DatabaseLoader
 from pycharter.etl_generator.loaders.file import FileLoader
-from pycharter.etl_generator.loaders.cloud import CloudStorageLoader
+from pycharter.etl_generator.loaders.cloud_storage import CloudStorageLoader
+from pycharter.etl_generator.loaders.mongodb import MongoDBLoader
 
 logger = logging.getLogger(__name__)
 
 
 class LoaderFactory:
     """
-    Factory for creating loader instances based on target type.
+    Factory for creating loader instances based on type.
     
     Supports:
     - Explicit 'type' field (recommended)
-    - Legacy 'target_type' field
-    - Auto-detection from config keys (for backward compatibility)
+    - Auto-detection from config keys
     
     Example:
         # With explicit type (recommended)
         config = {"type": "postgres", "table": "users", "database": {"url": "..."}}
         loader = LoaderFactory.create(config)
         
-        # Auto-detected (legacy)
-        config = {"table": "users", "connection_string": "postgresql://..."}
-        loader = LoaderFactory.create(config)  # Detected as postgres
+        # Auto-detected from config keys
+        config = {"file_path": "/path/to/output.json"}
+        loader = LoaderFactory.create(config)  # Detected as file
     """
     
     # Registry of loaders by target type
@@ -52,6 +52,8 @@ class LoaderFactory:
         "sqlite": DatabaseLoader,
         "file": FileLoader,
         "cloud_storage": CloudStorageLoader,
+        "mongodb": MongoDBLoader,
+        "mongo": MongoDBLoader,  # Alias
     }
     
     @classmethod
@@ -98,8 +100,8 @@ class LoaderFactory:
         Raises:
             ValueError: If type cannot be determined or is not registered
         """
-        # Get type from config (check 'type' first, then 'target_type' for legacy)
-        load_type = config.get("type") or config.get("target_type")
+        # Get type from config
+        load_type = config.get("type")
         
         # Auto-detect if not specified
         if not load_type:
@@ -139,7 +141,11 @@ class LoaderFactory:
         
         This is for backward compatibility. New configs should use explicit 'type'.
         """
-        # Database indicators
+        # MongoDB indicators (check first as it has specific 'mongodb' config key)
+        if "mongodb" in config:
+            return "mongodb"
+        
+        # Database indicators (SQL databases)
         if "table" in config:
             if "connection_string" in config or "database" in config:
                 # Check if it's SQLite
@@ -159,22 +165,3 @@ class LoaderFactory:
             return "cloud_storage"
         
         return None
-    
-    # Legacy method name for consistency with ExtractorFactory
-    @classmethod
-    def get_loader(cls, load_config: Dict[str, Any]) -> BaseLoader:
-        """Legacy method. Use create() instead."""
-        return cls.create(load_config)
-
-
-def get_loader(load_config: Dict[str, Any]) -> BaseLoader:
-    """
-    Convenience function to get loader instance.
-    
-    Args:
-        load_config: Load configuration dictionary
-    
-    Returns:
-        Loader instance
-    """
-    return LoaderFactory.create(load_config)

pycharter/etl_generator/loaders/file.py

@@ -1,15 +1,28 @@
 """
 File loader for ETL pipelines.
+
+Provides both class-based (FileLoader) and function-based (load_to_file) APIs.
+Supports JSON, CSV, Parquet, and JSONL formats.
 """
 
+import json
+import logging
 import time
 from pathlib import Path
 from typing import Any, Dict, List, Optional
 
 from pycharter.etl_generator.loaders.base import BaseLoader
-from pycharter.etl_generator.loaders.file_loader import load_to_file
 from pycharter.etl_generator.result import LoadResult
+from pycharter.utils.value_injector import resolve_values
+
+logger = logging.getLogger(__name__)
+
+SUPPORTED_FORMATS = ("json", "csv", "parquet", "jsonl")
+
 
+# =============================================================================
+# CLASS-BASED API (for programmatic use)
+# =============================================================================
 
 class FileLoader(BaseLoader):
     """
@@ -70,3 +83,124 @@ class FileLoader(BaseLoader):
                 error=str(e),
                 duration_seconds=time.time() - start_time,
             )
+
+
+# =============================================================================
+# FUNCTION-BASED API (for config-driven use)
+# =============================================================================
+
+def load_to_file(
+    data: List[Dict[str, Any]],
+    load_config: Dict[str, Any],
+    contract_dir: Optional[Any] = None,
+    config_context: Optional[Dict[str, Any]] = None,
+) -> Dict[str, Any]:
+    """
+    Write transformed data to a local file.
+
+    Load config (destination_type: file):
+      file_path: Path to output file (required). Supports ${VAR} resolution.
+      format: json | csv | parquet | jsonl (default: json)
+      write_mode: overwrite | append (default: overwrite).
+        append: for jsonl/csv, appends lines; for json, read-merge-write (array concat).
+
+    Returns:
+        Dict with keys: written, total, path, format
+    """
+    source_file = str(contract_dir / "load.yaml") if contract_dir else None
+    file_path = load_config.get("file_path")
+    if not file_path:
+        raise ValueError(
+            "File loader requires 'file_path' in load configuration. "
+            "Example: file_path: ./output/data.json"
+        )
+    file_path = resolve_values(
+        file_path, context=config_context, source_file=source_file
+    )
+    path = Path(file_path)
+
+    fmt = (load_config.get("format") or "json").lower()
+    if fmt not in SUPPORTED_FORMATS:
+        raise ValueError(
+            f"File loader format must be one of {SUPPORTED_FORMATS}, got '{fmt}'"
+        )
+    write_mode = (load_config.get("write_mode") or "overwrite").lower()
+    if write_mode not in ("overwrite", "append"):
+        raise ValueError(
+            "File loader write_mode must be 'overwrite' or 'append', "
+            f"got '{write_mode}'"
+        )
+
+    path.parent.mkdir(parents=True, exist_ok=True)
+
+    if fmt == "json":
+        _write_json(data, path, write_mode)
+    elif fmt == "jsonl":
+        _write_jsonl(data, path, write_mode)
+    elif fmt == "csv":
+        _write_csv(data, path, write_mode)
+    elif fmt == "parquet":
+        _write_parquet(data, path, write_mode)
+
+    logger.info(f"File loader wrote {len(data)} records to {path} ({fmt})")
+    return {"written": len(data), "total": len(data), "path": str(path), "format": fmt}
+
+
+# =============================================================================
+# INTERNAL HELPER FUNCTIONS
+# =============================================================================
+
+def _write_json(
+    data: List[Dict[str, Any]], path: Path, write_mode: str
+) -> None:
+    if write_mode == "append" and path.exists():
+        with open(path, "r", encoding="utf-8") as f:
+            existing = json.load(f)
+        if isinstance(existing, list):
+            data = existing + data
+        else:
+            data = [existing] + data
+    with open(path, "w", encoding="utf-8") as f:
+        json.dump(data, f, indent=2, default=str)
+
+
+def _write_jsonl(
+    data: List[Dict[str, Any]], path: Path, write_mode: str
+) -> None:
+    mode = "a" if write_mode == "append" and path.exists() else "w"
+    with open(path, mode, encoding="utf-8") as f:
+        for record in data:
+            f.write(json.dumps(record, default=str) + "\n")
+
+
+def _write_csv(
+    data: List[Dict[str, Any]], path: Path, write_mode: str
+) -> None:
+    if not data:
+        return
+    import csv
+
+    mode = "a" if write_mode == "append" and path.exists() else "w"
+    newfile = mode == "w"
+    with open(path, mode, encoding="utf-8", newline="") as f:
+        writer = csv.DictWriter(f, fieldnames=data[0].keys())
+        if newfile:
+            writer.writeheader()
+        writer.writerows(data)
+
+
+def _write_parquet(
+    data: List[Dict[str, Any]], path: Path, write_mode: str
+) -> None:
+    try:
+        import pandas as pd
+    except ImportError as e:
+        raise ImportError(
+            "pandas is required for Parquet file load. "
+            "Install with: pip install pandas pyarrow"
+        ) from e
+    df = pd.DataFrame(data)
+    if write_mode == "append" and path.exists():
+        existing = pd.read_parquet(path)
+        df = pd.concat([existing, df], ignore_index=True)
+    df.to_parquet(path, index=False)