pycharter 0.0.25__py3-none-any.whl → 0.0.26__py3-none-any.whl

pycharter/etl_generator/config_loader.py

@@ -1,394 +0,0 @@
-"""
-Unified configuration loader for ETL pipelines.
-
-Supports both single-file (pipeline.yaml) and multi-file (extract.yaml, transform.yaml, load.yaml) formats.
-Handles variable resolution and config validation.
-"""
-
-import os
-import re
-from pathlib import Path
-from typing import Any, Dict, Optional, Tuple, Union
-
-import yaml
-
-from pycharter.shared.errors import ConfigLoadError
-from pycharter.etl_generator.config_validator import (
-    ConfigValidator,
-    ConfigValidationError,
-)
-
-
-# Variable pattern: ${VAR_NAME} or ${VAR_NAME:-default} or ${VAR_NAME:?error}
-VARIABLE_PATTERN = re.compile(r'\$\{([^}:]+)(?::([?-])([^}]*))?\}')
-
-
-class PipelineConfig:
-    """
-    Loaded and validated pipeline configuration.
-    
-    Attributes:
-        extract: Extract configuration dict
-        transform: Transform configuration dict (may be empty)
-        load: Load configuration dict
-        name: Pipeline name (from config or directory name)
-        version: Pipeline version (if specified)
-        source_path: Path to the config file or directory
-    """
-    
-    def __init__(
-        self,
-        extract: Dict[str, Any],
-        transform: Dict[str, Any],
-        load: Dict[str, Any],
-        name: Optional[str] = None,
-        version: Optional[str] = None,
-        source_path: Optional[str] = None,
-    ):
-        self.extract = extract
-        self.transform = transform
-        self.load = load
-        self.name = name
-        self.version = version
-        self.source_path = source_path
-    
-    def __repr__(self) -> str:
-        return f"PipelineConfig(name={self.name!r}, source={self.source_path!r})"
-
-
-class ConfigLoader:
-    """
-    Loads ETL pipeline configurations from files or directories.
-    
-    Supports:
-    - Single-file format: pipeline.yaml with extract, transform, load sections
-    - Multi-file format: Directory with extract.yaml, transform.yaml, load.yaml
-    - Variable resolution: ${VAR}, ${VAR:-default}, ${VAR:?error}
-    - Config validation with clear error messages
-    
-    Usage:
-        loader = ConfigLoader()
-        
-        # Load from single file
-        config = loader.load("pipelines/users/pipeline.yaml")
-        
-        # Load from directory
-        config = loader.load("pipelines/users/")
-        
-        # Load with variables
-        config = loader.load("pipelines/users/", variables={"API_KEY": "xxx"})
-    """
-    
-    def __init__(
-        self,
-        validate: bool = True,
-        strict: bool = True,
-    ):
-        """
-        Initialize the config loader.
-        
-        Args:
-            validate: If True, validate configs against schemas
-            strict: If True, raise errors on validation failure
-        """
-        self.validate = validate
-        self.strict = strict
-        self._validator = ConfigValidator(strict=strict) if validate else None
-    
-    def load(
-        self,
-        path: Union[str, Path],
-        variables: Optional[Dict[str, str]] = None,
-    ) -> PipelineConfig:
-        """
-        Load pipeline configuration from a file or directory.
-        
-        Args:
-            path: Path to config file or directory
-            variables: Additional variables for resolution (in addition to env vars)
-            
-        Returns:
-            PipelineConfig with loaded and validated configuration
-            
-        Raises:
-            ConfigLoadError: If config files cannot be found or loaded
-            ConfigValidationError: If config validation fails
-        """
-        path = Path(path).resolve()
-        
-        if path.is_file():
-            return self._load_single_file(path, variables)
-        elif path.is_dir():
-            return self._load_multi_file(path, variables)
-        else:
-            # Check if it's a path without extension
-            for ext in (".yaml", ".yml"):
-                single_file = path.with_suffix(ext)
-                if single_file.exists():
-                    return self._load_single_file(single_file, variables)
-            
-            # Check if adding pipeline.yaml works
-            pipeline_file = path / "pipeline.yaml"
-            if pipeline_file.exists():
-                return self._load_single_file(pipeline_file, variables)
-            
-            raise ConfigLoadError(
-                f"Config not found: {path}. Expected a YAML file or directory with config files.",
-                str(path),
-            )
-    
-    def _load_single_file(
-        self,
-        path: Path,
-        variables: Optional[Dict[str, str]] = None,
-    ) -> PipelineConfig:
-        """Load configuration from a single pipeline.yaml file."""
-        config = self._load_yaml(path, variables, str(path.parent))
-        
-        if not isinstance(config, dict):
-            raise ConfigLoadError(
-                f"Invalid config format: expected a dict, got {type(config).__name__}",
-                str(path),
-            )
-        
-        # Extract sections
-        extract = config.get("extract")
-        transform = config.get("transform", {})
-        load = config.get("load")
-        
-        if extract is None:
-            raise ConfigLoadError(
-                "Missing 'extract' section in pipeline config",
-                str(path),
-            )
-        
-        if load is None:
-            raise ConfigLoadError(
-                "Missing 'load' section in pipeline config",
-                str(path),
-            )
-        
-        # Normalize transform config
-        transform = self._normalize_transform(transform)
-        
-        # Validate if enabled
-        if self._validator:
-            self._validator.validate_pipeline(config, str(path))
-        
-        return PipelineConfig(
-            extract=extract,
-            transform=transform,
-            load=load,
-            name=config.get("name", path.stem),
-            version=config.get("version"),
-            source_path=str(path),
-        )
-    
-    def _load_multi_file(
-        self,
-        directory: Path,
-        variables: Optional[Dict[str, str]] = None,
-    ) -> PipelineConfig:
-        """Load configuration from separate extract.yaml, transform.yaml, load.yaml files."""
-        contract_dir = str(directory)
-        
-        # Load extract config (required)
-        extract_path = self._find_config_file(directory, "extract")
-        if extract_path is None:
-            raise ConfigLoadError(
-                f"Missing extract config in {directory}. Expected extract.yaml or extract.yml",
-                str(directory),
-            )
-        extract = self._load_yaml(extract_path, variables, contract_dir)
-        
-        # Load transform config (optional)
-        transform_path = self._find_config_file(directory, "transform")
-        if transform_path:
-            transform = self._load_yaml(transform_path, variables, contract_dir)
-            transform = self._normalize_transform(transform)
-        else:
-            transform = {}
-        
-        # Load load config (required)
-        load_path = self._find_config_file(directory, "load")
-        if load_path is None:
-            raise ConfigLoadError(
-                f"Missing load config in {directory}. Expected load.yaml or load.yml",
-                str(directory),
-            )
-        load_config = self._load_yaml(load_path, variables, contract_dir)
-        
-        # Validate if enabled
-        if self._validator:
-            self._validator.validate_extract(extract, str(extract_path))
-            if transform:
-                self._validator.validate_transform(
-                    {"transform": transform} if isinstance(transform, list) else transform,
-                    str(transform_path) if transform_path else None,
-                )
-            self._validator.validate_load(load_config, str(load_path))
-        
-        return PipelineConfig(
-            extract=extract,
-            transform=transform,
-            load=load_config,
-            name=directory.name,
-            source_path=str(directory),
-        )
-    
-    def _find_config_file(self, directory: Path, name: str) -> Optional[Path]:
-        """Find a config file by name, checking for .yaml and .yml extensions."""
-        for ext in (".yaml", ".yml"):
-            path = directory / f"{name}{ext}"
-            if path.exists():
-                return path
-        return None
-    
-    def _load_yaml(
-        self,
-        path: Path,
-        variables: Optional[Dict[str, str]] = None,
-        contract_dir: Optional[str] = None,
-    ) -> Dict[str, Any]:
-        """Load and parse a YAML file with variable resolution."""
-        try:
-            with open(path) as f:
-                content = f.read()
-        except FileNotFoundError:
-            raise ConfigLoadError(f"Config file not found: {path}", str(path))
-        except IOError as e:
-            raise ConfigLoadError(f"Error reading config file: {e}", str(path))
-        
-        # Resolve variables in the YAML content
-        content = self._resolve_variables(content, variables, contract_dir)
-        
-        try:
-            config = yaml.safe_load(content)
-        except yaml.YAMLError as e:
-            raise ConfigLoadError(f"Invalid YAML: {e}", str(path))
-        
-        return config or {}
-    
-    def _resolve_variables(
-        self,
-        content: str,
-        variables: Optional[Dict[str, str]] = None,
-        contract_dir: Optional[str] = None,
-    ) -> str:
-        """
-        Resolve ${VAR} placeholders in content.
-        
-        Supports:
-        - ${VAR} - Use variable or env var
-        - ${VAR:-default} - Use default if not set
-        - ${VAR:?error message} - Raise error if not set
-        """
-        variables = variables or {}
-        
-        def replace_var(match):
-            var_name = match.group(1)
-            modifier = match.group(2)  # '-' for default, '?' for required
-            modifier_value = match.group(3)  # default value or error message
-            
-            # Check variables dict first, then environment
-            value = variables.get(var_name) or os.environ.get(var_name)
-            
-            if value:
-                return value
-            
-            # Handle modifiers
-            if modifier == "-":
-                return modifier_value if modifier_value is not None else ""
-            elif modifier == "?":
-                error_msg = modifier_value or f"Required variable {var_name} is not set"
-                raise ConfigLoadError(error_msg, path=None)
-            
-            # No modifier - return original placeholder (will be resolved later or cause error)
-            return match.group(0)
-        
-        return VARIABLE_PATTERN.sub(replace_var, content)
-    
-    def _normalize_transform(
-        self,
-        transform: Union[Dict[str, Any], list, None],
-    ) -> Union[Dict[str, Any], list]:
-        """
-        Normalize transform config to consistent format.
-        
-        Handles:
-        - Empty/None -> {}
-        - List (ordered) -> list (unchanged)
-        - Dict with 'transform' key -> unwrap
-        - Dict (legacy) -> dict (unchanged)
-        """
-        if transform is None:
-            return {}
-        
-        if isinstance(transform, list):
-            return transform
-        
-        if isinstance(transform, dict):
-            # Check if wrapped in 'transform' key
-            if "transform" in transform and len(transform) == 1:
-                return transform["transform"]
-            return transform
-        
-        return {}
-
-
-def load_pipeline_config(
-    path: Union[str, Path],
-    variables: Optional[Dict[str, str]] = None,
-    validate: bool = True,
-) -> PipelineConfig:
-    """
-    Convenience function to load a pipeline configuration.
-    
-    Args:
-        path: Path to config file or directory
-        variables: Additional variables for resolution
-        validate: If True, validate configs against schemas
-        
-    Returns:
-        PipelineConfig with loaded configuration
-    """
-    loader = ConfigLoader(validate=validate)
-    return loader.load(path, variables)
-
-
-def detect_config_format(path: Union[str, Path]) -> str:
-    """
-    Detect whether a path points to single-file or multi-file config.
-    
-    Args:
-        path: Path to check
-        
-    Returns:
-        "single" for single-file format, "multi" for multi-file format
-        
-    Raises:
-        ConfigLoadError: If config format cannot be determined
-    """
-    path = Path(path)
-    
-    if path.is_file():
-        return "single"
-    
-    if path.is_dir():
-        # Check for pipeline.yaml (single-file in directory)
-        if (path / "pipeline.yaml").exists() or (path / "pipeline.yml").exists():
-            return "single"
-        
-        # Check for extract.yaml (multi-file)
-        if (path / "extract.yaml").exists() or (path / "extract.yml").exists():
-            return "multi"
-    
-    # Check with extensions
-    for ext in (".yaml", ".yml"):
-        if path.with_suffix(ext).exists():
-            return "single"
-    
-    raise ConfigLoadError(
-        f"Cannot determine config format for: {path}",
-        str(path),
-    )

pycharter/etl_generator/loaders/cloud.py

@@ -1,87 +0,0 @@
-"""
-Cloud storage loader for ETL pipelines.
-"""
-
-import time
-from typing import Any, Dict, List, Optional
-
-from pycharter.etl_generator.loaders.base import BaseLoader
-from pycharter.etl_generator.loaders.cloud_storage_loader import load_to_cloud_storage
-from pycharter.etl_generator.result import LoadResult
-
-
-class CloudStorageLoader(BaseLoader):
-    """
-    Loader for cloud storage (S3, GCS, Azure).
-    
-    Supports JSON, CSV, Parquet, and JSONL formats.
-    
-    Example:
-        >>> loader = CloudStorageLoader(
-        ...     provider="s3",
-        ...     bucket="my-bucket",
-        ...     path="output/data.json",
-        ...     format="json",
-        ... )
-        >>> result = await loader.load(data)
-    """
-    
-    def __init__(
-        self,
-        provider: str,
-        bucket: str,
-        path: str,
-        credentials: Optional[Dict[str, Any]] = None,
-        file_format: str = "json",
-    ):
-        self.provider = provider
-        self.bucket = bucket
-        self.path = path
-        self.credentials = credentials
-        self.file_format = file_format
-    
-    @classmethod
-    def from_config(cls, config: Dict[str, Any]) -> "CloudStorageLoader":
-        """Create loader from configuration dict."""
-        storage_config = config.get("storage", {})
-        return cls(
-            provider=storage_config.get("provider") or config.get("provider"),
-            bucket=storage_config.get("bucket") or config.get("bucket"),
-            path=storage_config.get("path") or config.get("path"),
-            credentials=storage_config.get("credentials") or config.get("credentials"),
-            file_format=config.get("format", "json"),
-        )
-    
-    async def load(self, data: List[Dict[str, Any]], **params) -> LoadResult:
-        """Load data to cloud storage."""
-        start_time = time.time()
-        
-        if not data:
-            return LoadResult(success=True, rows_loaded=0)
-        
-        try:
-            load_config = {
-                "storage": {
-                    "provider": self.provider,
-                    "bucket": self.bucket,
-                    "path": self.path,
-                    "credentials": self.credentials,
-                },
-                "format": self.file_format,
-            }
-            
-            result = load_to_cloud_storage(data, load_config)
-            
-            duration = time.time() - start_time
-            return LoadResult(
-                success=True,
-                rows_loaded=result.get("written", 0),
-                duration_seconds=duration,
-            )
-        
-        except Exception as e:
-            return LoadResult(
-                success=False,
-                error=str(e),
-                duration_seconds=time.time() - start_time,
-            )

pycharter/etl_generator/loaders/file_loader.py

@@ -1,130 +0,0 @@
-"""
-File-based loader for ETL orchestrator.
-
-Writes transformed data to local files in JSON, CSV, Parquet, or JSONL format.
-"""
-
-import io
-import json
-import logging
-from pathlib import Path
-from typing import Any, Dict, List, Optional
-
-from pycharter.utils.value_injector import resolve_values
-
-logger = logging.getLogger(__name__)
-
-SUPPORTED_FORMATS = ("json", "csv", "parquet", "jsonl")
-
-
-def load_to_file(
-    data: List[Dict[str, Any]],
-    load_config: Dict[str, Any],
-    contract_dir: Optional[Any] = None,
-    config_context: Optional[Dict[str, Any]] = None,
-) -> Dict[str, Any]:
-    """
-    Write transformed data to a local file.
-
-    Load config (destination_type: file):
-      file_path: Path to output file (required). Supports ${VAR} resolution.
-      format: json | csv | parquet | jsonl (default: json)
-      write_mode: overwrite | append (default: overwrite).
-        append: for jsonl/csv, appends lines; for json, read-merge-write (array concat).
-
-    Returns:
-        Dict with keys: written, total, path, format
-    """
-    source_file = str(contract_dir / "load.yaml") if contract_dir else None
-    file_path = load_config.get("file_path")
-    if not file_path:
-        raise ValueError(
-            "File loader requires 'file_path' in load configuration. "
-            "Example: file_path: ./output/data.json"
-        )
-    file_path = resolve_values(
-        file_path, context=config_context, source_file=source_file
-    )
-    path = Path(file_path)
-
-    fmt = (load_config.get("format") or "json").lower()
-    if fmt not in SUPPORTED_FORMATS:
-        raise ValueError(
-            f"File loader format must be one of {SUPPORTED_FORMATS}, got '{fmt}'"
-        )
-    write_mode = (load_config.get("write_mode") or "overwrite").lower()
-    if write_mode not in ("overwrite", "append"):
-        raise ValueError(
-            "File loader write_mode must be 'overwrite' or 'append', "
-            f"got '{write_mode}'"
-        )
-
-    path.parent.mkdir(parents=True, exist_ok=True)
-
-    if fmt == "json":
-        _write_json(data, path, write_mode)
-    elif fmt == "jsonl":
-        _write_jsonl(data, path, write_mode)
-    elif fmt == "csv":
-        _write_csv(data, path, write_mode)
-    elif fmt == "parquet":
-        _write_parquet(data, path, write_mode)
-
-    logger.info(f"File loader wrote {len(data)} records to {path} ({fmt})")
-    return {"written": len(data), "total": len(data), "path": str(path), "format": fmt}
-
-
-def _write_json(
-    data: List[Dict[str, Any]], path: Path, write_mode: str
-) -> None:
-    if write_mode == "append" and path.exists():
-        with open(path, "r", encoding="utf-8") as f:
-            existing = json.load(f)
-        if isinstance(existing, list):
-            data = existing + data
-        else:
-            data = [existing] + data
-    with open(path, "w", encoding="utf-8") as f:
-        json.dump(data, f, indent=2, default=str)
-
-
-def _write_jsonl(
-    data: List[Dict[str, Any]], path: Path, write_mode: str
-) -> None:
-    mode = "a" if write_mode == "append" and path.exists() else "w"
-    with open(path, mode, encoding="utf-8") as f:
-        for record in data:
-            f.write(json.dumps(record, default=str) + "\n")
-
-
-def _write_csv(
-    data: List[Dict[str, Any]], path: Path, write_mode: str
-) -> None:
-    if not data:
-        return
-    import csv
-
-    mode = "a" if write_mode == "append" and path.exists() else "w"
-    newfile = mode == "w"
-    with open(path, mode, encoding="utf-8", newline="") as f:
-        writer = csv.DictWriter(f, fieldnames=data[0].keys())
-        if newfile:
-            writer.writeheader()
-        writer.writerows(data)
-
-
-def _write_parquet(
-    data: List[Dict[str, Any]], path: Path, write_mode: str
-) -> None:
-    try:
-        import pandas as pd
-    except ImportError as e:
-        raise ImportError(
-            "pandas is required for Parquet file load. "
-            "Install with: pip install pandas pyarrow"
-        ) from e
-    df = pd.DataFrame(data)
-    if write_mode == "append" and path.exists():
-        existing = pd.read_parquet(path)
-        df = pd.concat([existing, df], ignore_index=True)
-    df.to_parquet(path, index=False)