pycharter 0.0.22__py3-none-any.whl → 0.0.24__py3-none-any.whl

pycharter/etl_generator/protocols.py

@@ -0,0 +1,54 @@
+"""
+Protocol definitions for ETL components.
+
+Uses Python's Protocol for structural subtyping (duck typing with type hints).
+"""
+
+from typing import Any, AsyncIterator, Dict, List, Protocol, runtime_checkable
+
+from pycharter.etl_generator.result import LoadResult
+
+
+@runtime_checkable
+class Extractor(Protocol):
+    """
+    Protocol for data extractors.
+    
+    Extractors read data from sources (HTTP, files, databases, cloud storage)
+    and yield batches of records.
+    """
+    
+    async def extract(self, **params) -> AsyncIterator[List[Dict[str, Any]]]:
+        """
+        Extract data from the source.
+        
+        Yields:
+            Batches of records (list of dicts)
+        """
+        ...
+
+
+@runtime_checkable
+class Transformer(Protocol):
+    """
+    Protocol for data transformers.
+    
+    Transformers process batches of records. They can be chained with |.
+    """
+    
+    def transform(self, data: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+        """Transform a batch of records."""
+        ...
+
+
+@runtime_checkable
+class Loader(Protocol):
+    """
+    Protocol for data loaders.
+    
+    Loaders write data to destinations (databases, files, cloud storage).
+    """
+    
+    async def load(self, data: List[Dict[str, Any]], **params) -> LoadResult:
+        """Load data to the destination."""
+        ...

pycharter/etl_generator/result.py

@@ -0,0 +1,63 @@
+"""
+Result classes for ETL operations.
+"""
+
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from typing import Any, Dict, List, Optional
+
+
+@dataclass
+class LoadResult:
+    """Result from a load operation."""
+    success: bool = True
+    rows_loaded: int = 0
+    rows_failed: int = 0
+    error: Optional[str] = None
+    duration_seconds: Optional[float] = None
+
+
+@dataclass
+class BatchResult:
+    """Result from processing a single batch."""
+    batch_index: int
+    rows_in: int = 0
+    rows_out: int = 0
+    rows_failed: int = 0
+    errors: List[str] = field(default_factory=list)
+    
+    @property
+    def success(self) -> bool:
+        return len(self.errors) == 0 and self.rows_failed == 0
+
+
+@dataclass
+class PipelineResult:
+    """Complete result from running an ETL pipeline."""
+    success: bool = True
+    rows_extracted: int = 0
+    rows_transformed: int = 0
+    rows_loaded: int = 0
+    rows_failed: int = 0
+    start_time: Optional[datetime] = None
+    end_time: Optional[datetime] = None
+    duration_seconds: Optional[float] = None
+    batches_processed: int = 0
+    batch_results: List[BatchResult] = field(default_factory=list)
+    errors: List[str] = field(default_factory=list)
+    pipeline_name: Optional[str] = None
+    run_id: Optional[str] = None
+    
+    def to_dict(self) -> Dict[str, Any]:
+        return {
+            "success": self.success,
+            "rows_extracted": self.rows_extracted,
+            "rows_transformed": self.rows_transformed,
+            "rows_loaded": self.rows_loaded,
+            "rows_failed": self.rows_failed,
+            "duration_seconds": self.duration_seconds,
+            "batches_processed": self.batches_processed,
+            "errors": self.errors,
+            "pipeline_name": self.pipeline_name,
+            "run_id": self.run_id,
+        }

pycharter/etl_generator/schemas/__init__.py

@@ -0,0 +1,49 @@
+"""
+JSON Schemas for ETL Pipeline Configuration.
+
+Provides validation schemas for extract, transform, load, and pipeline configs.
+"""
+
+import json
+from pathlib import Path
+from typing import Dict, Any
+
+SCHEMA_DIR = Path(__file__).parent
+
+
+def load_schema(name: str) -> Dict[str, Any]:
+    """Load a JSON schema by name."""
+    schema_path = SCHEMA_DIR / f"{name}.json"
+    if not schema_path.exists():
+        raise FileNotFoundError(f"Schema not found: {schema_path}")
+    with open(schema_path) as f:
+        return json.load(f)
+
+
+def get_extract_schema() -> Dict[str, Any]:
+    """Get the extract config schema."""
+    return load_schema("extract")
+
+
+def get_transform_schema() -> Dict[str, Any]:
+    """Get the transform config schema."""
+    return load_schema("transform")
+
+
+def get_load_schema() -> Dict[str, Any]:
+    """Get the load config schema."""
+    return load_schema("load")
+
+
+def get_pipeline_schema() -> Dict[str, Any]:
+    """Get the combined pipeline config schema."""
+    return load_schema("pipeline")
+
+
+__all__ = [
+    "load_schema",
+    "get_extract_schema",
+    "get_transform_schema",
+    "get_load_schema",
+    "get_pipeline_schema",
+]

pycharter/etl_generator/transformers/__init__.py

@@ -0,0 +1,49 @@
+"""
+Transform stage for ETL pipelines.
+
+Two APIs:
+1. Config-driven: apply_transforms(data, config) - uses YAML config
+2. Programmatic: Rename(...) | AddField(...) | Filter(...) - chainable
+
+Pipeline order for config: Simple operations → JSONata → Custom function.
+"""
+
+# Config-driven API
+from pycharter.etl_generator.transformers.pipeline import apply_transforms
+
+# Chainable transformers
+from pycharter.etl_generator.transformers.base import (
+    BaseTransformer,
+    TransformerChain,
+)
+from pycharter.etl_generator.transformers.operations import (
+    Rename,
+    AddField,
+    Drop,
+    Select,
+    Filter,
+    Convert,
+    Default,
+    Map,
+    FlatMap,
+    CustomFunction,
+)
+
+__all__ = [
+    # Config-driven
+    "apply_transforms",
+    # Base classes
+    "BaseTransformer",
+    "TransformerChain",
+    # Operations
+    "Rename",
+    "AddField",
+    "Drop",
+    "Select",
+    "Filter",
+    "Convert",
+    "Default",
+    "Map",
+    "FlatMap",
+    "CustomFunction",
+]

pycharter/etl_generator/transformers/base.py

@@ -0,0 +1,63 @@
+"""
+Base transformer class with | operator support.
+"""
+
+from abc import ABC, abstractmethod
+from typing import Any, Dict, List, Optional
+
+
+class BaseTransformer(ABC):
+    """
+    Base class for chainable transformers.
+    
+    Supports | operator for chaining:
+        >>> chain = Rename({"a": "b"}) | AddField("c", "value") | Drop(["d"])
+        >>> result = chain.transform(data)
+    """
+    
+    @abstractmethod
+    def transform(self, data: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+        """Transform a batch of records."""
+        ...
+    
+    def __or__(self, other: "BaseTransformer") -> "TransformerChain":
+        """Chain transformers with | operator."""
+        if isinstance(other, TransformerChain):
+            return TransformerChain([self] + other.transformers)
+        return TransformerChain([self, other])
+    
+    def __ror__(self, other: "BaseTransformer") -> "TransformerChain":
+        """Support other | self."""
+        if isinstance(other, TransformerChain):
+            return TransformerChain(other.transformers + [self])
+        return TransformerChain([other, self])
+
+
+class TransformerChain(BaseTransformer):
+    """
+    Chain of transformers that processes data through each in sequence.
+    
+    Created automatically when using | operator.
+    """
+    
+    def __init__(self, transformers: Optional[List[BaseTransformer]] = None):
+        self.transformers: List[BaseTransformer] = list(transformers) if transformers else []
+    
+    def transform(self, data: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+        """Transform data through all transformers."""
+        result = data
+        for transformer in self.transformers:
+            result = transformer.transform(result)
+        return result
+    
+    def __or__(self, other: BaseTransformer) -> "TransformerChain":
+        """Add transformer to chain."""
+        if isinstance(other, TransformerChain):
+            return TransformerChain(self.transformers + other.transformers)
+        return TransformerChain(self.transformers + [other])
+    
+    def __len__(self) -> int:
+        return len(self.transformers)
+    
+    def __iter__(self):
+        return iter(self.transformers)

pycharter/etl_generator/transformers/config.py

@@ -0,0 +1,45 @@
+"""
+Normalize transform configuration for use by transformer modules.
+
+Supports both the canonical shape (transform: { rename, convert, ... }) and
+legacy top-level keys; outputs a single normalized dict for each step.
+"""
+
+from typing import Any, Dict
+
+
+def normalize_transform_config(raw: Dict[str, Any]) -> Dict[str, Any]:
+    """
+    Normalize transform config so transformers see a single shape.
+
+    Canonical: transform: { rename, convert, defaults, add, select, drop }
+    Legacy: rename, convert, ... at top level (when 'transform' not in config)
+
+    Returns a dict with keys: simple_ops, jsonata, custom_function.
+    Each is a dict or None if not configured.
+    """
+    out: Dict[str, Any] = {
+        "simple_ops": None,
+        "jsonata": None,
+        "custom_function": None,
+    }
+
+    # Simple operations: merge from transform.X or top-level X
+    simple_ops: Dict[str, Any] = {}
+    if "transform" in raw:
+        simple_ops = dict(raw.get("transform") or {})
+    for key in ("rename", "convert", "defaults", "add", "select", "drop"):
+        if key in raw and "transform" not in raw:
+            val = raw.get(key)
+            if val is not None:
+                simple_ops[key] = val
+    if simple_ops:
+        out["simple_ops"] = simple_ops
+
+    if raw.get("jsonata"):
+        out["jsonata"] = dict(raw["jsonata"])
+
+    if raw.get("custom_function"):
+        out["custom_function"] = dict(raw["custom_function"])
+
+    return out

pycharter/etl_generator/transformers/custom_function.py

@@ -0,0 +1,101 @@
+"""
+Custom Python function transformation.
+
+Dynamically imports and invokes a module/function or callable path,
+with optional class instantiation (optimize/run/__call__).
+"""
+
+import importlib
+import logging
+from typing import Any, Dict, List
+
+logger = logging.getLogger(__name__)
+
+
+def apply_custom_function(
+    data: List[Dict[str, Any]], config: Dict[str, Any], **kwargs: Any
+) -> List[Dict[str, Any]]:
+    """
+    Run a custom Python function for transformation.
+
+    Args:
+        data: Input data.
+        config: 'callable' ("module.func") or 'module' + 'function'.
+                Optional 'mode': "batch" (default) or "record".
+                Optional 'kwargs': dict merged with **kwargs.
+        **kwargs: Runtime kwargs merged with config['kwargs'].
+
+    Returns:
+        Transformed list of records.
+
+    Example config:
+        custom_function:
+          module: "pyoptima"
+          function: "optimize_from_etl_inputs"
+          mode: "batch"
+          kwargs:
+            method: "min_volatility"
+    """
+    callable_path = config.get("callable")
+    module_path = config.get("module")
+    func_name = config.get("function")
+
+    if callable_path:
+        parts = callable_path.rsplit(".", 1)
+        if len(parts) != 2:
+            raise ValueError(
+                f"Invalid callable path: {callable_path}. "
+                "Use 'module.function' format."
+            )
+        module_path, func_name = parts
+
+    if not module_path or not func_name:
+        raise ValueError(
+            "custom_function requires either 'callable' or 'module' + 'function'"
+        )
+
+    try:
+        module = importlib.import_module(module_path)
+        func = getattr(module, func_name)
+    except ImportError as e:
+        raise ValueError(f"Cannot import module '{module_path}': {e}") from e
+    except AttributeError as e:
+        raise ValueError(
+            f"Function '{func_name}' not found in module '{module_path}'"
+        ) from e
+
+    if isinstance(func, type):
+        instance = func()
+        if hasattr(instance, "optimize"):
+            func = instance.optimize
+        elif hasattr(instance, "run"):
+            func = instance.run
+        elif hasattr(instance, "__call__"):
+            func = instance
+        else:
+            raise ValueError(
+                f"Class '{func_name}' has no 'optimize', 'run', or '__call__'"
+            )
+
+    mode = config.get("mode", "batch")
+    func_kwargs = config.get("kwargs", {})
+    merged_kwargs = {**func_kwargs, **kwargs}
+
+    try:
+        if mode == "batch":
+            result = func(data, **merged_kwargs)
+            if result is None:
+                return []
+            return result if isinstance(result, list) else [result]
+        results = []
+        for record in data:
+            record_result = func(record, **merged_kwargs)
+            if record_result is not None:
+                if isinstance(record_result, list):
+                    results.extend(record_result)
+                else:
+                    results.append(record_result)
+        return results
+    except Exception as e:
+        logger.error("Custom function %r failed: %s", func_name, e)
+        raise ValueError(f"Custom function error: {e}") from e

pycharter/etl_generator/transformers/jsonata_transformer.py

@@ -0,0 +1,56 @@
+"""
+JSONata-based transformation.
+
+Applies a JSONata expression to data in batch or record mode.
+"""
+
+import logging
+from typing import Any, Dict, List
+
+import jsonata
+
+logger = logging.getLogger(__name__)
+
+
+def apply_jsonata(
+    data: List[Dict[str, Any]], config: Dict[str, Any]
+) -> List[Dict[str, Any]]:
+    """
+    Apply a JSONata expression to transform data.
+
+    Args:
+        data: Input data (list of records).
+        config: Must have 'expression'. Optional 'mode': "batch" (default) or "record".
+
+    Returns:
+        Transformed list of records.
+
+    Example config:
+        jsonata:
+          expression: |
+            $.{"ticker": symbol, "avg_price": $average(prices)}
+          mode: "batch"
+    """
+    expression_str = config.get("expression")
+    if not expression_str:
+        return data
+
+    mode = config.get("mode", "batch")
+
+    try:
+        expr = jsonata.Jsonata(expression_str)
+
+        if mode == "batch":
+            result = expr.evaluate(data)
+            if result is None:
+                return []
+            return result if isinstance(result, list) else [result]
+        # record mode
+        return [
+            expr.evaluate(record)
+            for record in data
+            if expr.evaluate(record) is not None
+        ]
+    except Exception as e:
+        logger.error("JSONata transformation failed: %s", e)
+        raise ValueError(f"JSONata transformation error: {e}") from e

pycharter/etl_generator/transformers/operations.py

@@ -0,0 +1,218 @@
+"""
+Built-in transformer operations.
+"""
+
+from typing import Any, Callable, Dict, List, Optional, Union
+
+from pycharter.etl_generator.transformers.base import BaseTransformer
+from pycharter.etl_generator.expression import evaluate_expression, is_expression
+
+
+class Rename(BaseTransformer):
+    """Rename fields in records."""
+    
+    def __init__(self, mapping: Dict[str, str]):
+        self.mapping = mapping
+    
+    def transform(self, data: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+        return [
+            {self.mapping.get(k, k): v for k, v in record.items()}
+            for record in data
+        ]
+
+
+class AddField(BaseTransformer):
+    """
+    Add a new field to records.
+    
+    Supports:
+    - Static values: AddField("status", "active")
+    - Callable: AddField("full_name", lambda r: f"{r['first']} {r['last']}")
+    - Expressions: AddField("full_name", "${first_name} ${last_name}")
+    - Functions: AddField("id", "uuid()")
+    
+    Expression syntax:
+    - ${field_name} - Reference field value
+    - ${field_name:-default} - Field with default
+    - now() - Current timestamp
+    - uuid() - Generate UUID
+    - concat(${a}, " ", ${b}) - Concatenate values
+    """
+    
+    def __init__(
+        self,
+        field: str,
+        value: Union[Any, Callable[[Dict[str, Any]], Any]],
+        evaluate_expressions: bool = True,
+    ):
+        """
+        Initialize AddField transformer.
+        
+        Args:
+            field: Name of the field to add
+            value: Value, callable, or expression string
+            evaluate_expressions: If True, evaluate string expressions.
+                                 Set to False to use literal string values.
+        """
+        self.field = field
+        self.value = value
+        self.evaluate_expressions = evaluate_expressions
+        
+        # Pre-check if value is an expression to optimize
+        self._is_expression = (
+            evaluate_expressions 
+            and isinstance(value, str) 
+            and is_expression(value)
+        )
+    
+    def transform(self, data: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+        result = []
+        for record in data:
+            new_record = dict(record)
+            
+            if callable(self.value):
+                # Callable - call with record
+                new_record[self.field] = self.value(record)
+            elif self._is_expression:
+                # Expression - evaluate in record context
+                new_record[self.field] = evaluate_expression(self.value, record)
+            else:
+                # Static value
+                new_record[self.field] = self.value
+            
+            result.append(new_record)
+        return result
+
+
+class Drop(BaseTransformer):
+    """Drop fields from records."""
+    
+    def __init__(self, fields: List[str]):
+        self.fields = set(fields)
+    
+    def transform(self, data: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+        return [
+            {k: v for k, v in record.items() if k not in self.fields}
+            for record in data
+        ]
+
+
+class Select(BaseTransformer):
+    """Select only specific fields."""
+    
+    def __init__(self, fields: List[str]):
+        self.fields = set(fields)
+    
+    def transform(self, data: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+        return [
+            {k: v for k, v in record.items() if k in self.fields}
+            for record in data
+        ]
+
+
+class Filter(BaseTransformer):
+    """Filter records based on a predicate."""
+    
+    def __init__(self, predicate: Callable[[Dict[str, Any]], bool]):
+        self.predicate = predicate
+    
+    def transform(self, data: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+        return [record for record in data if self.predicate(record)]
+
+
+class Convert(BaseTransformer):
+    """Convert field types."""
+    
+    def __init__(self, conversions: Dict[str, Callable[[Any], Any]], errors: str = "ignore"):
+        self.conversions = conversions
+        self.errors = errors
+    
+    def transform(self, data: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+        result = []
+        for record in data:
+            new_record = dict(record)
+            for field, converter in self.conversions.items():
+                if field in new_record:
+                    try:
+                        new_record[field] = converter(new_record[field])
+                    except (ValueError, TypeError):
+                        if self.errors == "raise":
+                            raise
+                        elif self.errors == "null":
+                            new_record[field] = None
+            result.append(new_record)
+        return result
+
+
+class Default(BaseTransformer):
+    """Set default values for missing or null fields."""
+    
+    def __init__(self, defaults: Dict[str, Any], replace_null: bool = True):
+        self.defaults = defaults
+        self.replace_null = replace_null
+    
+    def transform(self, data: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+        result = []
+        for record in data:
+            new_record = dict(record)
+            for field, default in self.defaults.items():
+                if field not in new_record:
+                    new_record[field] = default
+                elif self.replace_null and new_record[field] is None:
+                    new_record[field] = default
+            result.append(new_record)
+        return result
+
+
+class Map(BaseTransformer):
+    """Apply a function to each record."""
+    
+    def __init__(self, func: Callable[[Dict[str, Any]], Dict[str, Any]]):
+        self.func = func
+    
+    def transform(self, data: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+        return [self.func(record) for record in data]
+
+
+class FlatMap(BaseTransformer):
+    """Apply a function that returns multiple records per input."""
+    
+    def __init__(self, func: Callable[[Dict[str, Any]], List[Dict[str, Any]]]):
+        self.func = func
+    
+    def transform(self, data: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+        result = []
+        for record in data:
+            result.extend(self.func(record))
+        return result
+
+
+class CustomFunction(BaseTransformer):
+    """Run a custom Python function on data."""
+    
+    def __init__(
+        self,
+        module: Optional[str] = None,
+        function: Optional[str] = None,
+        func: Optional[Callable[[List[Dict[str, Any]]], List[Dict[str, Any]]]] = None,
+        kwargs: Optional[Dict[str, Any]] = None,
+    ):
+        self.module = module
+        self.function = function
+        self._func = func
+        self.kwargs = kwargs or {}
+        
+        if func is not None:
+            self._resolved_func = func
+        elif module and function:
+            self._resolved_func = self._import_function(module, function)
+        else:
+            raise ValueError("Must provide either 'func' or both 'module' and 'function'")
+    
+    def _import_function(self, module: str, function: str) -> Callable:
+        import importlib
+        mod = importlib.import_module(module)
+        return getattr(mod, function)
+    
+    def transform(self, data: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+        return self._resolved_func(data, **self.kwargs)