pycharter 0.0.24__py3-none-any.whl → 0.0.26__py3-none-any.whl

pycharter/etl_generator/schemas/__init__.py

@@ -1,49 +1,78 @@
 """
-JSON Schemas for ETL Pipeline Configuration.
+ETL Pipeline Configuration Schemas.
 
-Provides validation schemas for extract, transform, load, and pipeline configs.
-"""
-
-import json
-from pathlib import Path
-from typing import Dict, Any
-
-SCHEMA_DIR = Path(__file__).parent
-
-
-def load_schema(name: str) -> Dict[str, Any]:
-    """Load a JSON schema by name."""
-    schema_path = SCHEMA_DIR / f"{name}.json"
-    if not schema_path.exists():
-        raise FileNotFoundError(f"Schema not found: {schema_path}")
-    with open(schema_path) as f:
-        return json.load(f)
-
-
-def get_extract_schema() -> Dict[str, Any]:
-    """Get the extract config schema."""
-    return load_schema("extract")
+This module previously provided JSON Schema validation.
+Now validation is handled by Pydantic models in config_models.py.
 
+For backward compatibility, this module re-exports the config model classes.
+"""
 
-def get_transform_schema() -> Dict[str, Any]:
-    """Get the transform config schema."""
-    return load_schema("transform")
-
-
-def get_load_schema() -> Dict[str, Any]:
-    """Get the load config schema."""
-    return load_schema("load")
-
-
-def get_pipeline_schema() -> Dict[str, Any]:
-    """Get the combined pipeline config schema."""
-    return load_schema("pipeline")
-
+from pycharter.etl_generator.config_models import (
+    # Config parsers
+    parse_extract_config,
+    parse_load_config,
+    parse_transform_config,
+    parse_settings_config,
+    # Extract configs
+    ExtractConfig,
+    HTTPExtractConfig,
+    FileExtractConfig,
+    DatabaseExtractConfig,
+    CloudStorageExtractConfig,
+    # Load configs
+    LoadConfig,
+    PostgresLoadConfig,
+    SQLiteLoadConfig,
+    FileLoadConfig,
+    CloudStorageLoadConfig,
+    # Transform configs
+    TransformConfig,
+    SimpleOpsConfig,
+    JSONataConfig,
+    CustomFunctionConfig,
+    # Settings
+    SettingsConfig,
+    # Validation
+    ExtractValidationConfig,
+    LoadValidationConfig,
+    DLQConfig,
+    ContractRef,
+    MetadataStoreConfig,
+    # Pipeline
+    PipelineConfig,
+)
 
 __all__ = [
-    "load_schema",
-    "get_extract_schema",
-    "get_transform_schema",
-    "get_load_schema",
-    "get_pipeline_schema",
+    # Parsers
+    "parse_extract_config",
+    "parse_load_config",
+    "parse_transform_config",
+    "parse_settings_config",
+    # Extract
+    "ExtractConfig",
+    "HTTPExtractConfig",
+    "FileExtractConfig",
+    "DatabaseExtractConfig",
+    "CloudStorageExtractConfig",
+    # Load
+    "LoadConfig",
+    "PostgresLoadConfig",
+    "SQLiteLoadConfig",
+    "FileLoadConfig",
+    "CloudStorageLoadConfig",
+    # Transform
+    "TransformConfig",
+    "SimpleOpsConfig",
+    "JSONataConfig",
+    "CustomFunctionConfig",
+    # Settings
+    "SettingsConfig",
+    # Validation
+    "ExtractValidationConfig",
+    "LoadValidationConfig",
+    "DLQConfig",
+    "ContractRef",
+    "MetadataStoreConfig",
+    # Pipeline
+    "PipelineConfig",
 ]

pycharter/etl_generator/transformers/config.py

@@ -12,7 +12,7 @@ def normalize_transform_config(raw: Dict[str, Any]) -> Dict[str, Any]:
     """
     Normalize transform config so transformers see a single shape.
 
-    Canonical: transform: { rename, convert, defaults, add, select, drop }
+    Canonical: transform: { rename, convert, defaults, add, select, drop, filter }
     Legacy: rename, convert, ... at top level (when 'transform' not in config)
 
     Returns a dict with keys: simple_ops, jsonata, custom_function.
@@ -25,10 +25,11 @@ def normalize_transform_config(raw: Dict[str, Any]) -> Dict[str, Any]:
     }
 
     # Simple operations: merge from transform.X or top-level X
+    # Order: rename, convert, defaults, add, select, drop, filter
     simple_ops: Dict[str, Any] = {}
     if "transform" in raw:
         simple_ops = dict(raw.get("transform") or {})
-    for key in ("rename", "convert", "defaults", "add", "select", "drop"):
+    for key in ("rename", "convert", "defaults", "add", "select", "drop", "filter"):
         if key in raw and "transform" not in raw:
             val = raw.get(key)
             if val is not None:

pycharter/etl_generator/transformers/simple_operations.py

@@ -1,12 +1,12 @@
 """
 Simple declarative transformation operations.
 
-Order: rename → convert → defaults → add → select → drop.
+Order: rename → convert → defaults → add → select → drop → filter.
 """
 
 import logging
 from datetime import date, datetime
-from typing import Any, Dict, List
+from typing import Any, Callable, Dict, List, Optional
 
 from pycharter.etl_generator.expression import evaluate_expression, ExpressionEvaluator
 
@@ -19,7 +19,7 @@ def apply_simple_operations(
     """
     Apply simple declarative operations to a list of records.
 
-    Operations (in order): rename, convert, defaults, add, select, drop.
+    Operations (in order): rename, convert, defaults, add, select, drop, filter.
     """
     if not data:
         return data
@@ -32,6 +32,7 @@ def apply_simple_operations(
     add_map = config.get("add") or {}
     select_fields = config.get("select")
     drop_fields = set(config.get("drop") or [])
+    filter_config = config.get("filter")
 
     if rename_map:
         missing = [k for k in rename_map if k not in available_fields]
@@ -42,6 +43,11 @@ def apply_simple_operations(
                 sorted(available_fields),
             )
 
+    # Build filter predicate if configured
+    filter_predicate: Optional[Callable[[Dict[str, Any]], bool]] = None
+    if filter_config and isinstance(filter_config, dict):
+        filter_predicate = _create_filter_predicate(filter_config)
+
     for record in data:
         row = dict(record)
         for old_name, new_name in rename_map.items():
@@ -72,11 +78,58 @@ def apply_simple_operations(
             row = {k: v for k, v in row.items() if k in select_fields}
         if drop_fields:
             row = {k: v for k, v in row.items() if k not in drop_fields}
-        result.append(row)
+        # Apply filter last - only include row if predicate passes (or no filter)
+        if filter_predicate is None or filter_predicate(row):
+            result.append(row)
 
     return result
 
 
+def _create_filter_predicate(
+    filter_config: Dict[str, Any]
+) -> Optional[Callable[[Dict[str, Any]], bool]]:
+    """
+    Create a filter predicate from config.
+
+    Config format:
+        filter:
+          field: status
+          operator: eq  # eq, ne, gt, gte, lt, lte, in, not_in, contains, not_contains, is_null, is_not_null
+          value: active
+
+    Returns callable that takes a record and returns True if it should be kept.
+    """
+    field = filter_config.get("field")
+    operator = filter_config.get("operator", "eq")
+    value = filter_config.get("value")
+
+    if not field:
+        logger.warning("Filter config missing 'field'. Filter will be skipped.")
+        return None
+
+    operators: Dict[str, Callable[[Dict[str, Any]], bool]] = {
+        "eq": lambda r: r.get(field) == value,
+        "ne": lambda r: r.get(field) != value,
+        "gt": lambda r: r.get(field) is not None and r.get(field) > value,
+        "gte": lambda r: r.get(field) is not None and r.get(field) >= value,
+        "lt": lambda r: r.get(field) is not None and r.get(field) < value,
+        "lte": lambda r: r.get(field) is not None and r.get(field) <= value,
+        "in": lambda r: r.get(field) in (value if isinstance(value, (list, tuple, set)) else [value]),
+        "not_in": lambda r: r.get(field) not in (value if isinstance(value, (list, tuple, set)) else [value]),
+        "contains": lambda r: value in str(r.get(field, "")),
+        "not_contains": lambda r: value not in str(r.get(field, "")),
+        "is_null": lambda r: r.get(field) is None,
+        "is_not_null": lambda r: r.get(field) is not None,
+    }
+
+    predicate = operators.get(operator)
+    if predicate is None:
+        logger.warning(f"Unknown filter operator: {operator}. Filter will be skipped.")
+        return None
+
+    return predicate
+
+
 def convert_type(value: Any, target_type: str) -> Any:
     """Convert a value to the given type (string, integer, float, boolean, datetime, date)."""
     if value is None: