etlplus 0.15.0__py3-none-any.whl → 0.16.6__py3-none-any.whl

etlplus/ops/validate.py

@@ -11,8 +11,8 @@ Highlights
 ----------
 - Centralized type map and helpers for clarity and reuse.
 - Consistent error wording; field and item paths like ``[2].email``.
-- Small, focused public API with ``load_data``, ``validate_field``,
-  ``validate``.
+- Small, focused public API with :func:`load_data`, :func:`validate_field`,
+    :func:`validate`.
 
 Examples
 --------
@@ -44,9 +44,9 @@ from .load import load_data
 
 
 __all__ = [
-    'FieldRules',
-    'FieldValidation',
-    'Validation',
+    'FieldRulesDict',
+    'FieldValidationDict',
+    'ValidationDict',
     'validate_field',
     'validate',
 ]
@@ -66,10 +66,10 @@ TYPE_MAP: Final[dict[str, type | tuple[type, ...]]] = {
 }
 
 
-# SECTION: CLASSES ========================================================== #
+# SECTION: TYPED DICTS ====================================================== #
 
 
-class FieldRules(TypedDict, total=False):
+class FieldRulesDict(TypedDict, total=False):
     """
     Validation rules for a single field.
 
@@ -93,7 +93,7 @@ class FieldRules(TypedDict, total=False):
     enum: list[Any]
 
 
-class FieldValidation(TypedDict):
+class FieldValidationDict(TypedDict):
     """
     Validation result for a single field.
 
@@ -109,7 +109,7 @@ class FieldValidation(TypedDict):
     errors: list[str]
 
 
-class Validation(TypedDict):
+class ValidationDict(TypedDict):
     """
     Validation result for a complete data structure.
 
@@ -134,7 +134,7 @@ class Validation(TypedDict):
 # SECTION: TYPE ALIASES ===================================================== #
 
 
-type RulesMap = Mapping[str, FieldRules]
+type RulesMap = Mapping[str, FieldRulesDict]
 
 
 # SECTION: INTERNAL FUNCTIONS ============================================== #
@@ -339,8 +339,8 @@ def _validate_record(
 
 def validate_field(
     value: Any,
-    rules: StrAnyMap | FieldRules,
-) -> FieldValidation:
+    rules: StrAnyMap | FieldRulesDict,
+) -> FieldValidationDict:
     """
     Validate a single value against field rules.
 
@@ -348,14 +348,14 @@ def validate_field(
     ----------
     value : Any
         The value to validate. ``None`` is treated as missing.
-    rules : StrAnyMap | FieldRules
+    rules : StrAnyMap | FieldRulesDict
         Rule dictionary. Supported keys include ``required``, ``type``,
         ``min``, ``max``, ``minLength``, ``maxLength``, ``pattern``, and
         ``enum``.
 
     Returns
     -------
-    FieldValidation
+    FieldValidationDict
         Result with ``valid`` and a list of ``errors``.
 
     Notes
@@ -438,7 +438,7 @@ def validate_field(
 def validate(
     source: StrPath | JSONData,
     rules: RulesMap | None = None,
-) -> Validation:
+) -> ValidationDict:
     """
     Validate data against rules.
 
@@ -452,7 +452,7 @@ def validate(
 
     Returns
     -------
-    Validation
+    ValidationDict
         Structured result with keys ``valid``, ``errors``, ``field_errors``,
         and ``data``. If loading fails, ``data`` is ``None`` and an error is
         reported in ``errors``.

etlplus/types.py

@@ -11,12 +11,13 @@ Notes
 
 See Also
 --------
-- :mod:`etlplus.api.types` for HTTP-specific aliases
-- :mod:`etlplus.config.types` for TypedDict surfaces
+- :mod:`etlplus.api.types` for HTTP-specific aliases and data classes
+- :mod:`etlplus.connector.types` for connector-specific aliases
 
 Examples
 --------
->>> from etlplus.types import JSONDict, PipelineConfig
+>>> from etlplus.types import JSONDict
+>>> from etlplus.ops.types import PipelineConfig
 >>> payload: JSONDict = {'id': 1, 'name': 'Ada'}
 >>> isinstance(payload, dict)
 True
@@ -53,33 +54,15 @@ __all__ = [
     'JSONRecords',
     # Type Aliases (File System)
     'StrPath',
-    # Type Aliases (Functions)
-    'AggregateFunc',
-    'OperatorFunc',
-    # Type Aliases (Records & Fields)
-    'FieldName',
-    'Fields',
     # Type Aliases (Transform Specs)
     'StrAnyMap',
     'StrSeqMap',
     'StrStrMap',
-    'AggregateSpec',
-    'FilterSpec',
-    'MapSpec',
-    'SelectSpec',
-    'SortSpec',
-    # Type Aliases (Pipelines)
-    'StepOrSteps',
-    'StepSeq',
-    'StepSpec',
-    'PipelineStepName',
-    'PipelineConfig',
-    # Type Aliases (Helpers)
-    'StepApplier',
-    'SortKey',
     # Type Aliases (Networking / Runtime)
     'Sleeper',
     'Timeout',
+    # Type Aliases (Templates)
+    'TemplateKey',
 ]
 
 
@@ -124,22 +107,6 @@ type JSONRecords = list[JSONRecord]
 # Path-like inputs accepted by file helpers.
 type StrPath = str | Path | PathLike[str]
 
-# -- Functions -- #
-
-# Callable reducing numeric collections into a summary value.
-type AggregateFunc = Callable[[list[float], int], Any]
-
-# Binary predicate consumed by filter operations.
-type OperatorFunc = Callable[[Any, Any], bool]
-
-# -- Records & Fields -- #
-
-# Individual field identifier referenced inside specs.
-type FieldName = str
-
-# Ordered list of :data:`FieldName` entries preserving projection order.
-type Fields = list[FieldName]
-
 # -- Transform Specs -- #
 
 # Kept intentionally broad for runtime-friendly validation in transform.py.
@@ -155,69 +122,6 @@ type StrStrMap = Mapping[str, str]
 # Mapping whose values are homogeneous sequences.
 type StrSeqMap = Mapping[str, Sequence[Any]]
 
-# Transform step specifications
-
-# Filtering spec expecting ``field``, ``op``, and ``value`` keys.
-type FilterSpec = StrAnyMap
-
-# Field renaming instructions mapping old keys to new ones.
-type MapSpec = StrStrMap
-
-# Projection spec as a field list or mapping with metadata.
-#
-# Examples
-# --------
-# >>> from etlplus.types import SelectSpec
-# >>> spec1: SelectSpec = ['a','b']
-# >>> spec2: SelectSpec = {'fields': [...]}
-type SelectSpec = Fields | StrSeqMap
-
-# Sort directive expressed as a field string or mapping with flags.
-#
-# Examples
-# --------
-# >>> from etlplus.types import SortSpec
-# >>> spec1: SortSpec = 'field'
-# >>> spec2: SortSpec = {'field': 'x', 'reverse': True}
-type SortSpec = str | StrAnyMap
-
-# Aggregate instruction covering ``field``, ``func``, and optional alias.
-#
-# Supported functions: ``avg``, ``count``, ``max``, ``min``, and ``sum``.
-# Examples
-# --------
-# >>> from etlplus.types import AggregateSpec
-# >>> spec: AggregateSpec = \
-# ...   {'field': 'x', 'func': 'sum' | 'avg' | ..., 'alias'?: '...'}
-type AggregateSpec = StrAnyMap
-
-# -- Pipelines-- #
-
-# Unified pipeline step spec consumed by :mod:`etlplus.ops.transform`.
-type StepSpec = AggregateSpec | FilterSpec | MapSpec | SelectSpec | SortSpec
-
-# Collections of steps
-
-# Ordered collection of :data:`StepSpec` entries.
-type StepSeq = Sequence[StepSpec]
-
-# Accepts either a single :data:`StepSpec` or a sequence of them.
-type StepOrSteps = StepSpec | StepSeq
-
-# Canonical literal names for supported transform stages.
-type PipelineStepName = Literal['filter', 'map', 'select', 'sort', 'aggregate']
-
-# Mapping from step name to its associated specification payload.
-type PipelineConfig = Mapping[PipelineStepName, StepOrSteps]
-
-# -- Helpers -- #
-
-# Callable that applies step configuration to a batch of records.
-type StepApplier = Callable[[JSONList, Any], JSONList]
-
-# Tuple combining stable sort index and computed sort value.
-type SortKey = tuple[int, Any]
-
 # -- Networking / Runtime -- #
 
 # Sleep function used by retry helpers.

etlplus/utils.py

@@ -8,6 +8,7 @@ from __future__ import annotations
 
 import json
 from collections.abc import Callable
+from collections.abc import Iterable
 from collections.abc import Mapping
 from typing import Any
 from typing import TypeVar
@@ -25,6 +26,7 @@ __all__ = [
     # Mapping utilities
     'cast_str_dict',
     'coerce_dict',
+    'deep_substitute',
     'maybe_mapping',
     # Float coercion
     'to_float',
@@ -39,7 +41,8 @@ __all__ = [
     # Generic number coercion
     'to_number',
     # Text processing
-    'normalized_str',
+    'normalize_choice',
+    'normalize_str',
 ]
 
 
@@ -56,6 +59,52 @@ Num = TypeVar('Num', int, float)
 # -- Data Utilities -- #
 
 
+def deep_substitute(
+    value: Any,
+    vars_map: StrAnyMap | None,
+    env_map: Mapping[str, str] | None,
+) -> Any:
+    """
+    Recursively substitute ``${VAR}`` tokens in nested structures.
+
+    Only strings are substituted; other types are returned as-is.
+
+    Parameters
+    ----------
+    value : Any
+        The value to perform substitutions on.
+    vars_map : StrAnyMap | None
+        Mapping of variable names to replacement values (lower precedence).
+    env_map : Mapping[str, str] | None
+        Mapping of environment variables overriding *vars_map* values (higher
+        precedence).
+
+    Returns
+    -------
+    Any
+        New structure with substitutions applied where tokens were found.
+    """
+    substitutions = _prepare_substitutions(vars_map, env_map)
+
+    def _apply(node: Any) -> Any:
+        match node:
+            case str():
+                return _replace_tokens(node, substitutions)
+            case Mapping():
+                return {k: _apply(v) for k, v in node.items()}
+            case list() | tuple() as seq:
+                apply = [_apply(item) for item in seq]
+                return apply if isinstance(seq, list) else tuple(apply)
+            case set():
+                return {_apply(item) for item in node}
+            case frozenset():
+                return frozenset(_apply(item) for item in node)
+            case _:
+                return node
+
+    return _apply(value)
+
+
 def cast_str_dict(
     mapping: StrAnyMap | None,
 ) -> dict[str, str]:
@@ -81,7 +130,7 @@ def coerce_dict(
     value: Any,
 ) -> dict[str, Any]:
     """
-    Return a ``dict`` copy when ``value`` is mapping-like.
+    Return a ``dict`` copy when *value* is mapping-like.
 
     Parameters
     ----------
@@ -91,7 +140,7 @@ def coerce_dict(
     Returns
     -------
     dict[str, Any]
-        Shallow copy of ``value`` converted to a standard ``dict``.
+        Shallow copy of *value* converted to a standard ``dict``.
     """
     return dict(value) if isinstance(value, Mapping) else {}
 
@@ -121,7 +170,7 @@ def maybe_mapping(
     value: Any,
 ) -> StrAnyMap | None:
     """
-    Return ``value`` when it is mapping-like; otherwise ``None``.
+    Return *value* when it is mapping-like; otherwise ``None``.
 
     Parameters
     ----------
@@ -140,7 +189,7 @@ def print_json(
     obj: Any,
 ) -> None:
     """
-    Pretty-print ``obj`` as UTF-8 JSON without ASCII escaping.
+    Pretty-print *obj* as UTF-8 JSON without ASCII escaping.
 
     Parameters
     ----------
@@ -165,12 +214,12 @@ def to_float(
     maximum: float | None = None,
 ) -> float | None:
     """
-    Coerce ``value`` to a float with optional fallback and bounds.
+    Coerce *value* to a float with optional fallback and bounds.
 
     Notes
     -----
     For strings, leading/trailing whitespace is ignored. Returns ``None``
-    when coercion fails and no ``default`` is provided.
+    when coercion fails and no *default* is provided.
     """
     return _normalize_number(
         _coerce_float,
@@ -186,7 +235,7 @@ def to_maximum_float(
     default: float,
 ) -> float:
     """
-    Return the greater of ``default`` and ``value`` after float coercion.
+    Return the greater of *default* and *value* after float coercion.
 
     Parameters
     ----------
@@ -198,7 +247,7 @@ def to_maximum_float(
     Returns
     -------
     float
-        ``default`` if coercion fails; else ``max(coerced, default)``.
+        *default* if coercion fails; else ``max(coerced, default)``.
     """
     result = to_float(value, default)
     return max(_value_or_default(result, default), default)
@@ -209,7 +258,7 @@ def to_minimum_float(
     default: float,
 ) -> float:
     """
-    Return the lesser of ``default`` and ``value`` after float coercion.
+    Return the lesser of *default* and *value* after float coercion.
 
     Parameters
     ----------
@@ -221,7 +270,7 @@ def to_minimum_float(
     Returns
     -------
     float
-        ``default`` if coercion fails; else ``min(coerced, default)``.
+        *default* if coercion fails; else ``min(coerced, default)``.
     """
     result = to_float(value, default)
     return min(_value_or_default(result, default), default)
@@ -257,12 +306,12 @@ def to_int(
     maximum: int | None = None,
 ) -> int | None:
     """
-    Coerce ``value`` to an integer with optional fallback and bounds.
+    Coerce *value* to an integer with optional fallback and bounds.
 
     Notes
     -----
     For strings, leading/trailing whitespace is ignored. Returns ``None``
-    when coercion fails and no ``default`` is provided.
+    when coercion fails and no *default* is provided.
     """
     return _normalize_number(
         _coerce_int,
@@ -278,7 +327,7 @@ def to_maximum_int(
     default: int,
 ) -> int:
     """
-    Return the greater of ``default`` and ``value`` after integer coercion.
+    Return the greater of *default* and *value* after integer coercion.
 
     Parameters
     ----------
@@ -290,7 +339,7 @@ def to_maximum_int(
     Returns
     -------
     int
-        ``default`` if coercion fails; else ``max(coerced, default)``.
+        *default* if coercion fails; else ``max(coerced, default)``.
     """
     result = to_int(value, default)
     return max(_value_or_default(result, default), default)
@@ -301,7 +350,7 @@ def to_minimum_int(
     default: int,
 ) -> int:
     """
-    Return the lesser of ``default`` and ``value`` after integer coercion.
+    Return the lesser of *default* and *value* after integer coercion.
 
     Parameters
     ----------
@@ -313,7 +362,7 @@ def to_minimum_int(
     Returns
     -------
     int
-        ``default`` if coercion fails; else ``min(coerced, default)``.
+        *default* if coercion fails; else ``min(coerced, default)``.
     """
     result = to_int(value, default)
     return min(_value_or_default(result, default), default)
@@ -326,21 +375,21 @@ def to_positive_int(
     minimum: int = 1,
 ) -> int:
     """
-    Return a positive integer, falling back to ``minimum`` when needed.
+    Return a positive integer, falling back to *minimum* when needed.
 
     Parameters
     ----------
     value : Any
         Candidate input coerced with :func:`to_int`.
     default : int
-        Fallback value when coercion fails; clamped by ``minimum``.
+        Fallback value when coercion fails; clamped by *minimum*.
     minimum : int
         Inclusive lower bound for the result. Defaults to ``1``.
 
     Returns
     -------
     int
-        Positive integer respecting ``minimum``.
+        Positive integer respecting *minimum*.
     """
     result = to_int(value, default, minimum=minimum)
     return _value_or_default(result, minimum)
@@ -353,7 +402,7 @@ def to_number(
     value: object,
 ) -> float | None:
     """
-    Coerce ``value`` to a ``float`` using the internal float coercer.
+    Coerce *value* to a ``float`` using the internal float coercer.
 
     Parameters
     ----------
@@ -372,7 +421,7 @@ def to_number(
 # -- Text Processing -- #
 
 
-def normalized_str(
+def normalize_str(
     value: str | None,
 ) -> str:
     """
@@ -392,6 +441,36 @@ def normalized_str(
     return (value or '').strip().lower()
 
 
+def normalize_choice(
+    value: str | None,
+    *,
+    mapping: Mapping[str, str],
+    default: str,
+    normalize: Callable[[str | None], str] = normalize_str,
+) -> str:
+    """
+    Normalize a string choice using a mapping and fallback.
+
+    Parameters
+    ----------
+    value : str | None
+        Input value to normalize.
+    mapping : Mapping[str, str]
+        Mapping of acceptable normalized inputs to output values.
+    default : str
+        Default return value when input is missing or unrecognized.
+    normalize : Callable[[str | None], str], optional
+        Normalization function applied to *value*. Defaults to
+        :func:`normalize_str`.
+
+    Returns
+    -------
+    str
+        Normalized mapped value or *default*.
+    """
+    return mapping.get(normalize(value), default)
+
+
 # SECTION: INTERNAL FUNCTIONS =============================================== #
 
 
@@ -401,7 +480,7 @@ def _clamp(
     maximum: Num | None,
 ) -> Num:
     """
-    Return ``value`` constrained to the interval ``[minimum, maximum]``.
+    Return *value* constrained to the interval ``[minimum, maximum]``.
 
     Parameters
     ----------
@@ -425,6 +504,61 @@ def _clamp(
     return value
 
 
+def _prepare_substitutions(
+    vars_map: StrAnyMap | None,
+    env_map: Mapping[str, Any] | None,
+) -> tuple[tuple[str, Any], ...]:
+    """
+    Merge variable and environment maps into an ordered substitutions list.
+
+    Parameters
+    ----------
+    vars_map : StrAnyMap | None
+        Mapping of variable names to replacement values (lower precedence).
+    env_map : Mapping[str, Any] | None
+        Environment-backed values that override entries from *vars_map*.
+
+    Returns
+    -------
+    tuple[tuple[str, Any], ...]
+        Immutable sequence of ``(name, value)`` pairs suitable for token
+        replacement.
+    """
+    if not vars_map and not env_map:
+        return ()
+    merged: dict[str, Any] = {**(vars_map or {}), **(env_map or {})}
+    return tuple(merged.items())
+
+
+def _replace_tokens(
+    text: str,
+    substitutions: Iterable[tuple[str, Any]],
+) -> str:
+    """
+    Replace ``${VAR}`` tokens in *text* using *substitutions*.
+
+    Parameters
+    ----------
+    text : str
+        Input string that may contain ``${VAR}`` tokens.
+    substitutions : Iterable[tuple[str, Any]]
+        Sequence of ``(name, value)`` pairs used for token replacement.
+
+    Returns
+    -------
+    str
+        Updated text with replacements applied.
+    """
+    if not substitutions:
+        return text
+    out = text
+    for name, replacement in substitutions:
+        token = f'${{{name}}}'
+        if token in out:
+            out = out.replace(token, str(replacement))
+    return out
+
+
 def _coerce_float(
     value: object,
 ) -> float | None:
@@ -502,7 +636,7 @@ def _integral_from_float(
     candidate: float | None,
 ) -> int | None:
     """
-    Return ``int(candidate)`` when ``candidate`` is integral.
+    Return ``int(candidate)`` when *candidate* is integral.
 
     Parameters
     ----------
@@ -512,7 +646,7 @@ def _integral_from_float(
     Returns
     -------
     int | None
-        Integer form of ``candidate``; else ``None`` if not integral.
+        Integer form of *candidate*; else ``None`` if not integral.
     """
     if candidate is None or not candidate.is_integer():
         return None
@@ -528,7 +662,7 @@ def _normalize_number(
     maximum: Num | None = None,
 ) -> Num | None:
     """
-    Coerce ``value`` with ``coercer`` and optionally clamp it.
+    Coerce *value* with *coercer* and optionally clamp it.
 
     Parameters
     ----------
@@ -561,7 +695,7 @@ def _validate_bounds(
     maximum: Num | None,
 ) -> tuple[Num | None, Num | None]:
     """
-    Ensure ``minimum`` does not exceed ``maximum``.
+    Ensure *minimum* does not exceed *maximum*.
 
     Parameters
     ----------
@@ -590,7 +724,7 @@ def _value_or_default(
     default: Num,
 ) -> Num:
     """
-    Return ``value`` if not ``None``; else ``default``.
+    Return *value* if not ``None``; else *default*.
 
     Parameters
     ----------
@@ -602,6 +736,6 @@ def _value_or_default(
     Returns
     -------
     Num
-        ``value`` or ``default``.
+        *value* or *default*.
     """
     return default if value is None else value

etlplus/workflow/README.md

@@ -12,8 +12,6 @@ Back to project overview: see the top-level [README](../../README.md).
 
 - [`etlplus.workflow` Subpackage](#etlplusworkflow-subpackage)
   - [Supported Configuration Types](#supported-configuration-types)
-  - [Loading and Validating Configs](#loading-and-validating-configs)
-  - [Example: Loading a Pipeline Config](#example-loading-a-pipeline-config)
   - [See Also](#see-also)
 
 ## Supported Configuration Types
@@ -23,28 +21,6 @@ Back to project overview: see the top-level [README](../../README.md).
 - **Pipeline**: End-to-end pipeline configuration
 - **Profile**: User or environment-specific settings
 
-## Loading and Validating Configs
-
-Use the provided classes to load and validate configuration files:
-
-```python
-from etlplus.workflow import PipelineConfig
-
-cfg = PipelineConfig.from_yaml("pipeline.yml")
-```
-
-- Supports YAML and JSON formats
-- Validates against expected schema
-
-## Example: Loading a Pipeline Config
-
-```python
-from etlplus.workflow import PipelineConfig
-
-pipeline = PipelineConfig.from_yaml("configs/pipeline.yml")
-print(pipeline)
-```
-
 ## See Also
 
 - Top-level CLI and library usage in the main [README](../../README.md)