etlplus 0.14.3__py3-none-any.whl → 0.16.0__py3-none-any.whl

etlplus/utils.py

@@ -8,6 +8,7 @@ from __future__ import annotations
 
 import json
 from collections.abc import Callable
+from collections.abc import Iterable
 from collections.abc import Mapping
 from typing import Any
 from typing import TypeVar
@@ -25,6 +26,7 @@ __all__ = [
     # Mapping utilities
     'cast_str_dict',
     'coerce_dict',
+    'deep_substitute',
     'maybe_mapping',
     # Float coercion
     'to_float',
@@ -39,7 +41,8 @@ __all__ = [
     # Generic number coercion
     'to_number',
     # Text processing
-    'normalized_str',
+    'normalize_choice',
+    'normalize_str',
 ]
 
 
@@ -56,6 +59,52 @@ Num = TypeVar('Num', int, float)
 # -- Data Utilities -- #
 
 
+def deep_substitute(
+    value: Any,
+    vars_map: StrAnyMap | None,
+    env_map: Mapping[str, str] | None,
+) -> Any:
+    """
+    Recursively substitute ``${VAR}`` tokens in nested structures.
+
+    Only strings are substituted; other types are returned as-is.
+
+    Parameters
+    ----------
+    value : Any
+        The value to perform substitutions on.
+    vars_map : StrAnyMap | None
+        Mapping of variable names to replacement values (lower precedence).
+    env_map : Mapping[str, str] | None
+        Mapping of environment variables overriding *vars_map* values (higher
+        precedence).
+
+    Returns
+    -------
+    Any
+        New structure with substitutions applied where tokens were found.
+    """
+    substitutions = _prepare_substitutions(vars_map, env_map)
+
+    def _apply(node: Any) -> Any:
+        match node:
+            case str():
+                return _replace_tokens(node, substitutions)
+            case Mapping():
+                return {k: _apply(v) for k, v in node.items()}
+            case list() | tuple() as seq:
+                apply = [_apply(item) for item in seq]
+                return apply if isinstance(seq, list) else tuple(apply)
+            case set():
+                return {_apply(item) for item in node}
+            case frozenset():
+                return frozenset(_apply(item) for item in node)
+            case _:
+                return node
+
+    return _apply(value)
+
+
 def cast_str_dict(
     mapping: StrAnyMap | None,
 ) -> dict[str, str]:
@@ -81,7 +130,7 @@ def coerce_dict(
     value: Any,
 ) -> dict[str, Any]:
     """
-    Return a ``dict`` copy when ``value`` is mapping-like.
+    Return a ``dict`` copy when *value* is mapping-like.
 
     Parameters
     ----------
@@ -91,7 +140,7 @@ def coerce_dict(
     Returns
     -------
     dict[str, Any]
-        Shallow copy of ``value`` converted to a standard ``dict``.
+        Shallow copy of *value* converted to a standard ``dict``.
     """
     return dict(value) if isinstance(value, Mapping) else {}
 
@@ -121,7 +170,7 @@ def maybe_mapping(
     value: Any,
 ) -> StrAnyMap | None:
     """
-    Return ``value`` when it is mapping-like; otherwise ``None``.
+    Return *value* when it is mapping-like; otherwise ``None``.
 
     Parameters
     ----------
@@ -140,7 +189,7 @@ def print_json(
     obj: Any,
 ) -> None:
     """
-    Pretty-print ``obj`` as UTF-8 JSON without ASCII escaping.
+    Pretty-print *obj* as UTF-8 JSON without ASCII escaping.
 
     Parameters
     ----------
@@ -165,12 +214,12 @@ def to_float(
     maximum: float | None = None,
 ) -> float | None:
     """
-    Coerce ``value`` to a float with optional fallback and bounds.
+    Coerce *value* to a float with optional fallback and bounds.
 
     Notes
     -----
     For strings, leading/trailing whitespace is ignored. Returns ``None``
-    when coercion fails and no ``default`` is provided.
+    when coercion fails and no *default* is provided.
     """
     return _normalize_number(
         _coerce_float,
@@ -186,7 +235,7 @@ def to_maximum_float(
     default: float,
 ) -> float:
     """
-    Return the greater of ``default`` and ``value`` after float coercion.
+    Return the greater of *default* and *value* after float coercion.
 
     Parameters
     ----------
@@ -198,7 +247,7 @@ def to_maximum_float(
     Returns
     -------
     float
-        ``default`` if coercion fails; else ``max(coerced, default)``.
+        *default* if coercion fails; else ``max(coerced, default)``.
     """
     result = to_float(value, default)
     return max(_value_or_default(result, default), default)
@@ -209,7 +258,7 @@ def to_minimum_float(
     default: float,
 ) -> float:
     """
-    Return the lesser of ``default`` and ``value`` after float coercion.
+    Return the lesser of *default* and *value* after float coercion.
 
     Parameters
     ----------
@@ -221,7 +270,7 @@ def to_minimum_float(
     Returns
     -------
     float
-        ``default`` if coercion fails; else ``min(coerced, default)``.
+        *default* if coercion fails; else ``min(coerced, default)``.
     """
     result = to_float(value, default)
     return min(_value_or_default(result, default), default)
@@ -257,12 +306,12 @@ def to_int(
     maximum: int | None = None,
 ) -> int | None:
     """
-    Coerce ``value`` to an integer with optional fallback and bounds.
+    Coerce *value* to an integer with optional fallback and bounds.
 
     Notes
     -----
     For strings, leading/trailing whitespace is ignored. Returns ``None``
-    when coercion fails and no ``default`` is provided.
+    when coercion fails and no *default* is provided.
     """
     return _normalize_number(
         _coerce_int,
@@ -278,7 +327,7 @@ def to_maximum_int(
     default: int,
 ) -> int:
     """
-    Return the greater of ``default`` and ``value`` after integer coercion.
+    Return the greater of *default* and *value* after integer coercion.
 
     Parameters
     ----------
@@ -290,7 +339,7 @@ def to_maximum_int(
     Returns
     -------
     int
-        ``default`` if coercion fails; else ``max(coerced, default)``.
+        *default* if coercion fails; else ``max(coerced, default)``.
     """
     result = to_int(value, default)
     return max(_value_or_default(result, default), default)
@@ -301,7 +350,7 @@ def to_minimum_int(
     default: int,
 ) -> int:
     """
-    Return the lesser of ``default`` and ``value`` after integer coercion.
+    Return the lesser of *default* and *value* after integer coercion.
 
     Parameters
     ----------
@@ -313,7 +362,7 @@ def to_minimum_int(
     Returns
     -------
     int
-        ``default`` if coercion fails; else ``min(coerced, default)``.
+        *default* if coercion fails; else ``min(coerced, default)``.
     """
     result = to_int(value, default)
     return min(_value_or_default(result, default), default)
@@ -326,21 +375,21 @@ def to_positive_int(
     minimum: int = 1,
 ) -> int:
     """
-    Return a positive integer, falling back to ``minimum`` when needed.
+    Return a positive integer, falling back to *minimum* when needed.
 
     Parameters
     ----------
     value : Any
         Candidate input coerced with :func:`to_int`.
     default : int
-        Fallback value when coercion fails; clamped by ``minimum``.
+        Fallback value when coercion fails; clamped by *minimum*.
     minimum : int
         Inclusive lower bound for the result. Defaults to ``1``.
 
     Returns
     -------
     int
-        Positive integer respecting ``minimum``.
+        Positive integer respecting *minimum*.
     """
     result = to_int(value, default, minimum=minimum)
     return _value_or_default(result, minimum)
@@ -353,7 +402,7 @@ def to_number(
     value: object,
 ) -> float | None:
     """
-    Coerce ``value`` to a ``float`` using the internal float coercer.
+    Coerce *value* to a ``float`` using the internal float coercer.
 
     Parameters
     ----------
@@ -372,7 +421,7 @@ def to_number(
 # -- Text Processing -- #
 
 
-def normalized_str(
+def normalize_str(
     value: str | None,
 ) -> str:
     """
@@ -392,6 +441,36 @@ def normalized_str(
     return (value or '').strip().lower()
 
 
+def normalize_choice(
+    value: str | None,
+    *,
+    mapping: Mapping[str, str],
+    default: str,
+    normalize: Callable[[str | None], str] = normalize_str,
+) -> str:
+    """
+    Normalize a string choice using a mapping and fallback.
+
+    Parameters
+    ----------
+    value : str | None
+        Input value to normalize.
+    mapping : Mapping[str, str]
+        Mapping of acceptable normalized inputs to output values.
+    default : str
+        Default return value when input is missing or unrecognized.
+    normalize : Callable[[str | None], str], optional
+        Normalization function applied to *value*. Defaults to
+        :func:`normalize_str`.
+
+    Returns
+    -------
+    str
+        Normalized mapped value or *default*.
+    """
+    return mapping.get(normalize(value), default)
+
+
 # SECTION: INTERNAL FUNCTIONS =============================================== #
 
 
@@ -401,7 +480,7 @@ def _clamp(
     maximum: Num | None,
 ) -> Num:
     """
-    Return ``value`` constrained to the interval ``[minimum, maximum]``.
+    Return *value* constrained to the interval ``[minimum, maximum]``.
 
     Parameters
     ----------
@@ -425,6 +504,61 @@ def _clamp(
     return value
 
 
+def _prepare_substitutions(
+    vars_map: StrAnyMap | None,
+    env_map: Mapping[str, Any] | None,
+) -> tuple[tuple[str, Any], ...]:
+    """
+    Merge variable and environment maps into an ordered substitutions list.
+
+    Parameters
+    ----------
+    vars_map : StrAnyMap | None
+        Mapping of variable names to replacement values (lower precedence).
+    env_map : Mapping[str, Any] | None
+        Environment-backed values that override entries from *vars_map*.
+
+    Returns
+    -------
+    tuple[tuple[str, Any], ...]
+        Immutable sequence of ``(name, value)`` pairs suitable for token
+        replacement.
+    """
+    if not vars_map and not env_map:
+        return ()
+    merged: dict[str, Any] = {**(vars_map or {}), **(env_map or {})}
+    return tuple(merged.items())
+
+
+def _replace_tokens(
+    text: str,
+    substitutions: Iterable[tuple[str, Any]],
+) -> str:
+    """
+    Replace ``${VAR}`` tokens in *text* using *substitutions*.
+
+    Parameters
+    ----------
+    text : str
+        Input string that may contain ``${VAR}`` tokens.
+    substitutions : Iterable[tuple[str, Any]]
+        Sequence of ``(name, value)`` pairs used for token replacement.
+
+    Returns
+    -------
+    str
+        Updated text with replacements applied.
+    """
+    if not substitutions:
+        return text
+    out = text
+    for name, replacement in substitutions:
+        token = f'${{{name}}}'
+        if token in out:
+            out = out.replace(token, str(replacement))
+    return out
+
+
 def _coerce_float(
     value: object,
 ) -> float | None:
@@ -502,7 +636,7 @@ def _integral_from_float(
     candidate: float | None,
 ) -> int | None:
     """
-    Return ``int(candidate)`` when ``candidate`` is integral.
+    Return ``int(candidate)`` when *candidate* is integral.
 
     Parameters
     ----------
@@ -512,7 +646,7 @@ def _integral_from_float(
     Returns
     -------
     int | None
-        Integer form of ``candidate``; else ``None`` if not integral.
+        Integer form of *candidate*; else ``None`` if not integral.
     """
     if candidate is None or not candidate.is_integer():
         return None
@@ -528,7 +662,7 @@ def _normalize_number(
     maximum: Num | None = None,
 ) -> Num | None:
     """
-    Coerce ``value`` with ``coercer`` and optionally clamp it.
+    Coerce *value* with *coercer* and optionally clamp it.
 
     Parameters
     ----------
@@ -561,7 +695,7 @@ def _validate_bounds(
     maximum: Num | None,
 ) -> tuple[Num | None, Num | None]:
     """
-    Ensure ``minimum`` does not exceed ``maximum``.
+    Ensure *minimum* does not exceed *maximum*.
 
     Parameters
     ----------
@@ -590,7 +724,7 @@ def _value_or_default(
     default: Num,
 ) -> Num:
     """
-    Return ``value`` if not ``None``; else ``default``.
+    Return *value* if not ``None``; else *default*.
 
     Parameters
     ----------
@@ -602,6 +736,6 @@ def _value_or_default(
     Returns
     -------
     Num
-        ``value`` or ``default``.
+        *value* or *default*.
     """
     return default if value is None else value

etlplus/{config → workflow}/README.md

@@ -1,7 +1,7 @@
-# etlplus.config subpackage
+# `etlplus.workflow` Subpackage
 
-Documentation for the `etlplus.config` subpackage: configuration helpers for connectors, pipelines,
-jobs, and profiles.
+Documentation for the `etlplus.workflow` subpackage: configuration helpers for connectors,
+pipelines, jobs, and profiles.
 
 - Provides classes and utilities for managing ETL pipeline configuration
 - Supports YAML/JSON config loading and validation
@@ -10,7 +10,7 @@ jobs, and profiles.
 
 Back to project overview: see the top-level [README](../../README.md).
 
-- [etlplus.config subpackage](#etlplusconfig-subpackage)
+- [`etlplus.workflow` Subpackage](#etlplusworkflow-subpackage)
   - [Supported Configuration Types](#supported-configuration-types)
   - [Loading and Validating Configs](#loading-and-validating-configs)
   - [Example: Loading a Pipeline Config](#example-loading-a-pipeline-config)
@@ -28,7 +28,7 @@ Back to project overview: see the top-level [README](../../README.md).
 Use the provided classes to load and validate configuration files:
 
 ```python
-from etlplus.config import PipelineConfig
+from etlplus.workflow import PipelineConfig
 
 cfg = PipelineConfig.from_yaml("pipeline.yml")
 ```
@@ -39,7 +39,7 @@ cfg = PipelineConfig.from_yaml("pipeline.yml")
 ## Example: Loading a Pipeline Config
 
 ```python
-from etlplus.config import PipelineConfig
+from etlplus.workflow import PipelineConfig
 
 pipeline = PipelineConfig.from_yaml("configs/pipeline.yml")
 print(pipeline)

etlplus/workflow/__init__.py

@@ -0,0 +1,32 @@
+"""
+:mod:`etlplus.workflow` package.
+
+Job workflow helpers.
+"""
+
+from __future__ import annotations
+
+from .dag import topological_sort_jobs
+from .jobs import ExtractRef
+from .jobs import JobConfig
+from .jobs import LoadRef
+from .jobs import TransformRef
+from .jobs import ValidationRef
+from .pipeline import PipelineConfig
+from .pipeline import load_pipeline_config
+
+# SECTION: EXPORTS ========================================================== #
+
+
+__all__ = [
+    # Data Classes
+    'ExtractRef',
+    'JobConfig',
+    'LoadRef',
+    'PipelineConfig',
+    'TransformRef',
+    'ValidationRef',
+    # Functions
+    'load_pipeline_config',
+    'topological_sort_jobs',
+]

etlplus/{dag.py → workflow/dag.py}

@@ -1,8 +1,8 @@
 """
-:mod:`etlplus.dag` module.
+:mod:`etlplus.workflow.dag` module.
 
 Lightweight directed acyclic graph (DAG) helpers for ordering jobs based on
-``depends_on``.
+:attr:`depends_on`.
 """
 
 from __future__ import annotations
@@ -10,13 +10,15 @@ from __future__ import annotations
 from collections import deque
 from dataclasses import dataclass
 
-from .config.jobs import JobConfig
+from .jobs import JobConfig
 
 # SECTION: EXPORTS ========================================================== #
 
 
 __all__ = [
+    # Errors
     'DagError',
+    # Functions
     'topological_sort_jobs',
 ]
 
@@ -52,7 +54,7 @@ def topological_sort_jobs(
     jobs: list[JobConfig],
 ) -> list[JobConfig]:
     """
-    Return jobs in topological order based on ``depends_on``.
+    Return jobs in topological order based on :attr:`depends_on`.
 
     Parameters
     ----------