etlplus 0.15.0__py3-none-any.whl → 0.16.0__py3-none-any.whl

etlplus/ops/load.py

@@ -14,7 +14,7 @@ from typing import cast
 
 from ..api import HttpMethod
 from ..api.utils import resolve_request
-from ..enums import DataConnectorType
+from ..connector import DataConnectorType
 from ..file import File
 from ..file import FileFormat
 from ..types import JSONData
@@ -23,6 +23,19 @@ from ..types import JSONList
 from ..types import StrPath
 from ..utils import count_records
 
+# SECTION: EXPORTS ========================================================== #
+
+
+__all__ = [
+    # Functions
+    'load',
+    'load_data',
+    'load_to_api',
+    'load_to_database',
+    'load_to_file',
+]
+
+
 # SECTION: INTERNAL FUNCTIONS ============================================== #
 
 
@@ -30,7 +43,7 @@ def _parse_json_string(
     raw: str,
 ) -> JSONData:
     """
-    Parse JSON data from ``raw`` text.
+    Parse JSON data from *raw* text.
 
     Parameters
     ----------

etlplus/ops/run.py

@@ -20,7 +20,7 @@ from ..api import RequestOptions
 from ..api import compose_api_request_env
 from ..api import compose_api_target_env
 from ..api import paginate_with_client
-from ..enums import DataConnectorType
+from ..connector import DataConnectorType
 from ..file import FileFormat
 from ..types import JSONData
 from ..types import JSONDict
@@ -94,7 +94,7 @@ def run(
     Run a pipeline job defined in a YAML configuration.
 
     By default it reads the configuration from ``in/pipeline.yml``, but callers
-    can provide an explicit ``config_path`` to override this.
+    can provide an explicit *config_path* to override this.
 
     Parameters
     ----------
@@ -328,11 +328,11 @@ def run_pipeline(
     Parameters
     ----------
     source_type : DataConnectorType | str | None, optional
-        Connector type for extraction. When ``None``, ``source`` is assumed
+        Connector type for extraction. When ``None``, *source* is assumed
         to be pre-loaded data and extraction is skipped.
     source : StrPath | JSONData | None, optional
         Data source for extraction or the pre-loaded payload when
-        ``source_type`` is ``None``.
+        *source_type* is ``None``.
     operations : PipelineConfig | None, optional
         Transform configuration passed to :func:`etlplus.ops.transform`.
     target_type : DataConnectorType | str | None, optional

etlplus/ops/transform.py

@@ -110,7 +110,7 @@ def _agg_count(
     present: int,
 ) -> int:
     """
-    Return the provided presence count ``present``.
+    Return the provided presence count *present*.
 
     Parameters
     ----------
@@ -120,7 +120,7 @@ def _agg_count(
     Returns
     -------
     int
-        The provided presence count ``present``.
+        The provided presence count *present*.
     """
     return present
 

etlplus/ops/utils.py

@@ -7,13 +7,11 @@ The helpers defined here embrace a "high cohesion, low coupling" design by
 isolating normalization, configuration, and logging responsibilities. The
 resulting surface keeps ``maybe_validate`` focused on orchestration while
 offloading ancillary concerns to composable helpers.
-
 """
 
 from __future__ import annotations
 
 from collections.abc import Callable
-from collections.abc import Mapping
 from dataclasses import dataclass
 from types import MappingProxyType
 from typing import Any
@@ -23,7 +21,7 @@ from typing import TypedDict
 from typing import cast
 
 from ..types import StrAnyMap
-from ..utils import normalized_str
+from ..utils import normalize_choice
 
 # SECTION: TYPED DICTIONARIES =============================================== #
 
@@ -207,14 +205,14 @@ def maybe_validate(
     Returns
     -------
     Any
-        ``payload`` when validation is skipped or when severity is ``"warn"``
+        *payload* when validation is skipped or when severity is ``"warn"``
         and the validation fails. Returns the validator ``data`` payload when
         validation succeeds.
 
     Raises
     ------
     ValueError
-        Raised when validation fails and ``severity`` is ``"error"``.
+        Raised when validation fails and *severity* is ``"error"``.
 
     Examples
     --------
@@ -320,7 +318,7 @@ def _normalize_phase(
     """
     return cast(
         ValidationPhase,
-        _normalize_choice(
+        normalize_choice(
             value,
             mapping=_PHASE_CHOICES,
             default='before_transform',
@@ -346,7 +344,7 @@ def _normalize_severity(
     """
     return cast(
         ValidationSeverity,
-        _normalize_choice(
+        normalize_choice(
             value,
             mapping=_SEVERITY_CHOICES,
             default='error',
@@ -372,7 +370,7 @@ def _normalize_window(
     """
     return cast(
         ValidationWindow,
-        _normalize_choice(
+        normalize_choice(
             value,
             mapping=_WINDOW_CHOICES,
             default='both',
@@ -380,33 +378,6 @@ def _normalize_window(
     )
 
 
-def _normalize_choice(
-    value: str | None,
-    *,
-    mapping: Mapping[str, str],
-    default: str,
-) -> str:
-    """
-    Normalize a text value against a mapping with a default fallback.
-
-    Parameters
-    ----------
-    value : str | None
-        Input text to normalize.
-    mapping : Mapping[str, str]
-        Mapping of accepted values to normalized outputs.
-    default : str
-        Default to return when input is missing or unrecognized.
-
-    Returns
-    -------
-    str
-        Normalized value.
-    """
-    normalized = normalized_str(value)
-    return mapping.get(normalized, default)
-
-
 def _rule_name(
     rules: Ruleset,
 ) -> str | None:

etlplus/ops/validate.py

@@ -11,8 +11,8 @@ Highlights
 ----------
 - Centralized type map and helpers for clarity and reuse.
 - Consistent error wording; field and item paths like ``[2].email``.
-- Small, focused public API with ``load_data``, ``validate_field``,
-  ``validate``.
+- Small, focused public API with :func:`load_data`, :func:`validate_field`,
+    :func:`validate`.
 
 Examples
 --------
@@ -66,7 +66,7 @@ TYPE_MAP: Final[dict[str, type | tuple[type, ...]]] = {
 }
 
 
-# SECTION: CLASSES ========================================================== #
+# SECTION: TYPED DICTS ====================================================== #
 
 
 class FieldRules(TypedDict, total=False):

etlplus/types.py

@@ -11,8 +11,9 @@ Notes
 
 See Also
 --------
-- :mod:`etlplus.api.types` for HTTP-specific aliases
-- :mod:`etlplus.config.types` for TypedDict surfaces
+- :mod:`etlplus.api.types` for HTTP-specific aliases and data classes
+- :mod:`etlplus.connector.types` for connector-specific aliases and TypedDict
+    surfaces
 
 Examples
 --------

etlplus/utils.py

@@ -8,6 +8,7 @@ from __future__ import annotations
 
 import json
 from collections.abc import Callable
+from collections.abc import Iterable
 from collections.abc import Mapping
 from typing import Any
 from typing import TypeVar
@@ -25,6 +26,7 @@ __all__ = [
     # Mapping utilities
     'cast_str_dict',
     'coerce_dict',
+    'deep_substitute',
     'maybe_mapping',
     # Float coercion
     'to_float',
@@ -39,7 +41,8 @@ __all__ = [
     # Generic number coercion
     'to_number',
     # Text processing
-    'normalized_str',
+    'normalize_choice',
+    'normalize_str',
 ]
 
 
@@ -56,6 +59,52 @@ Num = TypeVar('Num', int, float)
 # -- Data Utilities -- #
 
 
+def deep_substitute(
+    value: Any,
+    vars_map: StrAnyMap | None,
+    env_map: Mapping[str, str] | None,
+) -> Any:
+    """
+    Recursively substitute ``${VAR}`` tokens in nested structures.
+
+    Only strings are substituted; other types are returned as-is.
+
+    Parameters
+    ----------
+    value : Any
+        The value to perform substitutions on.
+    vars_map : StrAnyMap | None
+        Mapping of variable names to replacement values (lower precedence).
+    env_map : Mapping[str, str] | None
+        Mapping of environment variables overriding *vars_map* values (higher
+        precedence).
+
+    Returns
+    -------
+    Any
+        New structure with substitutions applied where tokens were found.
+    """
+    substitutions = _prepare_substitutions(vars_map, env_map)
+
+    def _apply(node: Any) -> Any:
+        match node:
+            case str():
+                return _replace_tokens(node, substitutions)
+            case Mapping():
+                return {k: _apply(v) for k, v in node.items()}
+            case list() | tuple() as seq:
+                apply = [_apply(item) for item in seq]
+                return apply if isinstance(seq, list) else tuple(apply)
+            case set():
+                return {_apply(item) for item in node}
+            case frozenset():
+                return frozenset(_apply(item) for item in node)
+            case _:
+                return node
+
+    return _apply(value)
+
+
 def cast_str_dict(
     mapping: StrAnyMap | None,
 ) -> dict[str, str]:
@@ -81,7 +130,7 @@ def coerce_dict(
     value: Any,
 ) -> dict[str, Any]:
     """
-    Return a ``dict`` copy when ``value`` is mapping-like.
+    Return a ``dict`` copy when *value* is mapping-like.
 
     Parameters
     ----------
@@ -91,7 +140,7 @@ def coerce_dict(
     Returns
     -------
     dict[str, Any]
-        Shallow copy of ``value`` converted to a standard ``dict``.
+        Shallow copy of *value* converted to a standard ``dict``.
     """
     return dict(value) if isinstance(value, Mapping) else {}
 
@@ -121,7 +170,7 @@ def maybe_mapping(
     value: Any,
 ) -> StrAnyMap | None:
     """
-    Return ``value`` when it is mapping-like; otherwise ``None``.
+    Return *value* when it is mapping-like; otherwise ``None``.
 
     Parameters
     ----------
@@ -140,7 +189,7 @@ def print_json(
     obj: Any,
 ) -> None:
     """
-    Pretty-print ``obj`` as UTF-8 JSON without ASCII escaping.
+    Pretty-print *obj* as UTF-8 JSON without ASCII escaping.
 
     Parameters
     ----------
@@ -165,12 +214,12 @@ def to_float(
     maximum: float | None = None,
 ) -> float | None:
     """
-    Coerce ``value`` to a float with optional fallback and bounds.
+    Coerce *value* to a float with optional fallback and bounds.
 
     Notes
     -----
     For strings, leading/trailing whitespace is ignored. Returns ``None``
-    when coercion fails and no ``default`` is provided.
+    when coercion fails and no *default* is provided.
     """
     return _normalize_number(
         _coerce_float,
@@ -186,7 +235,7 @@ def to_maximum_float(
     default: float,
 ) -> float:
     """
-    Return the greater of ``default`` and ``value`` after float coercion.
+    Return the greater of *default* and *value* after float coercion.
 
     Parameters
     ----------
@@ -198,7 +247,7 @@ def to_maximum_float(
     Returns
     -------
     float
-        ``default`` if coercion fails; else ``max(coerced, default)``.
+        *default* if coercion fails; else ``max(coerced, default)``.
     """
     result = to_float(value, default)
     return max(_value_or_default(result, default), default)
@@ -209,7 +258,7 @@ def to_minimum_float(
     default: float,
 ) -> float:
     """
-    Return the lesser of ``default`` and ``value`` after float coercion.
+    Return the lesser of *default* and *value* after float coercion.
 
     Parameters
     ----------
@@ -221,7 +270,7 @@ def to_minimum_float(
     Returns
     -------
     float
-        ``default`` if coercion fails; else ``min(coerced, default)``.
+        *default* if coercion fails; else ``min(coerced, default)``.
     """
     result = to_float(value, default)
     return min(_value_or_default(result, default), default)
@@ -257,12 +306,12 @@ def to_int(
     maximum: int | None = None,
 ) -> int | None:
     """
-    Coerce ``value`` to an integer with optional fallback and bounds.
+    Coerce *value* to an integer with optional fallback and bounds.
 
     Notes
     -----
     For strings, leading/trailing whitespace is ignored. Returns ``None``
-    when coercion fails and no ``default`` is provided.
+    when coercion fails and no *default* is provided.
     """
     return _normalize_number(
         _coerce_int,
@@ -278,7 +327,7 @@ def to_maximum_int(
     default: int,
 ) -> int:
     """
-    Return the greater of ``default`` and ``value`` after integer coercion.
+    Return the greater of *default* and *value* after integer coercion.
 
     Parameters
     ----------
@@ -290,7 +339,7 @@ def to_maximum_int(
     Returns
     -------
     int
-        ``default`` if coercion fails; else ``max(coerced, default)``.
+        *default* if coercion fails; else ``max(coerced, default)``.
     """
     result = to_int(value, default)
     return max(_value_or_default(result, default), default)
@@ -301,7 +350,7 @@ def to_minimum_int(
     default: int,
 ) -> int:
     """
-    Return the lesser of ``default`` and ``value`` after integer coercion.
+    Return the lesser of *default* and *value* after integer coercion.
 
     Parameters
     ----------
@@ -313,7 +362,7 @@ def to_minimum_int(
     Returns
     -------
     int
-        ``default`` if coercion fails; else ``min(coerced, default)``.
+        *default* if coercion fails; else ``min(coerced, default)``.
     """
     result = to_int(value, default)
     return min(_value_or_default(result, default), default)
@@ -326,21 +375,21 @@ def to_positive_int(
     minimum: int = 1,
 ) -> int:
     """
-    Return a positive integer, falling back to ``minimum`` when needed.
+    Return a positive integer, falling back to *minimum* when needed.
 
     Parameters
     ----------
     value : Any
         Candidate input coerced with :func:`to_int`.
     default : int
-        Fallback value when coercion fails; clamped by ``minimum``.
+        Fallback value when coercion fails; clamped by *minimum*.
     minimum : int
         Inclusive lower bound for the result. Defaults to ``1``.
 
     Returns
     -------
     int
-        Positive integer respecting ``minimum``.
+        Positive integer respecting *minimum*.
     """
     result = to_int(value, default, minimum=minimum)
     return _value_or_default(result, minimum)
@@ -353,7 +402,7 @@ def to_number(
     value: object,
 ) -> float | None:
     """
-    Coerce ``value`` to a ``float`` using the internal float coercer.
+    Coerce *value* to a ``float`` using the internal float coercer.
 
     Parameters
     ----------
@@ -372,7 +421,7 @@ def to_number(
 # -- Text Processing -- #
 
 
-def normalized_str(
+def normalize_str(
     value: str | None,
 ) -> str:
     """
@@ -392,6 +441,36 @@ def normalized_str(
     return (value or '').strip().lower()
 
 
+def normalize_choice(
+    value: str | None,
+    *,
+    mapping: Mapping[str, str],
+    default: str,
+    normalize: Callable[[str | None], str] = normalize_str,
+) -> str:
+    """
+    Normalize a string choice using a mapping and fallback.
+
+    Parameters
+    ----------
+    value : str | None
+        Input value to normalize.
+    mapping : Mapping[str, str]
+        Mapping of acceptable normalized inputs to output values.
+    default : str
+        Default return value when input is missing or unrecognized.
+    normalize : Callable[[str | None], str], optional
+        Normalization function applied to *value*. Defaults to
+        :func:`normalize_str`.
+
+    Returns
+    -------
+    str
+        Normalized mapped value or *default*.
+    """
+    return mapping.get(normalize(value), default)
+
+
 # SECTION: INTERNAL FUNCTIONS =============================================== #
 
 
@@ -401,7 +480,7 @@ def _clamp(
     maximum: Num | None,
 ) -> Num:
     """
-    Return ``value`` constrained to the interval ``[minimum, maximum]``.
+    Return *value* constrained to the interval ``[minimum, maximum]``.
 
     Parameters
     ----------
@@ -425,6 +504,61 @@ def _clamp(
     return value
 
 
+def _prepare_substitutions(
+    vars_map: StrAnyMap | None,
+    env_map: Mapping[str, Any] | None,
+) -> tuple[tuple[str, Any], ...]:
+    """
+    Merge variable and environment maps into an ordered substitutions list.
+
+    Parameters
+    ----------
+    vars_map : StrAnyMap | None
+        Mapping of variable names to replacement values (lower precedence).
+    env_map : Mapping[str, Any] | None
+        Environment-backed values that override entries from *vars_map*.
+
+    Returns
+    -------
+    tuple[tuple[str, Any], ...]
+        Immutable sequence of ``(name, value)`` pairs suitable for token
+        replacement.
+    """
+    if not vars_map and not env_map:
+        return ()
+    merged: dict[str, Any] = {**(vars_map or {}), **(env_map or {})}
+    return tuple(merged.items())
+
+
+def _replace_tokens(
+    text: str,
+    substitutions: Iterable[tuple[str, Any]],
+) -> str:
+    """
+    Replace ``${VAR}`` tokens in *text* using *substitutions*.
+
+    Parameters
+    ----------
+    text : str
+        Input string that may contain ``${VAR}`` tokens.
+    substitutions : Iterable[tuple[str, Any]]
+        Sequence of ``(name, value)`` pairs used for token replacement.
+
+    Returns
+    -------
+    str
+        Updated text with replacements applied.
+    """
+    if not substitutions:
+        return text
+    out = text
+    for name, replacement in substitutions:
+        token = f'${{{name}}}'
+        if token in out:
+            out = out.replace(token, str(replacement))
+    return out
+
+
 def _coerce_float(
     value: object,
 ) -> float | None:
@@ -502,7 +636,7 @@ def _integral_from_float(
     candidate: float | None,
 ) -> int | None:
     """
-    Return ``int(candidate)`` when ``candidate`` is integral.
+    Return ``int(candidate)`` when *candidate* is integral.
 
     Parameters
     ----------
@@ -512,7 +646,7 @@ def _integral_from_float(
     Returns
     -------
     int | None
-        Integer form of ``candidate``; else ``None`` if not integral.
+        Integer form of *candidate*; else ``None`` if not integral.
     """
     if candidate is None or not candidate.is_integer():
         return None
@@ -528,7 +662,7 @@ def _normalize_number(
     maximum: Num | None = None,
 ) -> Num | None:
     """
-    Coerce ``value`` with ``coercer`` and optionally clamp it.
+    Coerce *value* with *coercer* and optionally clamp it.
 
     Parameters
     ----------
@@ -561,7 +695,7 @@ def _validate_bounds(
     maximum: Num | None,
 ) -> tuple[Num | None, Num | None]:
     """
-    Ensure ``minimum`` does not exceed ``maximum``.
+    Ensure *minimum* does not exceed *maximum*.
 
     Parameters
     ----------
@@ -590,7 +724,7 @@ def _value_or_default(
     default: Num,
 ) -> Num:
     """
-    Return ``value`` if not ``None``; else ``default``.
+    Return *value* if not ``None``; else *default*.
 
     Parameters
     ----------
@@ -602,6 +736,6 @@ def _value_or_default(
     Returns
     -------
     Num
-        ``value`` or ``default``.
+        *value* or *default*.
     """
     return default if value is None else value

etlplus/workflow/__init__.py

@@ -6,11 +6,6 @@ Job workflow helpers.
 
 from __future__ import annotations
 
-from .connector import Connector
-from .connector import ConnectorApi
-from .connector import ConnectorDb
-from .connector import ConnectorFile
-from .connector import parse_connector
 from .dag import topological_sort_jobs
 from .jobs import ExtractRef
 from .jobs import JobConfig
@@ -25,9 +20,6 @@ from .pipeline import load_pipeline_config
 
 __all__ = [
     # Data Classes
-    'ConnectorApi',
-    'ConnectorDb',
-    'ConnectorFile',
     'ExtractRef',
     'JobConfig',
     'LoadRef',
@@ -36,8 +28,5 @@ __all__ = [
     'ValidationRef',
     # Functions
     'load_pipeline_config',
-    'parse_connector',
     'topological_sort_jobs',
-    # Type Aliases
-    'Connector',
 ]