etlplus 0.14.3__py3-none-any.whl → 0.15.2__py3-none-any.whl

etlplus/file/yaml.py

@@ -18,13 +18,11 @@ Notes
 from __future__ import annotations
 
 from pathlib import Path
-from typing import cast
 
 from ..types import JSONData
-from ..types import JSONDict
-from ..types import JSONList
 from ..utils import count_records
 from ._imports import get_yaml
+from ._io import coerce_record_payload
 
 # SECTION: EXPORTS ========================================================== #
 
@@ -64,17 +62,7 @@ def read(
     with path.open('r', encoding='utf-8') as handle:
         loaded = get_yaml().safe_load(handle)
 
-    if isinstance(loaded, dict):
-        return cast(JSONDict, loaded)
-    if isinstance(loaded, list):
-        if all(isinstance(item, dict) for item in loaded):
-            return cast(JSONList, loaded)
-        raise TypeError(
-            'YAML array must contain only objects (dicts) when loading',
-        )
-    raise TypeError(
-        'YAML root must be an object or an array of objects when loading',
-    )
+    return coerce_record_payload(loaded, format_name='YAML')
 
 
 def write(

etlplus/ops/run.py

@@ -20,7 +20,6 @@ from ..api import RequestOptions
 from ..api import compose_api_request_env
 from ..api import compose_api_target_env
 from ..api import paginate_with_client
-from ..config import load_pipeline_config
 from ..enums import DataConnectorType
 from ..file import FileFormat
 from ..types import JSONData
@@ -29,6 +28,7 @@ from ..types import PipelineConfig
 from ..types import StrPath
 from ..types import Timeout
 from ..utils import print_json
+from ..workflow import load_pipeline_config
 from .extract import extract
 from .load import load
 from .transform import transform
@@ -162,9 +162,12 @@ def run(
                 # can monkeypatch this class on etlplus.ops.run.
                 ClientClass = EndpointClient  # noqa: N806
                 client = ClientClass(
-                    base_url=cast(str, env['base_url']),
+                    base_url=cast(str, env.get('base_url')),
                     base_path=cast(str | None, env.get('base_path')),
-                    endpoints=cast(dict[str, str], env['endpoints_map']),
+                    endpoints=cast(
+                        dict[str, str],
+                        env.get('endpoints_map', {}),
+                    ),
                     retry=env.get('retry'),
                     retry_network_errors=bool(
                         env.get('retry_network_errors', False),
@@ -173,7 +176,7 @@ def run(
                 )
                 data = paginate_with_client(
                     client,
-                    cast(str, env['endpoint_key']),
+                    cast(str, env.get('endpoint_key')),
                     env.get('params'),
                     env.get('headers'),
                     env.get('timeout'),
@@ -276,12 +279,14 @@ def run(
             if not url_t:
                 raise ValueError('API target missing "url"')
             kwargs_t: dict[str, Any] = {}
-            if env_t.get('headers'):
-                kwargs_t['headers'] = cast(dict[str, str], env_t['headers'])
+            headers = env_t.get('headers')
+            if headers:
+                kwargs_t['headers'] = cast(dict[str, str], headers)
             if env_t.get('timeout') is not None:
-                kwargs_t['timeout'] = env_t['timeout']
-            if env_t.get('session') is not None:
-                kwargs_t['session'] = env_t['session']
+                kwargs_t['timeout'] = env_t.get('timeout')
+            session = env_t.get('session')
+            if session is not None:
+                kwargs_t['session'] = session
             result = load(
                 data,
                 'api',

etlplus/ops/utils.py

@@ -7,13 +7,11 @@ The helpers defined here embrace a "high cohesion, low coupling" design by
 isolating normalization, configuration, and logging responsibilities. The
 resulting surface keeps ``maybe_validate`` focused on orchestration while
 offloading ancillary concerns to composable helpers.
-
 """
 
 from __future__ import annotations
 
 from collections.abc import Callable
-from collections.abc import Mapping
 from dataclasses import dataclass
 from types import MappingProxyType
 from typing import Any
@@ -23,7 +21,7 @@ from typing import TypedDict
 from typing import cast
 
 from ..types import StrAnyMap
-from ..utils import normalized_str
+from ..utils import normalize_choice
 
 # SECTION: TYPED DICTIONARIES =============================================== #
 
@@ -320,7 +318,7 @@ def _normalize_phase(
     """
     return cast(
         ValidationPhase,
-        _normalize_choice(
+        normalize_choice(
             value,
             mapping=_PHASE_CHOICES,
             default='before_transform',
@@ -346,7 +344,7 @@ def _normalize_severity(
     """
     return cast(
         ValidationSeverity,
-        _normalize_choice(
+        normalize_choice(
             value,
             mapping=_SEVERITY_CHOICES,
             default='error',
@@ -372,7 +370,7 @@ def _normalize_window(
     """
     return cast(
         ValidationWindow,
-        _normalize_choice(
+        normalize_choice(
             value,
             mapping=_WINDOW_CHOICES,
             default='both',
@@ -380,33 +378,6 @@ def _normalize_window(
     )
 
 
-def _normalize_choice(
-    value: str | None,
-    *,
-    mapping: Mapping[str, str],
-    default: str,
-) -> str:
-    """
-    Normalize a text value against a mapping with a default fallback.
-
-    Parameters
-    ----------
-    value : str | None
-        Input text to normalize.
-    mapping : Mapping[str, str]
-        Mapping of accepted values to normalized outputs.
-    default : str
-        Default to return when input is missing or unrecognized.
-
-    Returns
-    -------
-    str
-        Normalized value.
-    """
-    normalized = normalized_str(value)
-    return mapping.get(normalized, default)
-
-
 def _rule_name(
     rules: Ruleset,
 ) -> str | None:

etlplus/ops/validate.py

@@ -11,8 +11,8 @@ Highlights
 ----------
 - Centralized type map and helpers for clarity and reuse.
 - Consistent error wording; field and item paths like ``[2].email``.
-- Small, focused public API with ``load_data``, ``validate_field``,
-  ``validate``.
+- Small, focused public API with :func:`load_data`, :func:`validate_field`,
+    :func:`validate`.
 
 Examples
 --------
@@ -66,7 +66,7 @@ TYPE_MAP: Final[dict[str, type | tuple[type, ...]]] = {
 }
 
 
-# SECTION: CLASSES ========================================================== #
+# SECTION: TYPED DICTS ====================================================== #
 
 
 class FieldRules(TypedDict, total=False):

etlplus/templates/README.md

@@ -1,4 +1,4 @@
-# etlplus.templates subpackage
+# `etlplus.templates` Subpackage
 
 Documentation for the `etlplus.templates` subpackage: SQL and DDL template helpers.
 
@@ -8,7 +8,7 @@ Documentation for the `etlplus.templates` subpackage: SQL and DDL template helpe
 
 Back to project overview: see the top-level [README](../../README.md).
 
-- [etlplus.templates subpackage](#etlpustemplates-subpackage)
+- [`etlplus.templates` Subpackage](#etlplus-templates-subpackage)
     - [Available Templates](#available-templates)
     - [Rendering Templates](#rendering-templates)
     - [Example: Rendering a DDL Template](#example-rendering-a-ddl-template)

etlplus/types.py

@@ -11,8 +11,9 @@ Notes
 
 See Also
 --------
-- :mod:`etlplus.api.types` for HTTP-specific aliases
-- :mod:`etlplus.config.types` for TypedDict surfaces
+- :mod:`etlplus.api.types` for HTTP-specific aliases and data classes
+- :mod:`etlplus.workflow.types` for workflow-specific aliases and TypedDict
+    surfaces
 
 Examples
 --------

etlplus/utils.py

@@ -8,6 +8,7 @@ from __future__ import annotations
 
 import json
 from collections.abc import Callable
+from collections.abc import Iterable
 from collections.abc import Mapping
 from typing import Any
 from typing import TypeVar
@@ -25,6 +26,7 @@ __all__ = [
     # Mapping utilities
     'cast_str_dict',
     'coerce_dict',
+    'deep_substitute',
     'maybe_mapping',
     # Float coercion
     'to_float',
@@ -39,7 +41,8 @@ __all__ = [
     # Generic number coercion
     'to_number',
     # Text processing
-    'normalized_str',
+    'normalize_choice',
+    'normalize_str',
 ]
 
 
@@ -56,6 +59,52 @@ Num = TypeVar('Num', int, float)
 # -- Data Utilities -- #
 
 
+def deep_substitute(
+    value: Any,
+    vars_map: StrAnyMap | None,
+    env_map: Mapping[str, str] | None,
+) -> Any:
+    """
+    Recursively substitute ``${VAR}`` tokens in nested structures.
+
+    Only strings are substituted; other types are returned as-is.
+
+    Parameters
+    ----------
+    value : Any
+        The value to perform substitutions on.
+    vars_map : StrAnyMap | None
+        Mapping of variable names to replacement values (lower precedence).
+    env_map : Mapping[str, str] | None
+        Mapping of environment variables overriding ``vars_map`` values
+        (higher precedence).
+
+    Returns
+    -------
+    Any
+        New structure with substitutions applied where tokens were found.
+    """
+    substitutions = _prepare_substitutions(vars_map, env_map)
+
+    def _apply(node: Any) -> Any:
+        match node:
+            case str():
+                return _replace_tokens(node, substitutions)
+            case Mapping():
+                return {k: _apply(v) for k, v in node.items()}
+            case list() | tuple() as seq:
+                apply = [_apply(item) for item in seq]
+                return apply if isinstance(seq, list) else tuple(apply)
+            case set():
+                return {_apply(item) for item in node}
+            case frozenset():
+                return frozenset(_apply(item) for item in node)
+            case _:
+                return node
+
+    return _apply(value)
+
+
 def cast_str_dict(
     mapping: StrAnyMap | None,
 ) -> dict[str, str]:
@@ -372,7 +421,7 @@ def to_number(
 # -- Text Processing -- #
 
 
-def normalized_str(
+def normalize_str(
     value: str | None,
 ) -> str:
     """
@@ -392,6 +441,36 @@ def normalized_str(
     return (value or '').strip().lower()
 
 
+def normalize_choice(
+    value: str | None,
+    *,
+    mapping: Mapping[str, str],
+    default: str,
+    normalize: Callable[[str | None], str] = normalize_str,
+) -> str:
+    """
+    Normalize a string choice using a mapping and fallback.
+
+    Parameters
+    ----------
+    value : str | None
+        Input value to normalize.
+    mapping : Mapping[str, str]
+        Mapping of acceptable normalized inputs to output values.
+    default : str
+        Default return value when input is missing or unrecognized.
+    normalize : Callable[[str | None], str], optional
+        Normalization function applied to *value*. Defaults to
+        :func:`normalize_str`.
+
+    Returns
+    -------
+    str
+        Normalized mapped value or ``default``.
+    """
+    return mapping.get(normalize(value), default)
+
+
 # SECTION: INTERNAL FUNCTIONS =============================================== #
 
 
@@ -425,6 +504,61 @@ def _clamp(
     return value
 
 
+def _prepare_substitutions(
+    vars_map: StrAnyMap | None,
+    env_map: Mapping[str, Any] | None,
+) -> tuple[tuple[str, Any], ...]:
+    """
+    Merge variable and environment maps into an ordered substitutions list.
+
+    Parameters
+    ----------
+    vars_map : StrAnyMap | None
+        Mapping of variable names to replacement values (lower precedence).
+    env_map : Mapping[str, Any] | None
+        Environment-backed values that override entries from ``vars_map``.
+
+    Returns
+    -------
+    tuple[tuple[str, Any], ...]
+        Immutable sequence of ``(name, value)`` pairs suitable for token
+        replacement.
+    """
+    if not vars_map and not env_map:
+        return ()
+    merged: dict[str, Any] = {**(vars_map or {}), **(env_map or {})}
+    return tuple(merged.items())
+
+
+def _replace_tokens(
+    text: str,
+    substitutions: Iterable[tuple[str, Any]],
+) -> str:
+    """
+    Replace ``${VAR}`` tokens in ``text`` using ``substitutions``.
+
+    Parameters
+    ----------
+    text : str
+        Input string that may contain ``${VAR}`` tokens.
+    substitutions : Iterable[tuple[str, Any]]
+        Sequence of ``(name, value)`` pairs used for token replacement.
+
+    Returns
+    -------
+    str
+        Updated text with replacements applied.
+    """
+    if not substitutions:
+        return text
+    out = text
+    for name, replacement in substitutions:
+        token = f'${{{name}}}'
+        if token in out:
+            out = out.replace(token, str(replacement))
+    return out
+
+
 def _coerce_float(
     value: object,
 ) -> float | None:

etlplus/{config → workflow}/README.md

@@ -1,7 +1,7 @@
-# etlplus.config subpackage
+# `etlplus.workflow` Subpackage
 
-Documentation for the `etlplus.config` subpackage: configuration helpers for connectors, pipelines,
-jobs, and profiles.
+Documentation for the `etlplus.workflow` subpackage: configuration helpers for connectors,
+pipelines, jobs, and profiles.
 
 - Provides classes and utilities for managing ETL pipeline configuration
 - Supports YAML/JSON config loading and validation
@@ -10,7 +10,7 @@ jobs, and profiles.
 
 Back to project overview: see the top-level [README](../../README.md).
 
-- [etlplus.config subpackage](#etlplusconfig-subpackage)
+- [`etlplus.workflow` Subpackage](#etlplusworkflow-subpackage)
   - [Supported Configuration Types](#supported-configuration-types)
   - [Loading and Validating Configs](#loading-and-validating-configs)
   - [Example: Loading a Pipeline Config](#example-loading-a-pipeline-config)
@@ -28,7 +28,7 @@ Back to project overview: see the top-level [README](../../README.md).
 Use the provided classes to load and validate configuration files:
 
 ```python
-from etlplus.config import PipelineConfig
+from etlplus.workflow import PipelineConfig
 
 cfg = PipelineConfig.from_yaml("pipeline.yml")
 ```
@@ -39,7 +39,7 @@ cfg = PipelineConfig.from_yaml("pipeline.yml")
 ## Example: Loading a Pipeline Config
 
 ```python
-from etlplus.config import PipelineConfig
+from etlplus.workflow import PipelineConfig
 
 pipeline = PipelineConfig.from_yaml("configs/pipeline.yml")
 print(pipeline)

etlplus/{config → workflow}/__init__.py

@@ -1,17 +1,7 @@
 """
-:mod:`etlplus.config` package.
+:mod:`etlplus.workflow` package.
 
-Configuration models and helpers for ETLPlus.
-
-This package defines models for data sources/targets ("connectors"), APIs,
-pagination/rate limits, pipeline orchestration, and related utilities. The
-parsers are permissive (accepting ``Mapping[str, Any]``) and normalize to
-concrete types without raising on unknown/optional fields.
-
-Notes
------
-- The models use ``@dataclass(slots=True)`` and avoid mutating inputs.
-- TypedDicts are editor/type-checking hints and are not enforced at runtime.
+Job workflow helpers.
 """
 
 from __future__ import annotations
@@ -21,6 +11,7 @@ from .connector import ConnectorApi
 from .connector import ConnectorDb
 from .connector import ConnectorFile
 from .connector import parse_connector
+from .dag import topological_sort_jobs
 from .jobs import ExtractRef
 from .jobs import JobConfig
 from .jobs import LoadRef
@@ -28,29 +19,25 @@ from .jobs import TransformRef
 from .jobs import ValidationRef
 from .pipeline import PipelineConfig
 from .pipeline import load_pipeline_config
-from .profile import ProfileConfig
-from .types import ConnectorType
 
 # SECTION: EXPORTS ========================================================== #
 
 
 __all__ = [
-    # Connectors
-    'Connector',
-    'ConnectorType',
+    # Data Classes
     'ConnectorApi',
     'ConnectorDb',
     'ConnectorFile',
-    'parse_connector',
-    # Jobs / Refs
     'ExtractRef',
     'JobConfig',
     'LoadRef',
+    'PipelineConfig',
     'TransformRef',
     'ValidationRef',
-    # Pipeline
-    'PipelineConfig',
+    # Functions
     'load_pipeline_config',
-    # Profile
-    'ProfileConfig',
+    'parse_connector',
+    'topological_sort_jobs',
+    # Type Aliases
+    'Connector',
 ]