etlplus 0.14.3__py3-none-any.whl → 0.15.2__py3-none-any.whl

etlplus/{config → workflow}/connector.py

@@ -1,5 +1,5 @@
 """
-:mod:`etlplus.config.connector` module.
+:mod:`etlplus.workflow.connector` module.
 
 A module defining configuration types for data source/target connectors in ETL
 pipelines. A "connector" is any I/O endpoint:
@@ -11,25 +11,26 @@ pipelines. A "connector" is any I/O endpoint:
 
 Examples
 --------
-- Use ``ConnectorApi``/``ConnectorFile``/``ConnectorDb`` when you want the
-  concrete dataclasses.
-- Use the ``Connector`` union for typing a value that can be any connector.
-- Use ``parse_connector(obj)`` to construct a connector instance from a generic
-  mapping that includes a ``type`` key.
+- Use :class:`ConnectorApi`/:class:`ConnectorFile`/:class:`ConnectorDb` when
+    you want the concrete dataclasses.
+- Use the :class:`Connector` union for typing a value that can be any
+    connector.
+- Use :func:`parse_connector(obj)` to construct a connector instance from a
+    generic mapping that includes a *type* key.
 
 Notes
 -----
 - TypedDict shapes are editor hints; runtime parsing remains permissive
-  (from_obj accepts Mapping[str, Any]).
-- TypedDicts referenced in :mod:`etlplus.config.types` remain editor hints.
-  Runtime parsing stays permissive and tolerant.
+    (from_obj accepts Mapping[str, Any]).
+- TypedDicts referenced in :mod:`etlplus.workflow.types` remain editor hints.
+    Runtime parsing stays permissive and tolerant.
 
 See Also
 --------
 - TypedDict shapes for editor hints (not enforced at runtime):
-    :mod:`etlplus.config.types.ConnectorApiConfigMap`,
-    :mod:`etlplus.config.types.ConnectorDbConfigMap`,
-    :mod:`etlplus.config.types.ConnectorFileConfigMap`.
+    :mod:`etlplus.workflow.types.ConnectorApiConfigMap`,
+    :mod:`etlplus.workflow.types.ConnectorDbConfigMap`,
+    :mod:`etlplus.workflow.types.ConnectorFileConfigMap`.
 """
 
 from __future__ import annotations
@@ -59,7 +60,7 @@ if TYPE_CHECKING:  # Editor-only typing hints to avoid runtime imports
 
 
 __all__ = [
-    # Classes
+    # Data Classes
     'ConnectorApi',
     'ConnectorDb',
     'ConnectorFile',
@@ -70,6 +71,40 @@ __all__ = [
 ]
 
 
+# SECTION: INTERNAL FUNCTIONS ============================================== #
+
+
+def _require_name(
+    obj: StrAnyMap,
+    *,
+    kind: str,
+) -> str:
+    """
+    Extract and validate the ``name`` field from connector mappings.
+
+    Parameters
+    ----------
+    obj : StrAnyMap
+        Connector mapping with a ``name`` entry.
+    kind : str
+        Connector kind used in the error message.
+
+    Returns
+    -------
+    str
+        Valid connector name.
+
+    Raises
+    ------
+    TypeError
+        If ``name`` is missing or not a string.
+    """
+    name = obj.get('name')
+    if not isinstance(name, str):
+        raise TypeError(f'Connector{kind} requires a "name" (str)')
+    return name
+
+
 # SECTION: DATA CLASSES ===================================================== #
 
 
@@ -83,12 +118,12 @@ class ConnectorApi:
     name : str
         Unique connector name.
     type : ConnectorType
-        Connector kind literal, always ``"api"``.
+        Connector kind literal, always ``'api'``.
     url : str | None
         Direct absolute URL (when not using ``service``/``endpoint`` refs).
     method : str | None
         Optional HTTP method; typically omitted for sources (defaults to
-        GET) and used for targets (e.g., ``"post"``).
+        GET) and used for targets (e.g., ``'post'``).
     headers : dict[str, str]
         Additional request headers.
     query_params : dict[str, Any]
@@ -111,7 +146,7 @@ class ConnectorApi:
 
     # Direct form
     url: str | None = None
-    # Optional HTTP method; typically omitted for sources (defaults to GET
+    # Optional HTTP method; typically omitted for sources (defaults to GET)
     # at runtime) and used for targets (e.g., 'post', 'put').
     method: str | None = None
     headers: dict[str, str] = field(default_factory=dict)
@@ -150,15 +185,8 @@ class ConnectorApi:
         -------
         Self
             Parsed connector instance.
-
-        Raises
-        ------
-        TypeError
-            If ``name`` is missing or invalid.
         """
-        name = obj.get('name')
-        if not isinstance(name, str):
-            raise TypeError('ConnectorApi requires a "name" (str)')
+        name = _require_name(obj, kind='Api')
         headers = cast_str_dict(obj.get('headers'))
 
         return cls(
@@ -185,7 +213,7 @@ class ConnectorDb:
     name : str
         Unique connector name.
     type : ConnectorType
-        Connector kind literal, always ``"database"``.
+        Connector kind literal, always ``'database'``.
     connection_string : str | None
         Connection string/DSN for the database.
     query : str | None
@@ -193,7 +221,7 @@ class ConnectorDb:
     table : str | None
         Target/source table name (optional).
     mode : str | None
-        Load mode hint (e.g., ``"append"``, ``"replace"``) — future use.
+        Load mode hint (e.g., ``'append'``, ``'replace'``) — future use.
     """
 
     # -- Attributes -- #
@@ -232,15 +260,8 @@ class ConnectorDb:
         -------
         Self
             Parsed connector instance.
-
-        Raises
-        ------
-        TypeError
-            If ``name`` is missing or invalid.
         """
-        name = obj.get('name')
-        if not isinstance(name, str):
-            raise TypeError('ConnectorDb requires a "name" (str)')
+        name = _require_name(obj, kind='Db')
 
         return cls(
             name=name,
@@ -262,9 +283,9 @@ class ConnectorFile:
     name : str
         Unique connector name.
     type : ConnectorType
-        Connector kind literal, always ``"file"``.
+        Connector kind literal, always ``'file'``.
     format : str | None
-        File format (e.g., ``"json"``, ``"csv"``).
+        File format (e.g., ``'json'``, ``'csv'``).
     path : str | None
         File path or URI.
     options : dict[str, Any]
@@ -306,15 +327,8 @@ class ConnectorFile:
         -------
         Self
             Parsed connector instance.
-
-        Raises
-        ------
-        TypeError
-            If ``name`` is missing or invalid.
         """
-        name = obj.get('name')
-        if not isinstance(name, str):
-            raise TypeError('ConnectorFile requires a "name" (str)')
+        name = _require_name(obj, kind='File')
 
         return cls(
             name=name,

etlplus/{dag.py → workflow/dag.py}

@@ -1,8 +1,8 @@
 """
-:mod:`etlplus.dag` module.
+:mod:`etlplus.workflow.dag` module.
 
 Lightweight directed acyclic graph (DAG) helpers for ordering jobs based on
-``depends_on``.
+:attr:`depends_on`.
 """
 
 from __future__ import annotations
@@ -10,13 +10,15 @@ from __future__ import annotations
 from collections import deque
 from dataclasses import dataclass
 
-from .config.jobs import JobConfig
+from .jobs import JobConfig
 
 # SECTION: EXPORTS ========================================================== #
 
 
 __all__ = [
+    # Errors
     'DagError',
+    # Functions
     'topological_sort_jobs',
 ]
 
@@ -52,7 +54,7 @@ def topological_sort_jobs(
     jobs: list[JobConfig],
 ) -> list[JobConfig]:
     """
-    Return jobs in topological order based on ``depends_on``.
+    Return jobs in topological order based on :attr:`depends_on`.
 
     Parameters
     ----------

etlplus/{config → workflow}/jobs.py

@@ -1,12 +1,12 @@
 """
-:mod:`etlplus.config.jobs` module.
+:mod:`etlplus.workflow.jobs` module.
 
 Data classes modeling job orchestration references (extract, validate,
 transform, load).
 
 Notes
 -----
-- Lightweight references used inside ``PipelineConfig`` to avoid storing
+- Lightweight references used inside :class:`PipelineConfig` to avoid storing
     large nested structures.
 - All attributes are simple and optional where appropriate, keeping parsing
     tolerant.
@@ -19,6 +19,7 @@ from dataclasses import field
 from typing import Any
 from typing import Self
 
+from ..types import StrAnyMap
 from ..utils import coerce_dict
 from ..utils import maybe_mapping
 
@@ -26,6 +27,7 @@ from ..utils import maybe_mapping
 
 
 __all__ = [
+    # Data Classes
     'ExtractRef',
     'JobConfig',
     'LoadRef',
@@ -34,6 +36,75 @@ __all__ = [
 ]
 
 
+# SECTION: INTERNAL FUNCTIONS =============================================== #
+
+
+def _coerce_optional_str(value: Any) -> str | None:
+    """
+    Normalize optional string values, coercing non-strings when needed.
+
+    Parameters
+    ----------
+    value : Any
+        Optional value to normalize.
+
+    Returns
+    -------
+    str | None
+        ``None`` when ``value`` is ``None``; otherwise a string value.
+    """
+    if value is None:
+        return None
+    return value if isinstance(value, str) else str(value)
+
+
+def _parse_depends_on(
+    value: Any,
+) -> list[str]:
+    """
+    Normalize dependency declarations into a string list.
+
+    Parameters
+    ----------
+    value : Any
+        Input dependency specification (string or list of strings).
+
+    Returns
+    -------
+    list[str]
+        Normalized dependency list.
+    """
+    if isinstance(value, str):
+        return [value]
+    if isinstance(value, list):
+        return [entry for entry in value if isinstance(entry, str)]
+    return []
+
+
+def _require_str(
+    # data: dict[str, Any],
+    data: StrAnyMap,
+    key: str,
+) -> str | None:
+    """
+    Extract a required string field from a mapping.
+
+    Parameters
+    ----------
+    data : StrAnyMap
+        Mapping containing the target field.
+    key : str
+        Field name to extract.
+
+    Returns
+    -------
+    str | None
+        The string value when present and valid; otherwise ``None``.
+    """
+    value = data.get(key)
+    return value if isinstance(value, str) else None
+
+
 # SECTION: DATA CLASSES ===================================================== #
 
 
@@ -62,12 +133,13 @@ class ExtractRef:
         cls,
         obj: Any,
     ) -> Self | None:
-        """Parse a mapping into an :class:`ExtractRef` instance.
+        """
+        Parse a mapping into an :class:`ExtractRef` instance.
 
         Parameters
         ----------
         obj : Any
-            Mapping with ``source`` and optional ``options``.
+            Mapping with :attr:`source` and optional :attr:`options`.
 
         Returns
         -------
@@ -77,8 +149,8 @@ class ExtractRef:
         data = maybe_mapping(obj)
         if not data:
             return None
-        source = data.get('source')
-        if not isinstance(source, str):
+        source = _require_str(data, 'source')
+        if source is None:
             return None
         return cls(
             source=source,
@@ -126,7 +198,8 @@ class JobConfig:
         cls,
         obj: Any,
     ) -> Self | None:
-        """Parse a mapping into a :class:`JobConfig` instance.
+        """
+        Parse a mapping into a :class:`JobConfig` instance.
 
         Parameters
         ----------
@@ -141,22 +214,13 @@ class JobConfig:
         data = maybe_mapping(obj)
         if not data:
             return None
-        name = data.get('name')
-        if not isinstance(name, str):
+        name = _require_str(data, 'name')
+        if name is None:
             return None
 
-        description = data.get('description')
-        if description is not None and not isinstance(description, str):
-            description = str(description)
+        description = _coerce_optional_str(data.get('description'))
 
-        depends_raw = data.get('depends_on')
-        depends_on: list[str] = []
-        if isinstance(depends_raw, str):
-            depends_on = [depends_raw]
-        elif isinstance(depends_raw, list):
-            for entry in depends_raw:
-                if isinstance(entry, str):
-                    depends_on.append(entry)
+        depends_on = _parse_depends_on(data.get('depends_on'))
 
         return cls(
             name=name,
@@ -194,12 +258,13 @@ class LoadRef:
         cls,
         obj: Any,
     ) -> Self | None:
-        """Parse a mapping into a :class:`LoadRef` instance.
+        """
+        Parse a mapping into a :class:`LoadRef` instance.
 
         Parameters
         ----------
         obj : Any
-            Mapping with ``target`` and optional ``overrides``.
+            Mapping with :attr:`target` and optional :attr:`overrides`.
 
         Returns
         -------
@@ -209,8 +274,8 @@ class LoadRef:
         data = maybe_mapping(obj)
         if not data:
             return None
-        target = data.get('target')
-        if not isinstance(target, str):
+        target = _require_str(data, 'target')
+        if target is None:
             return None
         return cls(
             target=target,
@@ -240,12 +305,13 @@ class TransformRef:
         cls,
         obj: Any,
     ) -> Self | None:
-        """Parse a mapping into a :class:`TransformRef` instance.
+        """
+        Parse a mapping into a :class:`TransformRef` instance.
 
         Parameters
         ----------
         obj : Any
-            Mapping with ``pipeline``.
+            Mapping with :attr:`pipeline`.
 
         Returns
         -------
@@ -255,8 +321,8 @@ class TransformRef:
         data = maybe_mapping(obj)
         if not data:
             return None
-        pipeline = data.get('pipeline')
-        if not isinstance(pipeline, str):
+        pipeline = _require_str(data, 'pipeline')
+        if pipeline is None:
             return None
         return cls(pipeline=pipeline)
 
@@ -290,12 +356,13 @@ class ValidationRef:
         cls,
         obj: Any,
     ) -> Self | None:
-        """Parse a mapping into a :class:`ValidationRef` instance.
+        """
+        Parse a mapping into a :class:`ValidationRef` instance.
 
         Parameters
         ----------
         obj : Any
-            Mapping with ``ruleset`` plus optional metadata.
+            Mapping with :attr:`ruleset` plus optional metadata.
 
         Returns
         -------
@@ -305,15 +372,11 @@ class ValidationRef:
         data = maybe_mapping(obj)
         if not data:
             return None
-        ruleset = data.get('ruleset')
-        if not isinstance(ruleset, str):
+        ruleset = _require_str(data, 'ruleset')
+        if ruleset is None:
             return None
-        severity = data.get('severity')
-        if severity is not None and not isinstance(severity, str):
-            severity = str(severity)
-        phase = data.get('phase')
-        if phase is not None and not isinstance(phase, str):
-            phase = str(phase)
+        severity = _coerce_optional_str(data.get('severity'))
+        phase = _coerce_optional_str(data.get('phase'))
         return cls(
             ruleset=ruleset,
             severity=severity,

etlplus/{config → workflow}/pipeline.py

@@ -1,5 +1,5 @@
 """
-:mod:`etlplus.config.pipeline` module.
+:mod:`etlplus.workflow.pipeline` module.
 
 Pipeline configuration model and helpers for job orchestration.
 
@@ -16,6 +16,7 @@ Notes
 from __future__ import annotations
 
 import os
+from collections.abc import Callable
 from collections.abc import Mapping
 from dataclasses import dataclass
 from dataclasses import field
@@ -28,68 +29,86 @@ from ..file import File
 from ..file import FileFormat
 from ..types import StrAnyMap
 from ..utils import coerce_dict
+from ..utils import deep_substitute
 from ..utils import maybe_mapping
 from .connector import Connector
 from .connector import parse_connector
 from .jobs import JobConfig
 from .profile import ProfileConfig
-from .utils import deep_substitute
 
 # SECTION: EXPORTS ========================================================== #
 
 
-__all__ = ['PipelineConfig', 'load_pipeline_config']
+__all__ = [
+    # Data Classes
+    'PipelineConfig',
+    # Functions
+    'load_pipeline_config',
+]
 
 
-def _build_jobs(
+# SECTION: INTERNAL FUNCTIONS =============================================== #
+
+
+def _collect_parsed[T](
     raw: StrAnyMap,
-) -> list[JobConfig]:
+    key: str,
+    parser: Callable[[Any], T | None],
+) -> list[T]:
     """
-    Return a list of ``JobConfig`` objects parsed from the mapping.
+    Collect parsed items from ``raw[key]`` using a tolerant parser.
 
     Parameters
     ----------
     raw : StrAnyMap
         Raw pipeline mapping.
+    key : str
+        Key pointing to a list-like payload.
+    parser : Callable[[Any], T | None]
+        Parser that returns an instance or ``None`` for invalid entries.
 
     Returns
     -------
-    list[JobConfig]
-        Parsed job configurations.
+    list[T]
+        Parsed items, excluding invalid entries.
     """
-    jobs: list[JobConfig] = []
-    for job_raw in raw.get('jobs', []) or []:
-        job_cfg = JobConfig.from_obj(job_raw)
-        if job_cfg is not None:
-            jobs.append(job_cfg)
-
-    return jobs
+    items: list[T] = []
+    for entry in raw.get(key, []) or []:
+        parsed = parser(entry)
+        if parsed is not None:
+            items.append(parsed)
+    return items
 
 
-def _build_sources(
-    raw: StrAnyMap,
-) -> list[Connector]:
+def _parse_connector_entry(
+    obj: Any,
+) -> Connector | None:
     """
-    Return a list of source connectors parsed from the mapping.
+    Parse a connector mapping into a concrete connector instance.
 
     Parameters
     ----------
-    raw : StrAnyMap
-        Raw pipeline mapping.
+    obj : Any
+        Candidate connector mapping.
 
     Returns
     -------
-    list[Connector]
-        Parsed source connectors.
+    Connector | None
+        Parsed connector instance or ``None`` when invalid.
     """
-    return _build_connectors(raw, 'sources')
+    if not (entry := maybe_mapping(obj)):
+        return None
+    try:
+        return parse_connector(entry)
+    except TypeError:
+        return None
 
 
-def _build_targets(
+def _build_sources(
     raw: StrAnyMap,
 ) -> list[Connector]:
     """
-    Return a list of target connectors parsed from the mapping.
+    Return a list of source connectors parsed from the mapping.
 
     Parameters
     ----------
@@ -99,43 +118,32 @@ def _build_targets(
     Returns
     -------
     list[Connector]
-        Parsed target connectors.
+        Parsed source connectors.
     """
-    return _build_connectors(raw, 'targets')
+    return list(
+        _collect_parsed(raw, 'sources', _parse_connector_entry),
+    )
 
 
-def _build_connectors(
+def _build_targets(
     raw: StrAnyMap,
-    key: str,
 ) -> list[Connector]:
     """
-    Return parsed connectors from ``raw[key]`` using tolerant parsing.
-
-    Unknown or malformed entries are skipped to preserve permissiveness.
+    Return a list of target connectors parsed from the mapping.
 
     Parameters
     ----------
     raw : StrAnyMap
         Raw pipeline mapping.
-    key : str
-        List-containing top-level key ("sources" or "targets").
 
     Returns
     -------
     list[Connector]
-        Constructed connector instances (malformed entries skipped).
+        Parsed target connectors.
     """
-    items: list[Connector] = []
-    for obj in raw.get(key, []) or []:
-        if not (entry := maybe_mapping(obj)):
-            continue
-        try:
-            items.append(parse_connector(entry))
-        except TypeError:
-            # Skip unsupported types or malformed entries
-            continue
-
-    return items
+    return list(
+        _collect_parsed(raw, 'targets', _parse_connector_entry),
+    )
 
 
 # SECTION: FUNCTIONS ======================================================== #
@@ -156,7 +164,7 @@ def load_pipeline_config(
     return PipelineConfig.from_yaml(path, substitute=substitute, env=env)
 
 
-# SECTION: CLASSES ========================================================== #
+# SECTION: DATA CLASSES ===================================================== #
 
 
 @dataclass(kw_only=True, slots=True)
@@ -313,7 +321,7 @@ class PipelineConfig:
         targets = _build_targets(raw)
 
         # Jobs
-        jobs = _build_jobs(raw)
+        jobs = _collect_parsed(raw, 'jobs', JobConfig.from_obj)
 
         # Table schemas (optional, tolerant pass-through structures).
         table_schemas: list[dict[str, Any]] = []