etlplus 0.9.2__py3-none-any.whl → 0.10.2__py3-none-any.whl

etlplus/{workflow → config}/jobs.py

@@ -1,12 +1,12 @@
 """
-:mod:`etlplus.workflow.jobs` module.
+:mod:`etlplus.config.jobs` module.
 
 Data classes modeling job orchestration references (extract, validate,
 transform, load).
 
 Notes
 -----
-- Lightweight references used inside :class:`PipelineConfig` to avoid storing
+- Lightweight references used inside ``PipelineConfig`` to avoid storing
     large nested structures.
 - All attributes are simple and optional where appropriate, keeping parsing
     tolerant.
@@ -19,7 +19,6 @@ from dataclasses import field
 from typing import Any
 from typing import Self
 
-from ..types import StrAnyMap
 from ..utils import coerce_dict
 from ..utils import maybe_mapping
 
@@ -27,7 +26,6 @@ from ..utils import maybe_mapping
 
 
 __all__ = [
-    # Data Classes
     'ExtractRef',
     'JobConfig',
     'LoadRef',
@@ -36,76 +34,10 @@ __all__ = [
 ]
 
 
-# SECTION: INTERNAL FUNCTIONS =============================================== #
+# SECTION: TYPE ALIASES ===================================================== #
 
 
-def _coerce_optional_str(value: Any) -> str | None:
-    """
-    Normalize optional string values, coercing non-strings when needed.
-
-    Parameters
-    ----------
-    value : Any
-        Optional value to normalize.
-
-    Returns
-    -------
-    str | None
-        ``None`` when ``value`` is ``None``; otherwise a string value.
-    """
-    if value is None:
-        return None
-    return value if isinstance(value, str) else str(value)
-
-
-def _parse_depends_on(
-    value: Any,
-) -> list[str]:
-    """
-    Normalize dependency declarations into a string list.
-
-    Parameters
-    ----------
-    value : Any
-        Input dependency specification (string or list of strings).
-
-    Returns
-    -------
-    list[str]
-        Normalized dependency list.
-    """
-    if isinstance(value, str):
-        return [value]
-    if isinstance(value, list):
-        return [entry for entry in value if isinstance(entry, str)]
-    return []
-
-
-def _require_str(
-    # data: dict[str, Any],
-    data: StrAnyMap,
-    key: str,
-) -> str | None:
-    """
-    Extract a required string field from a mapping.
-
-    Parameters
-    ----------
-    data : StrAnyMap
-        Mapping containing the target field.
-    key : str
-        Field name to extract.
-
-    Returns
-    -------
-    str | None
-        The string value when present and valid; otherwise ``None``.
-    """
-    value = data.get(key)
-    return value if isinstance(value, str) else None
-
-
-# SECTION: DATA CLASSES ===================================================== #
+# SECTION: CLASSES ========================================================== #
 
 
 @dataclass(kw_only=True, slots=True)
@@ -133,13 +65,12 @@ class ExtractRef:
         cls,
         obj: Any,
     ) -> Self | None:
-        """
-        Parse a mapping into an :class:`ExtractRef` instance.
+        """Parse a mapping into an :class:`ExtractRef` instance.
 
         Parameters
         ----------
         obj : Any
-            Mapping with :attr:`source` and optional :attr:`options`.
+            Mapping with ``source`` and optional ``options``.
 
         Returns
         -------
@@ -149,8 +80,8 @@ class ExtractRef:
         data = maybe_mapping(obj)
         if not data:
             return None
-        source = _require_str(data, 'source')
-        if source is None:
+        source = data.get('source')
+        if not isinstance(source, str):
             return None
         return cls(
             source=source,
@@ -169,8 +100,6 @@ class JobConfig:
         Unique job name.
     description : str | None
         Optional human-friendly description.
-    depends_on : list[str]
-        Optional job dependency list. Dependencies must refer to other jobs.
     extract : ExtractRef | None
         Extraction reference.
     validate : ValidationRef | None
@@ -185,7 +114,6 @@ class JobConfig:
 
     name: str
     description: str | None = None
-    depends_on: list[str] = field(default_factory=list)
     extract: ExtractRef | None = None
     validate: ValidationRef | None = None
     transform: TransformRef | None = None
@@ -198,8 +126,7 @@ class JobConfig:
         cls,
         obj: Any,
     ) -> Self | None:
-        """
-        Parse a mapping into a :class:`JobConfig` instance.
+        """Parse a mapping into a :class:`JobConfig` instance.
 
         Parameters
         ----------
@@ -214,18 +141,17 @@ class JobConfig:
         data = maybe_mapping(obj)
         if not data:
             return None
-        name = _require_str(data, 'name')
-        if name is None:
+        name = data.get('name')
+        if not isinstance(name, str):
             return None
 
-        description = _coerce_optional_str(data.get('description'))
-
-        depends_on = _parse_depends_on(data.get('depends_on'))
+        description = data.get('description')
+        if description is not None and not isinstance(description, str):
+            description = str(description)
 
         return cls(
             name=name,
             description=description,
-            depends_on=depends_on,
             extract=ExtractRef.from_obj(data.get('extract')),
             validate=ValidationRef.from_obj(data.get('validate')),
             transform=TransformRef.from_obj(data.get('transform')),
@@ -258,13 +184,12 @@ class LoadRef:
         cls,
         obj: Any,
     ) -> Self | None:
-        """
-        Parse a mapping into a :class:`LoadRef` instance.
+        """Parse a mapping into a :class:`LoadRef` instance.
 
         Parameters
         ----------
         obj : Any
-            Mapping with :attr:`target` and optional :attr:`overrides`.
+            Mapping with ``target`` and optional ``overrides``.
 
         Returns
         -------
@@ -274,8 +199,8 @@ class LoadRef:
         data = maybe_mapping(obj)
         if not data:
             return None
-        target = _require_str(data, 'target')
-        if target is None:
+        target = data.get('target')
+        if not isinstance(target, str):
             return None
         return cls(
             target=target,
@@ -305,13 +230,12 @@ class TransformRef:
         cls,
         obj: Any,
     ) -> Self | None:
-        """
-        Parse a mapping into a :class:`TransformRef` instance.
+        """Parse a mapping into a :class:`TransformRef` instance.
 
         Parameters
         ----------
         obj : Any
-            Mapping with :attr:`pipeline`.
+            Mapping with ``pipeline``.
 
         Returns
         -------
@@ -321,8 +245,8 @@ class TransformRef:
         data = maybe_mapping(obj)
         if not data:
             return None
-        pipeline = _require_str(data, 'pipeline')
-        if pipeline is None:
+        pipeline = data.get('pipeline')
+        if not isinstance(pipeline, str):
             return None
         return cls(pipeline=pipeline)
 
@@ -356,13 +280,12 @@ class ValidationRef:
         cls,
         obj: Any,
     ) -> Self | None:
-        """
-        Parse a mapping into a :class:`ValidationRef` instance.
+        """Parse a mapping into a :class:`ValidationRef` instance.
 
         Parameters
         ----------
         obj : Any
-            Mapping with :attr:`ruleset` plus optional metadata.
+            Mapping with ``ruleset`` plus optional metadata.
 
         Returns
         -------
@@ -372,11 +295,15 @@ class ValidationRef:
         data = maybe_mapping(obj)
         if not data:
             return None
-        ruleset = _require_str(data, 'ruleset')
-        if ruleset is None:
+        ruleset = data.get('ruleset')
+        if not isinstance(ruleset, str):
             return None
-        severity = _coerce_optional_str(data.get('severity'))
-        phase = _coerce_optional_str(data.get('phase'))
+        severity = data.get('severity')
+        if severity is not None and not isinstance(severity, str):
+            severity = str(severity)
+        phase = data.get('phase')
+        if phase is not None and not isinstance(phase, str):
+            phase = str(phase)
         return cls(
             ruleset=ruleset,
             severity=severity,

etlplus/{workflow → config}/pipeline.py

@@ -1,5 +1,5 @@
 """
-:mod:`etlplus.workflow.pipeline` module.
+:mod:`etlplus.config.pipeline` module.
 
 Pipeline configuration model and helpers for job orchestration.
 
@@ -16,7 +16,6 @@ Notes
 from __future__ import annotations
 
 import os
-from collections.abc import Callable
 from collections.abc import Mapping
 from dataclasses import dataclass
 from dataclasses import field
@@ -25,90 +24,72 @@ from typing import Any
 from typing import Self
 
 from ..api import ApiConfig
+from ..enums import FileFormat
 from ..file import File
-from ..file import FileFormat
 from ..types import StrAnyMap
 from ..utils import coerce_dict
-from ..utils import deep_substitute
 from ..utils import maybe_mapping
 from .connector import Connector
 from .connector import parse_connector
 from .jobs import JobConfig
 from .profile import ProfileConfig
+from .utils import deep_substitute
 
 # SECTION: EXPORTS ========================================================== #
 
 
-__all__ = [
-    # Data Classes
-    'PipelineConfig',
-    # Functions
-    'load_pipeline_config',
-]
+__all__ = ['PipelineConfig', 'load_pipeline_config']
 
 
-# SECTION: INTERNAL FUNCTIONS =============================================== #
-
-
-def _collect_parsed[T](
+def _build_jobs(
     raw: StrAnyMap,
-    key: str,
-    parser: Callable[[Any], T | None],
-) -> list[T]:
+) -> list[JobConfig]:
     """
-    Collect parsed items from ``raw[key]`` using a tolerant parser.
+    Return a list of ``JobConfig`` objects parsed from the mapping.
 
     Parameters
     ----------
     raw : StrAnyMap
         Raw pipeline mapping.
-    key : str
-        Key pointing to a list-like payload.
-    parser : Callable[[Any], T | None]
-        Parser that returns an instance or ``None`` for invalid entries.
 
     Returns
     -------
-    list[T]
-        Parsed items, excluding invalid entries.
+    list[JobConfig]
+        Parsed job configurations.
     """
-    items: list[T] = []
-    for entry in raw.get(key, []) or []:
-        parsed = parser(entry)
-        if parsed is not None:
-            items.append(parsed)
-    return items
+    jobs: list[JobConfig] = []
+    for job_raw in raw.get('jobs', []) or []:
+        job_cfg = JobConfig.from_obj(job_raw)
+        if job_cfg is not None:
+            jobs.append(job_cfg)
+
+    return jobs
 
 
-def _parse_connector_entry(
-    obj: Any,
-) -> Connector | None:
+def _build_sources(
+    raw: StrAnyMap,
+) -> list[Connector]:
     """
-    Parse a connector mapping into a concrete connector instance.
+    Return a list of source connectors parsed from the mapping.
 
     Parameters
     ----------
-    obj : Any
-        Candidate connector mapping.
+    raw : StrAnyMap
+        Raw pipeline mapping.
 
     Returns
     -------
-    Connector | None
-        Parsed connector instance or ``None`` when invalid.
+    list[Connector]
+        Parsed source connectors.
     """
-    if not (entry := maybe_mapping(obj)):
-        return None
-    try:
-        return parse_connector(entry)
-    except TypeError:
-        return None
+    return _build_connectors(raw, 'sources')
 
 
-def _build_sources(
+def _build_targets(
     raw: StrAnyMap,
 ) -> list[Connector]:
     """
-    Return a list of source connectors parsed from the mapping.
+    Return a list of target connectors parsed from the mapping.
 
     Parameters
     ----------
@@ -118,32 +99,43 @@ def _build_sources(
     Returns
     -------
     list[Connector]
-        Parsed source connectors.
+        Parsed target connectors.
     """
-    return list(
-        _collect_parsed(raw, 'sources', _parse_connector_entry),
-    )
+    return _build_connectors(raw, 'targets')
 
 
-def _build_targets(
+def _build_connectors(
     raw: StrAnyMap,
+    key: str,
 ) -> list[Connector]:
     """
-    Return a list of target connectors parsed from the mapping.
+    Return parsed connectors from ``raw[key]`` using tolerant parsing.
+
+    Unknown or malformed entries are skipped to preserve permissiveness.
 
     Parameters
     ----------
     raw : StrAnyMap
         Raw pipeline mapping.
+    key : str
+        List-containing top-level key ("sources" or "targets").
 
     Returns
     -------
     list[Connector]
-        Parsed target connectors.
+        Constructed connector instances (malformed entries skipped).
     """
-    return list(
-        _collect_parsed(raw, 'targets', _parse_connector_entry),
-    )
+    items: list[Connector] = []
+    for obj in raw.get(key, []) or []:
+        if not (entry := maybe_mapping(obj)):
+            continue
+        try:
+            items.append(parse_connector(entry))
+        except TypeError:
+            # Skip unsupported types or malformed entries
+            continue
+
+    return items
 
 
 # SECTION: FUNCTIONS ======================================================== #
@@ -164,7 +156,7 @@ def load_pipeline_config(
     return PipelineConfig.from_yaml(path, substitute=substitute, env=env)
 
 
-# SECTION: DATA CLASSES ===================================================== #
+# SECTION: CLASSES ========================================================== #
 
 
 @dataclass(kw_only=True, slots=True)
@@ -254,7 +246,7 @@ class PipelineConfig:
         TypeError
             If the YAML root is not a mapping/object.
         """
-        raw = File(Path(path), FileFormat.YAML).read()
+        raw = File(Path(path), FileFormat.YAML).read_yaml()
         if not isinstance(raw, dict):
             raise TypeError('Pipeline YAML must have a mapping/object root')
 
@@ -321,7 +313,7 @@ class PipelineConfig:
         targets = _build_targets(raw)
 
         # Jobs
-        jobs = _collect_parsed(raw, 'jobs', JobConfig.from_obj)
+        jobs = _build_jobs(raw)
 
         # Table schemas (optional, tolerant pass-through structures).
         table_schemas: list[dict[str, Any]] = []

etlplus/{workflow → config}/profile.py

@@ -1,5 +1,5 @@
 """
-:mod:`etlplus.workflow.profile` module.
+:mod:`etlplus.config.profile` module.
 
 Profile model for pipeline-level defaults and environment.
 
@@ -22,13 +22,10 @@ from ..utils import cast_str_dict
 # SECTION: EXPORTS ========================================================== #
 
 
-__all__ = [
-    # Data Classes
-    'ProfileConfig',
-]
+__all__ = ['ProfileConfig']
 
 
-# SECTION: DATA CLASSES ===================================================== #
+# SECTION: CLASSES ========================================================== #
 
 
 @dataclass(kw_only=True, slots=True)
@@ -56,7 +53,7 @@ class ProfileConfig:
         cls,
         obj: StrAnyMap | None,
     ) -> Self:
-        """Parse a mapping into a :class:`ProfileConfig` instance.
+        """Parse a mapping into a ``ProfileConfig`` instance.
 
         Parameters
         ----------
@@ -67,7 +64,7 @@ class ProfileConfig:
         -------
         Self
             Parsed profile configuration; non-mapping input yields a default
-            instance. All :attr:`env` values are coerced to strings.
+            instance. All ``env`` values are coerced to strings.
         """
         if not isinstance(obj, Mapping):
             return cls()