etlplus 0.16.0__py3-none-any.whl → 0.16.6__py3-none-any.whl

etlplus/cli/handlers.py

@@ -14,6 +14,7 @@ from typing import Any
 from typing import Literal
 from typing import cast
 
+from .. import Config
 from ..database import load_table_spec
 from ..database import render_tables
 from ..file import File
@@ -23,11 +24,9 @@ from ..ops import load
 from ..ops import run
 from ..ops import transform
 from ..ops import validate
-from ..ops.validate import FieldRules
+from ..ops.validate import FieldRulesDict
 from ..types import JSONData
 from ..types import TemplateKey
-from ..workflow import PipelineConfig
-from ..workflow import load_pipeline_config
 from . import io as cli_io
 
 # SECTION: EXPORTS ========================================================== #
@@ -73,14 +72,14 @@ def _collect_table_specs(
         specs.append(dict(load_table_spec(Path(spec_path))))
 
     if config_path:
-        cfg = load_pipeline_config(config_path, substitute=True)
+        cfg = Config.from_yaml(config_path, substitute=True)
         specs.extend(getattr(cfg, 'table_schemas', []))
 
     return specs
 
 
 def _check_sections(
-    cfg: PipelineConfig,
+    cfg: Config,
     *,
     jobs: bool,
     pipelines: bool,
@@ -93,7 +92,7 @@ def _check_sections(
 
     Parameters
     ----------
-    cfg : PipelineConfig
+    cfg : Config
         The loaded pipeline configuration.
     jobs : bool
         Whether to include job metadata.
@@ -133,14 +132,14 @@ def _check_sections(
 
 
 def _pipeline_summary(
-    cfg: PipelineConfig,
+    cfg: Config,
 ) -> dict[str, Any]:
     """
     Return a human-friendly snapshot of a pipeline config.
 
     Parameters
     ----------
-    cfg : PipelineConfig
+    cfg : Config
         The loaded pipeline configuration.
 
     Returns
@@ -229,7 +228,7 @@ def check_handler(
         Zero on success.
 
     """
-    cfg = load_pipeline_config(config, substitute=substitute)
+    cfg = Config.from_yaml(config, substitute=substitute)
     if summary:
         cli_io.emit_json(_pipeline_summary(cfg), pretty=True)
         return 0
@@ -514,7 +513,7 @@ def run_handler(
     int
         Zero on success.
     """
-    cfg = load_pipeline_config(config, substitute=True)
+    cfg = Config.from_yaml(config, substitute=True)
 
     job_name = job or pipeline
     if job_name:
@@ -662,7 +661,7 @@ def validate_handler(
     if not isinstance(rules_payload, dict):
         raise ValueError('rules must resolve to a mapping of field rules')
 
-    field_rules = cast(Mapping[str, FieldRules], rules_payload)
+    field_rules = cast(Mapping[str, FieldRulesDict], rules_payload)
     result = validate(payload, field_rules)
 
     if target and target != '-':
@@ -671,11 +670,11 @@ def validate_handler(
             cli_io.write_json_output(
                 validated_data,
                 target,
-                success_message='Validation result saved to',
+                success_message='ValidationDict result saved to',
             )
         else:
             print(
-                f'Validation failed, no data to save for {target}',
+                f'ValidationDict failed, no data to save for {target}',
                 file=sys.stderr,
             )
     else:

etlplus/{workflow/pipeline.py → config.py}

@@ -1,7 +1,7 @@
 """
-:mod:`etlplus.workflow.pipeline` module.
+:mod:`etlplus.config` module.
 
-Pipeline configuration model and helpers for job orchestration.
+Configuration model and helpers for job pipeline orchestration.
 
 Notes
 -----
@@ -24,46 +24,66 @@ from pathlib import Path
 from typing import Any
 from typing import Self
 
-from ..api import ApiConfig
-from ..connector import Connector
-from ..connector import parse_connector
-from ..file import File
-from ..file import FileFormat
-from ..types import StrAnyMap
-from ..utils import coerce_dict
-from ..utils import deep_substitute
-from ..utils import maybe_mapping
-from .jobs import JobConfig
-from .profile import ProfileConfig
+from .api import ApiConfig
+from .connector import Connector
+from .connector import parse_connector
+from .file import File
+from .file import FileFormat
+from .types import StrAnyMap
+from .utils import coerce_dict
+from .utils import deep_substitute
+from .utils import maybe_mapping
+from .workflow.jobs import JobConfig
+from .workflow.profile import ProfileConfig
 
 # SECTION: EXPORTS ========================================================== #
 
 
 __all__ = [
     # Data Classes
-    'PipelineConfig',
-    # Functions
-    'load_pipeline_config',
+    'Config',
 ]
 
 
 # SECTION: INTERNAL FUNCTIONS =============================================== #
 
 
-def _collect_parsed[T](
+def _build_connectors(
     raw: StrAnyMap,
+    *,
     key: str,
-    parser: Callable[[Any], T | None],
-) -> list[T]:
+) -> list[Connector]:
     """
-    Collect parsed items from ``raw[key]`` using a tolerant parser.
+    Parse connector entries from a list under ``raw[key]``.
 
     Parameters
     ----------
     raw : StrAnyMap
         Raw pipeline mapping.
     key : str
-        Key pointing to a list-like payload.
+        Key pointing to connector entries (e.g., ``"sources"``).
+
+    Returns
+    -------
+    list[Connector]
+        Parsed connector instances.
+    """
+    return list(
+        _collect_parsed(raw.get(key, []) or [], _parse_connector_entry),
+    )
+
+
+def _collect_parsed[T](
+    items: Any,
+    parser: Callable[[Any], T | None],
+) -> list[T]:
+    """
+    Collect parsed items from ``raw[key]`` using a tolerant parser.
+
+    Parameters
+    ----------
+    items : Any
+        List-like payload to parse.
     parser : Callable[[Any], T | None]
         Parser that returns an instance or ``None`` for invalid entries.
 
@@ -72,12 +92,12 @@ def _collect_parsed[T](
     list[T]
         Parsed items, excluding invalid entries.
     """
-    items: list[T] = []
-    for entry in raw.get(key, []) or []:
+    parsed_items: list[T] = []
+    for entry in items or []:
         parsed = parser(entry)
         if parsed is not None:
-            items.append(parsed)
-    return items
+            parsed_items.append(parsed)
+    return parsed_items
 
 
 def _parse_connector_entry(
@@ -104,71 +124,11 @@ def _parse_connector_entry(
         return None
 
 
-def _build_sources(
-    raw: StrAnyMap,
-) -> list[Connector]:
-    """
-    Return a list of source connectors parsed from the mapping.
-
-    Parameters
-    ----------
-    raw : StrAnyMap
-        Raw pipeline mapping.
-
-    Returns
-    -------
-    list[Connector]
-        Parsed source connectors.
-    """
-    return list(
-        _collect_parsed(raw, 'sources', _parse_connector_entry),
-    )
-
-
-def _build_targets(
-    raw: StrAnyMap,
-) -> list[Connector]:
-    """
-    Return a list of target connectors parsed from the mapping.
-
-    Parameters
-    ----------
-    raw : StrAnyMap
-        Raw pipeline mapping.
-
-    Returns
-    -------
-    list[Connector]
-        Parsed target connectors.
-    """
-    return list(
-        _collect_parsed(raw, 'targets', _parse_connector_entry),
-    )
-
-
-# SECTION: FUNCTIONS ======================================================== #
-
-
-def load_pipeline_config(
-    path: Path | str,
-    *,
-    substitute: bool = False,
-    env: Mapping[str, str] | None = None,
-) -> PipelineConfig:
-    """
-    Load a pipeline YAML file into a ``PipelineConfig`` instance.
-
-    Delegates to ``PipelineConfig.from_yaml`` for construction and optional
-    variable substitution.
-    """
-    return PipelineConfig.from_yaml(path, substitute=substitute, env=env)
-
-
 # SECTION: DATA CLASSES ===================================================== #
 
 
 @dataclass(kw_only=True, slots=True)
-class PipelineConfig:
+class Config:
     """
     Configuration for the data processing pipeline.
 
@@ -231,7 +191,7 @@ class PipelineConfig:
         env: Mapping[str, str] | None = None,
     ) -> Self:
         """
-        Parse a YAML file into a ``PipelineConfig`` instance.
+        Parse a YAML file into a ``Config`` instance.
 
         Parameters
         ----------
@@ -279,7 +239,7 @@ class PipelineConfig:
         raw: StrAnyMap,
     ) -> Self:
         """
-        Parse a mapping into a ``PipelineConfig`` instance.
+        Parse a mapping into a ``Config`` instance.
 
         Parameters
         ----------
@@ -311,17 +271,20 @@ class PipelineConfig:
         file_systems = coerce_dict(raw.get('file_systems'))
 
         # Sources
-        sources = _build_sources(raw)
+        sources = _build_connectors(raw, key='sources')
 
         # Validations/Transforms
         validations = coerce_dict(raw.get('validations'))
         transforms = coerce_dict(raw.get('transforms'))
 
         # Targets
-        targets = _build_targets(raw)
+        targets = _build_connectors(raw, key='targets')
 
         # Jobs
-        jobs = _collect_parsed(raw, 'jobs', JobConfig.from_obj)
+        jobs: list[JobConfig] = _collect_parsed(
+            raw.get('jobs', []) or [],
+            JobConfig.from_obj,
+        )
 
         # Table schemas (optional, tolerant pass-through structures).
         table_schemas: list[dict[str, Any]] = []

etlplus/connector/__init__.py

@@ -7,15 +7,15 @@ Connector configuration types and enums.
 from __future__ import annotations
 
 from .api import ConnectorApi
-from .api import ConnectorApiConfigMap
+from .api import ConnectorApiConfigDict
 from .connector import Connector
 from .core import ConnectorBase
 from .core import ConnectorProtocol
 from .database import ConnectorDb
-from .database import ConnectorDbConfigMap
+from .database import ConnectorDbConfigDict
 from .enums import DataConnectorType
 from .file import ConnectorFile
-from .file import ConnectorFileConfigMap
+from .file import ConnectorFileConfigDict
 from .types import ConnectorType
 from .utils import parse_connector
 
@@ -37,7 +37,7 @@ __all__ = [
     'ConnectorProtocol',
     'ConnectorType',
     # Typed Dicts
-    'ConnectorApiConfigMap',
-    'ConnectorDbConfigMap',
-    'ConnectorFileConfigMap',
+    'ConnectorApiConfigDict',
+    'ConnectorDbConfigDict',
+    'ConnectorFileConfigDict',
 ]

etlplus/connector/api.py

@@ -22,9 +22,9 @@ from typing import TypedDict
 from typing import overload
 
 from ..api import PaginationConfig
-from ..api import PaginationConfigMap
+from ..api import PaginationConfigDict
 from ..api import RateLimitConfig
-from ..api import RateLimitConfigMap
+from ..api import RateLimitConfigDict
 from ..types import StrAnyMap
 from ..types import StrStrMap
 from ..utils import cast_str_dict
@@ -39,14 +39,14 @@ from .types import ConnectorType
 
 __all__ = [
     'ConnectorApi',
-    'ConnectorApiConfigMap',
+    'ConnectorApiConfigDict',
 ]
 
 
 # SECTION: TYPED DICTS ====================================================== #
 
 
-class ConnectorApiConfigMap(TypedDict, total=False):
+class ConnectorApiConfigDict(TypedDict, total=False):
     """
     Shape accepted by :meth:`ConnectorApi.from_obj` (all keys optional).
 
@@ -61,8 +61,8 @@ class ConnectorApiConfigMap(TypedDict, total=False):
     method: str
     headers: StrStrMap
     query_params: StrAnyMap
-    pagination: PaginationConfigMap
-    rate_limit: RateLimitConfigMap
+    pagination: PaginationConfigDict
+    rate_limit: RateLimitConfigDict
     api: str
     endpoint: str
 
@@ -121,7 +121,7 @@ class ConnectorApi(ConnectorBase):
 
     @classmethod
     @overload
-    def from_obj(cls, obj: ConnectorApiConfigMap) -> Self: ...
+    def from_obj(cls, obj: ConnectorApiConfigDict) -> Self: ...
 
     @classmethod
     @overload

etlplus/connector/database.py

@@ -29,14 +29,14 @@ from .types import ConnectorType
 
 __all__ = [
     'ConnectorDb',
-    'ConnectorDbConfigMap',
+    'ConnectorDbConfigDict',
 ]
 
 
 # SECTION: TYPED DICTS ====================================================== #
 
 
-class ConnectorDbConfigMap(TypedDict, total=False):
+class ConnectorDbConfigDict(TypedDict, total=False):
     """
     Shape accepted by :meth:`ConnectorDb.from_obj` (all keys optional).
 
@@ -87,7 +87,7 @@ class ConnectorDb(ConnectorBase):
 
     @classmethod
     @overload
-    def from_obj(cls, obj: ConnectorDbConfigMap) -> Self: ...
+    def from_obj(cls, obj: ConnectorDbConfigDict) -> Self: ...
 
     @classmethod
     @overload

etlplus/connector/file.py

@@ -32,14 +32,14 @@ from .types import ConnectorType
 
 __all__ = [
     'ConnectorFile',
-    'ConnectorFileConfigMap',
+    'ConnectorFileConfigDict',
 ]
 
 
 # SECTION: TYPED DICTS ====================================================== #
 
 
-class ConnectorFileConfigMap(TypedDict, total=False):
+class ConnectorFileConfigDict(TypedDict, total=False):
     """
     Shape accepted by :meth:`ConnectorFile.from_obj` (all keys optional).
 
@@ -86,7 +86,7 @@ class ConnectorFile(ConnectorBase):
 
     @classmethod
     @overload
-    def from_obj(cls, obj: ConnectorFileConfigMap) -> Self: ...
+    def from_obj(cls, obj: ConnectorFileConfigDict) -> Self: ...
 
     @classmethod
     @overload

etlplus/connector/types.py

@@ -14,8 +14,8 @@ Examples
 >>>     "type": "database",
 >>>     "connection_string": "postgresql://user:pass@localhost/db",
 >>> }
->>> from etlplus.api import RetryPolicy
->>> rp: RetryPolicy = {"max_attempts": 3, "backoff": 0.5}
+>>> from etlplus.api import RetryPolicyDict
+>>> rp: RetryPolicyDict = {"max_attempts": 3, "backoff": 0.5}
 """
 
 from __future__ import annotations