etlplus 0.15.0__py3-none-any.whl → 0.16.6__py3-none-any.whl

etlplus/cli/handlers.py

@@ -14,6 +14,7 @@ from typing import Any
 from typing import Literal
 from typing import cast
 
+from .. import Config
 from ..database import load_table_spec
 from ..database import render_tables
 from ..file import File
@@ -23,11 +24,9 @@ from ..ops import load
 from ..ops import run
 from ..ops import transform
 from ..ops import validate
-from ..ops.validate import FieldRules
+from ..ops.validate import FieldRulesDict
 from ..types import JSONData
 from ..types import TemplateKey
-from ..workflow import PipelineConfig
-from ..workflow import load_pipeline_config
 from . import io as cli_io
 
 # SECTION: EXPORTS ========================================================== #
@@ -73,14 +72,14 @@ def _collect_table_specs(
         specs.append(dict(load_table_spec(Path(spec_path))))
 
     if config_path:
-        cfg = load_pipeline_config(config_path, substitute=True)
+        cfg = Config.from_yaml(config_path, substitute=True)
         specs.extend(getattr(cfg, 'table_schemas', []))
 
     return specs
 
 
 def _check_sections(
-    cfg: PipelineConfig,
+    cfg: Config,
     *,
     jobs: bool,
     pipelines: bool,
@@ -93,7 +92,7 @@ def _check_sections(
 
     Parameters
     ----------
-    cfg : PipelineConfig
+    cfg : Config
         The loaded pipeline configuration.
     jobs : bool
         Whether to include job metadata.
@@ -121,23 +120,26 @@ def _check_sections(
     if targets:
         sections['targets'] = [tgt.name for tgt in cfg.targets]
     if transforms:
-        sections['transforms'] = [
-            getattr(trf, 'name', None) for trf in cfg.transforms
-        ]
+        if isinstance(cfg.transforms, Mapping):
+            sections['transforms'] = list(cfg.transforms)
+        else:
+            sections['transforms'] = [
+                getattr(trf, 'name', None) for trf in cfg.transforms
+            ]
     if not sections:
         sections['jobs'] = _pipeline_summary(cfg)['jobs']
     return sections
 
 
 def _pipeline_summary(
-    cfg: PipelineConfig,
+    cfg: Config,
 ) -> dict[str, Any]:
     """
     Return a human-friendly snapshot of a pipeline config.
 
     Parameters
     ----------
-    cfg : PipelineConfig
+    cfg : Config
         The loaded pipeline configuration.
 
     Returns
@@ -157,6 +159,29 @@ def _pipeline_summary(
     }
 
 
+def _write_file_payload(
+    payload: JSONData,
+    target: str,
+    *,
+    format_hint: str | None,
+) -> None:
+    """
+    Write a JSON-like payload to a file path using an optional format hint.
+
+    Parameters
+    ----------
+    payload : JSONData
+        The structured data to write.
+    target : str
+        File path to write to.
+    format_hint : str | None
+        Optional format hint for :class:`FileFormat`.
+    """
+    file_path = Path(target)
+    file_format = FileFormat.coerce(format_hint) if format_hint else None
+    File(file_path, file_format=file_format).write(payload)
+
+
 # SECTION: FUNCTIONS ======================================================== #
 
 
@@ -203,7 +228,7 @@ def check_handler(
         Zero on success.
 
     """
-    cfg = load_pipeline_config(config, substitute=substitute)
+    cfg = Config.from_yaml(config, substitute=substitute)
     if summary:
         cli_io.emit_json(_pipeline_summary(cfg), pretty=True)
         return 0
@@ -479,7 +504,7 @@ def run_handler(
         Name of the job to run. If not provided, runs the entire pipeline.
         Default is ``None``.
     pipeline : str | None, optional
-        Alias for ``job``. Default is ``None``.
+        Alias for *job*. Default is ``None``.
     pretty : bool, optional
         Whether to pretty-print output. Default is ``True``.
 
@@ -488,7 +513,7 @@ def run_handler(
     int
         Zero on success.
     """
-    cfg = load_pipeline_config(config, substitute=True)
+    cfg = Config.from_yaml(config, substitute=True)
 
     job_name = job or pipeline
     if job_name:
@@ -572,15 +597,7 @@ def transform_handler(
 
     # TODO: Generalize to handle non-file targets.
     if target and target != '-':
-        # Convert target to Path and target_format to FileFormat if needed
-        file_path = Path(target)
-        file_format = None
-        if target_format is not None:
-            try:
-                file_format = FileFormat(target_format)
-            except ValueError:
-                file_format = None  # or handle error as appropriate
-        File(file_path, file_format=file_format).write(data)
+        _write_file_payload(data, target, format_hint=target_format)
         print(f'Data transformed and saved to {target}')
         return 0
 
@@ -644,7 +661,7 @@ def validate_handler(
     if not isinstance(rules_payload, dict):
         raise ValueError('rules must resolve to a mapping of field rules')
 
-    field_rules = cast(Mapping[str, FieldRules], rules_payload)
+    field_rules = cast(Mapping[str, FieldRulesDict], rules_payload)
     result = validate(payload, field_rules)
 
     if target and target != '-':
@@ -653,11 +670,11 @@ def validate_handler(
             cli_io.write_json_output(
                 validated_data,
                 target,
-                success_message='Validation result saved to',
+                success_message='ValidationDict result saved to',
             )
         else:
             print(
-                f'Validation failed, no data to save for {target}',
+                f'ValidationDict failed, no data to save for {target}',
                 file=sys.stderr,
             )
     else:

etlplus/cli/io.py

@@ -71,7 +71,7 @@ def emit_or_write(
     success_message: str,
 ) -> None:
     """
-    Emit JSON or persist to disk based on ``output_path``.
+    Emit JSON or persist to disk based on *output_path*.
 
     Parameters
     ----------
@@ -122,7 +122,7 @@ def materialize_file_payload(
     format_explicit: bool,
 ) -> JSONData | object:
     """
-    Return structured payloads when ``source`` references a file.
+    Return structured payloads when *source* references a file.
 
     Parameters
     ----------

etlplus/cli/main.py

@@ -44,13 +44,13 @@ def _emit_context_help(
     Returns
     -------
     bool
-        ``True`` when help was emitted, ``False`` when ``ctx`` was ``None``.
+        ``True`` when help was emitted, ``False`` when *ctx* was ``None``.
     """
     if ctx is None:
         return False
 
     with contextlib.redirect_stdout(sys.stderr):
-        ctx.get_help()
+        print(ctx.get_help())
     return True
 
 

etlplus/cli/state.py

@@ -15,6 +15,7 @@ from typing import Final
 
 import typer
 
+from ..utils import normalize_str
 from .constants import DATA_CONNECTORS
 
 # SECTION: EXPORTS ========================================================== #
@@ -322,14 +323,10 @@ def validate_choice(
     typer.BadParameter
         If the input value is not in the set of valid choices.
     """
-    v = str(value or '').strip().lower()
-    normalized_choices = {c.lower() for c in choices}
+    v = normalize_str(str(value or ''))
+    normalized_choices = {normalize_str(c): c for c in choices}
     if v in normalized_choices:
-        # Preserve original casing from choices when possible for messages
-        for choice in choices:
-            if choice.lower() == v:
-                return choice
-        return v
+        return normalized_choices[v]
     allowed = ', '.join(sorted(choices))
     raise typer.BadParameter(
         f"Invalid {label} '{value}'. Choose from: {allowed}",

etlplus/{workflow/pipeline.py → config.py}

@@ -1,7 +1,7 @@
 """
-:mod:`etlplus.workflow.pipeline` module.
+:mod:`etlplus.config` module.
 
-Pipeline configuration model and helpers for job orchestration.
+Configuration model and helpers for job pipeline orchestration.
 
 Notes
 -----
@@ -16,6 +16,7 @@ Notes
 from __future__ import annotations
 
 import os
+from collections.abc import Callable
 from collections.abc import Mapping
 from dataclasses import dataclass
 from dataclasses import field
@@ -23,152 +24,111 @@ from pathlib import Path
 from typing import Any
 from typing import Self
 
-from ..api import ApiConfig
-from ..file import File
-from ..file import FileFormat
-from ..types import StrAnyMap
-from ..utils import coerce_dict
-from ..utils import maybe_mapping
+from .api import ApiConfig
 from .connector import Connector
 from .connector import parse_connector
-from .jobs import JobConfig
-from .profile import ProfileConfig
+from .file import File
+from .file import FileFormat
+from .types import StrAnyMap
+from .utils import coerce_dict
 from .utils import deep_substitute
+from .utils import maybe_mapping
+from .workflow.jobs import JobConfig
+from .workflow.profile import ProfileConfig
 
 # SECTION: EXPORTS ========================================================== #
 
 
 __all__ = [
     # Data Classes
-    'PipelineConfig',
-    # Functions
-    'load_pipeline_config',
+    'Config',
 ]
 
 
 # SECTION: INTERNAL FUNCTIONS =============================================== #
 
 
-def _build_jobs(
-    raw: StrAnyMap,
-) -> list[JobConfig]:
-    """
-    Return a list of ``JobConfig`` objects parsed from the mapping.
-
-    Parameters
-    ----------
-    raw : StrAnyMap
-        Raw pipeline mapping.
-
-    Returns
-    -------
-    list[JobConfig]
-        Parsed job configurations.
-    """
-    jobs: list[JobConfig] = []
-    for job_raw in raw.get('jobs', []) or []:
-        job_cfg = JobConfig.from_obj(job_raw)
-        if job_cfg is not None:
-            jobs.append(job_cfg)
-
-    return jobs
-
-
-def _build_sources(
+def _build_connectors(
     raw: StrAnyMap,
+    *,
+    key: str,
 ) -> list[Connector]:
     """
-    Return a list of source connectors parsed from the mapping.
+    Parse connector entries from a list under ``raw[key]``.
 
     Parameters
     ----------
     raw : StrAnyMap
         Raw pipeline mapping.
+    key : str
+        Key pointing to connector entries (e.g., ``"sources"``).
 
     Returns
     -------
     list[Connector]
-        Parsed source connectors.
+        Parsed connector instances.
     """
-    return _build_connectors(raw, 'sources')
+    return list(
+        _collect_parsed(raw.get(key, []) or [], _parse_connector_entry),
+    )
 
 
-def _build_targets(
-    raw: StrAnyMap,
-) -> list[Connector]:
+def _collect_parsed[T](
+    items: Any,
+    parser: Callable[[Any], T | None],
+) -> list[T]:
     """
-    Return a list of target connectors parsed from the mapping.
+    Collect parsed items from ``raw[key]`` using a tolerant parser.
 
     Parameters
     ----------
-    raw : StrAnyMap
-        Raw pipeline mapping.
+    items : Any
+        List-like payload to parse.
+    parser : Callable[[Any], T | None]
+        Parser that returns an instance or ``None`` for invalid entries.
 
     Returns
     -------
-    list[Connector]
-        Parsed target connectors.
+    list[T]
+        Parsed items, excluding invalid entries.
     """
-    return _build_connectors(raw, 'targets')
+    parsed_items: list[T] = []
+    for entry in items or []:
+        parsed = parser(entry)
+        if parsed is not None:
+            parsed_items.append(parsed)
+    return parsed_items
 
 
-def _build_connectors(
-    raw: StrAnyMap,
-    key: str,
-) -> list[Connector]:
+def _parse_connector_entry(
+    obj: Any,
+) -> Connector | None:
     """
-    Return parsed connectors from ``raw[key]`` using tolerant parsing.
-
-    Unknown or malformed entries are skipped to preserve permissiveness.
+    Parse a connector mapping into a concrete connector instance.
 
     Parameters
     ----------
-    raw : StrAnyMap
-        Raw pipeline mapping.
-    key : str
-        List-containing top-level key ("sources" or "targets").
+    obj : Any
+        Candidate connector mapping.
 
     Returns
     -------
-    list[Connector]
-        Constructed connector instances (malformed entries skipped).
+    Connector | None
+        Parsed connector instance or ``None`` when invalid.
     """
-    items: list[Connector] = []
-    for obj in raw.get(key, []) or []:
-        if not (entry := maybe_mapping(obj)):
-            continue
-        try:
-            items.append(parse_connector(entry))
-        except TypeError:
-            # Skip unsupported types or malformed entries
-            continue
-
-    return items
-
-
-# SECTION: FUNCTIONS ======================================================== #
-
-
-def load_pipeline_config(
-    path: Path | str,
-    *,
-    substitute: bool = False,
-    env: Mapping[str, str] | None = None,
-) -> PipelineConfig:
-    """
-    Load a pipeline YAML file into a ``PipelineConfig`` instance.
-
-    Delegates to ``PipelineConfig.from_yaml`` for construction and optional
-    variable substitution.
-    """
-    return PipelineConfig.from_yaml(path, substitute=substitute, env=env)
+    if not (entry := maybe_mapping(obj)):
+        return None
+    try:
+        return parse_connector(entry)
+    except TypeError:
+        return None
 
 
 # SECTION: DATA CLASSES ===================================================== #
 
 
 @dataclass(kw_only=True, slots=True)
-class PipelineConfig:
+class Config:
     """
     Configuration for the data processing pipeline.
 
@@ -231,7 +191,7 @@ class PipelineConfig:
         env: Mapping[str, str] | None = None,
     ) -> Self:
         """
-        Parse a YAML file into a ``PipelineConfig`` instance.
+        Parse a YAML file into a ``Config`` instance.
 
         Parameters
         ----------
@@ -279,7 +239,7 @@ class PipelineConfig:
         raw: StrAnyMap,
     ) -> Self:
         """
-        Parse a mapping into a ``PipelineConfig`` instance.
+        Parse a mapping into a ``Config`` instance.
 
         Parameters
         ----------
@@ -311,17 +271,20 @@ class PipelineConfig:
         file_systems = coerce_dict(raw.get('file_systems'))
 
         # Sources
-        sources = _build_sources(raw)
+        sources = _build_connectors(raw, key='sources')
 
         # Validations/Transforms
         validations = coerce_dict(raw.get('validations'))
         transforms = coerce_dict(raw.get('transforms'))
 
         # Targets
-        targets = _build_targets(raw)
+        targets = _build_connectors(raw, key='targets')
 
         # Jobs
-        jobs = _build_jobs(raw)
+        jobs: list[JobConfig] = _collect_parsed(
+            raw.get('jobs', []) or [],
+            JobConfig.from_obj,
+        )
 
         # Table schemas (optional, tolerant pass-through structures).
         table_schemas: list[dict[str, Any]] = []

etlplus/connector/__init__.py

@@ -0,0 +1,43 @@
+"""
+:mod:`etlplus.connector` package.
+
+Connector configuration types and enums.
+"""
+
+from __future__ import annotations
+
+from .api import ConnectorApi
+from .api import ConnectorApiConfigDict
+from .connector import Connector
+from .core import ConnectorBase
+from .core import ConnectorProtocol
+from .database import ConnectorDb
+from .database import ConnectorDbConfigDict
+from .enums import DataConnectorType
+from .file import ConnectorFile
+from .file import ConnectorFileConfigDict
+from .types import ConnectorType
+from .utils import parse_connector
+
+# SECTION: EXPORTS ========================================================== #
+
+
+__all__ = [
+    # Data Classes
+    'ConnectorApi',
+    'ConnectorDb',
+    'ConnectorFile',
+    # Enums
+    'DataConnectorType',
+    # Functions
+    'parse_connector',
+    # Type Aliases
+    'Connector',
+    'ConnectorBase',
+    'ConnectorProtocol',
+    'ConnectorType',
+    # Typed Dicts
+    'ConnectorApiConfigDict',
+    'ConnectorDbConfigDict',
+    'ConnectorFileConfigDict',
+]

etlplus/connector/api.py

@@ -0,0 +1,161 @@
+"""
+:mod:`etlplus.connector.api` module.
+
+API connector configuration dataclass.
+
+Notes
+-----
+- TypedDicts in this module are intentionally ``total=False`` and are not
+    enforced at runtime.
+- :meth:`*.from_obj` constructors accept :class:`Mapping[str, Any]` and perform
+    tolerant parsing and light casting. This keeps the runtime permissive while
+    improving autocomplete and static analysis for contributors.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from dataclasses import field
+from typing import Any
+from typing import Self
+from typing import TypedDict
+from typing import overload
+
+from ..api import PaginationConfig
+from ..api import PaginationConfigDict
+from ..api import RateLimitConfig
+from ..api import RateLimitConfigDict
+from ..types import StrAnyMap
+from ..types import StrStrMap
+from ..utils import cast_str_dict
+from ..utils import coerce_dict
+from ..utils import maybe_mapping
+from .core import ConnectorBase
+from .enums import DataConnectorType
+from .types import ConnectorType
+
+# SECTION: EXPORTS ========================================================== #
+
+
+__all__ = [
+    'ConnectorApi',
+    'ConnectorApiConfigDict',
+]
+
+
+# SECTION: TYPED DICTS ====================================================== #
+
+
+class ConnectorApiConfigDict(TypedDict, total=False):
+    """
+    Shape accepted by :meth:`ConnectorApi.from_obj` (all keys optional).
+
+    See Also
+    --------
+    - :meth:`etlplus.connector.api.ConnectorApi.from_obj`
+    """
+
+    name: str
+    type: ConnectorType
+    url: str
+    method: str
+    headers: StrStrMap
+    query_params: StrAnyMap
+    pagination: PaginationConfigDict
+    rate_limit: RateLimitConfigDict
+    api: str
+    endpoint: str
+
+
+# SECTION: DATA CLASSES ===================================================== #
+
+
+@dataclass(kw_only=True, slots=True)
+class ConnectorApi(ConnectorBase):
+    """
+    Configuration for an API-based data connector.
+
+    Attributes
+    ----------
+    type : ConnectorType
+        Connector kind, always ``'api'``.
+    url : str | None
+        Direct absolute URL (when not using ``service``/``endpoint`` refs).
+    method : str | None
+        Optional HTTP method; typically omitted for sources (defaults to
+        GET) and used for targets (e.g., ``'post'``).
+    headers : dict[str, str]
+        Additional request headers.
+    query_params : dict[str, Any]
+        Default query parameters.
+    pagination : PaginationConfig | None
+        Pagination settings (optional).
+    rate_limit : RateLimitConfig | None
+        Rate limiting settings (optional).
+    api : str | None
+        Service reference into the pipeline ``apis`` block (a.k.a.
+        ``service``).
+    endpoint : str | None
+        Endpoint name within the referenced service.
+    """
+
+    # -- Attributes -- #
+
+    type: ConnectorType = DataConnectorType.API
+
+    # Direct form
+    url: str | None = None
+    # Optional HTTP method; typically omitted for sources (defaults to GET)
+    # at runtime) and used for targets (e.g., 'post', 'put').
+    method: str | None = None
+    headers: dict[str, str] = field(default_factory=dict)
+    query_params: dict[str, Any] = field(default_factory=dict)
+    pagination: PaginationConfig | None = None
+    rate_limit: RateLimitConfig | None = None
+
+    # Reference form (to top-level APIs/endpoints)
+    api: str | None = None
+    endpoint: str | None = None
+
+    # -- Class Methods -- #
+
+    @classmethod
+    @overload
+    def from_obj(cls, obj: ConnectorApiConfigDict) -> Self: ...
+
+    @classmethod
+    @overload
+    def from_obj(cls, obj: StrAnyMap) -> Self: ...
+
+    @classmethod
+    def from_obj(
+        cls,
+        obj: StrAnyMap,
+    ) -> Self:
+        """
+        Parse a mapping into a ``ConnectorApi`` instance.
+
+        Parameters
+        ----------
+        obj : StrAnyMap
+            Mapping with at least ``name``.
+
+        Returns
+        -------
+        Self
+            Parsed connector instance.
+        """
+        name = cls._require_name(obj, kind='Api')
+        headers = cast_str_dict(maybe_mapping(obj.get('headers')))
+
+        return cls(
+            name=name,
+            url=obj.get('url'),
+            method=obj.get('method'),
+            headers=headers,
+            query_params=coerce_dict(obj.get('query_params')),
+            pagination=PaginationConfig.from_obj(obj.get('pagination')),
+            rate_limit=RateLimitConfig.from_obj(obj.get('rate_limit')),
+            api=obj.get('api') or obj.get('service'),
+            endpoint=obj.get('endpoint'),
+        )