etlplus 0.14.3__py3-none-any.whl → 0.16.0__py3-none-any.whl

etlplus/cli/handlers.py

@@ -14,8 +14,6 @@ from typing import Any
 from typing import Literal
 from typing import cast
 
-from ..config import PipelineConfig
-from ..config import load_pipeline_config
 from ..database import load_table_spec
 from ..database import render_tables
 from ..file import File
@@ -28,6 +26,8 @@ from ..ops import validate
 from ..ops.validate import FieldRules
 from ..types import JSONData
 from ..types import TemplateKey
+from ..workflow import PipelineConfig
+from ..workflow import load_pipeline_config
 from . import io as cli_io
 
 # SECTION: EXPORTS ========================================================== #
@@ -121,9 +121,12 @@ def _check_sections(
     if targets:
         sections['targets'] = [tgt.name for tgt in cfg.targets]
     if transforms:
-        sections['transforms'] = [
-            getattr(trf, 'name', None) for trf in cfg.transforms
-        ]
+        if isinstance(cfg.transforms, Mapping):
+            sections['transforms'] = list(cfg.transforms)
+        else:
+            sections['transforms'] = [
+                getattr(trf, 'name', None) for trf in cfg.transforms
+            ]
     if not sections:
         sections['jobs'] = _pipeline_summary(cfg)['jobs']
     return sections
@@ -157,6 +160,29 @@ def _pipeline_summary(
     }
 
 
+def _write_file_payload(
+    payload: JSONData,
+    target: str,
+    *,
+    format_hint: str | None,
+) -> None:
+    """
+    Write a JSON-like payload to a file path using an optional format hint.
+
+    Parameters
+    ----------
+    payload : JSONData
+        The structured data to write.
+    target : str
+        File path to write to.
+    format_hint : str | None
+        Optional format hint for :class:`FileFormat`.
+    """
+    file_path = Path(target)
+    file_format = FileFormat.coerce(format_hint) if format_hint else None
+    File(file_path, file_format=file_format).write(payload)
+
+
 # SECTION: FUNCTIONS ======================================================== #
 
 
@@ -479,7 +505,7 @@ def run_handler(
         Name of the job to run. If not provided, runs the entire pipeline.
         Default is ``None``.
     pipeline : str | None, optional
-        Alias for ``job``. Default is ``None``.
+        Alias for *job*. Default is ``None``.
     pretty : bool, optional
         Whether to pretty-print output. Default is ``True``.
 
@@ -572,15 +598,7 @@ def transform_handler(
 
     # TODO: Generalize to handle non-file targets.
     if target and target != '-':
-        # Convert target to Path and target_format to FileFormat if needed
-        file_path = Path(target)
-        file_format = None
-        if target_format is not None:
-            try:
-                file_format = FileFormat(target_format)
-            except ValueError:
-                file_format = None  # or handle error as appropriate
-        File(file_path, file_format=file_format).write(data)
+        _write_file_payload(data, target, format_hint=target_format)
         print(f'Data transformed and saved to {target}')
         return 0
 

etlplus/cli/io.py

@@ -71,7 +71,7 @@ def emit_or_write(
     success_message: str,
 ) -> None:
     """
-    Emit JSON or persist to disk based on ``output_path``.
+    Emit JSON or persist to disk based on *output_path*.
 
     Parameters
     ----------
@@ -122,7 +122,7 @@ def materialize_file_payload(
     format_explicit: bool,
 ) -> JSONData | object:
     """
-    Return structured payloads when ``source`` references a file.
+    Return structured payloads when *source* references a file.
 
     Parameters
     ----------

etlplus/cli/main.py

@@ -44,13 +44,13 @@ def _emit_context_help(
     Returns
     -------
     bool
-        ``True`` when help was emitted, ``False`` when ``ctx`` was ``None``.
+        ``True`` when help was emitted, ``False`` when *ctx* was ``None``.
     """
     if ctx is None:
         return False
 
     with contextlib.redirect_stdout(sys.stderr):
-        ctx.get_help()
+        print(ctx.get_help())
     return True
 
 

etlplus/cli/state.py

@@ -15,6 +15,7 @@ from typing import Final
 
 import typer
 
+from ..utils import normalize_str
 from .constants import DATA_CONNECTORS
 
 # SECTION: EXPORTS ========================================================== #
@@ -322,14 +323,10 @@ def validate_choice(
     typer.BadParameter
         If the input value is not in the set of valid choices.
     """
-    v = str(value or '').strip().lower()
-    normalized_choices = {c.lower() for c in choices}
+    v = normalize_str(str(value or ''))
+    normalized_choices = {normalize_str(c): c for c in choices}
     if v in normalized_choices:
-        # Preserve original casing from choices when possible for messages
-        for choice in choices:
-            if choice.lower() == v:
-                return choice
-        return v
+        return normalized_choices[v]
     allowed = ', '.join(sorted(choices))
     raise typer.BadParameter(
         f"Invalid {label} '{value}'. Choose from: {allowed}",

etlplus/connector/__init__.py

@@ -0,0 +1,43 @@
+"""
+:mod:`etlplus.connector` package.
+
+Connector configuration types and enums.
+"""
+
+from __future__ import annotations
+
+from .api import ConnectorApi
+from .api import ConnectorApiConfigMap
+from .connector import Connector
+from .core import ConnectorBase
+from .core import ConnectorProtocol
+from .database import ConnectorDb
+from .database import ConnectorDbConfigMap
+from .enums import DataConnectorType
+from .file import ConnectorFile
+from .file import ConnectorFileConfigMap
+from .types import ConnectorType
+from .utils import parse_connector
+
+# SECTION: EXPORTS ========================================================== #
+
+
+__all__ = [
+    # Data Classes
+    'ConnectorApi',
+    'ConnectorDb',
+    'ConnectorFile',
+    # Enums
+    'DataConnectorType',
+    # Functions
+    'parse_connector',
+    # Type Aliases
+    'Connector',
+    'ConnectorBase',
+    'ConnectorProtocol',
+    'ConnectorType',
+    # Typed Dicts
+    'ConnectorApiConfigMap',
+    'ConnectorDbConfigMap',
+    'ConnectorFileConfigMap',
+]

etlplus/connector/api.py

@@ -0,0 +1,161 @@
+"""
+:mod:`etlplus.connector.api` module.
+
+API connector configuration dataclass.
+
+Notes
+-----
+- TypedDicts in this module are intentionally ``total=False`` and are not
+    enforced at runtime.
+- :meth:`*.from_obj` constructors accept :class:`Mapping[str, Any]` and perform
+    tolerant parsing and light casting. This keeps the runtime permissive while
+    improving autocomplete and static analysis for contributors.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from dataclasses import field
+from typing import Any
+from typing import Self
+from typing import TypedDict
+from typing import overload
+
+from ..api import PaginationConfig
+from ..api import PaginationConfigMap
+from ..api import RateLimitConfig
+from ..api import RateLimitConfigMap
+from ..types import StrAnyMap
+from ..types import StrStrMap
+from ..utils import cast_str_dict
+from ..utils import coerce_dict
+from ..utils import maybe_mapping
+from .core import ConnectorBase
+from .enums import DataConnectorType
+from .types import ConnectorType
+
+# SECTION: EXPORTS ========================================================== #
+
+
+__all__ = [
+    'ConnectorApi',
+    'ConnectorApiConfigMap',
+]
+
+
+# SECTION: TYPED DICTS ====================================================== #
+
+
+class ConnectorApiConfigMap(TypedDict, total=False):
+    """
+    Shape accepted by :meth:`ConnectorApi.from_obj` (all keys optional).
+
+    See Also
+    --------
+    - :meth:`etlplus.connector.api.ConnectorApi.from_obj`
+    """
+
+    name: str
+    type: ConnectorType
+    url: str
+    method: str
+    headers: StrStrMap
+    query_params: StrAnyMap
+    pagination: PaginationConfigMap
+    rate_limit: RateLimitConfigMap
+    api: str
+    endpoint: str
+
+
+# SECTION: DATA CLASSES ===================================================== #
+
+
+@dataclass(kw_only=True, slots=True)
+class ConnectorApi(ConnectorBase):
+    """
+    Configuration for an API-based data connector.
+
+    Attributes
+    ----------
+    type : ConnectorType
+        Connector kind, always ``'api'``.
+    url : str | None
+        Direct absolute URL (when not using ``service``/``endpoint`` refs).
+    method : str | None
+        Optional HTTP method; typically omitted for sources (defaults to
+        GET) and used for targets (e.g., ``'post'``).
+    headers : dict[str, str]
+        Additional request headers.
+    query_params : dict[str, Any]
+        Default query parameters.
+    pagination : PaginationConfig | None
+        Pagination settings (optional).
+    rate_limit : RateLimitConfig | None
+        Rate limiting settings (optional).
+    api : str | None
+        Service reference into the pipeline ``apis`` block (a.k.a.
+        ``service``).
+    endpoint : str | None
+        Endpoint name within the referenced service.
+    """
+
+    # -- Attributes -- #
+
+    type: ConnectorType = DataConnectorType.API
+
+    # Direct form
+    url: str | None = None
+    # Optional HTTP method; typically omitted for sources (defaults to GET)
+    # at runtime) and used for targets (e.g., 'post', 'put').
+    method: str | None = None
+    headers: dict[str, str] = field(default_factory=dict)
+    query_params: dict[str, Any] = field(default_factory=dict)
+    pagination: PaginationConfig | None = None
+    rate_limit: RateLimitConfig | None = None
+
+    # Reference form (to top-level APIs/endpoints)
+    api: str | None = None
+    endpoint: str | None = None
+
+    # -- Class Methods -- #
+
+    @classmethod
+    @overload
+    def from_obj(cls, obj: ConnectorApiConfigMap) -> Self: ...
+
+    @classmethod
+    @overload
+    def from_obj(cls, obj: StrAnyMap) -> Self: ...
+
+    @classmethod
+    def from_obj(
+        cls,
+        obj: StrAnyMap,
+    ) -> Self:
+        """
+        Parse a mapping into a ``ConnectorApi`` instance.
+
+        Parameters
+        ----------
+        obj : StrAnyMap
+            Mapping with at least ``name``.
+
+        Returns
+        -------
+        Self
+            Parsed connector instance.
+        """
+        name = cls._require_name(obj, kind='Api')
+        headers = cast_str_dict(maybe_mapping(obj.get('headers')))
+
+        return cls(
+            name=name,
+            url=obj.get('url'),
+            method=obj.get('method'),
+            headers=headers,
+            query_params=coerce_dict(obj.get('query_params')),
+            pagination=PaginationConfig.from_obj(obj.get('pagination')),
+            rate_limit=RateLimitConfig.from_obj(obj.get('rate_limit')),
+            api=obj.get('api') or obj.get('service'),
+            endpoint=obj.get('endpoint'),
+        )

etlplus/connector/connector.py

@@ -0,0 +1,26 @@
+"""
+:mod:`etlplus.connector.connector` module.
+
+Compatibility re-exports for connector configuration classes.
+"""
+
+from __future__ import annotations
+
+from .api import ConnectorApi
+from .database import ConnectorDb
+from .file import ConnectorFile
+
+# SECTION: EXPORTS ========================================================== #
+
+
+__all__ = [
+    # Type aliases
+    'Connector',
+]
+
+
+# SECTION: TYPED ALIASES ==================================================== #
+
+
+# Type alias representing any supported connector
+type Connector = ConnectorApi | ConnectorDb | ConnectorFile

etlplus/connector/core.py

@@ -0,0 +1,132 @@
+"""
+:mod:`etlplus.connector.core` module.
+
+Protocols and base classes for connector implementations.
+"""
+
+from __future__ import annotations
+
+from abc import ABC
+from abc import abstractmethod
+from dataclasses import dataclass
+from typing import Protocol
+from typing import Self
+from typing import runtime_checkable
+
+from ..types import StrAnyMap
+from .types import ConnectorType
+
+# SECTION: EXPORTS ========================================================== #
+
+
+__all__ = [
+    'ConnectorBase',
+    'ConnectorProtocol',
+]
+
+
+# SECTION: PROTOCOLS ======================================================== #
+
+
+@runtime_checkable
+class ConnectorProtocol(Protocol):
+    """
+    Structural contract for connector implementations.
+
+    Attributes
+    ----------
+    name : str
+        Unique connector name.
+    type : ConnectorType
+        Connector kind.
+    """
+
+    # -- Attributes -- #
+
+    name: str
+    type: ConnectorType
+
+    # -- Class Methods -- #
+
+    @classmethod
+    def from_obj(cls, obj: StrAnyMap) -> Self:
+        """
+        Parse a mapping into a connector instance.
+
+        Parameters
+        ----------
+        obj : StrAnyMap
+            Mapping with at least ``name``.
+
+        Returns
+        -------
+        Self
+            Parsed connector instance.
+        """
+        ...
+
+    # -- Internal Static Methods -- #
+
+    @staticmethod
+    def _require_name(obj: StrAnyMap, *, kind: str) -> str:
+        """
+        Extract and validate the ``name`` field from connector mappings.
+
+        Parameters
+        ----------
+        obj : StrAnyMap
+            Connector mapping with a ``name`` entry.
+        kind : str
+            Connector kind used in the error message.
+
+        Returns
+        -------
+        str
+            Valid connector name.
+
+        Raises
+        ------
+        TypeError
+            If ``name`` is missing or not a string.
+        """
+        name = obj.get('name')
+        if not isinstance(name, str):
+            raise TypeError(f'Connector{kind} requires a "name" (str)')
+        return name
+
+
+# SECTION: ABSTRACT BASE DATA CLASSES ======================================= #
+
+
+@dataclass(kw_only=True, slots=True)
+class ConnectorBase(ABC, ConnectorProtocol):
+    """
+    Abstract base class for connector implementations.
+
+    Attributes
+    ----------
+    name : str
+        Unique connector name.
+    type : ConnectorType
+        Connector kind.
+    """
+
+    name: str
+    type: ConnectorType
+
+    @classmethod
+    @abstractmethod
+    def from_obj(cls, obj: StrAnyMap) -> Self:
+        """
+        Parse a mapping into a connector instance.
+
+        Parameters
+        ----------
+        obj : StrAnyMap
+            Mapping with at least ``name``.
+
+        Returns
+        -------
+        Self
+            Parsed connector instance.
+        """

etlplus/connector/database.py

@@ -0,0 +1,122 @@
+"""
+:mod:`etlplus.connector.database` module.
+
+Database connector configuration dataclass.
+
+Notes
+-----
+- TypedDicts in this module are intentionally ``total=False`` and are not
+    enforced at runtime.
+- :meth:`*.from_obj` constructors accept :class:`Mapping[str, Any]` and perform
+    tolerant parsing and light casting. This keeps the runtime permissive while
+    improving autocomplete and static analysis for contributors.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Self
+from typing import TypedDict
+from typing import overload
+
+from ..types import StrAnyMap
+from .core import ConnectorBase
+from .enums import DataConnectorType
+from .types import ConnectorType
+
+# SECTION: EXPORTS ========================================================== #
+
+
+__all__ = [
+    'ConnectorDb',
+    'ConnectorDbConfigMap',
+]
+
+
+# SECTION: TYPED DICTS ====================================================== #
+
+
+class ConnectorDbConfigMap(TypedDict, total=False):
+    """
+    Shape accepted by :meth:`ConnectorDb.from_obj` (all keys optional).
+
+    See Also
+    --------
+    - :meth:`etlplus.connector.database.ConnectorDb.from_obj`
+    """
+
+    name: str
+    type: ConnectorType
+    connection_string: str
+    query: str
+    table: str
+    mode: str
+
+
+# SECTION: DATA CLASSES ===================================================== #
+
+
+@dataclass(kw_only=True, slots=True)
+class ConnectorDb(ConnectorBase):
+    """
+    Configuration for a database-based data connector.
+
+    Attributes
+    ----------
+    type : ConnectorType
+        Connector kind, always ``'database'``.
+    connection_string : str | None
+        Connection string/DSN for the database.
+    query : str | None
+        Query to execute for extraction (optional).
+    table : str | None
+        Target/source table name (optional).
+    mode : str | None
+        Load mode hint (e.g., ``'append'``, ``'replace'``) - future use.
+    """
+
+    # -- Attributes -- #
+
+    type: ConnectorType = DataConnectorType.DATABASE
+    connection_string: str | None = None
+    query: str | None = None
+    table: str | None = None
+    mode: str | None = None  # append|replace|upsert (future)
+
+    # -- Class Methods -- #
+
+    @classmethod
+    @overload
+    def from_obj(cls, obj: ConnectorDbConfigMap) -> Self: ...
+
+    @classmethod
+    @overload
+    def from_obj(cls, obj: StrAnyMap) -> Self: ...
+
+    @classmethod
+    def from_obj(
+        cls,
+        obj: StrAnyMap,
+    ) -> Self:
+        """
+        Parse a mapping into a ``ConnectorDb`` instance.
+
+        Parameters
+        ----------
+        obj : StrAnyMap
+            Mapping with at least ``name``.
+
+        Returns
+        -------
+        Self
+            Parsed connector instance.
+        """
+        name = cls._require_name(obj, kind='Db')
+
+        return cls(
+            name=name,
+            connection_string=obj.get('connection_string'),
+            query=obj.get('query'),
+            table=obj.get('table'),
+            mode=obj.get('mode'),
+        )

etlplus/connector/enums.py

@@ -0,0 +1,52 @@
+"""
+:mod:`etlplus.connector.enums` module.
+
+Connector enums and helpers.
+"""
+
+from __future__ import annotations
+
+from ..enums import CoercibleStrEnum
+from ..types import StrStrMap
+
+# SECTION: EXPORTS ========================================================= #
+
+
+__all__ = [
+    # Enums
+    'DataConnectorType',
+]
+
+
+# SECTION: ENUMS ============================================================ #
+
+
+class DataConnectorType(CoercibleStrEnum):
+    """Supported data connector types."""
+
+    # -- Constants -- #
+
+    API = 'api'
+    DATABASE = 'database'
+    FILE = 'file'
+
+    # -- Class Methods -- #
+
+    @classmethod
+    def aliases(cls) -> StrStrMap:
+        """
+        Return a mapping of common aliases for each enum member.
+
+        Returns
+        -------
+        StrStrMap
+            A mapping of alias names to their corresponding enum member names.
+        """
+        return {
+            'http': 'api',
+            'https': 'api',
+            'rest': 'api',
+            'db': 'database',
+            'filesystem': 'file',
+            'fs': 'file',
+        }