etlplus 0.15.5__tar.gz → 0.16.0__tar.gz

{etlplus-0.15.5/etlplus.egg-info → etlplus-0.16.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: etlplus
-Version: 0.15.5
+Version: 0.16.0
 Summary: A Swiss Army knife for simple ETL operations
 Home-page: https://github.com/Dagitali/ETLPlus
 Author: ETLPlus Team

{etlplus-0.15.5 → etlplus-0.16.0}/docs/pipeline-guide.md

@@ -377,9 +377,9 @@ Details:
 - Unknown or malformed entries are skipped rather than failing the whole load (keeping pipeline
   authoring permissive).
 - The connector kind is also available as a type-safe literal in code as
-  `etlplus.workflow.ConnectorType` (values: `"file" | "database" | "api"`).
+  `etlplus.connector.ConnectorType` (values: `"file" | "database" | "api"`).
 
-To add new connector kinds in the future, implement a new dataclass in `etlplus.workflow.connector`
+To add new connector kinds in the future, implement a new dataclass in `etlplus.connector`
 and extend the internal parser to handle its `type` value.
 
 ## Jobs

{etlplus-0.15.5 → etlplus-0.16.0}/etlplus/cli/constants.py

@@ -8,7 +8,7 @@ from __future__ import annotations
 
 from typing import Final
 
-from ..enums import DataConnectorType
+from ..connector import DataConnectorType
 from ..file import FileFormat
 
 # SECTION: EXPORTS ========================================================== #

etlplus-0.16.0/etlplus/connector/__init__.py

@@ -0,0 +1,43 @@
+"""
+:mod:`etlplus.connector` package.
+
+Connector configuration types and enums.
+"""
+
+from __future__ import annotations
+
+from .api import ConnectorApi
+from .api import ConnectorApiConfigMap
+from .connector import Connector
+from .core import ConnectorBase
+from .core import ConnectorProtocol
+from .database import ConnectorDb
+from .database import ConnectorDbConfigMap
+from .enums import DataConnectorType
+from .file import ConnectorFile
+from .file import ConnectorFileConfigMap
+from .types import ConnectorType
+from .utils import parse_connector
+
+# SECTION: EXPORTS ========================================================== #
+
+
+__all__ = [
+    # Data Classes
+    'ConnectorApi',
+    'ConnectorDb',
+    'ConnectorFile',
+    # Enums
+    'DataConnectorType',
+    # Functions
+    'parse_connector',
+    # Type Aliases
+    'Connector',
+    'ConnectorBase',
+    'ConnectorProtocol',
+    'ConnectorType',
+    # Typed Dicts
+    'ConnectorApiConfigMap',
+    'ConnectorDbConfigMap',
+    'ConnectorFileConfigMap',
+]

etlplus-0.16.0/etlplus/connector/api.py

@@ -0,0 +1,161 @@
+"""
+:mod:`etlplus.connector.api` module.
+
+API connector configuration dataclass.
+
+Notes
+-----
+- TypedDicts in this module are intentionally ``total=False`` and are not
+    enforced at runtime.
+- :meth:`*.from_obj` constructors accept :class:`Mapping[str, Any]` and perform
+    tolerant parsing and light casting. This keeps the runtime permissive while
+    improving autocomplete and static analysis for contributors.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from dataclasses import field
+from typing import Any
+from typing import Self
+from typing import TypedDict
+from typing import overload
+
+from ..api import PaginationConfig
+from ..api import PaginationConfigMap
+from ..api import RateLimitConfig
+from ..api import RateLimitConfigMap
+from ..types import StrAnyMap
+from ..types import StrStrMap
+from ..utils import cast_str_dict
+from ..utils import coerce_dict
+from ..utils import maybe_mapping
+from .core import ConnectorBase
+from .enums import DataConnectorType
+from .types import ConnectorType
+
+# SECTION: EXPORTS ========================================================== #
+
+
+__all__ = [
+    'ConnectorApi',
+    'ConnectorApiConfigMap',
+]
+
+
+# SECTION: TYPED DICTS ====================================================== #
+
+
+class ConnectorApiConfigMap(TypedDict, total=False):
+    """
+    Shape accepted by :meth:`ConnectorApi.from_obj` (all keys optional).
+
+    See Also
+    --------
+    - :meth:`etlplus.connector.api.ConnectorApi.from_obj`
+    """
+
+    name: str
+    type: ConnectorType
+    url: str
+    method: str
+    headers: StrStrMap
+    query_params: StrAnyMap
+    pagination: PaginationConfigMap
+    rate_limit: RateLimitConfigMap
+    api: str
+    endpoint: str
+
+
+# SECTION: DATA CLASSES ===================================================== #
+
+
+@dataclass(kw_only=True, slots=True)
+class ConnectorApi(ConnectorBase):
+    """
+    Configuration for an API-based data connector.
+
+    Attributes
+    ----------
+    type : ConnectorType
+        Connector kind, always ``'api'``.
+    url : str | None
+        Direct absolute URL (when not using ``service``/``endpoint`` refs).
+    method : str | None
+        Optional HTTP method; typically omitted for sources (defaults to
+        GET) and used for targets (e.g., ``'post'``).
+    headers : dict[str, str]
+        Additional request headers.
+    query_params : dict[str, Any]
+        Default query parameters.
+    pagination : PaginationConfig | None
+        Pagination settings (optional).
+    rate_limit : RateLimitConfig | None
+        Rate limiting settings (optional).
+    api : str | None
+        Service reference into the pipeline ``apis`` block (a.k.a.
+        ``service``).
+    endpoint : str | None
+        Endpoint name within the referenced service.
+    """
+
+    # -- Attributes -- #
+
+    type: ConnectorType = DataConnectorType.API
+
+    # Direct form
+    url: str | None = None
+    # Optional HTTP method; typically omitted for sources (defaults to GET)
+    # at runtime) and used for targets (e.g., 'post', 'put').
+    method: str | None = None
+    headers: dict[str, str] = field(default_factory=dict)
+    query_params: dict[str, Any] = field(default_factory=dict)
+    pagination: PaginationConfig | None = None
+    rate_limit: RateLimitConfig | None = None
+
+    # Reference form (to top-level APIs/endpoints)
+    api: str | None = None
+    endpoint: str | None = None
+
+    # -- Class Methods -- #
+
+    @classmethod
+    @overload
+    def from_obj(cls, obj: ConnectorApiConfigMap) -> Self: ...
+
+    @classmethod
+    @overload
+    def from_obj(cls, obj: StrAnyMap) -> Self: ...
+
+    @classmethod
+    def from_obj(
+        cls,
+        obj: StrAnyMap,
+    ) -> Self:
+        """
+        Parse a mapping into a ``ConnectorApi`` instance.
+
+        Parameters
+        ----------
+        obj : StrAnyMap
+            Mapping with at least ``name``.
+
+        Returns
+        -------
+        Self
+            Parsed connector instance.
+        """
+        name = cls._require_name(obj, kind='Api')
+        headers = cast_str_dict(maybe_mapping(obj.get('headers')))
+
+        return cls(
+            name=name,
+            url=obj.get('url'),
+            method=obj.get('method'),
+            headers=headers,
+            query_params=coerce_dict(obj.get('query_params')),
+            pagination=PaginationConfig.from_obj(obj.get('pagination')),
+            rate_limit=RateLimitConfig.from_obj(obj.get('rate_limit')),
+            api=obj.get('api') or obj.get('service'),
+            endpoint=obj.get('endpoint'),
+        )

etlplus-0.16.0/etlplus/connector/connector.py

@@ -0,0 +1,26 @@
+"""
+:mod:`etlplus.connector.connector` module.
+
+Compatibility re-exports for connector configuration classes.
+"""
+
+from __future__ import annotations
+
+from .api import ConnectorApi
+from .database import ConnectorDb
+from .file import ConnectorFile
+
+# SECTION: EXPORTS ========================================================== #
+
+
+__all__ = [
+    # Type aliases
+    'Connector',
+]
+
+
+# SECTION: TYPED ALIASES ==================================================== #
+
+
+# Type alias representing any supported connector
+type Connector = ConnectorApi | ConnectorDb | ConnectorFile

etlplus-0.16.0/etlplus/connector/core.py

@@ -0,0 +1,132 @@
+"""
+:mod:`etlplus.connector.core` module.
+
+Protocols and base classes for connector implementations.
+"""
+
+from __future__ import annotations
+
+from abc import ABC
+from abc import abstractmethod
+from dataclasses import dataclass
+from typing import Protocol
+from typing import Self
+from typing import runtime_checkable
+
+from ..types import StrAnyMap
+from .types import ConnectorType
+
+# SECTION: EXPORTS ========================================================== #
+
+
+__all__ = [
+    'ConnectorBase',
+    'ConnectorProtocol',
+]
+
+
+# SECTION: PROTOCOLS ======================================================== #
+
+
+@runtime_checkable
+class ConnectorProtocol(Protocol):
+    """
+    Structural contract for connector implementations.
+
+    Attributes
+    ----------
+    name : str
+        Unique connector name.
+    type : ConnectorType
+        Connector kind.
+    """
+
+    # -- Attributes -- #
+
+    name: str
+    type: ConnectorType
+
+    # -- Class Methods -- #
+
+    @classmethod
+    def from_obj(cls, obj: StrAnyMap) -> Self:
+        """
+        Parse a mapping into a connector instance.
+
+        Parameters
+        ----------
+        obj : StrAnyMap
+            Mapping with at least ``name``.
+
+        Returns
+        -------
+        Self
+            Parsed connector instance.
+        """
+        ...
+
+    # -- Internal Static Methods -- #
+
+    @staticmethod
+    def _require_name(obj: StrAnyMap, *, kind: str) -> str:
+        """
+        Extract and validate the ``name`` field from connector mappings.
+
+        Parameters
+        ----------
+        obj : StrAnyMap
+            Connector mapping with a ``name`` entry.
+        kind : str
+            Connector kind used in the error message.
+
+        Returns
+        -------
+        str
+            Valid connector name.
+
+        Raises
+        ------
+        TypeError
+            If ``name`` is missing or not a string.
+        """
+        name = obj.get('name')
+        if not isinstance(name, str):
+            raise TypeError(f'Connector{kind} requires a "name" (str)')
+        return name
+
+
+# SECTION: ABSTRACT BASE DATA CLASSES ======================================= #
+
+
+@dataclass(kw_only=True, slots=True)
+class ConnectorBase(ABC, ConnectorProtocol):
+    """
+    Abstract base class for connector implementations.
+
+    Attributes
+    ----------
+    name : str
+        Unique connector name.
+    type : ConnectorType
+        Connector kind.
+    """
+
+    name: str
+    type: ConnectorType
+
+    @classmethod
+    @abstractmethod
+    def from_obj(cls, obj: StrAnyMap) -> Self:
+        """
+        Parse a mapping into a connector instance.
+
+        Parameters
+        ----------
+        obj : StrAnyMap
+            Mapping with at least ``name``.
+
+        Returns
+        -------
+        Self
+            Parsed connector instance.
+        """

etlplus-0.16.0/etlplus/connector/database.py

@@ -0,0 +1,122 @@
+"""
+:mod:`etlplus.connector.database` module.
+
+Database connector configuration dataclass.
+
+Notes
+-----
+- TypedDicts in this module are intentionally ``total=False`` and are not
+    enforced at runtime.
+- :meth:`*.from_obj` constructors accept :class:`Mapping[str, Any]` and perform
+    tolerant parsing and light casting. This keeps the runtime permissive while
+    improving autocomplete and static analysis for contributors.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Self
+from typing import TypedDict
+from typing import overload
+
+from ..types import StrAnyMap
+from .core import ConnectorBase
+from .enums import DataConnectorType
+from .types import ConnectorType
+
+# SECTION: EXPORTS ========================================================== #
+
+
+__all__ = [
+    'ConnectorDb',
+    'ConnectorDbConfigMap',
+]
+
+
+# SECTION: TYPED DICTS ====================================================== #
+
+
+class ConnectorDbConfigMap(TypedDict, total=False):
+    """
+    Shape accepted by :meth:`ConnectorDb.from_obj` (all keys optional).
+
+    See Also
+    --------
+    - :meth:`etlplus.connector.database.ConnectorDb.from_obj`
+    """
+
+    name: str
+    type: ConnectorType
+    connection_string: str
+    query: str
+    table: str
+    mode: str
+
+
+# SECTION: DATA CLASSES ===================================================== #
+
+
+@dataclass(kw_only=True, slots=True)
+class ConnectorDb(ConnectorBase):
+    """
+    Configuration for a database-based data connector.
+
+    Attributes
+    ----------
+    type : ConnectorType
+        Connector kind, always ``'database'``.
+    connection_string : str | None
+        Connection string/DSN for the database.
+    query : str | None
+        Query to execute for extraction (optional).
+    table : str | None
+        Target/source table name (optional).
+    mode : str | None
+        Load mode hint (e.g., ``'append'``, ``'replace'``) - future use.
+    """
+
+    # -- Attributes -- #
+
+    type: ConnectorType = DataConnectorType.DATABASE
+    connection_string: str | None = None
+    query: str | None = None
+    table: str | None = None
+    mode: str | None = None  # append|replace|upsert (future)
+
+    # -- Class Methods -- #
+
+    @classmethod
+    @overload
+    def from_obj(cls, obj: ConnectorDbConfigMap) -> Self: ...
+
+    @classmethod
+    @overload
+    def from_obj(cls, obj: StrAnyMap) -> Self: ...
+
+    @classmethod
+    def from_obj(
+        cls,
+        obj: StrAnyMap,
+    ) -> Self:
+        """
+        Parse a mapping into a ``ConnectorDb`` instance.
+
+        Parameters
+        ----------
+        obj : StrAnyMap
+            Mapping with at least ``name``.
+
+        Returns
+        -------
+        Self
+            Parsed connector instance.
+        """
+        name = cls._require_name(obj, kind='Db')
+
+        return cls(
+            name=name,
+            connection_string=obj.get('connection_string'),
+            query=obj.get('query'),
+            table=obj.get('table'),
+            mode=obj.get('mode'),
+        )

etlplus-0.16.0/etlplus/connector/enums.py

@@ -0,0 +1,52 @@
+"""
+:mod:`etlplus.connector.enums` module.
+
+Connector enums and helpers.
+"""
+
+from __future__ import annotations
+
+from ..enums import CoercibleStrEnum
+from ..types import StrStrMap
+
+# SECTION: EXPORTS ========================================================= #
+
+
+__all__ = [
+    # Enums
+    'DataConnectorType',
+]
+
+
+# SECTION: ENUMS ============================================================ #
+
+
+class DataConnectorType(CoercibleStrEnum):
+    """Supported data connector types."""
+
+    # -- Constants -- #
+
+    API = 'api'
+    DATABASE = 'database'
+    FILE = 'file'
+
+    # -- Class Methods -- #
+
+    @classmethod
+    def aliases(cls) -> StrStrMap:
+        """
+        Return a mapping of common aliases for each enum member.
+
+        Returns
+        -------
+        StrStrMap
+            A mapping of alias names to their corresponding enum member names.
+        """
+        return {
+            'http': 'api',
+            'https': 'api',
+            'rest': 'api',
+            'db': 'database',
+            'filesystem': 'file',
+            'fs': 'file',
+        }

etlplus-0.16.0/etlplus/connector/file.py

@@ -0,0 +1,120 @@
+"""
+:mod:`etlplus.connector.file` module.
+
+File connector configuration dataclass.
+
+Notes
+-----
+- TypedDicts in this module are intentionally ``total=False`` and are not
+    enforced at runtime.
+- :meth:`*.from_obj` constructors accept :class:`Mapping[str, Any]` and perform
+    tolerant parsing and light casting. This keeps the runtime permissive while
+    improving autocomplete and static analysis for contributors.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from dataclasses import field
+from typing import Any
+from typing import Self
+from typing import TypedDict
+from typing import overload
+
+from ..types import StrAnyMap
+from ..utils import coerce_dict
+from .core import ConnectorBase
+from .enums import DataConnectorType
+from .types import ConnectorType
+
+# SECTION: EXPORTS ========================================================== #
+
+
+__all__ = [
+    'ConnectorFile',
+    'ConnectorFileConfigMap',
+]
+
+
+# SECTION: TYPED DICTS ====================================================== #
+
+
+class ConnectorFileConfigMap(TypedDict, total=False):
+    """
+    Shape accepted by :meth:`ConnectorFile.from_obj` (all keys optional).
+
+    See Also
+    --------
+    - :meth:`etlplus.connector.file.ConnectorFile.from_obj`
+    """
+
+    name: str
+    type: ConnectorType
+    format: str
+    path: str
+    options: StrAnyMap
+
+
+# SECTION: DATA CLASSES ===================================================== #
+
+
+@dataclass(kw_only=True, slots=True)
+class ConnectorFile(ConnectorBase):
+    """
+    Configuration for a file-based data connector.
+
+    Attributes
+    ----------
+    type : ConnectorType
+        Connector kind, always ``'file'``.
+    format : str | None
+        File format (e.g., ``'json'``, ``'csv'``).
+    path : str | None
+        File path or URI.
+    options : dict[str, Any]
+        Reader/writer format options.
+    """
+
+    # -- Attributes -- #
+
+    type: ConnectorType = DataConnectorType.FILE
+    format: str | None = None
+    path: str | None = None
+    options: dict[str, Any] = field(default_factory=dict)
+
+    # -- Class Methods -- #
+
+    @classmethod
+    @overload
+    def from_obj(cls, obj: ConnectorFileConfigMap) -> Self: ...
+
+    @classmethod
+    @overload
+    def from_obj(cls, obj: StrAnyMap) -> Self: ...
+
+    @classmethod
+    def from_obj(
+        cls,
+        obj: StrAnyMap,
+    ) -> Self:
+        """
+        Parse a mapping into a ``ConnectorFile`` instance.
+
+        Parameters
+        ----------
+        obj : StrAnyMap
+            Mapping with at least ``name``.
+
+        Returns
+        -------
+        Self
+            Parsed connector instance.
+        """
+        name = cls._require_name(obj, kind='File')
+
+        return cls(
+            name=name,
+            format=obj.get('format'),
+            path=obj.get('path'),
+            options=coerce_dict(obj.get('options')),
+        )