etlplus 0.15.4__tar.gz → 0.16.0__tar.gz

{etlplus-0.15.4/etlplus.egg-info → etlplus-0.16.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: etlplus
-Version: 0.15.4
+Version: 0.16.0
 Summary: A Swiss Army knife for simple ETL operations
 Home-page: https://github.com/Dagitali/ETLPlus
 Author: ETLPlus Team

{etlplus-0.15.4 → etlplus-0.16.0}/docs/pipeline-guide.md

@@ -377,9 +377,9 @@ Details:
 - Unknown or malformed entries are skipped rather than failing the whole load (keeping pipeline
   authoring permissive).
 - The connector kind is also available as a type-safe literal in code as
-  `etlplus.workflow.ConnectorType` (values: `"file" | "database" | "api"`).
+  `etlplus.connector.ConnectorType` (values: `"file" | "database" | "api"`).
 
-To add new connector kinds in the future, implement a new dataclass in `etlplus.workflow.connector`
+To add new connector kinds in the future, implement a new dataclass in `etlplus.connector`
 and extend the internal parser to handle its `type` value.
 
 ## Jobs

{etlplus-0.15.4 → etlplus-0.16.0}/etlplus/api/auth.py

@@ -136,7 +136,7 @@ def _truncate(
     limit: int = MAX_LOG_BODY,
 ) -> str:
     """
-    Return ``text`` shortened to ``limit`` characters for logging.
+    Return *text* shortened to *limit* characters for logging.
 
     Parameters
     ----------

{etlplus-0.15.4 → etlplus-0.16.0}/etlplus/api/config.py

@@ -68,7 +68,7 @@ def _effective_service_defaults(
     fallback_headers: dict[str, str],
 ) -> tuple[str, dict[str, str]]:
     """
-    Return ``(base_url, headers)`` using ``profiles`` when present.
+    Return ``(base_url, headers)`` using *profiles* when present.
 
     Parameters
     ----------
@@ -87,7 +87,7 @@ def _effective_service_defaults(
     Raises
     ------
     TypeError
-        If no profiles are defined and ``fallback_base`` is not a string.
+        If no profiles are defined and *fallback_base* is not a string.
     """
     if profiles:
         name = 'default' if 'default' in profiles else next(iter(profiles))

{etlplus-0.15.4 → etlplus-0.16.0}/etlplus/api/endpoint_client.py

@@ -759,8 +759,8 @@ class EndpointClient:
         Raises
         ------
         KeyError
-            If ``endpoint_key`` is unknown or a required placeholder in the
-            path has no corresponding entry in ``path_parameters``.
+            If *endpoint_key* is unknown or a required placeholder in the path
+            has no corresponding entry in *path_parameters*.
         ValueError
             If the path template is invalid.
 
@@ -836,7 +836,7 @@ class EndpointClient:
         """
         Sleep for the specified seconds if positive.
 
-        The optional ``sleeper`` is useful for tests (e.g., pass
+        The optional *sleeper* is useful for tests (e.g., pass
         ``lambda s: None``). Defaults to using time.sleep when not provided.
 
         Parameters
@@ -870,7 +870,7 @@ class EndpointClient:
         rate_limit : RateLimitConfigMap | None
             Client-wide rate limit configuration.
         overrides : RateLimitOverrides, optional
-            Per-call overrides that take precedence over ``rate_limit``.
+            Per-call overrides that take precedence over *rate_limit*.
 
         Returns
         -------

{etlplus-0.15.4 → etlplus-0.16.0}/etlplus/api/pagination/config.py

@@ -398,7 +398,7 @@ class PaginationConfig(BoundsWarningsMixin):
         Returns
         -------
         Self | None
-            Parsed pagination configuration, or ``None`` if ``obj`` isn't a
+            Parsed pagination configuration, or ``None`` if *obj* isn't a
             mapping.
 
         Notes

{etlplus-0.15.4 → etlplus-0.16.0}/etlplus/api/pagination/paginator.py

@@ -66,14 +66,14 @@ def _resolve_path(
     path: str | None,
 ) -> Any:
     """
-    Resolve dotted ``path`` within ``obj`` or return ``_MISSING``.
+    Resolve dotted *path* within *obj* or return ``_MISSING``.
 
     Parameters
     ----------
     obj : Any
         JSON payload from an API response.
     path : str | None
-        Dotted path to the target value within ``obj``.
+        Dotted path to the target value within *obj*.
 
     Returns
     -------
@@ -665,7 +665,7 @@ class Paginator:
         records_path : str | None
             Optional dotted path to the records within the payload.
         fallback_path : str | None
-            Secondary dotted path consulted when ``records_path`` resolves to
+            Secondary dotted path consulted when *records_path* resolves to
             ``None`` or an empty list.
 
         Returns
@@ -675,9 +675,8 @@ class Paginator:
 
         Notes
         -----
-        Supports dotted path extraction via ``records_path`` and handles
-        lists, mappings, and scalars by coercing non-dict items into
-        ``{"value": x}``.
+        Supports dotted path extraction via *records_path* and handles lists,
+        mappings, and scalars by coercing non-dict items into ``{"value": x}``.
         """
         resolver = partial(_resolve_path, x)
         data = resolver(records_path)
@@ -729,7 +728,7 @@ class Paginator:
         Returns
         -------
         PaginationType | None
-            Detected pagination type, or ``default`` if not found.
+            Detected pagination type, or *default* if not found.
         """
         if not config:
             return default

{etlplus-0.15.4 → etlplus-0.16.0}/etlplus/api/rate_limiting/config.py

@@ -84,14 +84,14 @@ def _merge_rate_limit(
     overrides: RateLimitOverrides = None,
 ) -> dict[str, Any]:
     """
-    Merge ``rate_limit`` and ``overrides`` honoring override precedence.
+    Merge *rate_limit* and *overrides* honoring override precedence.
 
     Parameters
     ----------
     rate_limit : StrAnyMap | None
         Base rate-limit configuration.
     overrides : RateLimitOverrides, optional
-        Override configuration with precedence over ``rate_limit``.
+        Override configuration with precedence over *rate_limit*.
 
     Returns
     -------
@@ -274,7 +274,7 @@ class RateLimitConfig(BoundsWarningsMixin):
         rate_limit : StrAnyMap | RateLimitConfig | None, optional
             Base rate-limit configuration to normalize.
         overrides : RateLimitOverrides, optional
-            Override values that take precedence over ``rate_limit``.
+            Override values that take precedence over *rate_limit*.
 
         Returns
         -------
@@ -330,7 +330,7 @@ class RateLimitConfig(BoundsWarningsMixin):
         Returns
         -------
         Self | None
-            Parsed instance, or ``None`` if ``obj`` isn't a mapping.
+            Parsed instance, or ``None`` if *obj* isn't a mapping.
         """
         if obj is None:
             return None

{etlplus-0.15.4 → etlplus-0.16.0}/etlplus/api/rate_limiting/rate_limiter.py

@@ -235,7 +235,7 @@ class RateLimiter:
             Base rate-limit configuration. May contain ``"sleep_seconds"`` or
             ``"max_per_sec"``.
         overrides : RateLimitOverrides, optional
-            Optional overrides with the same keys as ``rate_limit``.
+            Optional overrides with the same keys as *rate_limit*.
 
         Returns
         -------

{etlplus-0.15.4 → etlplus-0.16.0}/etlplus/api/retry_manager.py

@@ -278,7 +278,7 @@ class RetryManager:
         **kwargs: Any,
     ) -> JSONData:
         """
-        Execute ``func`` with exponential-backoff retries.
+        Execute *func* with exponential-backoff retries.
 
         Parameters
         ----------
@@ -287,7 +287,7 @@ class RetryManager:
         url : str
             URL for the API request.
         **kwargs : Any
-            Additional keyword arguments to pass to ``func``
+            Additional keyword arguments to pass to *func*
 
         Returns
         -------

{etlplus-0.15.4 → etlplus-0.16.0}/etlplus/api/transport.py

@@ -309,7 +309,7 @@ def build_session_with_adapters(
     adapters_cfg: Sequence[HTTPAdapterMountConfig],
 ) -> requests.Session:
     """
-    Mount adapters described by ``adapters_cfg`` onto a new session.
+    Mount adapters described by *adapters_cfg* onto a new session.
 
     Ignores invalid adapter configurations so that a usable session is always
     returned.

{etlplus-0.15.4 → etlplus-0.16.0}/etlplus/api/types.py

@@ -61,10 +61,12 @@ _UNSET = object()
 
 class ApiConfigMap(TypedDict, total=False):
     """
-    Top-level API config shape parsed by ApiConfig.from_obj.
+    Top-level API config shape parsed by
+    :meth:`etlplus.api.config.ApiConfig.from_obj`.
 
-    Either provide a ``base_url`` with optional ``headers`` and ``endpoints``,
-    or provide ``profiles`` with at least one profile having a ``base_url``.
+    Either provide a :attr:`base_url` with optional :attr:`headers` and
+    :attr:`endpoints`, or provide :attr:`profiles` with at least one profile
+    having a :attr:`base_url`.
 
     See Also
     --------
@@ -79,11 +81,14 @@ class ApiConfigMap(TypedDict, total=False):
 
 class ApiProfileConfigMap(TypedDict, total=False):
     """
-    Shape accepted for a profile entry under ApiConfigMap.profiles.
+    Shape accepted for a profile entry under
+    :meth:`etlplus.api.config.ApiConfig.from_obj`.
 
     Notes
     -----
-    ``base_url`` is required at runtime when profiles are provided.
+    - :attr:`base_url` is required at runtime when :attr:`profiles` key/value
+        pairs are provided.
+    - :meth:`etlplus.api.config.ApiProfileConfig.from_obj` parses this mapping.
 
     See Also
     --------
@@ -103,7 +108,12 @@ class ApiProfileDefaultsMap(TypedDict, total=False):
 
     Notes
     -----
-    Runtime expects header values to be str; typing remains permissive.
+    - Runtime expects header values to be ``str``; typing remains permissive.
+    - :meth:`etlplus.api.config.ApiProfileConfig.from_obj` consumes this block.
+    - :meth:`etlplus.api.pagination.PaginationConfig.from_obj` parses
+        :attr:`pagination`.
+    - :meth:`etlplus.api.rate_limiting.RateLimitConfig.from_obj` parses
+        :attr:`rate_limit`.
 
     See Also
     --------
@@ -113,15 +123,15 @@ class ApiProfileDefaultsMap(TypedDict, total=False):
     """
 
     headers: StrAnyMap
-    pagination: Any
-    rate_limit: Any
+    pagination: StrAnyMap  # PaginationConfigMap | StrAnyMap
+    rate_limit: StrAnyMap  # RateLimitConfigMap | StrAnyMap
 
 
 class EndpointMap(TypedDict, total=False):
     """
-    Shape accepted by EndpointConfig.from_obj.
+    Shape accepted by :meth:`etlplus.api.config.EndpointConfig.from_obj`.
 
-    One of ``path`` or ``url`` should be provided.
+    One of :attr:`path` or :attr:`url` should be provided.
 
     See Also
     --------
@@ -134,8 +144,8 @@ class EndpointMap(TypedDict, total=False):
     path_params: StrAnyMap
     query_params: StrAnyMap
     body: Any
-    pagination: Any
-    rate_limit: Any
+    pagination: StrAnyMap  # PaginationConfigMap | StrAnyMap
+    rate_limit: StrAnyMap  # RateLimitConfigMap | StrAnyMap
 
 
 # SECTION: DATA CLASSES ===================================================== #

{etlplus-0.15.4 → etlplus-0.16.0}/etlplus/api/utils.py

@@ -172,7 +172,7 @@ def _coalesce(
     *args: Any,
 ) -> Any | None:
     """
-    Return the first non-``None`` value from ``args``.
+    Return the first non-``None`` value from *args*.
 
     Parameters
     ----------

{etlplus-0.15.4 → etlplus-0.16.0}/etlplus/cli/constants.py

@@ -8,7 +8,7 @@ from __future__ import annotations
 
 from typing import Final
 
-from ..enums import DataConnectorType
+from ..connector import DataConnectorType
 from ..file import FileFormat
 
 # SECTION: EXPORTS ========================================================== #

{etlplus-0.15.4 → etlplus-0.16.0}/etlplus/cli/handlers.py

@@ -505,7 +505,7 @@ def run_handler(
         Name of the job to run. If not provided, runs the entire pipeline.
         Default is ``None``.
     pipeline : str | None, optional
-        Alias for ``job``. Default is ``None``.
+        Alias for *job*. Default is ``None``.
     pretty : bool, optional
         Whether to pretty-print output. Default is ``True``.
 

{etlplus-0.15.4 → etlplus-0.16.0}/etlplus/cli/io.py

@@ -71,7 +71,7 @@ def emit_or_write(
     success_message: str,
 ) -> None:
     """
-    Emit JSON or persist to disk based on ``output_path``.
+    Emit JSON or persist to disk based on *output_path*.
 
     Parameters
     ----------
@@ -122,7 +122,7 @@ def materialize_file_payload(
     format_explicit: bool,
 ) -> JSONData | object:
     """
-    Return structured payloads when ``source`` references a file.
+    Return structured payloads when *source* references a file.
 
     Parameters
     ----------

{etlplus-0.15.4 → etlplus-0.16.0}/etlplus/cli/main.py

@@ -44,7 +44,7 @@ def _emit_context_help(
     Returns
     -------
     bool
-        ``True`` when help was emitted, ``False`` when ``ctx`` was ``None``.
+        ``True`` when help was emitted, ``False`` when *ctx* was ``None``.
     """
     if ctx is None:
         return False

etlplus-0.16.0/etlplus/connector/__init__.py

@@ -0,0 +1,43 @@
+"""
+:mod:`etlplus.connector` package.
+
+Connector configuration types and enums.
+"""
+
+from __future__ import annotations
+
+from .api import ConnectorApi
+from .api import ConnectorApiConfigMap
+from .connector import Connector
+from .core import ConnectorBase
+from .core import ConnectorProtocol
+from .database import ConnectorDb
+from .database import ConnectorDbConfigMap
+from .enums import DataConnectorType
+from .file import ConnectorFile
+from .file import ConnectorFileConfigMap
+from .types import ConnectorType
+from .utils import parse_connector
+
+# SECTION: EXPORTS ========================================================== #
+
+
+__all__ = [
+    # Data Classes
+    'ConnectorApi',
+    'ConnectorDb',
+    'ConnectorFile',
+    # Enums
+    'DataConnectorType',
+    # Functions
+    'parse_connector',
+    # Type Aliases
+    'Connector',
+    'ConnectorBase',
+    'ConnectorProtocol',
+    'ConnectorType',
+    # Typed Dicts
+    'ConnectorApiConfigMap',
+    'ConnectorDbConfigMap',
+    'ConnectorFileConfigMap',
+]

etlplus-0.16.0/etlplus/connector/api.py

@@ -0,0 +1,161 @@
+"""
+:mod:`etlplus.connector.api` module.
+
+API connector configuration dataclass.
+
+Notes
+-----
+- TypedDicts in this module are intentionally ``total=False`` and are not
+    enforced at runtime.
+- :meth:`*.from_obj` constructors accept :class:`Mapping[str, Any]` and perform
+    tolerant parsing and light casting. This keeps the runtime permissive while
+    improving autocomplete and static analysis for contributors.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from dataclasses import field
+from typing import Any
+from typing import Self
+from typing import TypedDict
+from typing import overload
+
+from ..api import PaginationConfig
+from ..api import PaginationConfigMap
+from ..api import RateLimitConfig
+from ..api import RateLimitConfigMap
+from ..types import StrAnyMap
+from ..types import StrStrMap
+from ..utils import cast_str_dict
+from ..utils import coerce_dict
+from ..utils import maybe_mapping
+from .core import ConnectorBase
+from .enums import DataConnectorType
+from .types import ConnectorType
+
+# SECTION: EXPORTS ========================================================== #
+
+
+__all__ = [
+    'ConnectorApi',
+    'ConnectorApiConfigMap',
+]
+
+
+# SECTION: TYPED DICTS ====================================================== #
+
+
+class ConnectorApiConfigMap(TypedDict, total=False):
+    """
+    Shape accepted by :meth:`ConnectorApi.from_obj` (all keys optional).
+
+    See Also
+    --------
+    - :meth:`etlplus.connector.api.ConnectorApi.from_obj`
+    """
+
+    name: str
+    type: ConnectorType
+    url: str
+    method: str
+    headers: StrStrMap
+    query_params: StrAnyMap
+    pagination: PaginationConfigMap
+    rate_limit: RateLimitConfigMap
+    api: str
+    endpoint: str
+
+
+# SECTION: DATA CLASSES ===================================================== #
+
+
+@dataclass(kw_only=True, slots=True)
+class ConnectorApi(ConnectorBase):
+    """
+    Configuration for an API-based data connector.
+
+    Attributes
+    ----------
+    type : ConnectorType
+        Connector kind, always ``'api'``.
+    url : str | None
+        Direct absolute URL (when not using ``service``/``endpoint`` refs).
+    method : str | None
+        Optional HTTP method; typically omitted for sources (defaults to
+        GET) and used for targets (e.g., ``'post'``).
+    headers : dict[str, str]
+        Additional request headers.
+    query_params : dict[str, Any]
+        Default query parameters.
+    pagination : PaginationConfig | None
+        Pagination settings (optional).
+    rate_limit : RateLimitConfig | None
+        Rate limiting settings (optional).
+    api : str | None
+        Service reference into the pipeline ``apis`` block (a.k.a.
+        ``service``).
+    endpoint : str | None
+        Endpoint name within the referenced service.
+    """
+
+    # -- Attributes -- #
+
+    type: ConnectorType = DataConnectorType.API
+
+    # Direct form
+    url: str | None = None
+    # Optional HTTP method; typically omitted for sources (defaults to GET)
+    # at runtime) and used for targets (e.g., 'post', 'put').
+    method: str | None = None
+    headers: dict[str, str] = field(default_factory=dict)
+    query_params: dict[str, Any] = field(default_factory=dict)
+    pagination: PaginationConfig | None = None
+    rate_limit: RateLimitConfig | None = None
+
+    # Reference form (to top-level APIs/endpoints)
+    api: str | None = None
+    endpoint: str | None = None
+
+    # -- Class Methods -- #
+
+    @classmethod
+    @overload
+    def from_obj(cls, obj: ConnectorApiConfigMap) -> Self: ...
+
+    @classmethod
+    @overload
+    def from_obj(cls, obj: StrAnyMap) -> Self: ...
+
+    @classmethod
+    def from_obj(
+        cls,
+        obj: StrAnyMap,
+    ) -> Self:
+        """
+        Parse a mapping into a ``ConnectorApi`` instance.
+
+        Parameters
+        ----------
+        obj : StrAnyMap
+            Mapping with at least ``name``.
+
+        Returns
+        -------
+        Self
+            Parsed connector instance.
+        """
+        name = cls._require_name(obj, kind='Api')
+        headers = cast_str_dict(maybe_mapping(obj.get('headers')))
+
+        return cls(
+            name=name,
+            url=obj.get('url'),
+            method=obj.get('method'),
+            headers=headers,
+            query_params=coerce_dict(obj.get('query_params')),
+            pagination=PaginationConfig.from_obj(obj.get('pagination')),
+            rate_limit=RateLimitConfig.from_obj(obj.get('rate_limit')),
+            api=obj.get('api') or obj.get('service'),
+            endpoint=obj.get('endpoint'),
+        )

etlplus-0.16.0/etlplus/connector/connector.py

@@ -0,0 +1,26 @@
+"""
+:mod:`etlplus.connector.connector` module.
+
+Compatibility re-exports for connector configuration classes.
+"""
+
+from __future__ import annotations
+
+from .api import ConnectorApi
+from .database import ConnectorDb
+from .file import ConnectorFile
+
+# SECTION: EXPORTS ========================================================== #
+
+
+__all__ = [
+    # Type aliases
+    'Connector',
+]
+
+
+# SECTION: TYPED ALIASES ==================================================== #
+
+
+# Type alias representing any supported connector
+type Connector = ConnectorApi | ConnectorDb | ConnectorFile