etlplus 0.16.0__tar.gz → 0.16.3__tar.gz

{etlplus-0.16.0/etlplus.egg-info → etlplus-0.16.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: etlplus
-Version: 0.16.0
+Version: 0.16.3
 Summary: A Swiss Army knife for simple ETL operations
 Home-page: https://github.com/Dagitali/ETLPlus
 Author: ETLPlus Team

{etlplus-0.16.0 → etlplus-0.16.3}/etlplus/api/types.py

@@ -53,7 +53,31 @@ __all__ = [
 # SECTION: CONSTANTS ======================================================== #
 
 
-_UNSET = object()
+_UNSET: object = object()
+
+
+# SECTION: INTERNAL FUNCTIONS =============================================== #
+
+
+def _to_dict(
+    value: Mapping[str, Any] | object | None,
+) -> dict[str, Any] | None:
+    """
+    Return a defensive ``dict`` copy for mapping inputs.
+
+    Parameters
+    ----------
+    value : Mapping[str, Any] | object | None
+        Mapping to copy, or ``None``.
+
+    Returns
+    -------
+    dict[str, Any] | None
+        New ``dict`` instance or ``None`` when the input is ``None``.
+    """
+    if value is None:
+        return None
+    return cast(dict[str, Any], value)
 
 
 # SECTION: TYPED DICTS ====================================================== #
@@ -176,9 +200,9 @@ class RequestOptions:
 
     def __post_init__(self) -> None:
         if self.params is not None:
-            object.__setattr__(self, 'params', dict(self.params))
+            object.__setattr__(self, 'params', _to_dict(self.params))
         if self.headers is not None:
-            object.__setattr__(self, 'headers', dict(self.headers))
+            object.__setattr__(self, 'headers', _to_dict(self.headers))
 
     # -- Instance Methods -- #
 
@@ -224,23 +248,20 @@ class RequestOptions:
 
         Returns
         -------
-        RequestOptions
+        Self
             New snapshot reflecting the provided overrides.
         """
         if params is _UNSET:
             next_params = self.params
-        elif params is None:
-            next_params = None
         else:
-            next_params = cast(dict, params)
+            # next_params = _to_dict(params) if params is not None else None
+            next_params = _to_dict(params)
 
         if headers is _UNSET:
             next_headers = self.headers
-        elif headers is None:
-            next_headers = None
         else:
-            next_headers = cast(dict, headers)
-
+            # next_headers = _to_dict(headers) if headers is not None else None
+            next_headers = _to_dict(headers)
         if timeout is _UNSET:
             next_timeout = self.timeout
         else:

etlplus-0.16.3/etlplus/enums.py

@@ -0,0 +1,144 @@
+"""
+:mod:`etlplus.enums` module.
+
+Shared enumeration base class.
+"""
+
+from __future__ import annotations
+
+import enum
+from typing import Self
+
+from .types import StrStrMap
+
+# SECTION: EXPORTS ========================================================== #
+
+
+__all__ = [
+    # Enums
+    'CoercibleStrEnum',
+]
+
+
+# SECTION: CLASSES ========================================================== #
+
+
+class CoercibleStrEnum(enum.StrEnum):
+    """
+    StrEnum with ergonomic helpers.
+
+    Provides a DRY, class-level :meth:`coerce` that normalizes inputs and
+    produces consistent, informative error messages. Also exposes
+    :meth:`choices` for UI/validation and :meth:`try_coerce` for soft parsing.
+
+    Notes
+    -----
+    - Values are normalized via ``str(value).strip().casefold()``.
+    - If value matching fails, the raw string is tried as a member name.
+    - Error messages enumerate allowed values for easier debugging.
+    """
+
+    # -- Class Methods -- #
+
+    @classmethod
+    def aliases(cls) -> StrStrMap:
+        """
+        Return a mapping of common aliases for each enum member.
+
+        Subclasses may override this method to provide custom aliases.
+
+        Returns
+        -------
+        StrStrMap
+            A mapping of alias strings to their corresponding enum member
+            values or names.
+
+        Notes
+        -----
+        - Alias keys are normalized via ``str(key).strip().casefold()``.
+        - Alias values should be member values or member names.
+        """
+        return {}
+
+    @classmethod
+    def choices(cls) -> tuple[str, ...]:
+        """
+        Return the allowed string values for this enum.
+
+        Returns
+        -------
+        tuple[str, ...]
+            A tuple of allowed string values for this enum.
+        """
+        return tuple(member.value for member in cls)
+
+    @classmethod
+    def coerce(cls, value: Self | str | object) -> Self:
+        """
+        Convert an enum member or string-like input to a member of *cls*.
+
+        Parameters
+        ----------
+        value : Self | str | object
+            An existing enum member or a string-like value to normalize.
+
+        Returns
+        -------
+        Self
+            The corresponding enum member.
+
+        Raises
+        ------
+        ValueError
+            If the value cannot be coerced into a valid member.
+        """
+        if isinstance(value, cls):
+            return value
+        try:
+            raw = str(value).strip()
+            normalized = raw.casefold()
+            aliases = {
+                str(key).strip().casefold(): alias
+                for key, alias in cls.aliases().items()
+            }
+            resolved = aliases.get(normalized)
+            if resolved is None:
+                try:
+                    return cls(normalized)  # type: ignore[arg-type]
+                except (ValueError, TypeError):
+                    return cls[raw]  # type: ignore[index]
+            if isinstance(resolved, cls):
+                return resolved
+            try:
+                return cls(resolved)  # type: ignore[arg-type]
+            except (ValueError, TypeError):
+                # Allow aliases to reference member names.
+                return cls[resolved]  # type: ignore[index]
+        except (ValueError, TypeError, KeyError) as e:
+            allowed = ', '.join(cls.choices())
+            raise ValueError(
+                f'Invalid {cls.__name__} value: {value!r}. Allowed: {allowed}',
+            ) from e
+
+    @classmethod
+    def try_coerce(
+        cls,
+        value: Self | str | object,
+    ) -> Self | None:
+        """
+        Attempt to coerce a value into the enum; return ``None`` on failure.
+
+        Parameters
+        ----------
+        value : Self | str | object
+            An existing enum member or a string-like value to normalize.
+
+        Returns
+        -------
+        Self | None
+            The corresponding enum member, or ``None`` if coercion fails.
+        """
+        try:
+            return cls.coerce(value)
+        except (ValueError, TypeError, KeyError):
+            return None

{etlplus-0.16.0 → etlplus-0.16.3}/etlplus/ops/__init__.py

@@ -52,6 +52,7 @@ from .validate import validate
 
 
 __all__ = [
+    # Functions
     'extract',
     'load',
     'run',

{etlplus-0.16.0/etlplus → etlplus-0.16.3/etlplus/ops}/enums.py

@@ -1,133 +1,30 @@
 """
-:mod:`etlplus.enums` module.
+:mod:`etlplus.ops.enums` module.
 
-Shared enumeration types used across ETLPlus modules.
+Operation-specific enums and helpers.
 """
 
 from __future__ import annotations
 
-import enum
 import operator as _op
 from statistics import fmean
-from typing import Self
 
+from ..enums import CoercibleStrEnum
+from ..types import StrStrMap
 from .types import AggregateFunc
 from .types import OperatorFunc
-from .types import StrStrMap
 
-# SECTION: EXPORTS ========================================================== #
+# SECTION: EXPORTS ========================================================= #
 
 
 __all__ = [
     # Enums
     'AggregateName',
-    'CoercibleStrEnum',
     'OperatorName',
     'PipelineStep',
 ]
 
 
-# SECTION: CLASSES ========================================================== #
-
-
-class CoercibleStrEnum(enum.StrEnum):
-    """
-    StrEnum with ergonomic helpers.
-
-    Provides a DRY, class-level :meth:`coerce` that normalizes inputs and
-    produces consistent, informative error messages. Also exposes
-    :meth:`choices` for UI/validation and :meth:`try_coerce` for soft parsing.
-
-    Notes
-    -----
-    - Values are normalized via ``str(value).strip().casefold()``.
-    - Error messages enumerate allowed values for easier debugging.
-    """
-
-    # -- Class Methods -- #
-
-    @classmethod
-    def aliases(cls) -> StrStrMap:
-        """
-        Return a mapping of common aliases for each enum member.
-
-        Subclasses may override this method to provide custom aliases.
-
-        Returns
-        -------
-        StrStrMap
-            A mapping of alias names to their corresponding enum member names.
-        """
-        return {}
-
-    @classmethod
-    def choices(cls) -> tuple[str, ...]:
-        """
-        Return the allowed string values for this enum.
-
-        Returns
-        -------
-        tuple[str, ...]
-            A tuple of allowed string values for this enum.
-        """
-        return tuple(member.value for member in cls)
-
-    @classmethod
-    def coerce(cls, value: Self | str | object) -> Self:
-        """
-        Convert an enum member or string-like input to a member of *cls*.
-
-        Parameters
-        ----------
-        value : Self | str | object
-            An existing enum member or a text value to normalize.
-
-        Returns
-        -------
-        Self
-            The corresponding enum member.
-
-        Raises
-        ------
-        ValueError
-            If the value cannot be coerced into a valid member.
-        """
-        if isinstance(value, cls):
-            return value
-        try:
-            normalized = str(value).strip().casefold()
-            resolved = cls.aliases().get(normalized, normalized)
-            return cls(resolved)  # type: ignore[arg-type]
-        except (ValueError, TypeError) as e:
-            allowed = ', '.join(cls.choices())
-            raise ValueError(
-                f'Invalid {cls.__name__} value: {value!r}. Allowed: {allowed}',
-            ) from e
-
-    @classmethod
-    def try_coerce(
-        cls,
-        value: object,
-    ) -> Self | None:
-        """
-        Best-effort parse; return ``None`` on failure instead of raising.
-
-        Parameters
-        ----------
-        value : object
-            An existing enum member or a text value to normalize.
-
-        Returns
-        -------
-        Self | None
-            The corresponding enum member, or ``None`` if coercion fails.
-        """
-        try:
-            return cls.coerce(value)
-        except ValueError:
-            return None
-
-
 # SECTION: ENUMS ============================================================ #
 
 

{etlplus-0.16.0 → etlplus-0.16.3}/etlplus/ops/extract.py

@@ -6,11 +6,19 @@ Helpers to extract data from files, databases, and REST APIs.
 
 from __future__ import annotations
 
+from collections.abc import Mapping
 from pathlib import Path
 from typing import Any
 from typing import cast
+from urllib.parse import urlsplit
+from urllib.parse import urlunsplit
 
+from ..api import EndpointClient
 from ..api import HttpMethod
+from ..api import PaginationConfigMap
+from ..api import RequestOptions
+from ..api import compose_api_request_env
+from ..api import paginate_with_client
 from ..api.utils import resolve_request
 from ..connector import DataConnectorType
 from ..file import File
@@ -19,6 +27,7 @@ from ..types import JSONData
 from ..types import JSONDict
 from ..types import JSONList
 from ..types import StrPath
+from ..types import Timeout
 
 # SECTION: EXPORTS ========================================================== #
 
@@ -32,50 +41,164 @@ __all__ = [
 ]
 
 
-# SECTION: FUNCTIONS ======================================================== #
+# SECTION: INTERNAL FUNCTIONS =============================================== #
 
 
-def extract_from_api(
-    url: str,
-    method: HttpMethod | str = HttpMethod.GET,
-    **kwargs: Any,
+def _build_client(
+    *,
+    base_url: str,
+    base_path: str | None,
+    endpoints: dict[str, str],
+    retry: Any,
+    retry_network_errors: bool,
+    session: Any,
+) -> EndpointClient:
+    """
+    Construct an API client with shared defaults.
+
+    Parameters
+    ----------
+    base_url : str
+        API base URL.
+    base_path : str | None
+        Base path to prepend for endpoints.
+    endpoints : dict[str, str]
+        Endpoint name to path mappings.
+    retry : Any
+        Retry policy configuration.
+    retry_network_errors : bool
+        Whether to retry on network errors.
+    session : Any
+        Optional requests session.
+
+    Returns
+    -------
+    EndpointClient
+        Configured endpoint client instance.
+    """
+    ClientClass = EndpointClient  # noqa: N806
+    return ClientClass(
+        base_url=base_url,
+        base_path=base_path,
+        endpoints=endpoints,
+        retry=retry,
+        retry_network_errors=retry_network_errors,
+        session=session,
+    )
+
+
+def _extract_from_api_env(
+    env: Mapping[str, Any],
+    *,
+    use_client: bool,
 ) -> JSONData:
     """
-    Extract data from a REST API.
+    Extract API data from a normalized request environment.
 
     Parameters
     ----------
-    url : str
-        API endpoint URL.
-    method : HttpMethod | str, optional
-        HTTP method to use. Defaults to ``GET``.
-    **kwargs : Any
-        Extra arguments forwarded to the underlying ``requests`` call
-        (for example, ``timeout``). To use a pre-configured
-        :class:`requests.Session`, provide it via ``session``.
-        When omitted, ``timeout`` defaults to 10 seconds.
+    env : Mapping[str, Any]
+        Normalized environment describing API request parameters.
+    use_client : bool
+        Whether to use the endpoint client/pagination machinery.
 
     Returns
     -------
     JSONData
-        Parsed JSON payload, or a fallback object with raw text.
+        Extracted payload.
 
     Raises
     ------
-    TypeError
-        If a provided ``session`` does not expose the required HTTP
-        method (for example, ``get``).
+    ValueError
+        If required parameters are missing.
     """
-    timeout = kwargs.pop('timeout', None)
-    session = kwargs.pop('session', None)
+    if (
+        use_client
+        and env.get('use_endpoints')
+        and env.get('base_url')
+        and env.get('endpoints_map')
+        and env.get('endpoint_key')
+    ):
+        client = _build_client(
+            base_url=cast(str, env.get('base_url')),
+            base_path=cast(str | None, env.get('base_path')),
+            endpoints=cast(dict[str, str], env.get('endpoints_map', {})),
+            retry=env.get('retry'),
+            retry_network_errors=bool(env.get('retry_network_errors', False)),
+            session=env.get('session'),
+        )
+        return paginate_with_client(
+            client,
+            cast(str, env.get('endpoint_key')),
+            env.get('params'),
+            env.get('headers'),
+            env.get('timeout'),
+            env.get('pagination'),
+            cast(float | None, env.get('sleep_seconds')),
+        )
+
+    url = env.get('url')
+    if not url:
+        raise ValueError('API source missing URL')
+
+    if use_client:
+        parts = urlsplit(cast(str, url))
+        base = urlunsplit((parts.scheme, parts.netloc, '', '', ''))
+        client = _build_client(
+            base_url=base,
+            base_path=None,
+            endpoints={},
+            retry=env.get('retry'),
+            retry_network_errors=bool(env.get('retry_network_errors', False)),
+            session=env.get('session'),
+        )
+        request_options = RequestOptions(
+            params=cast(Mapping[str, Any] | None, env.get('params')),
+            headers=cast(Mapping[str, str] | None, env.get('headers')),
+            timeout=cast(Timeout | None, env.get('timeout')),
+        )
+
+        return client.paginate_url(
+            cast(str, url),
+            cast(PaginationConfigMap | None, env.get('pagination')),
+            request=request_options,
+            sleep_seconds=cast(float, env.get('sleep_seconds', 0.0)),
+        )
+
+    method = env.get('method', HttpMethod.GET)
+    timeout = env.get('timeout', None)
+    session = env.get('session', None)
+    request_kwargs = dict(env.get('request_kwargs') or {})
     request_callable, timeout, _ = resolve_request(
         method,
         session=session,
         timeout=timeout,
     )
-    response = request_callable(url, timeout=timeout, **kwargs)
+    response = request_callable(
+        cast(str, url),
+        timeout=timeout,
+        **request_kwargs,
+    )
     response.raise_for_status()
+    return _parse_api_response(response)
+
 
+def _parse_api_response(
+    response: Any,
+) -> JSONData:
+    """
+    Parse API responses into a consistent JSON payload.
+
+    Parameters
+    ----------
+    response : Any
+        HTTP response object exposing ``headers``, ``json()``, and ``text``.
+
+    Returns
+    -------
+    JSONData
+        Parsed JSON payload, or a fallback object with raw text.
+    """
     content_type = response.headers.get('content-type', '').lower()
     if 'application/json' in content_type:
         try:
@@ -99,6 +222,70 @@ def extract_from_api(
     return {'content': response.text, 'content_type': content_type}
 
 
+# SECTION: FUNCTIONS ======================================================== #
+
+
+def extract_from_api(
+    url: str,
+    method: HttpMethod | str = HttpMethod.GET,
+    **kwargs: Any,
+) -> JSONData:
+    """
+    Extract data from a REST API.
+
+    Parameters
+    ----------
+    url : str
+        API endpoint URL.
+    method : HttpMethod | str, optional
+        HTTP method to use. Defaults to ``GET``.
+    **kwargs : Any
+        Extra arguments forwarded to the underlying ``requests`` call
+        (for example, ``timeout``). To use a pre-configured
+        :class:`requests.Session`, provide it via ``session``.
+        When omitted, ``timeout`` defaults to 10 seconds.
+
+    Returns
+    -------
+    JSONData
+        Parsed JSON payload, or a fallback object with raw text.
+    """
+    env = {
+        'url': url,
+        'method': method,
+        'timeout': kwargs.pop('timeout', None),
+        'session': kwargs.pop('session', None),
+        'request_kwargs': kwargs,
+    }
+    return _extract_from_api_env(env, use_client=False)
+
+
+def extract_from_api_source(
+    cfg: Any,
+    source_obj: Any,
+    overrides: dict[str, Any],
+) -> JSONData:
+    """
+    Extract data from a REST API source connector.
+
+    Parameters
+    ----------
+    cfg : Any
+        Pipeline configuration.
+    source_obj : Any
+        Connector configuration.
+    overrides : dict[str, Any]
+        Extract-time overrides.
+
+    Returns
+    -------
+    JSONData
+        Extracted payload.
+    """
+    env = compose_api_request_env(cfg, source_obj, overrides)
+    return _extract_from_api_env(env, use_client=True)
+
+
 def extract_from_database(
     connection_string: str,
 ) -> JSONList: