etlplus 0.5.4__py3-none-any.whl

etlplus/api/errors.py

@@ -0,0 +1,170 @@
+"""
+:mod:`etlplus.api.errors` module.
+
+Exception types with rich context for debugging REST API failures.
+
+Summary
+-------
+Provides subclasses for request errors (``ApiRequestError``), auth failures
+(``ApiAuthError``), and pagination errors with page context
+(``PaginationError``).
+
+Examples
+--------
+>>> try:
+...     client.paginate("list", pagination={"type": "page", "page_size": 50})
+... except ApiAuthError as e:
+...     print("auth failed", e.status)
+... except PaginationError as e:
+...     print("page:", e.page, "attempts:", e.attempts)
+... except ApiRequestError as e:
+...     print("request failed", e.url)
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import TYPE_CHECKING
+from typing import Any
+
+import requests  # type: ignore[import]
+
+if TYPE_CHECKING:  # pragma: no cover - typing only
+    from .retry_manager import RetryPolicy
+
+
+# SECTION: EXPORTS ========================================================== #
+
+
+__all__ = ['ApiAuthError', 'ApiRequestError', 'PaginationError']
+
+
+# SECTION: CLASSES ========================================================== #
+
+
+@dataclass(slots=True, kw_only=True)
+class ApiRequestError(requests.RequestException):
+    """
+    Base error for API request failures with rich context.
+
+    Parameters
+    ----------
+    url : str
+        Absolute URL that was requested.
+    status : int | None, optional
+        HTTP status code when available.
+    attempts : int, optional
+        Number of attempts performed (defaults to ``1``).
+    retried : bool, optional
+        Whether any retry attempts were made.
+    retry_policy : RetryPolicy | None, optional
+        The retry policy in effect, if any.
+    cause : Exception | None, optional
+        Original underlying exception.
+
+    Attributes
+    ----------
+    url : str
+        Absolute URL that was requested.
+    status : int | None
+        HTTP status code when available.
+    attempts : int
+        Number of attempts performed.
+    retried : bool
+        Whether any retry attempts were made.
+    retry_policy : RetryPolicy | None
+        The retry policy in effect, if any.
+    cause : Exception | None
+        Original underlying exception.
+
+    Examples
+    --------
+    >>> try:
+    ...     raise ApiRequestError(url="https://api.example.com/x", status=500)
+    ... except ApiRequestError as e:
+    ...     print(e.status, e.attempts)
+    500 1
+
+    Notes
+    -----
+    The :meth:`as_dict` helper returns a structured payload suitable for
+    structured logging or telemetry.
+    """
+
+    # -- Attributes -- #
+
+    url: str
+    status: int | None = None
+    attempts: int = 1
+    retried: bool = False
+    retry_policy: RetryPolicy | None = None
+    cause: Exception | None = None
+
+    # -- Magic Methods (Object Representation) -- #
+
+    def __str__(self) -> str:  # pragma: no cover - formatting only
+        base = f'request failed url={self.url!r} status={self.status}'
+        meta = f' attempts={self.attempts} retried={self.retried}'
+
+        return f'ApiRequestError({base}{meta})'
+
+    # -- Instance Methods -- #
+
+    def as_dict(self) -> dict[str, Any]:
+        """Return structured error context for logging or telemetry."""
+        return {
+            'url': self.url,
+            'status': self.status,
+            'attempts': self.attempts,
+            'retried': self.retried,
+            'retry_policy': self.retry_policy,
+            'cause': repr(self.cause) if self.cause else None,
+        }
+
+
+class ApiAuthError(ApiRequestError):
+    """Authentication/authorization failure (e.g., 401/403)."""
+
+
+@dataclass(slots=True, kw_only=True)
+class PaginationError(ApiRequestError):
+    """
+    Error raised during pagination with page context.
+
+    Parameters
+    ----------
+    page : int | None, optional
+        Page number (1-based) or request count when applicable.
+    **kwargs
+        Remaining keyword arguments forwarded to ``ApiRequestError``.
+
+    Attributes
+    ----------
+    page : int | None
+        Stored page number.
+
+    Examples
+    --------
+    >>> err = PaginationError(url="u", status=400, page=3)
+    >>> str(err).startswith("PaginationError(")
+    True
+    """
+
+    # -- Attributes -- #
+
+    page: int | None = None
+
+    # -- Magic Methods (Object Representation) -- #
+
+    def __str__(self) -> str:  # pragma: no cover - formatting only
+        base = super().__str__()
+
+        return f'PaginationError({base} page={self.page})'
+
+    # -- Instance Methods -- #
+
+    def as_dict(self) -> dict[str, Any]:
+        """Extend base context with pagination metadata."""
+        payload = super().as_dict()
+        payload['page'] = self.page
+        return payload

etlplus/api/pagination/__init__.py

@@ -0,0 +1,47 @@
+"""
+:mod:`etlplus.api.pagination` package.
+
+Pagination configuration and runtime helpers for REST API responses.
+
+This package groups configuration shapes, paginator utilities, and a
+client-facing driver for traversing page-, offset-, and cursor-based JSON
+responses.
+
+Notes
+-----
+- Pagination defaults are centralized on :class:`EndpointClient` (``page``,
+    ``per_page``, ``cursor``, ``limit``; start page ``1``; page size ``100``).
+- Prefer :data:`JSONRecords` (list of :data:`JSONDict`) for paginated
+    responses; scalar/record aliases are exported for convenience.
+- The underlying :class:`Paginator` is exported for advanced scenarios that
+    need to stream pages manually.
+"""
+
+from __future__ import annotations
+
+from .client import PaginationClient
+from .config import CursorPaginationConfigMap
+from .config import PagePaginationConfigMap
+from .config import PaginationConfig
+from .config import PaginationConfigMap
+from .config import PaginationInput
+from .config import PaginationType
+from .paginator import Paginator
+
+# SECTION: EXPORTS ========================================================== #
+
+
+__all__ = [
+    # Classes
+    'PaginationClient',
+    'Paginator',
+    # Data Classes
+    'PaginationConfig',
+    # Enums
+    'PaginationType',
+    # Type Aliases
+    'CursorPaginationConfigMap',
+    'PagePaginationConfigMap',
+    'PaginationInput',
+    'PaginationConfigMap',
+]

etlplus/api/pagination/client.py

@@ -0,0 +1,188 @@
+"""
+:mod:`etlplus.api.client` module.
+
+Client-facing pagination driver for REST API responses.
+
+This module wires pagination configuration, fetch callbacks, and optional rate
+limiting into :class:`etlplus.api.pagination.Paginator` instances.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Generator
+from collections.abc import Mapping
+from dataclasses import dataclass
+from typing import Any
+from typing import cast
+
+from ...types import JSONDict
+from ...types import JSONRecords
+from ..rate_limiting import RateLimiter
+from ..types import FetchPageCallable
+from ..types import RequestOptions
+from ..types import Url
+from .config import PaginationConfig
+from .config import PaginationInput
+from .config import PaginationType
+from .paginator import Paginator
+
+# SECTION: EXPORTS ========================================================== #
+
+
+__all__ = [
+    # Classes
+    'PaginationClient',
+]
+
+
+# SECTION: CLASSES ========================================================== #
+
+
+@dataclass(slots=True, kw_only=True)
+class PaginationClient:
+    """
+    Drive :class:`Paginator` instances with shared guardrails.
+
+    Parameters
+    ----------
+    pagination : PaginationInput
+        Pagination configuration mapping or :class:`PaginationConfig`.
+    fetch : FetchPageCallable
+        Callback used to fetch a single page.
+    rate_limiter : RateLimiter | None, optional
+        Optional limiter invoked between page fetches.
+
+    Attributes
+    ----------
+    pagination : PaginationInput
+        Resolved pagination configuration.
+    fetch : FetchPageCallable
+        Stored fetch callback invoked by ``Paginator``.
+    rate_limiter : RateLimiter | None
+        Limiter applied between requests when configured.
+    """
+
+    # -- Attributes -- #
+
+    pagination: PaginationInput
+    fetch: FetchPageCallable
+    rate_limiter: RateLimiter | None = None
+
+    # -- Properties -- #
+
+    @property
+    def is_paginated(self) -> bool:
+        """Return ``True`` when a known pagination type is configured."""
+        return self.pagination_type is not None
+
+    @property
+    def pagination_type(self) -> PaginationType | None:
+        """Return the normalized pagination type when available."""
+        if isinstance(self.pagination, PaginationConfig):
+            return self.pagination.type
+        return Paginator.detect_type(
+            cast(Mapping[str, Any] | None, self.pagination),
+            default=None,
+        )
+
+    # -- Instance Methods -- #
+
+    def collect(
+        self,
+        url: Url,
+        *,
+        request: RequestOptions | None = None,
+    ) -> JSONRecords:
+        """
+        Collect records across pages into a list.
+
+        Parameters
+        ----------
+        url : Url
+            Base URL to fetch pages from.
+        request : RequestOptions | None, optional
+            Snapshot of request metadata (params/headers/timeout) to clone
+            for this invocation.
+
+        Returns
+        -------
+        JSONRecords
+            List of JSON records.
+        """
+        return list(self.iterate(url, request=request))
+
+    def iterate(
+        self,
+        url: Url,
+        *,
+        request: RequestOptions | None = None,
+    ) -> Generator[JSONDict]:
+        """
+        Yield records for the configured pagination strategy.
+
+        Parameters
+        ----------
+        url : Url
+            Base URL to fetch pages from.
+        request : RequestOptions | None, optional
+            Snapshot of request metadata (params/headers/timeout) to clone
+            for this invocation.
+
+        Yields
+        ------
+        Generator[JSONDict]
+            Iterator over JSON records from one or more pages.
+        """
+        effective_request = request or RequestOptions()
+
+        if not self.is_paginated:
+            yield from self._iterate_single_page(url, effective_request)
+            return
+
+        paginator = Paginator.from_config(
+            cast(PaginationInput, self.pagination),
+            fetch=self.fetch,
+            rate_limiter=self.rate_limiter,
+        )
+        yield from paginator.paginate_iter(
+            url,
+            request=effective_request,
+        )
+
+    # -- Internal Instance Methods -- #
+
+    def _iterate_single_page(
+        self,
+        url: Url,
+        request: RequestOptions,
+    ) -> Generator[JSONDict]:
+        """
+        Iterate records for non-paginated responses.
+
+        Parameters
+        ----------
+        url : Url
+            Base URL to fetch pages from.
+        request : RequestOptions
+            Request metadata to forward to the fetch callback.
+
+        Yields
+        ------
+        Generator[JSONDict]
+            JSON records from the response.
+        """
+        pg_records_path: str | None
+        pg_fallback_path: str | None
+        if isinstance(self.pagination, Mapping):
+            pg = cast(Mapping[str, Any], self.pagination)
+            pg_records_path = cast(str | None, pg.get('records_path'))
+            pg_fallback_path = cast(str | None, pg.get('fallback_path'))
+        else:
+            pg_records_path = getattr(self.pagination, 'records_path', None)
+            pg_fallback_path = getattr(self.pagination, 'fallback_path', None)
+        page_data = self.fetch(url, request, None)
+        yield from Paginator.coalesce_records(
+            page_data,
+            pg_records_path,
+            pg_fallback_path,
+        )