etlplus 0.5.4__py3-none-any.whl

etlplus/__init__.py

@@ -0,0 +1,43 @@
+"""
+:mod:`etlplus` package.
+
+Top-level facade for the ETLPlus toolkit.
+
+Importing :mod:`etlplus` exposes the handful of coarse-grained helpers most
+users care about: ``extract``, ``transform``, ``load``, ``validate``, and
+``run``. Each helper delegates to the richer modules under ``etlplus.*`` while
+presenting a compact public API surface.
+
+Examples
+--------
+>>> from etlplus import extract, transform
+>>> raw = extract('file', 'input.json')
+>>> curated = transform(raw, {'select': ['id', 'name']})
+
+See Also
+--------
+- :mod:`etlplus.cli` for the command-line interface
+- :mod:`etlplus.run` for orchestrating pipeline jobs
+"""
+
+from .__version__ import __version__
+
+__author__ = 'ETLPlus Team'
+
+from .extract import extract
+from .load import load
+from .run import run
+from .transform import transform
+from .validate import validate
+
+# SECTION: EXPORTS ========================================================== #
+
+
+__all__ = [
+    '__version__',
+    'extract',
+    'load',
+    'run',
+    'transform',
+    'validate',
+]

etlplus/__main__.py

@@ -0,0 +1,22 @@
+"""
+:mod:`etlplus.__main__` module.
+
+Thin wrapper supporting `python -m etlplus` by delegating to the CLI
+entrypoint.
+"""
+
+from .cli import main
+
+# SECTION: INTERNAL FUNCTIONS =============================================== #
+
+
+def _run() -> int:
+    """Return the exit status."""
+    return main()
+
+
+# SECTION: MAIN EXECUTION =================================================== #
+
+
+if __name__ == '__main__':  # pragma: no cover - exercised via CLI
+    raise SystemExit(_run())

etlplus/__version__.py

@@ -0,0 +1,14 @@
+"""
+etlplus.__version__ module.
+
+Expose the installed ETLPlus version.
+"""
+
+from importlib import metadata as _metadata
+
+try:
+    __version__ = _metadata.version('etlplus')
+except _metadata.PackageNotFoundError:
+    # Local editable installs without metadata fallback to an obvious
+    # placeholder.
+    __version__ = '0.0.0'

etlplus/api/README.md

@@ -0,0 +1,237 @@
+# etlplus.api module.
+
+Focused documentation for the `etlplus.api` subpackage: a lightweight HTTP client and helpers for
+paginated REST endpoints.
+
+- Provides a small `EndpointClient` for calling JSON APIs
+- Supports page-, offset-, and cursor-based pagination via `PaginationConfig`
+- Simple bearer-auth credentials via `EndpointCredentialsBearer`
+- Convenience helpers to extract records from nested JSON payloads
+- Returns the shared `JSONRecords` alias (a list of `JSONDict`) for paginated responses, matching
+  the rest of the library.
+
+Back to project overview: see the top-level [README](../../README.md).
+
+## Installation
+
+`etlplus.api` ships as part of the `etlplus` package. Install the package as usual:
+
+```bash
+pip install etlplus
+# or for development
+pip install -e ".[dev]"
+```
+
+## Quickstart
+
+```python
+import requests
+from etlplus.api import (
+  EndpointClient,
+  EndpointCredentialsBearer,
+  JSONRecords,
+)
+
+auth = EndpointCredentialsBearer(
+  token_url="https://auth.example.com/oauth2/token",
+  client_id="CLIENT_ID",
+  client_secret="CLIENT_SECRET",
+  scope="read:items",
+)
+
+session = requests.Session()
+session.auth = auth
+
+client = EndpointClient(
+  base_url="https://api.example.com/v1",
+  endpoints={
+    "list": "/items",  # you can add more named endpoints here
+  },
+  retry={"max_attempts": 4, "backoff": 0.5},
+  retry_network_errors=True,
+  session=session,
+)
+
+# Page-based pagination
+pg: PaginationConfig = {"type": "page", "page_size": 100}
+rows: JSONRecords = client.paginate("list", pagination=pg)
+for row in rows:
+  print(row)
+```
+
+### Overriding rate limits per call
+
+When a client is constructed with ``rate_limit`` metadata you can still tweak the pacing for
+individual calls by passing ``rate_limit_overrides`` to ``paginate``/``paginate_iter``. The
+overrides share the same shape as the base configuration and take precedence over the client
+defaults.
+
+```python
+client = EndpointClient(
+  base_url="https://api.example.com/v1",
+  endpoints={"list": "/items"},
+  rate_limit={"max_per_sec": 2},  # ~0.5s between calls when unspecified
+)
+
+rows = client.paginate(
+  "list",
+  pagination={"type": "page", "page_size": 100},
+  rate_limit_overrides={"sleep_seconds": 0.1},  # per-call override
+)
+```
+
+Precedence is ``overrides.sleep_seconds`` > ``overrides.max_per_sec`` > the same keys from
+``client.rate_limit``. When no override is supplied the base settings are used.
+
+## Choosing `records_path` and `cursor_path`
+
+If the API responds like this:
+
+```json
+{
+  "data": {
+    "items": [{"id": 1}, {"id": 2}],
+    "nextCursor": "abc123"
+  }
+}
+```
+
+- `records_path` should be `data.items`
+- `cursor_path` should be `data.nextCursor`
+
+If the response is a list at the top level, you can omit `records_path`.
+
+## Cursor-based pagination example
+
+```python
+from etlplus.api import EndpointClient, PaginationConfig, JSONRecords
+
+client = EndpointClient(
+    base_url="https://api.example.com/v1",
+    endpoints={"list": "/items"},
+)
+
+pg: PaginationConfig = {
+    "type": "cursor",
+    # Where records live in the JSON payload (dot path or top-level key)
+    "records_path": "data.items",
+    # Query parameter name that carries the cursor
+    "cursor_param": "cursor",
+    # Dot path in the response JSON that holds the next cursor value
+    "cursor_path": "data.nextCursor",
+    # Optional: limit per page
+    "page_size": 100,
+    # Optional: start from a specific cursor value
+    # "start_cursor": "abc123",
+}
+
+rows: JSONRecords = client.paginate("list", pagination=pg)
+for row in rows:
+    process(row)
+```
+
+## Offset-based pagination example
+
+```python
+from etlplus.api import EndpointClient, PaginationConfig
+
+client = EndpointClient(
+    base_url="https://api.example.com/v1",
+    endpoints={"list": "/items"},
+)
+
+pg: PaginationConfig = {
+  "type": "offset",
+  # Key holding the offset value on each request
+  "page_param": "offset",
+  # Key holding the page size (limit) on each request
+  "size_param": "limit",
+  # Starting offset (0 is common for offset-based APIs)
+  "start_page": 0,
+  # Number of records per page
+  "page_size": 100,
+  # Optional: where records live in the JSON payload
+  # "records_path": "data.items",
+  # Optional caps
+  # "max_records": 1000,
+}
+
+rows = client.paginate("list", pagination=pg)
+for row in rows:
+    process(row)
+```
+
+## Authentication
+
+Use bearer tokens with `EndpointCredentialsBearer` (OAuth2 client credentials flow). Attach it to a
+`requests.Session` and pass that session to the client:
+
+```python
+import requests
+from etlplus.api import EndpointClient, EndpointCredentialsBearer
+
+auth = EndpointCredentialsBearer(
+    token_url="https://auth.example.com/oauth2/token",
+    client_id="CLIENT_ID",
+    client_secret="CLIENT_SECRET",
+    scope="read:items",
+)
+
+session = requests.Session()
+session.auth = auth
+
+client = EndpointClient(
+    base_url="https://api.example.com/v1",
+    endpoints={"list": "/items"},
+    session=session,
+)
+```
+
+`EndpointCredentialsBearer` refreshes tokens automatically, applies a 15-second default timeout
+(`DEFAULT_TOKEN_TIMEOUT`), and omits the optional `scope` field when not provided so identity
+providers can fall back to their own defaults. If you already possess a static token, attach it to a
+`requests.Session` manually rather than instantiating `EndpointCredentialsBearer`.
+
+## Errors and rate limiting
+
+- Errors: `ApiRequestError`, `ApiAuthError`, and `PaginationError` (in `etlplus/api/errors.py`)
+  include an `as_dict()` helper for structured logs.
+- Rate limiting: `RateLimiter` and its `resolve_sleep_seconds` helper (in
+  `etlplus/api/rate_limiter.py`) derives fixed sleeps or `max_per_sec` windows. The paginator now
+  builds a `RateLimiter` whenever the effective delay comes from
+  `rate_limit`/`rate_limit_overrides`, so each page fetch sleeps before making another HTTP call.
+  Passing `rate_limit_overrides` to `paginate*` lets you momentarily speed up or slow down a single
+  request without mutating the client-wide defaults.
+
+## Types and transport
+
+- Types: pagination config helpers live in `etlplus/api/paginator.py`; retry helpers (including
+  `RetryPolicy`) live in `etlplus/api/retry_manager.py`; rate-limit helpers live in
+  `etlplus/api/rate_limiter.py`. These are all re-exported from `etlplus.api` for convenience.
+- Transport/session: `etlplus/api/transport.py` contains the HTTP adapter helpers and
+  `etlplus/api/request_manager.py` wraps `requests` sessions plus retry orchestration. Advanced
+  users may consult those modules to adapt behavior.
+
+## Supporting modules
+
+- `etlplus.api.types` collects friendly aliases such as `Headers`, `Params`, `Url`, and
+  `RateLimitOverrides` (whose values accept numeric override inputs) so endpoint helpers share the
+  same type vocabulary.
+- `etlplus.utils` exposes lightweight helpers used across the project, including CLI-friendly
+  functions like `json_type`/`print_json` plus numeric coercion utilities (`to_float`,
+  `to_positive_int`, etc.).
+
+## Minimal contract
+
+- Inputs
+  - `base_url: str`, `endpoints: dict[str, str]`
+  - optional `credentials`
+  - `pagination: PaginationConfig` for `paginate()`
+- Outputs
+  - `paginate(name, ...)` yields an iterator of JSON-like rows
+- Errors
+  - Network/HTTP errors raise exceptions; consult `errors.py`
+
+## See also
+
+- Top-level CLI and library usage in the main [README](../../README.md)

etlplus/api/__init__.py

@@ -0,0 +1,136 @@
+"""
+:mod:`etlplus.api` package.
+
+High-level helpers for building REST API clients with pagination, retry,
+rate limiting, and transport configuration.
+
+Summary
+-------
+Use :class:`etlplus.api.EndpointClient` to register relative endpoint paths
+under a base URL and paginate responses. The client can apply rate limits
+between requests and perform exponential-backoff retries with full jitter.
+
+Examples
+--------
+Page-based pagination
+^^^^^^^^^^^^^^^^^^^^^
+>>> from etlplus.api import EndpointClient
+>>> client = EndpointClient(
+...     base_url="https://api.example.com/v1",
+...     endpoints={"list_users": "/users"},
+... )
+>>> page_cfg = {
+...     "type": "page",               # or "offset"
+...     "records_path": "data.items", # dotted path into payload
+...     "page_param": "page",
+...     "size_param": "per_page",
+...     "start_page": 1,
+...     "page_size": 100,
+... }
+>>> rows = client.paginate(
+...     "list_users",
+...     query_parameters={"active": "true"},
+...     pagination=page_cfg,
+... )
+
+Retries and network errors
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+>>> client = EndpointClient(
+...     base_url="https://api.example.com/v1",
+...     endpoints={"list": "/items"},
+...     retry={"max_attempts": 5, "backoff": 0.5, "retry_on": [429, 503]},
+...     retry_network_errors=True,
+... )
+>>> items = client.paginate(
+...     "list", pagination={"type": "page", "page_size": 50}
+... )
+
+Absolute URLs
+^^^^^^^^^^^^^
+Use :meth:`EndpointClient.paginate_url` for an already composed absolute URL.
+It accepts the same pagination config and returns either the raw JSON object
+(no pagination) or a list of record dicts aggregated across pages.
+
+Notes
+-----
+- ``EndpointClient.endpoints`` is read-only at runtime.
+- Pagination defaults are centralized on the client (``page``, ``per_page``,
+    ``cursor``, ``limit``; start page ``1``; page size ``100``).
+- Retries are opt-in via the ``retry`` parameter; backoff uses jitter.
+- Use ``retry_network_errors=True`` to also retry timeouts/connection errors.
+- Prefer :data:`JSONRecords` (list of :data:`JSONDict`) for paginated
+    responses; scalar/record aliases are exported for convenience.
+- The underlying :class:`Paginator` is exported for advanced scenarios that
+    need to stream pages manually.
+
+See Also
+--------
+- :mod:`etlplus.api.rate_limiting` for rate-limit helpers and config shapes
+- :mod:`etlplus.api.pagination` for pagination helpers and config shapes
+- :mod:`etlplus.api.retry_manager` for retry policies
+- :mod:`etlplus.api.transport` for HTTPAdapter helpers
+"""
+
+from __future__ import annotations
+
+from .auth import EndpointCredentialsBearer
+from .config import ApiConfig
+from .config import ApiProfileConfig
+from .config import EndpointConfig
+from .endpoint_client import EndpointClient
+from .pagination import CursorPaginationConfigMap
+from .pagination import PagePaginationConfigMap
+from .pagination import PaginationClient
+from .pagination import PaginationConfig
+from .pagination import PaginationConfigMap
+from .pagination import PaginationType
+from .pagination import Paginator
+from .rate_limiting import RateLimitConfig
+from .rate_limiting import RateLimitConfigMap
+from .rate_limiting import RateLimiter
+from .retry_manager import RetryManager
+from .retry_manager import RetryPolicy
+from .retry_manager import RetryStrategy
+from .transport import HTTPAdapterMountConfig
+from .transport import HTTPAdapterRetryConfig
+from .transport import build_http_adapter
+from .types import Headers
+from .types import Params
+from .types import RequestOptions
+from .types import Url
+
+# SECTION: EXPORTS ========================================================== #
+
+
+__all__ = [
+    # Classes
+    'EndpointClient',
+    'EndpointCredentialsBearer',
+    'Paginator',
+    'RateLimiter',
+    'RetryManager',
+    # Data Classes
+    'ApiConfig',
+    'ApiProfileConfig',
+    'EndpointConfig',
+    'PaginationClient',
+    'PaginationConfig',
+    'RateLimitConfig',
+    'RequestOptions',
+    'RetryStrategy',
+    # Enums
+    'PaginationType',
+    # Functions
+    'build_http_adapter',
+    # Type Aliases
+    'CursorPaginationConfigMap',
+    'Headers',
+    'HTTPAdapterMountConfig',
+    'HTTPAdapterRetryConfig',
+    'PagePaginationConfigMap',
+    'PaginationConfigMap',
+    'Params',
+    'RateLimitConfigMap',
+    'RetryPolicy',
+    'Url',
+]