etlplus 0.15.0__py3-none-any.whl → 0.16.6__py3-none-any.whl

etlplus/README.md

@@ -13,12 +13,12 @@ Back to project overview: see the top-level [README](../README.md).
 
 - [etlplus.api](api/README.md): Lightweight HTTP client and paginated REST helpers
 - [etlplus.file](file/README.md): Unified file format support and helpers
-- [etlplus.config](config/README.md): Configuration helpers for connectors, pipelines, jobs, and
-  profiles
-- [etlplus.cli](cli/README.md): Command-line interface for ETLPlus workflows
+- [etlplus.cli](cli/README.md): Command-line interface definitions for `etlplus`
 - [etlplus.database](database/README.md): Database engine, schema, and ORM helpers
 - [etlplus.templates](templates/README.md): SQL and DDL template helpers
 - [etlplus.validation](validation/README.md): Data validation utilities and helpers
+- [etlplus.workflow](etlplus/workflow/README.md): Helpers for data connectors, pipelines, jobs, and
+  profiles
 
 ## Quickstart
 
@@ -31,6 +31,28 @@ assert validate(filtered, {"age": {"type": "number", "min": 0}})["valid"]
 load(filtered, "file", "output.json", file_format="json")
 ```
 
+## Loading and Validating Configs
+
+Use the provided classes to load and validate configuration files:
+
+```python
+from etlplus.workflow import Config
+
+cfg = Config.from_yaml("pipeline.yml")
+```
+
+- Supports YAML and JSON formats
+- Validates against expected schema
+
+## Example: Loading a Pipeline Config
+
+```python
+from etlplus.workflow import Config
+
+config = Config.from_yaml("configs/pipeline.yml")
+print(config)
+```
+
 ## See Also
 
 - [Top-level project README](../README.md)

etlplus/__init__.py

@@ -5,6 +5,7 @@ Top-level facade for the ETLPlus toolkit.
 """
 
 from .__version__ import __version__
+from .config import Config
 
 __author__ = 'ETLPlus Team'
 
@@ -15,4 +16,5 @@ __author__ = 'ETLPlus Team'
 __all__ = [
     '__author__',
     '__version__',
+    'Config',
 ]

etlplus/api/README.md

@@ -22,6 +22,7 @@ Back to project overview: see the top-level [README](../../README.md).
   - [Authentication](#authentication)
   - [Errors and Rate Limiting](#errors-and-rate-limiting)
   - [Types and Transport](#types-and-transport)
+  - [Config Schemas](#config-schemas)
   - [Supporting Modules](#supporting-modules)
   - [Minimal Contract](#minimal-contract)
   - [See also](#see-also)
@@ -225,6 +226,36 @@ providers can fall back to their own defaults. If you already possess a static t
   `etlplus/api/request_manager.py` wraps `requests` sessions plus retry orchestration. Advanced
   users may consult those modules to adapt behavior.
 
+## Config Schemas
+
+`etlplus.api.types` defines TypedDict-based configuration shapes for API profiles and endpoints.
+Runtime parsing remains permissive in `etlplus.api.config`, but these types improve IDE
+autocomplete and static analysis.
+
+Exported types:
+
+- `ApiConfigMap`: top-level API config shape
+- `ApiProfileConfigMap`: per-profile API config shape
+- `ApiProfileDefaultsMap`: defaults block within a profile
+- `EndpointMap`: endpoint config shape
+
+Example:
+
+```python
+from etlplus.api import ApiConfigMap
+
+api_cfg: ApiConfigMap = {
+    "base_url": "https://example.test",
+    "headers": {"Authorization": "Bearer token"},
+    "endpoints": {
+        "users": {
+            "path": "/users",
+            "method": "GET",
+        },
+    },
+}
+```
+
 ## Supporting Modules
 
 - `etlplus.api.types` collects friendly aliases such as `Headers`, `Params`, `Url`, and

etlplus/api/__init__.py

@@ -79,21 +79,21 @@ from .config import ApiProfileConfig
 from .config import EndpointConfig
 from .endpoint_client import EndpointClient
 from .enums import HttpMethod
-from .pagination import CursorPaginationConfigMap
-from .pagination import PagePaginationConfigMap
+from .pagination import CursorPaginationConfigDict
+from .pagination import PagePaginationConfigDict
 from .pagination import PaginationClient
 from .pagination import PaginationConfig
-from .pagination import PaginationConfigMap
+from .pagination import PaginationConfigDict
 from .pagination import PaginationType
 from .pagination import Paginator
 from .rate_limiting import RateLimitConfig
-from .rate_limiting import RateLimitConfigMap
+from .rate_limiting import RateLimitConfigDict
 from .rate_limiting import RateLimiter
 from .retry_manager import RetryManager
-from .retry_manager import RetryPolicy
+from .retry_manager import RetryPolicyDict
 from .retry_manager import RetryStrategy
-from .transport import HTTPAdapterMountConfig
-from .transport import HTTPAdapterRetryConfig
+from .transport import HTTPAdapterMountConfigDict
+from .transport import HTTPAdapterRetryConfigDict
 from .transport import build_http_adapter
 from .types import Headers
 from .types import Params
@@ -133,14 +133,14 @@ __all__ = [
     'paginate_with_client',
     'resolve_request',
     # Type Aliases
-    'CursorPaginationConfigMap',
+    'CursorPaginationConfigDict',
     'Headers',
-    'HTTPAdapterMountConfig',
-    'HTTPAdapterRetryConfig',
-    'PagePaginationConfigMap',
-    'PaginationConfigMap',
+    'HTTPAdapterMountConfigDict',
+    'HTTPAdapterRetryConfigDict',
+    'PagePaginationConfigDict',
+    'PaginationConfigDict',
     'Params',
-    'RateLimitConfigMap',
-    'RetryPolicy',
+    'RateLimitConfigDict',
+    'RetryPolicyDict',
     'Url',
 ]

etlplus/api/auth.py

@@ -76,13 +76,16 @@ FORM_HEADERS = MappingProxyType(
 # SECTION: TYPED DICTS ====================================================== #
 
 
-class _TokenResponse(TypedDict):
+class _TokenResponseDict(TypedDict):
     """Minimal shape of an OAuth token response body."""
 
     access_token: str
     expires_in: int | float
 
 
+# SECTION: PROTOCOLS ======================================================== #
+
+
 class _TokenHttpClient(Protocol):
     """Protocol for objects that expose a ``post`` helper like ``requests``."""
 
@@ -136,7 +139,7 @@ def _truncate(
     limit: int = MAX_LOG_BODY,
 ) -> str:
     """
-    Return ``text`` shortened to ``limit`` characters for logging.
+    Return *text* shortened to *limit* characters for logging.
 
     Parameters
     ----------
@@ -284,7 +287,7 @@ class EndpointCredentialsBearer(AuthBase):
     def _parse_token_response(
         self,
         resp: Response,
-    ) -> _TokenResponse:
+    ) -> _TokenResponseDict:
         """
         Validate the JSON token response and return a typed mapping.
 
@@ -295,7 +298,7 @@ class EndpointCredentialsBearer(AuthBase):
 
         Returns
         -------
-        _TokenResponse
+        _TokenResponseDict
             Parsed token response mapping.
 
         Raises
@@ -335,15 +338,15 @@ class EndpointCredentialsBearer(AuthBase):
         except (TypeError, ValueError):
             ttl = float(DEFAULT_TOKEN_TTL)
 
-        return _TokenResponse(access_token=token, expires_in=ttl)
+        return _TokenResponseDict(access_token=token, expires_in=ttl)
 
-    def _request_token(self) -> _TokenResponse:
+    def _request_token(self) -> _TokenResponseDict:
         """
         Execute the OAuth2 token request and parse the response.
 
         Returns
         -------
-        _TokenResponse
+        _TokenResponseDict
             Parsed token response mapping.
 
         Raises

etlplus/api/config.py

@@ -3,11 +3,6 @@
 
 Configuration dataclasses for REST API services, profiles, and endpoints.
 
-These models used to live under :mod:`etlplus.config`, but they belong in the
-API layer because they compose runtime types such as
-:class:`etlplus.api.EndpointClient`, :class:`etlplus.api.PaginationConfig`, and
-:class:`etlplus.api.RateLimitConfig`.
-
 Notes
 -----
 - TypedDict references remain editor hints only; :meth:`from_obj` accepts
@@ -41,9 +36,9 @@ from .pagination import PaginationConfig
 from .rate_limiting import RateLimitConfig
 
 if TYPE_CHECKING:
-    from ..config.types import ApiConfigMap
-    from ..config.types import ApiProfileConfigMap
-    from ..config.types import EndpointMap
+    from .types import ApiConfigDict
+    from .types import ApiProfileConfigDict
+    from .types import EndpointConfigDict
 
 
 # SECTION: EXPORTS ========================================================== #
@@ -73,7 +68,7 @@ def _effective_service_defaults(
     fallback_headers: dict[str, str],
 ) -> tuple[str, dict[str, str]]:
     """
-    Return ``(base_url, headers)`` using ``profiles`` when present.
+    Return ``(base_url, headers)`` using *profiles* when present.
 
     Parameters
     ----------
@@ -92,7 +87,7 @@ def _effective_service_defaults(
     Raises
     ------
     TypeError
-        If no profiles are defined and ``fallback_base`` is not a string.
+        If no profiles are defined and *fallback_base* is not a string.
     """
     if profiles:
         name = 'default' if 'default' in profiles else next(iter(profiles))
@@ -269,7 +264,7 @@ class ApiProfileConfig:
     @overload
     def from_obj(
         cls,
-        obj: ApiProfileConfigMap,
+        obj: ApiProfileConfigDict,
     ) -> Self: ...
 
     @classmethod
@@ -465,7 +460,7 @@ class ApiConfig:
     @overload
     def from_obj(
         cls,
-        obj: ApiConfigMap,
+        obj: ApiConfigDict,
     ) -> Self: ...
 
     @classmethod
@@ -582,7 +577,7 @@ class EndpointConfig:
     @overload
     def from_obj(
         cls,
-        obj: EndpointMap,
+        obj: EndpointConfigDict,
     ) -> Self: ...
 
     @classmethod

etlplus/api/endpoint_client.py

@@ -11,8 +11,8 @@ Notes
 -----
 - Retry-related types live in :mod:`etlplus.api.retry_manager`.
 - Pagination requires a ``PaginationConfig``; see
-    :class:`PagePaginationConfigMap` and :class:`CursorPaginationConfigMap` for
-    the accepted shapes.
+    :class:`PagePaginationConfigDict` and :class:`CursorPaginationConfigDict`
+    for the accepted shapes.
 
 Examples
 --------
@@ -65,14 +65,14 @@ from .errors import PaginationError
 from .pagination import PaginationClient
 from .pagination import PaginationInput
 from .pagination import Paginator
-from .rate_limiting import RateLimitConfigMap
+from .rate_limiting import RateLimitConfigDict
 from .rate_limiting import RateLimiter
 from .rate_limiting import RateLimitOverrides
 from .request_manager import RequestManager
 from .retry_manager import RetryManager
-from .retry_manager import RetryPolicy
+from .retry_manager import RetryPolicyDict
 from .retry_manager import RetryStrategy
-from .transport import HTTPAdapterMountConfig
+from .transport import HTTPAdapterMountConfigDict
 from .types import RequestOptions
 from .types import Url
 
@@ -101,13 +101,13 @@ class EndpointClient:
     base_path : str | None, optional
         Optional base path prefix (``/v2``) prepended to all endpoint
         paths when building URLs.
-    retry : RetryPolicy | None, optional
+    retry : RetryPolicyDict | None, optional
         Optional retry policy. When provided, failed requests matching
         ``retry_on`` statuses are retried with full jitter.
     retry_network_errors : bool, optional
         When ``True``, also retry on network errors (timeouts, connection
         resets). Defaults to ``False``.
-    rate_limit : RateLimitConfigMap | None, optional
+    rate_limit : RateLimitConfigDict | None, optional
         Optional client-wide rate limit used to derive an inter-request
         delay when an explicit ``sleep_seconds`` isn't supplied.
     session : requests.Session | None, optional
@@ -115,7 +115,7 @@ class EndpointClient:
     session_factory : Callable[[], requests.Session] | None, optional
         Factory used to lazily create a session. Ignored if ``session`` is
         provided.
-    session_adapters : Sequence[HTTPAdapterMountConfig] | None, optional
+    session_adapters : Sequence[HTTPAdapterMountConfigDict] | None, optional
         Adapter mount configuration(s) used to build a session lazily when
         neither ``session`` nor ``session_factory`` is supplied.
 
@@ -128,17 +128,17 @@ class EndpointClient:
         (``MappingProxyType``).
     base_path : str | None
         Optional base path prefix appended after ``base_url``.
-    retry : RetryPolicy | None
+    retry : RetryPolicyDict | None
         Retry policy reference (may be ``None``).
     retry_network_errors : bool
         Whether network errors are retried in addition to HTTP statuses.
-    rate_limit : RateLimitConfigMap | None
+    rate_limit : RateLimitConfigDict | None
         Client-wide rate limit configuration (may be ``None``).
     session : requests.Session | None
         Explicit HTTP session used for requests when provided.
     session_factory : Callable[[], requests.Session] | None
         Lazily invoked factory producing a session when needed.
-    session_adapters : Sequence[HTTPAdapterMountConfig] | None
+    session_adapters : Sequence[HTTPAdapterMountConfigDict] | None
         Adapter mount configuration(s) for connection pooling / retries.
     DEFAULT_PAGE_PARAM : ClassVar[str]
         Default page parameter name.
@@ -212,10 +212,10 @@ class EndpointClient:
     base_path: str | None = None
 
     # Optional retry configuration (constructor parameter; object is frozen)
-    retry: RetryPolicy | None = None
+    retry: RetryPolicyDict | None = None
     retry_network_errors: bool = False
     # Optional client-wide rate limit configuration
-    rate_limit: RateLimitConfigMap | None = None
+    rate_limit: RateLimitConfigDict | None = None
 
     # Optional HTTP session or factory
     session: requests.Session | None = None
@@ -225,7 +225,7 @@ class EndpointClient:
     # and connection pooling. If provided and neither `session` nor
     # `session_factory` is supplied, a factory is synthesized to create a
     # Session and mount the configured adapters lazily.
-    session_adapters: Sequence[HTTPAdapterMountConfig] | None = None
+    session_adapters: Sequence[HTTPAdapterMountConfigDict] | None = None
 
     # Internal: context-managed session and ownership flag.
     _request_manager: RequestManager = field(
@@ -759,8 +759,8 @@ class EndpointClient:
         Raises
         ------
         KeyError
-            If ``endpoint_key`` is unknown or a required placeholder in the
-            path has no corresponding entry in ``path_parameters``.
+            If *endpoint_key* is unknown or a required placeholder in the path
+            has no corresponding entry in *path_parameters*.
         ValueError
             If the path template is invalid.
 
@@ -836,7 +836,7 @@ class EndpointClient:
         """
         Sleep for the specified seconds if positive.
 
-        The optional ``sleeper`` is useful for tests (e.g., pass
+        The optional *sleeper* is useful for tests (e.g., pass
         ``lambda s: None``). Defaults to using time.sleep when not provided.
 
         Parameters
@@ -857,7 +857,7 @@ class EndpointClient:
     @staticmethod
     def _resolve_sleep_seconds(
         explicit: float,
-        rate_limit: RateLimitConfigMap | None,
+        rate_limit: RateLimitConfigDict | None,
         overrides: RateLimitOverrides = None,
     ) -> float:
         """
@@ -867,10 +867,10 @@ class EndpointClient:
         ----------
         explicit : float
             Explicit sleep seconds provided by the caller.
-        rate_limit : RateLimitConfigMap | None
+        rate_limit : RateLimitConfigDict | None
             Client-wide rate limit configuration.
         overrides : RateLimitOverrides, optional
-            Per-call overrides that take precedence over ``rate_limit``.
+            Per-call overrides that take precedence over *rate_limit*.
 
         Returns
         -------

etlplus/api/errors.py

@@ -30,7 +30,7 @@ from typing import Any
 import requests  # type: ignore[import]
 
 if TYPE_CHECKING:  # pragma: no cover - typing only
-    from .retry_manager import RetryPolicy
+    from .retry_manager import RetryPolicyDict
 
 
 # SECTION: EXPORTS ========================================================== #
@@ -57,7 +57,7 @@ class ApiRequestError(requests.RequestException):
         Number of attempts performed (defaults to ``1``).
     retried : bool, optional
         Whether any retry attempts were made.
-    retry_policy : RetryPolicy | None, optional
+    retry_policy : RetryPolicyDict | None, optional
         The retry policy in effect, if any.
     cause : Exception | None, optional
         Original underlying exception.
@@ -72,7 +72,7 @@ class ApiRequestError(requests.RequestException):
         Number of attempts performed.
     retried : bool
         Whether any retry attempts were made.
-    retry_policy : RetryPolicy | None
+    retry_policy : RetryPolicyDict | None
         The retry policy in effect, if any.
     cause : Exception | None
         Original underlying exception.
@@ -97,7 +97,7 @@ class ApiRequestError(requests.RequestException):
     status: int | None = None
     attempts: int = 1
     retried: bool = False
-    retry_policy: RetryPolicy | None = None
+    retry_policy: RetryPolicyDict | None = None
     cause: Exception | None = None
 
     # -- Magic Methods (Object Representation) -- #

etlplus/api/pagination/__init__.py

@@ -20,10 +20,10 @@ Notes
 from __future__ import annotations
 
 from .client import PaginationClient
-from .config import CursorPaginationConfigMap
-from .config import PagePaginationConfigMap
+from .config import CursorPaginationConfigDict
+from .config import PagePaginationConfigDict
 from .config import PaginationConfig
-from .config import PaginationConfigMap
+from .config import PaginationConfigDict
 from .config import PaginationInput
 from .config import PaginationType
 from .paginator import Paginator
@@ -40,8 +40,8 @@ __all__ = [
     # Enums
     'PaginationType',
     # Type Aliases
-    'CursorPaginationConfigMap',
-    'PagePaginationConfigMap',
+    'CursorPaginationConfigDict',
+    'PagePaginationConfigDict',
     'PaginationInput',
-    'PaginationConfigMap',
+    'PaginationConfigDict',
 ]

etlplus/api/pagination/config.py

@@ -8,7 +8,7 @@ by :mod:`etlplus.api.pagination`. It exposes:
 
 - :class:`PaginationType` – enumeration of supported pagination modes.
 - :class:`PaginationConfig` – normalized configuration container.
-- ``*PaginationConfigMap`` TypedDicts – loose, user-facing config mappings.
+- ``*PaginationConfigDict`` TypedDicts – loose, user-facing config mappings.
 
 Notes
 -----
@@ -44,11 +44,11 @@ __all__ = [
     # Enums
     'PaginationType',
     # Type Aliases
-    'PaginationConfigMap',
+    'PaginationConfigDict',
     'PaginationInput',
     # Typed Dicts
-    'CursorPaginationConfigMap',
-    'PagePaginationConfigMap',
+    'CursorPaginationConfigDict',
+    'PagePaginationConfigDict',
 ]
 
 
@@ -74,7 +74,7 @@ class PaginationType(CoercibleStrEnum):
 # SECTION: TYPED DICTS ====================================================== #
 
 
-class CursorPaginationConfigMap(TypedDict, total=False):
+class CursorPaginationConfigDict(TypedDict, total=False):
     """
     Configuration mapping for cursor-based REST API response pagination.
 
@@ -131,7 +131,7 @@ class CursorPaginationConfigMap(TypedDict, total=False):
     limit_param: str
 
 
-class PagePaginationConfigMap(TypedDict, total=False):
+class PagePaginationConfigDict(TypedDict, total=False):
     """
     Configuration mapping for page-based and offset-based REST API response
     pagination.
@@ -379,7 +379,7 @@ class PaginationConfig(BoundsWarningsMixin):
     @overload
     def from_obj(
         cls,
-        obj: PaginationConfigMap,
+        obj: PaginationConfigDict,
     ) -> Self: ...
 
     @classmethod
@@ -398,7 +398,7 @@ class PaginationConfig(BoundsWarningsMixin):
         Returns
         -------
         Self | None
-            Parsed pagination configuration, or ``None`` if ``obj`` isn't a
+            Parsed pagination configuration, or ``None`` if *obj* isn't a
             mapping.
 
         Notes
@@ -429,12 +429,14 @@ class PaginationConfig(BoundsWarningsMixin):
 # SECTION: TYPE ALIASES ===================================================== #
 
 
-type PaginationConfigMap = PagePaginationConfigMap | CursorPaginationConfigMap
+type PaginationConfigDict = (
+    PagePaginationConfigDict | CursorPaginationConfigDict
+)
 
 # External callers may pass either a raw mapping-shaped config or an already
 # constructed PaginationConfig instance, or omit pagination entirely. Accept a
 # loose mapping here to reflect the runtime behavior while still providing
 # stronger TypedDict hints for common shapes.
 type PaginationInput = (
-    PaginationConfigMap | PaginationConfig | StrAnyMap | None
+    PaginationConfigDict | PaginationConfig | StrAnyMap | None
 )

etlplus/api/pagination/paginator.py

@@ -66,14 +66,14 @@ def _resolve_path(
     path: str | None,
 ) -> Any:
     """
-    Resolve dotted ``path`` within ``obj`` or return ``_MISSING``.
+    Resolve dotted *path* within *obj* or return ``_MISSING``.
 
     Parameters
     ----------
     obj : Any
         JSON payload from an API response.
     path : str | None
-        Dotted path to the target value within ``obj``.
+        Dotted path to the target value within *obj*.
 
     Returns
     -------
@@ -665,7 +665,7 @@ class Paginator:
         records_path : str | None
             Optional dotted path to the records within the payload.
         fallback_path : str | None
-            Secondary dotted path consulted when ``records_path`` resolves to
+            Secondary dotted path consulted when *records_path* resolves to
             ``None`` or an empty list.
 
         Returns
@@ -675,9 +675,8 @@ class Paginator:
 
         Notes
         -----
-        Supports dotted path extraction via ``records_path`` and handles
-        lists, mappings, and scalars by coercing non-dict items into
-        ``{"value": x}``.
+        Supports dotted path extraction via *records_path* and handles lists,
+        mappings, and scalars by coercing non-dict items into ``{"value": x}``.
         """
         resolver = partial(_resolve_path, x)
         data = resolver(records_path)
@@ -729,7 +728,7 @@ class Paginator:
         Returns
         -------
         PaginationType | None
-            Detected pagination type, or ``default`` if not found.
+            Detected pagination type, or *default* if not found.
         """
         if not config:
             return default

etlplus/api/rate_limiting/__init__.py

@@ -19,7 +19,7 @@ Notes
 from __future__ import annotations
 
 from .config import RateLimitConfig
-from .config import RateLimitConfigMap
+from .config import RateLimitConfigDict
 from .config import RateLimitOverrides
 from .rate_limiter import RateLimiter
 
@@ -34,5 +34,5 @@ __all__ = [
     # Type Aliases
     'RateLimitOverrides',
     # Type Dicts
-    'RateLimitConfigMap',
+    'RateLimitConfigDict',
 ]