etlplus 0.16.0__py3-none-any.whl → 0.16.7__py3-none-any.whl

etlplus/api/transport.py

@@ -41,8 +41,8 @@ from ..utils import to_positive_int
 
 __all__ = [
     # Classes
-    'HTTPAdapterMountConfig',
-    'HTTPAdapterRetryConfig',
+    'HTTPAdapterMountConfigDict',
+    'HTTPAdapterRetryConfigDict',
     # Functions
     'build_http_adapter',
     'build_session_with_adapters',
@@ -52,7 +52,7 @@ __all__ = [
 # SECTION: TYPED DICTS ====================================================== #
 
 
-class HTTPAdapterRetryConfig(TypedDict, total=False):
+class HTTPAdapterRetryConfigDict(TypedDict, total=False):
     """
     Retry configuration for urllib3 ``Retry``.
 
@@ -89,7 +89,7 @@ class HTTPAdapterRetryConfig(TypedDict, total=False):
 
     Examples
     --------
-    >>> retry_cfg: HTTPAdapterRetryConfig = {
+    >>> retry_cfg: HTTPAdapterRetryConfigDict = {
     ...     'total': 5,
     ...     'backoff_factor': 0.5,
     ...     'status_forcelist': [429, 503],
@@ -111,7 +111,7 @@ class HTTPAdapterRetryConfig(TypedDict, total=False):
     respect_retry_after_header: bool
 
 
-class HTTPAdapterMountConfig(TypedDict, total=False):
+class HTTPAdapterMountConfigDict(TypedDict, total=False):
     """
     Configuration mapping for mounting an ``HTTPAdapter`` on a ``Session``.
 
@@ -132,13 +132,13 @@ class HTTPAdapterMountConfig(TypedDict, total=False):
     pool_block : bool
         Whether the pool should block for connections instead of creating new
         ones.
-    max_retries : int | HTTPAdapterRetryConfig
+    max_retries : int | HTTPAdapterRetryConfigDict
         Retry configuration passed to ``HTTPAdapter`` (int) or converted to
         ``Retry``.
 
     Examples
     --------
-    >>> adapter_cfg: HTTPAdapterMountConfig = {
+    >>> adapter_cfg: HTTPAdapterMountConfigDict = {
     ...     'prefix': 'https://',
     ...     'pool_connections': 10,
     ...     'pool_maxsize': 10,
@@ -156,7 +156,7 @@ class HTTPAdapterMountConfig(TypedDict, total=False):
     pool_connections: int
     pool_maxsize: int
     pool_block: bool
-    max_retries: int | HTTPAdapterRetryConfig
+    max_retries: int | HTTPAdapterRetryConfigDict
 
 
 # SECTION: INTERNAL FUNCTIONS ============================================== #
@@ -306,7 +306,7 @@ def build_http_adapter(
 
 
 def build_session_with_adapters(
-    adapters_cfg: Sequence[HTTPAdapterMountConfig],
+    adapters_cfg: Sequence[HTTPAdapterMountConfigDict],
 ) -> requests.Session:
     """
     Mount adapters described by *adapters_cfg* onto a new session.
@@ -316,7 +316,7 @@ def build_session_with_adapters(
 
     Parameters
     ----------
-    adapters_cfg : Sequence[HTTPAdapterMountConfig]
+    adapters_cfg : Sequence[HTTPAdapterMountConfigDict]
         Configuration mappings describing the adapter prefix, pooling
         values, and retry policy for each mounted adapter.
 

etlplus/api/types.py

@@ -43,23 +43,47 @@ __all__ = [
     'Params',
     'Url',
     # Typed Dicts
-    'ApiConfigMap',
-    'ApiProfileConfigMap',
-    'ApiProfileDefaultsMap',
-    'EndpointMap',
+    'ApiConfigDict',
+    'ApiProfileConfigDict',
+    'ApiProfileDefaultsDict',
+    'EndpointConfigDict',
 ]
 
 
 # SECTION: CONSTANTS ======================================================== #
 
 
-_UNSET = object()
+_UNSET: object = object()
+
+
+# SECTION: INTERNAL FUNCTIONS =============================================== #
+
+
+def _to_dict(
+    value: Mapping[str, Any] | object | None,
+) -> dict[str, Any] | None:
+    """
+    Return a defensive ``dict`` copy for mapping inputs.
+
+    Parameters
+    ----------
+    value : Mapping[str, Any] | object | None
+        Mapping to copy, or ``None``.
+
+    Returns
+    -------
+    dict[str, Any] | None
+        New ``dict`` instance or ``None`` when the input is ``None``.
+    """
+    if value is None:
+        return None
+    return cast(dict[str, Any], value)
 
 
 # SECTION: TYPED DICTS ====================================================== #
 
 
-class ApiConfigMap(TypedDict, total=False):
+class ApiConfigDict(TypedDict, total=False):
     """
     Top-level API config shape parsed by
     :meth:`etlplus.api.config.ApiConfig.from_obj`.
@@ -75,11 +99,11 @@ class ApiConfigMap(TypedDict, total=False):
 
     base_url: str
     headers: StrAnyMap
-    endpoints: Mapping[str, EndpointMap | str]
-    profiles: Mapping[str, ApiProfileConfigMap]
+    endpoints: Mapping[str, EndpointConfigDict | str]
+    profiles: Mapping[str, ApiProfileConfigDict]
 
 
-class ApiProfileConfigMap(TypedDict, total=False):
+class ApiProfileConfigDict(TypedDict, total=False):
     """
     Shape accepted for a profile entry under
     :meth:`etlplus.api.config.ApiConfig.from_obj`.
@@ -99,10 +123,10 @@ class ApiProfileConfigMap(TypedDict, total=False):
     headers: StrAnyMap
     base_path: str
     auth: StrAnyMap
-    defaults: ApiProfileDefaultsMap
+    defaults: ApiProfileDefaultsDict
 
 
-class ApiProfileDefaultsMap(TypedDict, total=False):
+class ApiProfileDefaultsDict(TypedDict, total=False):
     """
     Defaults block available under a profile (all keys optional).
 
@@ -123,11 +147,11 @@ class ApiProfileDefaultsMap(TypedDict, total=False):
     """
 
     headers: StrAnyMap
-    pagination: StrAnyMap  # PaginationConfigMap | StrAnyMap
-    rate_limit: StrAnyMap  # RateLimitConfigMap | StrAnyMap
+    pagination: StrAnyMap  # PaginationConfigDict | StrAnyMap
+    rate_limit: StrAnyMap  # RateLimitConfigDict | StrAnyMap
 
 
-class EndpointMap(TypedDict, total=False):
+class EndpointConfigDict(TypedDict, total=False):
     """
     Shape accepted by :meth:`etlplus.api.config.EndpointConfig.from_obj`.
 
@@ -144,8 +168,8 @@ class EndpointMap(TypedDict, total=False):
     path_params: StrAnyMap
     query_params: StrAnyMap
     body: Any
-    pagination: StrAnyMap  # PaginationConfigMap | StrAnyMap
-    rate_limit: StrAnyMap  # RateLimitConfigMap | StrAnyMap
+    pagination: StrAnyMap  # PaginationConfigDict | StrAnyMap
+    rate_limit: StrAnyMap  # RateLimitConfigDict | StrAnyMap
 
 
 # SECTION: DATA CLASSES ===================================================== #
@@ -176,9 +200,9 @@ class RequestOptions:
 
     def __post_init__(self) -> None:
         if self.params is not None:
-            object.__setattr__(self, 'params', dict(self.params))
+            object.__setattr__(self, 'params', _to_dict(self.params))
         if self.headers is not None:
-            object.__setattr__(self, 'headers', dict(self.headers))
+            object.__setattr__(self, 'headers', _to_dict(self.headers))
 
     # -- Instance Methods -- #
 
@@ -224,23 +248,20 @@ class RequestOptions:
 
         Returns
         -------
-        RequestOptions
+        Self
             New snapshot reflecting the provided overrides.
         """
         if params is _UNSET:
             next_params = self.params
-        elif params is None:
-            next_params = None
         else:
-            next_params = cast(dict, params)
+            # next_params = _to_dict(params) if params is not None else None
+            next_params = _to_dict(params)
 
         if headers is _UNSET:
             next_headers = self.headers
-        elif headers is None:
-            next_headers = None
         else:
-            next_headers = cast(dict, headers)
-
+            # next_headers = _to_dict(headers) if headers is not None else None
+            next_headers = _to_dict(headers)
         if timeout is _UNSET:
             next_timeout = self.timeout
         else:

etlplus/api/utils.py

@@ -22,11 +22,11 @@ from .config import EndpointConfig
 from .endpoint_client import EndpointClient
 from .enums import HttpMethod
 from .pagination import PaginationConfig
-from .pagination import PaginationConfigMap
+from .pagination import PaginationConfigDict
 from .rate_limiting import RateLimitConfig
-from .rate_limiting import RateLimitConfigMap
+from .rate_limiting import RateLimitConfigDict
 from .rate_limiting import RateLimiter
-from .retry_manager import RetryPolicy
+from .retry_manager import RetryPolicyDict
 from .types import Headers
 from .types import Params
 from .types import Url
@@ -53,16 +53,16 @@ __all__ = [
     'paginate_with_client',
     'resolve_request',
     # Typed Dicts
-    'ApiRequestEnv',
-    'ApiTargetEnv',
-    'SessionConfig',
+    'ApiRequestEnvDict',
+    'ApiTargetEnvDict',
+    'SessionConfigDict',
 ]
 
 
 # SECTION: TYPED DICTS ====================================================== #
 
 
-class BaseApiHttpEnv(TypedDict, total=False):
+class BaseApiHttpEnvDict(TypedDict, total=False):
     """
     Common HTTP request environment for API interactions.
 
@@ -78,7 +78,7 @@ class BaseApiHttpEnv(TypedDict, total=False):
     session: requests.Session | None
 
 
-class ApiRequestEnv(BaseApiHttpEnv, total=False):
+class ApiRequestEnvDict(BaseApiHttpEnvDict, total=False):
     """
     Composed HTTP request environment configuration for REST API sources.
 
@@ -96,15 +96,15 @@ class ApiRequestEnv(BaseApiHttpEnv, total=False):
 
     # Request
     params: dict[str, Any]
-    pagination: PaginationConfigMap | None
+    pagination: PaginationConfigDict | None
     sleep_seconds: float
 
     # Reliability
-    retry: RetryPolicy | None
+    retry: RetryPolicyDict | None
     retry_network_errors: bool
 
 
-class ApiTargetEnv(BaseApiHttpEnv, total=False):
+class ApiTargetEnvDict(BaseApiHttpEnvDict, total=False):
     """
     Composed HTTP request environment configuration for REST API targets.
 
@@ -126,7 +126,7 @@ class ApiTargetEnv(BaseApiHttpEnv, total=False):
     method: str | None
 
 
-class SessionConfig(TypedDict, total=False):
+class SessionConfigDict(TypedDict, total=False):
     """
     Minimal session configuration schema accepted by the
     :class:`requests.Session` runner.
@@ -148,14 +148,14 @@ class SessionConfig(TypedDict, total=False):
 
 
 def _build_session_optional(
-    cfg: SessionConfig | None,
+    cfg: SessionConfigDict | None,
 ) -> requests.Session | None:
     """
     Return a configured session when *cfg* is a mapping.
 
     Parameters
     ----------
-    cfg : SessionConfig | None
+    cfg : SessionConfigDict | None
         Session configuration mapping.
 
     Returns
@@ -164,7 +164,7 @@ def _build_session_optional(
         Configured session or ``None``.
     """
     if isinstance(cfg, Mapping):
-        return build_session(cast(SessionConfig, cfg))
+        return build_session(cast(SessionConfigDict, cfg))
     return None
 
 
@@ -233,9 +233,9 @@ def _inherit_http_from_api_endpoint(
     ep: EndpointConfig,
     url: Url | None,
     headers: dict[str, str],
-    session_cfg: SessionConfig | None,
+    session_cfg: SessionConfigDict | None,
     force_url: bool = False,
-) -> tuple[Url | None, dict[str, str], SessionConfig | None]:
+) -> tuple[Url | None, dict[str, str], SessionConfigDict | None]:
     """
     Return HTTP settings inherited from API + endpoint definitions.
 
@@ -249,14 +249,14 @@ def _inherit_http_from_api_endpoint(
         Existing URL to use when not forcing endpoint URL.
     headers : dict[str, str]
         Existing headers to augment.
-    session_cfg : SessionConfig | None
+    session_cfg : SessionConfigDict | None
         Existing session configuration to augment.
     force_url : bool, optional
         Whether to always use the endpoint URL.
 
     Returns
     -------
-    tuple[Url | None, dict[str, str], SessionConfig | None]
+    tuple[Url | None, dict[str, str], SessionConfigDict | None]
         Resolved URL, headers, and session configuration.
     """
     if force_url or not url:
@@ -269,8 +269,8 @@ def _inherit_http_from_api_endpoint(
 def _merge_session_cfg_three(
     api_cfg: ApiConfig,
     ep: EndpointConfig,
-    source_session_cfg: SessionConfig | None,
-) -> SessionConfig | None:
+    source_session_cfg: SessionConfigDict | None,
+) -> SessionConfigDict | None:
     """
     Merge session configurations from API, endpoint, and source.
 
@@ -280,12 +280,12 @@ def _merge_session_cfg_three(
         API configuration.
     ep : EndpointConfig
         Endpoint configuration.
-    source_session_cfg : SessionConfig | None
+    source_session_cfg : SessionConfigDict | None
         Source session configuration.
 
     Returns
     -------
-    SessionConfig | None
+    SessionConfigDict | None
         Merged session configuration.
     """
     api_sess = getattr(api_cfg, 'session', None)
@@ -297,7 +297,7 @@ def _merge_session_cfg_three(
         merged.update(ep_sess)
     if isinstance(source_session_cfg, Mapping):
         merged.update(source_session_cfg)
-    return cast(SessionConfig | None, (merged or None))
+    return cast(SessionConfigDict | None, (merged or None))
 
 
 def _update_mapping(
@@ -361,7 +361,7 @@ def compose_api_request_env(
     cfg: Any,
     source_obj: Any,
     ex_opts: Mapping[str, Any] | None,
-) -> ApiRequestEnv:
+) -> ApiRequestEnvDict:
     """
     Compose the API request environment.
 
@@ -376,7 +376,7 @@ def compose_api_request_env(
 
     Returns
     -------
-    ApiRequestEnv
+    ApiRequestEnvDict
         The composed API request environment.
     """
     ex_opts = ex_opts or {}
@@ -393,13 +393,13 @@ def compose_api_request_env(
     headers: dict[str, str] = cast(dict[str, str], coerce_dict(source_headers))
     pagination = getattr(source_obj, 'pagination', None)
     rate_limit = getattr(source_obj, 'rate_limit', None)
-    retry: RetryPolicy | None = cast(
-        RetryPolicy | None,
+    retry: RetryPolicyDict | None = cast(
+        RetryPolicyDict | None,
         getattr(source_obj, 'retry', None),
     )
     retry_network_errors = getattr(source_obj, 'retry_network_errors', None)
     session_cfg = cast(
-        SessionConfig | None,
+        SessionConfigDict | None,
         getattr(source_obj, 'session', None),
     )
     api_name = getattr(source_obj, 'api', None)
@@ -435,7 +435,7 @@ def compose_api_request_env(
             api_cfg.effective_rate_limit_defaults(),
         )
         retry = cast(
-            RetryPolicy | None,
+            RetryPolicyDict | None,
             _coalesce(
                 retry,
                 getattr(ep, 'retry', None),
@@ -465,8 +465,8 @@ def compose_api_request_env(
     timeout: Timeout = ex_opts.get('timeout')
     pag_ov = ex_opts.get('pagination', {})
     rl_ov = ex_opts.get('rate_limit', {})
-    rty_ov: RetryPolicy | None = cast(
-        RetryPolicy | None,
+    rty_ov: RetryPolicyDict | None = cast(
+        RetryPolicyDict | None,
         (ex_opts.get('retry') if 'retry' in ex_opts else None),
     )
     rne_ov = (
@@ -474,7 +474,7 @@ def compose_api_request_env(
         if 'retry_network_errors' in ex_opts
         else None
     )
-    sess_ov = cast(SessionConfig | None, ex_opts.get('session'))
+    sess_ov = cast(SessionConfigDict | None, ex_opts.get('session'))
     sleep_s = compute_rl_sleep_seconds(rate_limit, rl_ov) or 0.0
     if rty_ov is not None:
         retry = rty_ov
@@ -485,8 +485,8 @@ def compose_api_request_env(
             cast(Mapping[str, Any], session_cfg or {}),
         )
         base_cfg.update(sess_ov)
-        session_cfg = cast(SessionConfig, base_cfg)
-    pag_cfg: PaginationConfigMap | None = build_pagination_cfg(
+        session_cfg = cast(SessionConfigDict, base_cfg)
+    pag_cfg: PaginationConfigDict | None = build_pagination_cfg(
         pagination,
         pag_ov,
     )
@@ -513,7 +513,7 @@ def compose_api_target_env(
     cfg: Any,
     target_obj: Any,
     overrides: Mapping[str, Any] | None,
-) -> ApiTargetEnv:
+) -> ApiTargetEnvDict:
     """
     Compose the API target environment.
 
@@ -528,7 +528,7 @@ def compose_api_target_env(
 
     Returns
     -------
-    ApiTargetEnv
+    ApiTargetEnvDict
         Composed API target environment.
     """
     ov = overrides or {}
@@ -553,8 +553,8 @@ def compose_api_target_env(
     timeout: Timeout = (
         cast(Timeout, ov.get('timeout')) if 'timeout' in ov else None
     )
-    sess_cfg: SessionConfig | None = cast(
-        SessionConfig | None,
+    sess_cfg: SessionConfigDict | None = cast(
+        SessionConfigDict | None,
         ov.get('session'),
     )
     api_name = getattr(target_obj, 'api', None)
@@ -583,7 +583,7 @@ def compose_api_target_env(
 def build_pagination_cfg(
     pagination: PaginationConfig | None,
     overrides: Mapping[str, Any] | None,
-) -> PaginationConfigMap | None:
+) -> PaginationConfigDict | None:
     """
     Build pagination configuration.
 
@@ -596,7 +596,7 @@ def build_pagination_cfg(
 
     Returns
     -------
-    PaginationConfigMap | None
+    PaginationConfigDict | None
         Pagination configuration.
     """
     ptype: str | None = None
@@ -683,7 +683,7 @@ def build_pagination_cfg(
         case _:
             pass
 
-    return cast(PaginationConfigMap, cfg)
+    return cast(PaginationConfigDict, cfg)
 
 
 def paginate_with_client(
@@ -692,7 +692,7 @@ def paginate_with_client(
     params: Params | None,
     headers: Headers | None,
     timeout: Timeout,
-    pagination: PaginationConfigMap | None,
+    pagination: PaginationConfigDict | None,
     sleep_seconds: float | None,
 ) -> Any:
     """
@@ -710,7 +710,7 @@ def paginate_with_client(
         Headers to include in the API request.
     timeout : Timeout
         Timeout configuration for the API request.
-    pagination : PaginationConfigMap | None
+    pagination : PaginationConfigDict | None
         Pagination configuration for the API request.
     sleep_seconds : float | None
         Sleep duration between API requests.
@@ -771,9 +771,9 @@ def compute_rl_sleep_seconds(
     else:
         rl_map = cast(Mapping[str, Any] | None, rate_limit)
 
-    rl_mapping = cast(RateLimitConfigMap | None, rl_map)
+    rl_mapping = cast(RateLimitConfigDict | None, rl_map)
 
-    typed_override: RateLimitConfigMap | None = None
+    typed_override: RateLimitConfigDict | None = None
     if overrides:
         filtered: dict[str, float | None] = {}
         if 'sleep_seconds' in overrides:
@@ -787,7 +787,7 @@ def compute_rl_sleep_seconds(
                 overrides.get('max_per_sec'),
             )
         if filtered:
-            typed_override = cast(RateLimitConfigMap, filtered)
+            typed_override = cast(RateLimitConfigDict, filtered)
 
     return RateLimiter.resolve_sleep_seconds(
         rate_limit=rl_mapping,
@@ -796,14 +796,14 @@ def compute_rl_sleep_seconds(
 
 
 def build_session(
-    cfg: SessionConfig | None,
+    cfg: SessionConfigDict | None,
 ) -> requests.Session:
     """
     Build a requests.Session object with the given configuration.
 
     Parameters
     ----------
-    cfg : SessionConfig | None
+    cfg : SessionConfigDict | None
         Session configuration.
 
     Returns

etlplus/cli/README.md

@@ -3,7 +3,7 @@
 Documentation for the `etlplus.cli` subpackage: command-line interface for ETLPlus workflows.
 
 - Provides a CLI for running ETL pipelines, jobs, and utilities
-- Supports commands for running, validating, and inspecting pipelines
+- Supports commands for extracting, transforming, validating data, and running pipelines
 - Includes options for configuration, state, and output control
 - Exposes handlers for custom command integration
 
@@ -17,20 +17,22 @@ Back to project overview: see the top-level [README](../../README.md).
 
 ## Available Commands
 
+- **check**: Inspect pipeline configuration (jobs, sources, targets)
+- **extract**: Extract data from files/APIs/databases
+- **transform**: Transform records
+- **load**: Load data to files/APIs/databases
+- **render**: Render SQL DDL from table specs
+- **validate**: Validate data against rules
 - **run**: Execute a pipeline or job
-- **validate**: Validate pipeline or config files
-- **inspect**: Show pipeline/job details
 
 ## Command Options
 
-- `--config`: Path to config file
-- `--state`: Path to state file
-- `--output`: Output file or format
+Use `etlplus <command> --help` for the exact options supported by each command.
 
 ## Example: Running a Pipeline
 
 ```bash
-etlplus run --config configs/pipeline.yml --output results.json
+etlplus run --config configs/pipeline.yml --job file_to_file_customers
 ```
 
 ## See Also