etlplus 0.12.10__py3-none-any.whl → 0.14.3__py3-none-any.whl

etlplus/{run_helpers.py → api/utils.py}

@@ -1,30 +1,13 @@
 """
-:mod:`etlplus.run_helpers` module.
-
-Helper functions and small utilities used by ``etlplus.run`` to compose API
-request/load environments, pagination configs, session objects, and endpoint
-clients. Extracted to keep ``run.py`` focused on orchestration while enabling
-reuse and testability.
-
-Public (re-export safe) helpers:
-- build_pagination_cfg(pagination, overrides)
-- build_session(cfg)
-- compose_api_request_env(cfg, source_obj, extract_opts)
-- compose_api_target_env(cfg, target_obj, overrides)
-- build_endpoint_client(base_url, base_path, endpoints, env)
-- compute_rl_sleep_seconds(rate_limit, overrides)
-- paginate_with_client(client, endpoint_key, params, headers,
-    timeout, pagination, sleep_seconds)
-
-Notes
------
-These helpers intentionally accept permissive ``Any``/``Mapping`` inputs to
-avoid tight coupling with config dataclasses while keeping runtime flexible.
+:mod:`etlplus.api.utils` module.
+
+Shared HTTP helpers for API clients that communicate with REST endpoints.
 """
 
 from __future__ import annotations
 
 import inspect
+from collections.abc import Callable
 from collections.abc import Mapping
 from typing import Any
 from typing import TypedDict
@@ -32,24 +15,34 @@ from typing import cast
 
 import requests  # type: ignore[import]
 
-from .api import ApiConfig
-from .api import EndpointClient
-from .api import EndpointConfig
-from .api import Headers
-from .api import PaginationConfig
-from .api import PaginationConfigMap
-from .api import Params
-from .api import RateLimitConfig
-from .api import RateLimitConfigMap
-from .api import RateLimiter
-from .api import RetryPolicy
-from .api import Url
-from .types import Timeout
+from ..types import Timeout
+from ..utils import coerce_dict
+from .config import ApiConfig
+from .config import EndpointConfig
+from .endpoint_client import EndpointClient
+from .enums import HttpMethod
+from .pagination import PaginationConfig
+from .pagination import PaginationConfigMap
+from .rate_limiting import RateLimitConfig
+from .rate_limiting import RateLimitConfigMap
+from .rate_limiting import RateLimiter
+from .retry_manager import RetryPolicy
+from .types import Headers
+from .types import Params
+from .types import Url
+
+# SECTION: CONSTANTS ======================================================== #
+
+
+DEFAULT_TIMEOUT: float = 10.0
+
 
 # SECTION: EXPORTS ========================================================== #
 
 
 __all__ = [
+    # Constants
+    'DEFAULT_TIMEOUT',
     # Functions
     'build_endpoint_client',
     'build_pagination_cfg',
@@ -58,6 +51,7 @@ __all__ = [
     'compose_api_target_env',
     'compute_rl_sleep_seconds',
     'paginate_with_client',
+    'resolve_request',
     # Typed Dicts
     'ApiRequestEnv',
     'ApiTargetEnv',
@@ -68,43 +62,83 @@ __all__ = [
 # SECTION: TYPED DICTS ====================================================== #
 
 
-class ApiRequestEnv(TypedDict, total=False):
-    """API request environment configuration."""
+class BaseApiHttpEnv(TypedDict, total=False):
+    """
+    Common HTTP request environment for API interactions.
+
+    Fields shared by both source-side and target-side API operations.
+    """
 
+    # Request details
     url: Url | None
     headers: dict[str, str]
     timeout: Timeout
+
+    # Session
     session: requests.Session | None
+
+
+class ApiRequestEnv(BaseApiHttpEnv, total=False):
+    """
+    Composed HTTP request environment configuration for REST API sources.
+
+    Returned by :func:`compose_api_request_env` and consumed by the API extract
+    branch. Values are fully merged with endpoint/API defaults and job-level
+    overrides, preserving the original precedence and behavior.
+    """
+
+    # Client
     use_endpoints: bool
     base_url: str | None
     base_path: str | None
     endpoints_map: dict[str, str] | None
     endpoint_key: str | None
+
+    # Request
     params: dict[str, Any]
     pagination: PaginationConfigMap | None
     sleep_seconds: float
+
+    # Reliability
     retry: RetryPolicy | None
     retry_network_errors: bool
 
 
-class ApiTargetEnv(TypedDict, total=False):
-    """API target environment configuration."""
+class ApiTargetEnv(BaseApiHttpEnv, total=False):
+    """
+    Composed HTTP request environment configuration for REST API targets.
+
+    Returned by :func:`compose_api_target_env` and consumed by the API load
+    branch. Values are merged from the target object, optional API/endpoint
+    reference, and job-level overrides, preserving original precedence and
+    behavior.
+
+    Notes
+    -----
+    - Precedence for inherited values matches original logic:
+        overrides -> target -> API profile defaults.
+    - Target composition does not include pagination/rate-limit/retry since
+        loads are single-request operations; only headers/timeout/session
+        apply.
+    """
 
-    url: Url | None
-    headers: dict[str, str]
-    timeout: Timeout
-    session: requests.Session | None
+    # Request
     method: str | None
 
 
 class SessionConfig(TypedDict, total=False):
-    """Configuration for requests.Session."""
+    """
+    Minimal session configuration schema accepted by the
+    :class:`requests.Session` runner.
+
+    Keys mirror common :class:`requests.Session` options; all are optional.
+    """
 
     headers: Mapping[str, Any]
     params: Mapping[str, Any]
-    auth: Any
+    auth: Any  # (user, pass) tuple or requests-compatible auth object
     verify: bool | str
-    cert: Any
+    cert: Any  # str or (cert, key)
     proxies: Mapping[str, Any]
     cookies: Mapping[str, Any]
     trust_env: bool
@@ -113,7 +147,47 @@ class SessionConfig(TypedDict, total=False):
 # SECTION: INTERNAL FUNCTIONS ============================================== #
 
 
-# -- API Environment Composition -- #
+def _build_session_optional(
+    cfg: SessionConfig | None,
+) -> requests.Session | None:
+    """
+    Return a configured session when *cfg* is a mapping.
+
+    Parameters
+    ----------
+    cfg : SessionConfig | None
+        Session configuration mapping.
+
+    Returns
+    -------
+    requests.Session | None
+        Configured session or ``None``.
+    """
+    if isinstance(cfg, Mapping):
+        return build_session(cast(SessionConfig, cfg))
+    return None
+
+
+def _coalesce(
+    *args: Any,
+) -> Any | None:
+    """
+    Return the first non-``None`` value from ``args``.
+
+    Parameters
+    ----------
+    *args : Any
+        Candidate values in descending precedence order.
+
+    Returns
+    -------
+    Any | None
+        The first non-``None`` value, or ``None`` if all are ``None``.
+    """
+    for arg in args:
+        if arg is not None:
+            return arg
+    return None
 
 
 def _get_api_cfg_and_endpoint(
@@ -217,37 +291,15 @@ def _merge_session_cfg_three(
     api_sess = getattr(api_cfg, 'session', None)
     ep_sess = getattr(ep, 'session', None)
     merged: dict[str, Any] = {}
-    if isinstance(api_sess, dict):
+    if isinstance(api_sess, Mapping):
         merged.update(api_sess)
-    if isinstance(ep_sess, dict):
+    if isinstance(ep_sess, Mapping):
         merged.update(ep_sess)
-    if isinstance(source_session_cfg, dict):
+    if isinstance(source_session_cfg, Mapping):
         merged.update(source_session_cfg)
     return cast(SessionConfig | None, (merged or None))
 
 
-# -- Mapping Helpers -- #
-
-
-def _copy_mapping(
-    mapping: Mapping[str, Any] | None,
-) -> dict[str, Any]:
-    """
-    Return a shallow copy of *mapping* or an empty dict.
-
-    Parameters
-    ----------
-    mapping : Mapping[str, Any] | None
-        The mapping to copy.
-
-    Returns
-    -------
-    dict[str, Any]
-        A shallow copy of the mapping or an empty dict.
-    """
-    return dict(mapping) if isinstance(mapping, Mapping) else {}
-
-
 def _update_mapping(
     target: dict[str, Any],
     extra: Mapping[str, Any] | None,
@@ -266,37 +318,9 @@ def _update_mapping(
         target.update(extra)
 
 
-# -- Session -- #
-
-
-def _build_session_optional(
-    cfg: SessionConfig | None,
-) -> requests.Session | None:
-    """
-    Return a configured session when *cfg* is a mapping.
-
-    Parameters
-    ----------
-    cfg : SessionConfig | None
-        Session configuration mapping.
-
-    Returns
-    -------
-    requests.Session | None
-        Configured session or ``None``.
-    """
-
-    if isinstance(cfg, dict):
-        return build_session(cfg)
-    return None
-
-
 # SECTION: FUNCTIONS ======================================================== #
 
 
-# -- API Environment Composition -- #
-
-
 def build_endpoint_client(
     *,
     base_url: str,
@@ -323,15 +347,7 @@ def build_endpoint_client(
     EndpointClient
         The constructed endpoint client.
     """
-    # Allow tests to monkeypatch etlplus.run.EndpointClient and have it
-    # propagate here by preferring the class on the run module if present.
-    try:
-        from . import run as run_mod  # local import to avoid cycles
-
-        ClientClass = getattr(run_mod, 'EndpointClient', EndpointClient)
-    except (ImportError, AttributeError):  # pragma: no cover - fallback path
-        ClientClass = EndpointClient
-    return ClientClass(
+    return EndpointClient(
         base_url=base_url,
         base_path=base_path,
         endpoints=endpoints,
@@ -369,21 +385,19 @@ def compose_api_request_env(
         Mapping[str, Any] | None,
         getattr(source_obj, 'query_params', None),
     )
-    params: dict[str, Any] = _copy_mapping(source_params)
+    params: dict[str, Any] = coerce_dict(source_params)
     source_headers = cast(
         Mapping[str, str] | None,
         getattr(source_obj, 'headers', None),
     )
-    headers: dict[str, str] = _copy_mapping(source_headers)
+    headers: dict[str, str] = cast(dict[str, str], coerce_dict(source_headers))
     pagination = getattr(source_obj, 'pagination', None)
     rate_limit = getattr(source_obj, 'rate_limit', None)
     retry: RetryPolicy | None = cast(
         RetryPolicy | None,
         getattr(source_obj, 'retry', None),
     )
-    retry_network_errors = bool(
-        getattr(source_obj, 'retry_network_errors', False),
-    )
+    retry_network_errors = getattr(source_obj, 'retry_network_errors', None)
     session_cfg = cast(
         SessionConfig | None,
         getattr(source_obj, 'session', None),
@@ -405,33 +419,33 @@ def compose_api_request_env(
             session_cfg,
             force_url=True,
         )
-        ep_params: dict[str, Any] = _copy_mapping(
+        ep_params: dict[str, Any] = coerce_dict(
             cast(Mapping[str, Any] | None, getattr(ep, 'query_params', None)),
         )
         _update_mapping(ep_params, params)
         params = ep_params
-        pagination = (
-            pagination
-            or ep.pagination
-            or api_cfg.effective_pagination_defaults()
+        pagination = _coalesce(
+            pagination,
+            ep.pagination,
+            api_cfg.effective_pagination_defaults(),
         )
-        rate_limit = (
-            rate_limit
-            or ep.rate_limit
-            or api_cfg.effective_rate_limit_defaults()
+        rate_limit = _coalesce(
+            rate_limit,
+            ep.rate_limit,
+            api_cfg.effective_rate_limit_defaults(),
         )
         retry = cast(
             RetryPolicy | None,
-            (
-                retry
-                or getattr(ep, 'retry', None)
-                or getattr(api_cfg, 'retry', None)
+            _coalesce(
+                retry,
+                getattr(ep, 'retry', None),
+                getattr(api_cfg, 'retry', None),
             ),
         )
-        retry_network_errors = (
-            retry_network_errors
-            or bool(getattr(ep, 'retry_network_errors', False))
-            or bool(getattr(api_cfg, 'retry_network_errors', False))
+        retry_network_errors = _coalesce(
+            retry_network_errors,
+            getattr(ep, 'retry_network_errors', None),
+            getattr(api_cfg, 'retry_network_errors', None),
         )
         use_client_endpoints = True
         client_base_url = api_cfg.base_url
@@ -466,8 +480,10 @@ def compose_api_request_env(
         retry = rty_ov
     if rne_ov is not None:
         retry_network_errors = bool(rne_ov)
-    if isinstance(sess_ov, dict):
-        base_cfg: dict[str, Any] = dict(cast(dict, session_cfg or {}))
+    if isinstance(sess_ov, Mapping):
+        base_cfg: dict[str, Any] = dict(
+            cast(Mapping[str, Any], session_cfg or {}),
+        )
         base_cfg.update(sess_ov)
         session_cfg = cast(SessionConfig, base_cfg)
     pag_cfg: PaginationConfigMap | None = build_pagination_cfg(
@@ -488,7 +504,7 @@ def compose_api_request_env(
         'pagination': pag_cfg,
         'sleep_seconds': sleep_s,
         'retry': retry,
-        'retry_network_errors': retry_network_errors,
+        'retry_network_errors': bool(retry_network_errors),
         'session': sess_obj,
     }
 
@@ -524,8 +540,14 @@ def compose_api_target_env(
         str | None,
         ov.get('method') or getattr(target_obj, 'method', 'post'),
     )
-    headers = _copy_mapping(
-        cast(Mapping[str, str] | None, getattr(target_obj, 'headers', None)),
+    headers = cast(
+        dict[str, str],
+        coerce_dict(
+            cast(
+                Mapping[str, str] | None,
+                getattr(target_obj, 'headers', None),
+            ),
+        ),
     )
     _update_mapping(headers, cast(Mapping[str, str] | None, ov.get('headers')))
     timeout: Timeout = (
@@ -558,9 +580,6 @@ def compose_api_target_env(
     }
 
 
-# -- Pagination -- #
-
-
 def build_pagination_cfg(
     pagination: PaginationConfig | None,
     overrides: Mapping[str, Any] | None,
@@ -667,9 +686,6 @@ def build_pagination_cfg(
     return cast(PaginationConfigMap, cfg)
 
 
-# -- Pagination Invocation -- #
-
-
 def paginate_with_client(
     client: Any,
     endpoint_key: str,
@@ -727,9 +743,6 @@ def paginate_with_client(
     return client.paginate(endpoint_key, **kw_pag)
 
 
-# -- Rate Limit -- #
-
-
 def compute_rl_sleep_seconds(
     rate_limit: RateLimitConfig | Mapping[str, Any] | None,
     overrides: Mapping[str, Any] | None,
@@ -782,9 +795,6 @@ def compute_rl_sleep_seconds(
     )
 
 
-# -- Session -- #
-
-
 def build_session(
     cfg: SessionConfig | None,
 ) -> requests.Session:
@@ -805,12 +815,12 @@ def build_session(
     if not cfg:
         return s
     headers = cfg.get('headers')
-    if isinstance(headers, dict):
+    if isinstance(headers, Mapping):
         s.headers.update(headers)
     params = cfg.get('params')
-    if isinstance(params, dict):
+    if isinstance(params, Mapping):
         try:
-            s.params = params
+            s.params = dict(params)
         except (AttributeError, TypeError):
             pass
     auth = cfg.get('auth')
@@ -825,12 +835,12 @@ def build_session(
     if cert is not None:
         s.cert = cert  # type: ignore[assignment]
     proxies = cfg.get('proxies')
-    if isinstance(proxies, dict):
+    if isinstance(proxies, Mapping):
         s.proxies.update(proxies)
     cookies = cfg.get('cookies')
-    if isinstance(cookies, dict):
+    if isinstance(cookies, Mapping):
         try:
-            s.cookies.update(cookies)
+            s.cookies.update(cast(Mapping[str, Any], cookies))
         except (TypeError, ValueError):
             pass
     if 'trust_env' in cfg:
@@ -841,3 +851,45 @@ def build_session(
             pass
 
     return s
+
+
+def resolve_request(
+    method: HttpMethod | str,
+    *,
+    session: Any | None = None,
+    timeout: Timeout = None,
+) -> tuple[Callable[..., requests.Response], float, HttpMethod]:
+    """
+    Resolve a request callable and effective timeout for an HTTP method.
+
+    Parameters
+    ----------
+    method : HttpMethod | str
+        HTTP method to execute.
+    session : Any | None, optional
+        Requests-compatible session object. Defaults to module-level
+        ``requests``.
+    timeout : Timeout, optional
+        Timeout in seconds for the request. Uses ``DEFAULT_TIMEOUT`` when
+        omitted.
+
+    Returns
+    -------
+    tuple[Callable[..., requests.Response], float, HttpMethod]
+        Tuple of (callable, timeout_seconds, resolved_method).
+
+    Raises
+    ------
+    TypeError
+        If the session object does not expose the requested HTTP method.
+    """
+    http_method = HttpMethod.coerce(method)
+    request_timeout = DEFAULT_TIMEOUT if timeout is None else timeout
+    requester = session or requests
+    request_callable = getattr(requester, http_method.value, None)
+    if not callable(request_callable):
+        raise TypeError(
+            'Session object must supply a callable '
+            f'"{http_method.value}" method',
+        )
+    return request_callable, request_timeout, http_method

etlplus/cli/handlers.py

@@ -18,15 +18,16 @@ from ..config import PipelineConfig
 from ..config import load_pipeline_config
 from ..database import load_table_spec
 from ..database import render_tables
-from ..extract import extract
 from ..file import File
-from ..load import load
-from ..run import run
-from ..transform import transform
+from ..file import FileFormat
+from ..ops import extract
+from ..ops import load
+from ..ops import run
+from ..ops import transform
+from ..ops import validate
+from ..ops.validate import FieldRules
 from ..types import JSONData
 from ..types import TemplateKey
-from ..validate import FieldRules
-from ..validate import validate
 from . import io as cli_io
 
 # SECTION: EXPORTS ========================================================== #
@@ -569,8 +570,17 @@ def transform_handler(
 
     data = transform(payload, cast(TransformOperations, operations_payload))
 
+    # TODO: Generalize to handle non-file targets.
     if target and target != '-':
-        File(target, file_format=target_format).write(data)
+        # Convert target to Path and target_format to FileFormat if needed
+        file_path = Path(target)
+        file_format = None
+        if target_format is not None:
+            try:
+                file_format = FileFormat(target_format)
+            except ValueError:
+                file_format = None  # or handle error as appropriate
+        File(file_path, file_format=file_format).write(data)
         print(f'Data transformed and saved to {target}')
         return 0
 

etlplus/config/jobs.py

@@ -34,10 +34,7 @@ __all__ = [
 ]
 
 
-# SECTION: TYPE ALIASES ===================================================== #
-
-
-# SECTION: CLASSES ========================================================== #
+# SECTION: DATA CLASSES ===================================================== #
 
 
 @dataclass(kw_only=True, slots=True)
@@ -100,6 +97,8 @@ class JobConfig:
         Unique job name.
     description : str | None
         Optional human-friendly description.
+    depends_on : list[str]
+        Optional job dependency list. Dependencies must refer to other jobs.
     extract : ExtractRef | None
         Extraction reference.
     validate : ValidationRef | None
@@ -114,6 +113,7 @@ class JobConfig:
 
     name: str
     description: str | None = None
+    depends_on: list[str] = field(default_factory=list)
     extract: ExtractRef | None = None
     validate: ValidationRef | None = None
     transform: TransformRef | None = None
@@ -149,9 +149,19 @@ class JobConfig:
         if description is not None and not isinstance(description, str):
             description = str(description)
 
+        depends_raw = data.get('depends_on')
+        depends_on: list[str] = []
+        if isinstance(depends_raw, str):
+            depends_on = [depends_raw]
+        elif isinstance(depends_raw, list):
+            for entry in depends_raw:
+                if isinstance(entry, str):
+                    depends_on.append(entry)
+
         return cls(
             name=name,
             description=description,
+            depends_on=depends_on,
             extract=ExtractRef.from_obj(data.get('extract')),
             validate=ValidationRef.from_obj(data.get('validate')),
             transform=TransformRef.from_obj(data.get('transform')),