etlplus 0.12.13__py3-none-any.whl → 0.14.0__py3-none-any.whl

etlplus/README.md

@@ -23,7 +23,7 @@ Back to project overview: see the top-level [README](../README.md).
 ## Quickstart
 
 ```python
-from etlplus import extract, validate, transform, load
+from etlplus.ops import extract, validate, transform, load
 
 data = extract("file", "input.csv")
 filtered = transform(data, {"filter": {"field": "age", "op": "gt", "value": 25}})

etlplus/__init__.py

@@ -2,42 +2,17 @@
 :mod:`etlplus` package.
 
 Top-level facade for the ETLPlus toolkit.
-
-Importing :mod:`etlplus` exposes the handful of coarse-grained helpers most
-users care about: ``extract``, ``transform``, ``load``, ``validate``, and
-``run``. Each helper delegates to the richer modules under ``etlplus.*`` while
-presenting a compact public API surface.
-
-Examples
---------
->>> from etlplus import extract, transform
->>> raw = extract('file', 'input.json')
->>> curated = transform(raw, {'select': ['id', 'name']})
-
-See Also
---------
-- :mod:`etlplus.cli` for the command-line interface
-- :mod:`etlplus.run` for orchestrating pipeline jobs
 """
 
 from .__version__ import __version__
 
 __author__ = 'ETLPlus Team'
 
-from .extract import extract
-from .load import load
-from .run import run
-from .transform import transform
-from .validate import validate
 
 # SECTION: EXPORTS ========================================================== #
 
 
 __all__ = [
+    '__author__',
     '__version__',
-    'extract',
-    'load',
-    'run',
-    'transform',
-    'validate',
 ]

etlplus/api/__init__.py

@@ -98,6 +98,10 @@ from .types import Headers
 from .types import Params
 from .types import RequestOptions
 from .types import Url
+from .utils import compose_api_request_env
+from .utils import compose_api_target_env
+from .utils import paginate_with_client
+from .utils import resolve_request
 
 # SECTION: EXPORTS ========================================================== #
 
@@ -122,6 +126,10 @@ __all__ = [
     'PaginationType',
     # Functions
     'build_http_adapter',
+    'compose_api_request_env',
+    'compose_api_target_env',
+    'paginate_with_client',
+    'resolve_request',
     # Type Aliases
     'CursorPaginationConfigMap',
     'Headers',

etlplus/api/endpoint_client.py

@@ -455,7 +455,7 @@ class EndpointClient:
         -------
         JSONData
             Parsed JSON payload or fallback structure matching
-            :func:`etlplus.extract.extract_from_api` semantics.
+            :func:`etlplus.ops.extract.extract_from_api` semantics.
         """
         return self._request_manager.get(url, **kwargs)
 
@@ -479,7 +479,7 @@ class EndpointClient:
         -------
         JSONData
             Parsed JSON payload or fallback structure matching
-            :func:`etlplus.extract.extract_from_api` semantics.
+            :func:`etlplus.ops.extract.extract_from_api` semantics.
         """
         return self._request_manager.post(url, **kwargs)
 
@@ -506,7 +506,7 @@ class EndpointClient:
         -------
         JSONData
             Parsed JSON payload or fallback structure matching
-            :func:`etlplus.extract.extract_from_api` semantics.
+            :func:`etlplus.ops.extract.extract_from_api` semantics.
         """
         return self._request_manager.request(method, url, **kwargs)
 

etlplus/{run_helpers.py → api/utils.py}

@@ -1,30 +1,13 @@
 """
-:mod:`etlplus.run_helpers` module.
-
-Helper functions and small utilities used by ``etlplus.run`` to compose API
-request/load environments, pagination configs, session objects, and endpoint
-clients. Extracted to keep ``run.py`` focused on orchestration while enabling
-reuse and testability.
-
-Public (re-export safe) helpers:
-- build_pagination_cfg(pagination, overrides)
-- build_session(cfg)
-- compose_api_request_env(cfg, source_obj, extract_opts)
-- compose_api_target_env(cfg, target_obj, overrides)
-- build_endpoint_client(base_url, base_path, endpoints, env)
-- compute_rl_sleep_seconds(rate_limit, overrides)
-- paginate_with_client(client, endpoint_key, params, headers,
-    timeout, pagination, sleep_seconds)
-
-Notes
------
-These helpers intentionally accept permissive ``Any``/``Mapping`` inputs to
-avoid tight coupling with config dataclasses while keeping runtime flexible.
+:mod:`etlplus.api.utils` module.
+
+Shared HTTP helpers for API clients that communicate with REST endpoints.
 """
 
 from __future__ import annotations
 
 import inspect
+from collections.abc import Callable
 from collections.abc import Mapping
 from typing import Any
 from typing import TypedDict
@@ -32,24 +15,33 @@ from typing import cast
 
 import requests  # type: ignore[import]
 
-from .api import ApiConfig
-from .api import EndpointClient
-from .api import EndpointConfig
-from .api import Headers
-from .api import PaginationConfig
-from .api import PaginationConfigMap
-from .api import Params
-from .api import RateLimitConfig
-from .api import RateLimitConfigMap
-from .api import RateLimiter
-from .api import RetryPolicy
-from .api import Url
-from .types import Timeout
+from ..enums import HttpMethod
+from ..types import Timeout
+from .config import ApiConfig
+from .config import EndpointConfig
+from .endpoint_client import EndpointClient
+from .pagination import PaginationConfig
+from .pagination import PaginationConfigMap
+from .rate_limiting import RateLimitConfig
+from .rate_limiting import RateLimitConfigMap
+from .rate_limiting import RateLimiter
+from .retry_manager import RetryPolicy
+from .types import Headers
+from .types import Params
+from .types import Url
+
+# SECTION: CONSTANTS ======================================================== #
+
+
+DEFAULT_TIMEOUT: float = 10.0
+
 
 # SECTION: EXPORTS ========================================================== #
 
 
 __all__ = [
+    # Constants
+    'DEFAULT_TIMEOUT',
     # Functions
     'build_endpoint_client',
     'build_pagination_cfg',
@@ -58,6 +50,7 @@ __all__ = [
     'compose_api_target_env',
     'compute_rl_sleep_seconds',
     'paginate_with_client',
+    'resolve_request',
     # Typed Dicts
     'ApiRequestEnv',
     'ApiTargetEnv',
@@ -68,43 +61,83 @@ __all__ = [
 # SECTION: TYPED DICTS ====================================================== #
 
 
-class ApiRequestEnv(TypedDict, total=False):
-    """API request environment configuration."""
+class BaseApiHttpEnv(TypedDict, total=False):
+    """
+    Common HTTP request environment for API interactions.
+
+    Fields shared by both source-side and target-side API operations.
+    """
 
+    # Request details
     url: Url | None
     headers: dict[str, str]
     timeout: Timeout
+
+    # Session
     session: requests.Session | None
+
+
+class ApiRequestEnv(BaseApiHttpEnv, total=False):
+    """
+    Composed HTTP request environment configuration for REST API sources.
+
+    Returned by :func:`compose_api_request_env` and consumed by the API extract
+    branch. Values are fully merged with endpoint/API defaults and job-level
+    overrides, preserving the original precedence and behavior.
+    """
+
+    # Client
     use_endpoints: bool
     base_url: str | None
     base_path: str | None
     endpoints_map: dict[str, str] | None
     endpoint_key: str | None
+
+    # Request
     params: dict[str, Any]
     pagination: PaginationConfigMap | None
     sleep_seconds: float
+
+    # Reliability
     retry: RetryPolicy | None
     retry_network_errors: bool
 
 
-class ApiTargetEnv(TypedDict, total=False):
-    """API target environment configuration."""
+class ApiTargetEnv(BaseApiHttpEnv, total=False):
+    """
+    Composed HTTP request environment configuration for REST API targets.
+
+    Returned by :func:`compose_api_target_env` and consumed by the API load
+    branch. Values are merged from the target object, optional API/endpoint
+    reference, and job-level overrides, preserving original precedence and
+    behavior.
+
+    Notes
+    -----
+    - Precedence for inherited values matches original logic:
+        overrides -> target -> API profile defaults.
+    - Target composition does not include pagination/rate-limit/retry since
+        loads are single-request operations; only headers/timeout/session
+        apply.
+    """
 
-    url: Url | None
-    headers: dict[str, str]
-    timeout: Timeout
-    session: requests.Session | None
+    # Request
     method: str | None
 
 
 class SessionConfig(TypedDict, total=False):
-    """Configuration for requests.Session."""
+    """
+    Minimal session configuration schema accepted by the
+    :class:`requests.Session` runner.
+
+    Keys mirror common :class:`requests.Session` options; all are optional.
+    """
 
     headers: Mapping[str, Any]
     params: Mapping[str, Any]
-    auth: Any
+    auth: Any  # (user, pass) tuple or requests-compatible auth object
     verify: bool | str
-    cert: Any
+    cert: Any  # str or (cert, key)
     proxies: Mapping[str, Any]
     cookies: Mapping[str, Any]
     trust_env: bool
@@ -113,9 +146,6 @@ class SessionConfig(TypedDict, total=False):
 # SECTION: INTERNAL FUNCTIONS ============================================== #
 
 
-# -- API Environment Composition -- #
-
-
 def _get_api_cfg_and_endpoint(
     cfg: Any,
     api_name: str,
@@ -226,9 +256,6 @@ def _merge_session_cfg_three(
     return cast(SessionConfig | None, (merged or None))
 
 
-# -- Mapping Helpers -- #
-
-
 def _copy_mapping(
     mapping: Mapping[str, Any] | None,
 ) -> dict[str, Any]:
@@ -266,9 +293,6 @@ def _update_mapping(
         target.update(extra)
 
 
-# -- Session -- #
-
-
 def _build_session_optional(
     cfg: SessionConfig | None,
 ) -> requests.Session | None:
@@ -285,7 +309,6 @@ def _build_session_optional(
     requests.Session | None
         Configured session or ``None``.
     """
-
     if isinstance(cfg, dict):
         return build_session(cfg)
     return None
@@ -294,9 +317,6 @@ def _build_session_optional(
 # SECTION: FUNCTIONS ======================================================== #
 
 
-# -- API Environment Composition -- #
-
-
 def build_endpoint_client(
     *,
     base_url: str,
@@ -323,15 +343,7 @@ def build_endpoint_client(
     EndpointClient
         The constructed endpoint client.
     """
-    # Allow tests to monkeypatch etlplus.run.EndpointClient and have it
-    # propagate here by preferring the class on the run module if present.
-    try:
-        from . import run as run_mod  # local import to avoid cycles
-
-        ClientClass = getattr(run_mod, 'EndpointClient', EndpointClient)
-    except (ImportError, AttributeError):  # pragma: no cover - fallback path
-        ClientClass = EndpointClient
-    return ClientClass(
+    return EndpointClient(
         base_url=base_url,
         base_path=base_path,
         endpoints=endpoints,
@@ -558,9 +570,6 @@ def compose_api_target_env(
     }
 
 
-# -- Pagination -- #
-
-
 def build_pagination_cfg(
     pagination: PaginationConfig | None,
     overrides: Mapping[str, Any] | None,
@@ -667,9 +676,6 @@ def build_pagination_cfg(
     return cast(PaginationConfigMap, cfg)
 
 
-# -- Pagination Invocation -- #
-
-
 def paginate_with_client(
     client: Any,
     endpoint_key: str,
@@ -727,9 +733,6 @@ def paginate_with_client(
     return client.paginate(endpoint_key, **kw_pag)
 
 
-# -- Rate Limit -- #
-
-
 def compute_rl_sleep_seconds(
     rate_limit: RateLimitConfig | Mapping[str, Any] | None,
     overrides: Mapping[str, Any] | None,
@@ -782,9 +785,6 @@ def compute_rl_sleep_seconds(
     )
 
 
-# -- Session -- #
-
-
 def build_session(
     cfg: SessionConfig | None,
 ) -> requests.Session:
@@ -841,3 +841,45 @@ def build_session(
             pass
 
     return s
+
+
+def resolve_request(
+    method: HttpMethod | str,
+    *,
+    session: Any | None = None,
+    timeout: Timeout = None,
+) -> tuple[Callable[..., requests.Response], float, HttpMethod]:
+    """
+    Resolve a request callable and effective timeout for an HTTP method.
+
+    Parameters
+    ----------
+    method : HttpMethod | str
+        HTTP method to execute.
+    session : Any | None, optional
+        Requests-compatible session object. Defaults to module-level
+        ``requests``.
+    timeout : Timeout, optional
+        Timeout in seconds for the request. Uses ``DEFAULT_TIMEOUT`` when
+        omitted.
+
+    Returns
+    -------
+    tuple[Callable[..., requests.Response], float, HttpMethod]
+        Tuple of (callable, timeout_seconds, resolved_method).
+
+    Raises
+    ------
+    TypeError
+        If the session object does not expose the requested HTTP method.
+    """
+    http_method = HttpMethod.coerce(method)
+    request_timeout = DEFAULT_TIMEOUT if timeout is None else timeout
+    requester = session or requests
+    request_callable = getattr(requester, http_method.value, None)
+    if not callable(request_callable):
+        raise TypeError(
+            'Session object must supply a callable '
+            f'"{http_method.value}" method',
+        )
+    return request_callable, request_timeout, http_method

etlplus/cli/handlers.py

@@ -18,15 +18,16 @@ from ..config import PipelineConfig
 from ..config import load_pipeline_config
 from ..database import load_table_spec
 from ..database import render_tables
-from ..extract import extract
 from ..file import File
-from ..load import load
-from ..run import run
-from ..transform import transform
+from ..file import FileFormat
+from ..ops import extract
+from ..ops import load
+from ..ops import run
+from ..ops import transform
+from ..ops import validate
+from ..ops.validate import FieldRules
 from ..types import JSONData
 from ..types import TemplateKey
-from ..validate import FieldRules
-from ..validate import validate
 from . import io as cli_io
 
 # SECTION: EXPORTS ========================================================== #
@@ -569,8 +570,17 @@ def transform_handler(
 
     data = transform(payload, cast(TransformOperations, operations_payload))
 
+    # TODO: Generalize to handle non-file targets.
     if target and target != '-':
-        File(target, file_format=target_format).write(data)
+        # Convert target to Path and target_format to FileFormat if needed
+        file_path = Path(target)
+        file_format = None
+        if target_format is not None:
+            try:
+                file_format = FileFormat(target_format)
+            except ValueError:
+                file_format = None  # or handle error as appropriate
+        File(file_path, file_format=file_format).write(data)
         print(f'Data transformed and saved to {target}')
         return 0
 

etlplus/config/jobs.py

@@ -34,10 +34,7 @@ __all__ = [
 ]
 
 
-# SECTION: TYPE ALIASES ===================================================== #
-
-
-# SECTION: CLASSES ========================================================== #
+# SECTION: DATA CLASSES ===================================================== #
 
 
 @dataclass(kw_only=True, slots=True)
@@ -100,6 +97,8 @@ class JobConfig:
         Unique job name.
     description : str | None
         Optional human-friendly description.
+    depends_on : list[str]
+        Optional job dependency list. Dependencies must refer to other jobs.
     extract : ExtractRef | None
         Extraction reference.
     validate : ValidationRef | None
@@ -114,6 +113,7 @@ class JobConfig:
 
     name: str
     description: str | None = None
+    depends_on: list[str] = field(default_factory=list)
     extract: ExtractRef | None = None
     validate: ValidationRef | None = None
     transform: TransformRef | None = None
@@ -149,9 +149,19 @@ class JobConfig:
         if description is not None and not isinstance(description, str):
             description = str(description)
 
+        depends_raw = data.get('depends_on')
+        depends_on: list[str] = []
+        if isinstance(depends_raw, str):
+            depends_on = [depends_raw]
+        elif isinstance(depends_raw, list):
+            for entry in depends_raw:
+                if isinstance(entry, str):
+                    depends_on.append(entry)
+
         return cls(
             name=name,
             description=description,
+            depends_on=depends_on,
             extract=ExtractRef.from_obj(data.get('extract')),
             validate=ValidationRef.from_obj(data.get('validate')),
             transform=TransformRef.from_obj(data.get('transform')),

etlplus/dag.py

@@ -0,0 +1,103 @@
+"""
+:mod:`etlplus.dag` module.
+
+Lightweight directed acyclic graph (DAG) helpers for ordering jobs based on
+``depends_on``.
+"""
+
+from __future__ import annotations
+
+from collections import deque
+from dataclasses import dataclass
+
+from .config.jobs import JobConfig
+
+# SECTION: EXPORTS ========================================================== #
+
+
+__all__ = [
+    'DagError',
+    'topological_sort_jobs',
+]
+
+
+# SECTION: ERRORS =========================================================== #
+
+
+@dataclass(slots=True)
+class DagError(ValueError):
+    """
+    Raised when the job dependency graph is invalid.
+
+    Attributes
+    ----------
+    message : str
+        Error message.
+    """
+
+    # -- Attributes -- #
+
+    message: str
+
+    # -- Magic Methods (Object Representation) -- #
+
+    def __str__(self) -> str:
+        return self.message
+
+
+# SECTION: FUNCTIONS ======================================================== #
+
+
+def topological_sort_jobs(
+    jobs: list[JobConfig],
+) -> list[JobConfig]:
+    """
+    Return jobs in topological order based on ``depends_on``.
+
+    Parameters
+    ----------
+    jobs : list[JobConfig]
+        List of job configurations to sort.
+
+    Returns
+    -------
+    list[JobConfig]
+        Jobs sorted in topological order.
+
+    Raises
+    ------
+    DagError
+        If a dependency is missing, self-referential, or when a cycle is
+        detected.
+    """
+    index = {job.name: job for job in jobs}
+    edges: dict[str, set[str]] = {name: set() for name in index}
+    indegree: dict[str, int] = {name: 0 for name in index}
+
+    for job in jobs:
+        for dep in job.depends_on:
+            if dep not in index:
+                raise DagError(
+                    f'Unknown dependency "{dep}" in job "{job.name}"',
+                )
+            if dep == job.name:
+                raise DagError(f'Job "{job.name}" depends on itself')
+            if job.name not in edges[dep]:
+                edges[dep].add(job.name)
+                indegree[job.name] += 1
+
+    queue = deque(sorted(name for name, deg in indegree.items() if deg == 0))
+    ordered: list[str] = []
+
+    while queue:
+        name = queue.popleft()
+        ordered.append(name)
+        for child in sorted(edges[name]):
+            indegree[child] -= 1
+            if indegree[child] == 0:
+                queue.append(child)
+
+    if len(ordered) != len(jobs):
+        raise DagError('Dependency cycle detected')
+
+    return [index[name] for name in ordered]

etlplus/{validation → ops}/README.md

@@ -1,4 +1,4 @@
-# etlplus.validation subpackage
+# etlplus.ops subpackage
 
 Documentation for the `etlplus.validation` subpackage: data validation utilities and helpers.
 
@@ -8,7 +8,7 @@ Documentation for the `etlplus.validation` subpackage: data validation utilities
 
 Back to project overview: see the top-level [README](../../README.md).
 
-- [etlplus.validation subpackage](#etlplusvalidation-subpackage)
+- [etlplus.ops subpackage](#etlplusops-subpackage)
   - [Validation Features](#validation-features)
   - [Defining Validation Rules](#defining-validation-rules)
   - [Example: Validating Data](#example-validating-data)

etlplus/ops/__init__.py

@@ -0,0 +1,57 @@
+"""
+:mod:`etlplus.ops` package.
+
+Data operations helpers.
+
+Importing :mod:`etlplus.ops` exposes the coarse-grained helpers most users care
+about: ``extract``, ``transform``, ``load``, ``validate``, and ``run``. Each
+helper delegates to the richer modules under ``etlplus.ops.*`` while
+presenting a compact public API surface. Conditional validation orchestration
+is available via :func:`etlplus.ops.utils.maybe_validate`.
+
+Examples
+--------
+>>> from etlplus.ops import extract, transform
+>>> raw = extract('file', 'input.json')
+>>> curated = transform(raw, {'select': ['id', 'name']})
+
+>>> from etlplus.ops.utils import maybe_validate
+>>> payload = {'name': 'Alice'}
+>>> rules = {'required': ['name']}
+>>> def validator(data, config):
+...     missing = [field for field in config['required'] if field not in data]
+...     return {'valid': not missing, 'errors': missing, 'data': data}
+>>> maybe_validate(
+...     payload,
+...     when='both',
+...     enabled=True,
+...     rules=rules,
+...     phase='before_transform',
+...     severity='warn',
+...     validate_fn=validator,
+...     print_json_fn=lambda message: message,
+... )
+{'name': 'Alice'}
+
+See Also
+--------
+:mod:`etlplus.ops.run`
+:mod:`etlplus.ops.utils`
+"""
+
+from .extract import extract
+from .load import load
+from .run import run
+from .transform import transform
+from .validate import validate
+
+# SECTION: EXPORTS ========================================================== #
+
+
+__all__ = [
+    'extract',
+    'load',
+    'run',
+    'transform',
+    'validate',
+]