etlplus 0.15.0__py3-none-any.whl → 0.16.6__py3-none-any.whl

etlplus/ops/enums.py

@@ -0,0 +1,173 @@
+"""
+:mod:`etlplus.ops.enums` module.
+
+Operation-specific enums and helpers.
+"""
+
+from __future__ import annotations
+
+import operator as _op
+from statistics import fmean
+
+from ..enums import CoercibleStrEnum
+from ..types import StrStrMap
+from .types import AggregateFunc
+from .types import OperatorFunc
+
+# SECTION: EXPORTS ========================================================= #
+
+
+__all__ = [
+    # Enums
+    'AggregateName',
+    'OperatorName',
+    'PipelineStep',
+]
+
+
+# SECTION: ENUMS ============================================================ #
+
+
+class AggregateName(CoercibleStrEnum):
+    """Supported aggregations with helpers."""
+
+    # -- Constants -- #
+
+    AVG = 'avg'
+    COUNT = 'count'
+    MAX = 'max'
+    MIN = 'min'
+    SUM = 'sum'
+
+    # -- Class Methods -- #
+
+    @property
+    def func(self) -> AggregateFunc:
+        """
+        Get the aggregation function for this aggregation type.
+
+        Returns
+        -------
+        AggregateFunc
+            The aggregation function corresponding to this aggregation type.
+        """
+        if self is AggregateName.COUNT:
+            return lambda xs, n: n
+        if self is AggregateName.MAX:
+            return lambda xs, n: (max(xs) if xs else None)
+        if self is AggregateName.MIN:
+            return lambda xs, n: (min(xs) if xs else None)
+        if self is AggregateName.SUM:
+            return lambda xs, n: sum(xs)
+
+        # AVG
+        return lambda xs, n: (fmean(xs) if xs else 0.0)
+
+
+class OperatorName(CoercibleStrEnum):
+    """Supported comparison operators with helpers."""
+
+    # -- Constants -- #
+
+    EQ = 'eq'
+    NE = 'ne'
+    GT = 'gt'
+    GTE = 'gte'
+    LT = 'lt'
+    LTE = 'lte'
+    IN = 'in'
+    CONTAINS = 'contains'
+
+    # -- Getters -- #
+
+    @property
+    def func(self) -> OperatorFunc:
+        """
+        Get the comparison function for this operator.
+
+        Returns
+        -------
+        OperatorFunc
+            The comparison function corresponding to this operator.
+        """
+        match self:
+            case OperatorName.EQ:
+                return _op.eq
+            case OperatorName.NE:
+                return _op.ne
+            case OperatorName.GT:
+                return _op.gt
+            case OperatorName.GTE:
+                return _op.ge
+            case OperatorName.LT:
+                return _op.lt
+            case OperatorName.LTE:
+                return _op.le
+            case OperatorName.IN:
+                return lambda a, b: a in b
+            case OperatorName.CONTAINS:
+                return lambda a, b: b in a
+
+    # -- Class Methods -- #
+
+    @classmethod
+    def aliases(cls) -> StrStrMap:
+        """
+        Return a mapping of common aliases for each enum member.
+
+        Returns
+        -------
+        StrStrMap
+            A mapping of alias names to their corresponding enum member names.
+        """
+        return {
+            '==': 'eq',
+            '=': 'eq',
+            '!=': 'ne',
+            '<>': 'ne',
+            '>=': 'gte',
+            '≥': 'gte',
+            '<=': 'lte',
+            '≤': 'lte',
+            '>': 'gt',
+            '<': 'lt',
+        }
+
+
+class PipelineStep(CoercibleStrEnum):
+    """Pipeline step names as an enum for internal orchestration."""
+
+    # -- Constants -- #
+
+    FILTER = 'filter'
+    MAP = 'map'
+    SELECT = 'select'
+    SORT = 'sort'
+    AGGREGATE = 'aggregate'
+
+    # -- Getters -- #
+
+    @property
+    def order(self) -> int:
+        """
+        Get the execution order of this pipeline step.
+
+        Returns
+        -------
+        int
+            The execution order of this pipeline step.
+        """
+        return _PIPELINE_ORDER_INDEX[self]
+
+
+# SECTION: INTERNAL CONSTANTS ============================================== #
+
+
+# Precomputed order index for PipelineStep; avoids recomputing on each access.
+_PIPELINE_ORDER_INDEX: dict[PipelineStep, int] = {
+    PipelineStep.FILTER: 0,
+    PipelineStep.MAP: 1,
+    PipelineStep.SELECT: 2,
+    PipelineStep.SORT: 3,
+    PipelineStep.AGGREGATE: 4,
+}

etlplus/ops/extract.py

@@ -6,64 +6,199 @@ Helpers to extract data from files, databases, and REST APIs.
 
 from __future__ import annotations
 
+from collections.abc import Mapping
 from pathlib import Path
 from typing import Any
 from typing import cast
+from urllib.parse import urlsplit
+from urllib.parse import urlunsplit
 
+from ..api import EndpointClient
 from ..api import HttpMethod
+from ..api import PaginationConfigDict
+from ..api import RequestOptions
+from ..api import compose_api_request_env
+from ..api import paginate_with_client
 from ..api.utils import resolve_request
-from ..enums import DataConnectorType
+from ..connector import DataConnectorType
 from ..file import File
 from ..file import FileFormat
 from ..types import JSONData
 from ..types import JSONDict
 from ..types import JSONList
 from ..types import StrPath
+from ..types import Timeout
 
-# SECTION: FUNCTIONS ======================================================== #
+# SECTION: EXPORTS ========================================================== #
 
 
-def extract_from_api(
-    url: str,
-    method: HttpMethod | str = HttpMethod.GET,
-    **kwargs: Any,
+__all__ = [
+    # Functions
+    'extract',
+    'extract_from_api',
+    'extract_from_database',
+    'extract_from_file',
+]
+
+
+# SECTION: INTERNAL FUNCTIONS =============================================== #
+
+
+def _build_client(
+    *,
+    base_url: str,
+    base_path: str | None,
+    endpoints: dict[str, str],
+    retry: Any,
+    retry_network_errors: bool,
+    session: Any,
+) -> EndpointClient:
+    """
+    Construct an API client with shared defaults.
+
+    Parameters
+    ----------
+    base_url : str
+        API base URL.
+    base_path : str | None
+        Base path to prepend for endpoints.
+    endpoints : dict[str, str]
+        Endpoint name to path mappings.
+    retry : Any
+        Retry policy configuration.
+    retry_network_errors : bool
+        Whether to retry on network errors.
+    session : Any
+        Optional requests session.
+
+    Returns
+    -------
+    EndpointClient
+        Configured endpoint client instance.
+    """
+    ClientClass = EndpointClient  # noqa: N806
+    return ClientClass(
+        base_url=base_url,
+        base_path=base_path,
+        endpoints=endpoints,
+        retry=retry,
+        retry_network_errors=retry_network_errors,
+        session=session,
+    )
+
+
+def _extract_from_api_env(
+    env: Mapping[str, Any],
+    *,
+    use_client: bool,
 ) -> JSONData:
     """
-    Extract data from a REST API.
+    Extract API data from a normalized request environment.
 
     Parameters
     ----------
-    url : str
-        API endpoint URL.
-    method : HttpMethod | str, optional
-        HTTP method to use. Defaults to ``GET``.
-    **kwargs : Any
-        Extra arguments forwarded to the underlying ``requests`` call
-        (for example, ``timeout``). To use a pre-configured
-        :class:`requests.Session`, provide it via ``session``.
-        When omitted, ``timeout`` defaults to 10 seconds.
+    env : Mapping[str, Any]
+        Normalized environment describing API request parameters.
+    use_client : bool
+        Whether to use the endpoint client/pagination machinery.
 
     Returns
     -------
     JSONData
-        Parsed JSON payload, or a fallback object with raw text.
+        Extracted payload.
 
     Raises
     ------
-    TypeError
-        If a provided ``session`` does not expose the required HTTP
-        method (for example, ``get``).
+    ValueError
+        If required parameters are missing.
     """
-    timeout = kwargs.pop('timeout', None)
-    session = kwargs.pop('session', None)
+    if (
+        use_client
+        and env.get('use_endpoints')
+        and env.get('base_url')
+        and env.get('endpoints_map')
+        and env.get('endpoint_key')
+    ):
+        client = _build_client(
+            base_url=cast(str, env.get('base_url')),
+            base_path=cast(str | None, env.get('base_path')),
+            endpoints=cast(dict[str, str], env.get('endpoints_map', {})),
+            retry=env.get('retry'),
+            retry_network_errors=bool(env.get('retry_network_errors', False)),
+            session=env.get('session'),
+        )
+        return paginate_with_client(
+            client,
+            cast(str, env.get('endpoint_key')),
+            env.get('params'),
+            env.get('headers'),
+            env.get('timeout'),
+            env.get('pagination'),
+            cast(float | None, env.get('sleep_seconds')),
+        )
+
+    url = env.get('url')
+    if not url:
+        raise ValueError('API source missing URL')
+
+    if use_client:
+        parts = urlsplit(cast(str, url))
+        base = urlunsplit((parts.scheme, parts.netloc, '', '', ''))
+        client = _build_client(
+            base_url=base,
+            base_path=None,
+            endpoints={},
+            retry=env.get('retry'),
+            retry_network_errors=bool(env.get('retry_network_errors', False)),
+            session=env.get('session'),
+        )
+        request_options = RequestOptions(
+            params=cast(Mapping[str, Any] | None, env.get('params')),
+            headers=cast(Mapping[str, str] | None, env.get('headers')),
+            timeout=cast(Timeout | None, env.get('timeout')),
+        )
+
+        return client.paginate_url(
+            cast(str, url),
+            cast(PaginationConfigDict | None, env.get('pagination')),
+            request=request_options,
+            sleep_seconds=cast(float, env.get('sleep_seconds', 0.0)),
+        )
+
+    method = env.get('method', HttpMethod.GET)
+    timeout = env.get('timeout', None)
+    session = env.get('session', None)
+    request_kwargs = dict(env.get('request_kwargs') or {})
     request_callable, timeout, _ = resolve_request(
         method,
         session=session,
         timeout=timeout,
     )
-    response = request_callable(url, timeout=timeout, **kwargs)
+    response = request_callable(
+        cast(str, url),
+        timeout=timeout,
+        **request_kwargs,
+    )
     response.raise_for_status()
+    return _parse_api_response(response)
 
+
+def _parse_api_response(
+    response: Any,
+) -> JSONData:
+    """
+    Parse API responses into a consistent JSON payload.
+
+    Parameters
+    ----------
+    response : Any
+        HTTP response object exposing ``headers``, ``json()``, and ``text``.
+
+    Returns
+    -------
+    JSONData
+        Parsed JSON payload, or a fallback object with raw text.
+    """
     content_type = response.headers.get('content-type', '').lower()
     if 'application/json' in content_type:
         try:
@@ -87,6 +222,70 @@ def extract_from_api(
     return {'content': response.text, 'content_type': content_type}
 
 
+# SECTION: FUNCTIONS ======================================================== #
+
+
+def extract_from_api(
+    url: str,
+    method: HttpMethod | str = HttpMethod.GET,
+    **kwargs: Any,
+) -> JSONData:
+    """
+    Extract data from a REST API.
+
+    Parameters
+    ----------
+    url : str
+        API endpoint URL.
+    method : HttpMethod | str, optional
+        HTTP method to use. Defaults to ``GET``.
+    **kwargs : Any
+        Extra arguments forwarded to the underlying ``requests`` call
+        (for example, ``timeout``). To use a pre-configured
+        :class:`requests.Session`, provide it via ``session``.
+        When omitted, ``timeout`` defaults to 10 seconds.
+
+    Returns
+    -------
+    JSONData
+        Parsed JSON payload, or a fallback object with raw text.
+    """
+    env = {
+        'url': url,
+        'method': method,
+        'timeout': kwargs.pop('timeout', None),
+        'session': kwargs.pop('session', None),
+        'request_kwargs': kwargs,
+    }
+    return _extract_from_api_env(env, use_client=False)
+
+
+def extract_from_api_source(
+    cfg: Any,
+    source_obj: Any,
+    overrides: dict[str, Any],
+) -> JSONData:
+    """
+    Extract data from a REST API source connector.
+
+    Parameters
+    ----------
+    cfg : Any
+        Pipeline configuration.
+    source_obj : Any
+        Connector configuration.
+    overrides : dict[str, Any]
+        Extract-time overrides.
+
+    Returns
+    -------
+    JSONData
+        Extracted payload.
+    """
+    env = compose_api_request_env(cfg, source_obj, overrides)
+    return _extract_from_api_env(env, use_client=True)
+
+
 def extract_from_database(
     connection_string: str,
 ) -> JSONList:

etlplus/ops/load.py

@@ -8,13 +8,15 @@ from __future__ import annotations
 
 import json
 import sys
+from collections.abc import Mapping
 from pathlib import Path
 from typing import Any
 from typing import cast
 
 from ..api import HttpMethod
+from ..api import compose_api_target_env
 from ..api.utils import resolve_request
-from ..enums import DataConnectorType
+from ..connector import DataConnectorType
 from ..file import File
 from ..file import FileFormat
 from ..types import JSONData
@@ -23,14 +25,129 @@ from ..types import JSONList
 from ..types import StrPath
 from ..utils import count_records
 
+# SECTION: EXPORTS ========================================================== #
+
+
+__all__ = [
+    # Functions
+    'load',
+    'load_data',
+    'load_to_api',
+    'load_to_database',
+    'load_to_file',
+]
+
+
 # SECTION: INTERNAL FUNCTIONS ============================================== #
 
 
+def _load_data_from_str(
+    source: str,
+) -> JSONData:
+    """
+    Load JSON data from a string or file path.
+
+    Parameters
+    ----------
+    source : str
+        Input string representing a file path or JSON payload.
+
+    Returns
+    -------
+    JSONData
+        Parsed JSON payload.
+    """
+    # Special case: '-' means read JSON from STDIN (Unix convention).
+    if source == '-':
+        raw = sys.stdin.read()
+        return _parse_json_string(raw)
+
+    candidate = Path(source)
+    if candidate.exists():
+        try:
+            return File(candidate, FileFormat.JSON).read()
+        except (OSError, json.JSONDecodeError, ValueError):
+            # Fall back to treating the string as raw JSON content.
+            pass
+    return _parse_json_string(source)
+
+
+def _load_to_api_env(
+    data: JSONData,
+    env: Mapping[str, Any],
+) -> JSONDict:
+    """
+    Load data to an API target using a normalized environment.
+
+    Parameters
+    ----------
+    data : JSONData
+        Payload to load.
+    env : Mapping[str, Any]
+        Normalized request environment.
+
+    Returns
+    -------
+    JSONDict
+        Load result payload.
+
+    Raises
+    ------
+    ValueError
+        If required parameters are missing.
+    """
+    url = env.get('url')
+    if not url:
+        raise ValueError('API target missing "url"')
+    method = env.get('method') or 'post'
+    kwargs: dict[str, Any] = {}
+    headers = env.get('headers')
+    if headers:
+        kwargs['headers'] = cast(dict[str, str], headers)
+    if env.get('timeout') is not None:
+        kwargs['timeout'] = env.get('timeout')
+    session = env.get('session')
+    if session is not None:
+        kwargs['session'] = session
+    extra_kwargs = env.get('request_kwargs')
+    if isinstance(extra_kwargs, Mapping):
+        kwargs.update(extra_kwargs)
+    timeout = kwargs.pop('timeout', 10.0)
+    session = kwargs.pop('session', None)
+    request_callable, timeout, http_method = resolve_request(
+        method,
+        session=session,
+        timeout=timeout,
+    )
+    response = request_callable(
+        cast(str, url),
+        json=data,
+        timeout=timeout,
+        **kwargs,
+    )
+    response.raise_for_status()
+
+    # Try JSON first, fall back to text.
+    try:
+        payload: Any = response.json()
+    except ValueError:
+        payload = response.text
+
+    return {
+        'status': 'success',
+        'status_code': response.status_code,
+        'message': f'Data loaded to {url}',
+        'response': payload,
+        'records': count_records(data),
+        'method': http_method.value.upper(),
+    }
+
+
 def _parse_json_string(
     raw: str,
 ) -> JSONData:
     """
-    Parse JSON data from ``raw`` text.
+    Parse JSON data from *raw* text.
 
     Parameters
     ----------
@@ -100,18 +217,7 @@ def load_data(
         return File(source, FileFormat.JSON).read()
 
     if isinstance(source, str):
-        # Special case: '-' means read JSON from STDIN (Unix convention).
-        if source == '-':
-            raw = sys.stdin.read()
-            return _parse_json_string(raw)
-        candidate = Path(source)
-        if candidate.exists():
-            try:
-                return File(candidate, FileFormat.JSON).read()
-            except (OSError, json.JSONDecodeError, ValueError):
-                # Fall back to treating the string as raw JSON content.
-                pass
-        return _parse_json_string(source)
+        return _load_data_from_str(source)
 
     raise TypeError(
         'source must be a mapping, sequence of mappings, path, or JSON string',
@@ -145,30 +251,43 @@ def load_to_api(
         Result dictionary including response payload or text.
     """
     # Apply a conservative timeout to guard against hanging requests.
-    timeout = kwargs.pop('timeout', 10.0)
-    session = kwargs.pop('session', None)
-    request_callable, timeout, http_method = resolve_request(
-        method,
-        session=session,
-        timeout=timeout,
-    )
-    response = request_callable(url, json=data, timeout=timeout, **kwargs)
-    response.raise_for_status()
+    env = {
+        'url': url,
+        'method': method,
+        'timeout': kwargs.pop('timeout', 10.0),
+        'session': kwargs.pop('session', None),
+        'request_kwargs': kwargs,
+    }
+    return _load_to_api_env(data, env)
 
-    # Try JSON first, fall back to text.
-    try:
-        payload: Any = response.json()
-    except ValueError:
-        payload = response.text
 
-    return {
-        'status': 'success',
-        'status_code': response.status_code,
-        'message': f'Data loaded to {url}',
-        'response': payload,
-        'records': count_records(data),
-        'method': http_method.value.upper(),
-    }
+def load_to_api_target(
+    cfg: Any,
+    target_obj: Any,
+    overrides: dict[str, Any],
+    data: JSONData,
+) -> JSONDict:
+    """
+    Load data to an API target connector.
+
+    Parameters
+    ----------
+    cfg : Any
+        Pipeline configuration.
+    target_obj : Any
+        Connector configuration.
+    overrides : dict[str, Any]
+        Load-time overrides.
+    data : JSONData
+        Payload to load.
+
+    Returns
+    -------
+    JSONDict
+        Load result.
+    """
+    env = compose_api_target_env(cfg, target_obj, overrides)
+    return _load_to_api_env(data, env)
 
 
 def load_to_database(