etlplus 0.15.5__py3-none-any.whl → 0.16.2__py3-none-any.whl

etlplus/connector/file.py

@@ -0,0 +1,120 @@
+"""
+:mod:`etlplus.connector.file` module.
+
+File connector configuration dataclass.
+
+Notes
+-----
+- TypedDicts in this module are intentionally ``total=False`` and are not
+    enforced at runtime.
+- :meth:`*.from_obj` constructors accept :class:`Mapping[str, Any]` and perform
+    tolerant parsing and light casting. This keeps the runtime permissive while
+    improving autocomplete and static analysis for contributors.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from dataclasses import field
+from typing import Any
+from typing import Self
+from typing import TypedDict
+from typing import overload
+
+from ..types import StrAnyMap
+from ..utils import coerce_dict
+from .core import ConnectorBase
+from .enums import DataConnectorType
+from .types import ConnectorType
+
+# SECTION: EXPORTS ========================================================== #
+
+
+__all__ = [
+    'ConnectorFile',
+    'ConnectorFileConfigMap',
+]
+
+
+# SECTION: TYPED DICTS ====================================================== #
+
+
+class ConnectorFileConfigMap(TypedDict, total=False):
+    """
+    Shape accepted by :meth:`ConnectorFile.from_obj` (all keys optional).
+
+    See Also
+    --------
+    - :meth:`etlplus.connector.file.ConnectorFile.from_obj`
+    """
+
+    name: str
+    type: ConnectorType
+    format: str
+    path: str
+    options: StrAnyMap
+
+
+# SECTION: DATA CLASSES ===================================================== #
+
+
+@dataclass(kw_only=True, slots=True)
+class ConnectorFile(ConnectorBase):
+    """
+    Configuration for a file-based data connector.
+
+    Attributes
+    ----------
+    type : ConnectorType
+        Connector kind, always ``'file'``.
+    format : str | None
+        File format (e.g., ``'json'``, ``'csv'``).
+    path : str | None
+        File path or URI.
+    options : dict[str, Any]
+        Reader/writer format options.
+    """
+
+    # -- Attributes -- #
+
+    type: ConnectorType = DataConnectorType.FILE
+    format: str | None = None
+    path: str | None = None
+    options: dict[str, Any] = field(default_factory=dict)
+
+    # -- Class Methods -- #
+
+    @classmethod
+    @overload
+    def from_obj(cls, obj: ConnectorFileConfigMap) -> Self: ...
+
+    @classmethod
+    @overload
+    def from_obj(cls, obj: StrAnyMap) -> Self: ...
+
+    @classmethod
+    def from_obj(
+        cls,
+        obj: StrAnyMap,
+    ) -> Self:
+        """
+        Parse a mapping into a ``ConnectorFile`` instance.
+
+        Parameters
+        ----------
+        obj : StrAnyMap
+            Mapping with at least ``name``.
+
+        Returns
+        -------
+        Self
+            Parsed connector instance.
+        """
+        name = cls._require_name(obj, kind='File')
+
+        return cls(
+            name=name,
+            format=obj.get('format'),
+            path=obj.get('path'),
+            options=coerce_dict(obj.get('options')),
+        )

etlplus/connector/types.py

@@ -0,0 +1,40 @@
+"""
+:mod:`etlplus.connector.types` module.
+
+Connector type aliases for :mod:`etlplus.connector`.
+
+Examples
+--------
+>>> from etlplus.connector import Connector
+>>> src: Connector = {
+>>>     "type": "file",
+>>>     "path": "/data/input.csv",
+>>> }
+>>> tgt: Connector = {
+>>>     "type": "database",
+>>>     "connection_string": "postgresql://user:pass@localhost/db",
+>>> }
+>>> from etlplus.api import RetryPolicy
+>>> rp: RetryPolicy = {"max_attempts": 3, "backoff": 0.5}
+"""
+
+from __future__ import annotations
+
+from typing import Literal
+
+from .enums import DataConnectorType
+
+# SECTION: EXPORTS  ========================================================= #
+
+
+__all__ = [
+    # Type Aliases
+    'ConnectorType',
+]
+
+
+# SECTION: TYPE ALIASES ===================================================== #
+
+
+# Literal type for supported connector kinds (strings or enum members)
+type ConnectorType = DataConnectorType | Literal['api', 'database', 'file']

etlplus/connector/utils.py

@@ -0,0 +1,122 @@
+"""
+:mod:`etlplus.connector.utils` module.
+
+Shared connector parsing helpers.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from typing import Any
+
+from .api import ConnectorApi
+from .connector import Connector
+from .database import ConnectorDb
+from .enums import DataConnectorType
+from .file import ConnectorFile
+
+# SECTION: EXPORTS ========================================================== #
+
+
+__all__ = [
+    # Functions
+    'parse_connector',
+]
+
+
+# SECTION: INTERNAL FUNCTIONS =============================================== #
+
+
+def _coerce_connector_type(
+    obj: Mapping[str, Any],
+) -> DataConnectorType:
+    """
+    Normalize and validate the connector ``type`` field.
+
+    Parameters
+    ----------
+    obj : Mapping[str, Any]
+        Mapping with a ``type`` entry.
+
+    Returns
+    -------
+    DataConnectorType
+        Normalized connector type enum.
+
+    Raises
+    ------
+    TypeError
+        If ``type`` is missing or unsupported.
+    """
+    if 'type' not in obj:
+        raise TypeError('Connector requires a "type"')
+    try:
+        return DataConnectorType.coerce(obj.get('type'))
+    except ValueError as exc:
+        allowed = ', '.join(DataConnectorType.choices())
+        raise TypeError(
+            f'Unsupported connector type: {obj.get("type")!r}. '
+            f'Expected one of {allowed}.',
+        ) from exc
+
+
+def _load_connector(
+    kind: DataConnectorType,
+) -> type[Connector]:
+    """
+    Resolve the connector class for the requested kind.
+
+    Parameters
+    ----------
+    kind : DataConnectorType
+        Connector kind enum.
+
+    Returns
+    -------
+    type[Connector]
+        Connector class corresponding to *kind*.
+    """
+    match kind:
+        case DataConnectorType.API:
+            return ConnectorApi
+        case DataConnectorType.DATABASE:
+            return ConnectorDb
+        case DataConnectorType.FILE:
+            return ConnectorFile
+
+
+# SECTION: FUNCTIONS ======================================================== #
+
+
+def parse_connector(
+    obj: Mapping[str, Any],
+) -> Connector:
+    """
+    Dispatch to a concrete connector constructor based on ``type``.
+
+    Parameters
+    ----------
+    obj : Mapping[str, Any]
+        Mapping with at least ``name`` and ``type``.
+
+    Returns
+    -------
+    Connector
+        Concrete connector instance.
+
+    Raises
+    ------
+    TypeError
+        If the mapping is invalid or the connector type is unsupported.
+
+    Notes
+    -----
+    Delegates to the tolerant ``from_obj`` constructors for each connector
+    kind. Connector types are normalized via
+    :class:`etlplus.connector.enums.DataConnectorType`, so common aliases
+    (e.g., ``'db'`` or ``'http'``) are accepted.
+    """
+    if not isinstance(obj, Mapping):
+        raise TypeError('Connector configuration must be a mapping.')
+    connector_cls = _load_connector(_coerce_connector_type(obj))
+    return connector_cls.from_obj(obj)

etlplus/enums.py

@@ -22,7 +22,6 @@ __all__ = [
     # Enums
     'AggregateName',
     'CoercibleStrEnum',
-    'DataConnectorType',
     'OperatorName',
     'PipelineStep',
 ]
@@ -168,37 +167,6 @@ class AggregateName(CoercibleStrEnum):
         return lambda xs, n: (fmean(xs) if xs else 0.0)
 
 
-class DataConnectorType(CoercibleStrEnum):
-    """Supported data connector types."""
-
-    # -- Constants -- #
-
-    API = 'api'
-    DATABASE = 'database'
-    FILE = 'file'
-
-    # -- Class Methods -- #
-
-    @classmethod
-    def aliases(cls) -> StrStrMap:
-        """
-        Return a mapping of common aliases for each enum member.
-
-        Returns
-        -------
-        StrStrMap
-            A mapping of alias names to their corresponding enum member names.
-        """
-        return {
-            'http': 'api',
-            'https': 'api',
-            'rest': 'api',
-            'db': 'database',
-            'filesystem': 'file',
-            'fs': 'file',
-        }
-
-
 class OperatorName(CoercibleStrEnum):
     """Supported comparison operators with helpers."""
 

etlplus/ops/extract.py

@@ -6,19 +6,28 @@ Helpers to extract data from files, databases, and REST APIs.
 
 from __future__ import annotations
 
+from collections.abc import Mapping
 from pathlib import Path
 from typing import Any
 from typing import cast
+from urllib.parse import urlsplit
+from urllib.parse import urlunsplit
 
+from ..api import EndpointClient
 from ..api import HttpMethod
+from ..api import PaginationConfigMap
+from ..api import RequestOptions
+from ..api import compose_api_request_env
+from ..api import paginate_with_client
 from ..api.utils import resolve_request
-from ..enums import DataConnectorType
+from ..connector import DataConnectorType
 from ..file import File
 from ..file import FileFormat
 from ..types import JSONData
 from ..types import JSONDict
 from ..types import JSONList
 from ..types import StrPath
+from ..types import Timeout
 
 # SECTION: EXPORTS ========================================================== #
 
@@ -32,50 +41,164 @@ __all__ = [
 ]
 
 
-# SECTION: FUNCTIONS ======================================================== #
+# SECTION: INTERNAL FUNCTIONS =============================================== #
 
 
-def extract_from_api(
-    url: str,
-    method: HttpMethod | str = HttpMethod.GET,
-    **kwargs: Any,
+def _build_client(
+    *,
+    base_url: str,
+    base_path: str | None,
+    endpoints: dict[str, str],
+    retry: Any,
+    retry_network_errors: bool,
+    session: Any,
+) -> EndpointClient:
+    """
+    Construct an API client with shared defaults.
+
+    Parameters
+    ----------
+    base_url : str
+        API base URL.
+    base_path : str | None
+        Base path to prepend for endpoints.
+    endpoints : dict[str, str]
+        Endpoint name to path mappings.
+    retry : Any
+        Retry policy configuration.
+    retry_network_errors : bool
+        Whether to retry on network errors.
+    session : Any
+        Optional requests session.
+
+    Returns
+    -------
+    EndpointClient
+        Configured endpoint client instance.
+    """
+    ClientClass = EndpointClient  # noqa: N806
+    return ClientClass(
+        base_url=base_url,
+        base_path=base_path,
+        endpoints=endpoints,
+        retry=retry,
+        retry_network_errors=retry_network_errors,
+        session=session,
+    )
+
+
+def _extract_from_api_env(
+    env: Mapping[str, Any],
+    *,
+    use_client: bool,
 ) -> JSONData:
     """
-    Extract data from a REST API.
+    Extract API data from a normalized request environment.
 
     Parameters
     ----------
-    url : str
-        API endpoint URL.
-    method : HttpMethod | str, optional
-        HTTP method to use. Defaults to ``GET``.
-    **kwargs : Any
-        Extra arguments forwarded to the underlying ``requests`` call
-        (for example, ``timeout``). To use a pre-configured
-        :class:`requests.Session`, provide it via ``session``.
-        When omitted, ``timeout`` defaults to 10 seconds.
+    env : Mapping[str, Any]
+        Normalized environment describing API request parameters.
+    use_client : bool
+        Whether to use the endpoint client/pagination machinery.
 
     Returns
     -------
     JSONData
-        Parsed JSON payload, or a fallback object with raw text.
+        Extracted payload.
 
     Raises
     ------
-    TypeError
-        If a provided ``session`` does not expose the required HTTP
-        method (for example, ``get``).
+    ValueError
+        If required parameters are missing.
     """
-    timeout = kwargs.pop('timeout', None)
-    session = kwargs.pop('session', None)
+    if (
+        use_client
+        and env.get('use_endpoints')
+        and env.get('base_url')
+        and env.get('endpoints_map')
+        and env.get('endpoint_key')
+    ):
+        client = _build_client(
+            base_url=cast(str, env.get('base_url')),
+            base_path=cast(str | None, env.get('base_path')),
+            endpoints=cast(dict[str, str], env.get('endpoints_map', {})),
+            retry=env.get('retry'),
+            retry_network_errors=bool(env.get('retry_network_errors', False)),
+            session=env.get('session'),
+        )
+        return paginate_with_client(
+            client,
+            cast(str, env.get('endpoint_key')),
+            env.get('params'),
+            env.get('headers'),
+            env.get('timeout'),
+            env.get('pagination'),
+            cast(float | None, env.get('sleep_seconds')),
+        )
+
+    url = env.get('url')
+    if not url:
+        raise ValueError('API source missing URL')
+
+    if use_client:
+        parts = urlsplit(cast(str, url))
+        base = urlunsplit((parts.scheme, parts.netloc, '', '', ''))
+        client = _build_client(
+            base_url=base,
+            base_path=None,
+            endpoints={},
+            retry=env.get('retry'),
+            retry_network_errors=bool(env.get('retry_network_errors', False)),
+            session=env.get('session'),
+        )
+        request_options = RequestOptions(
+            params=cast(Mapping[str, Any] | None, env.get('params')),
+            headers=cast(Mapping[str, str] | None, env.get('headers')),
+            timeout=cast(Timeout | None, env.get('timeout')),
+        )
+
+        return client.paginate_url(
+            cast(str, url),
+            cast(PaginationConfigMap | None, env.get('pagination')),
+            request=request_options,
+            sleep_seconds=cast(float, env.get('sleep_seconds', 0.0)),
+        )
+
+    method = env.get('method', HttpMethod.GET)
+    timeout = env.get('timeout', None)
+    session = env.get('session', None)
+    request_kwargs = dict(env.get('request_kwargs') or {})
     request_callable, timeout, _ = resolve_request(
         method,
         session=session,
         timeout=timeout,
     )
-    response = request_callable(url, timeout=timeout, **kwargs)
+    response = request_callable(
+        cast(str, url),
+        timeout=timeout,
+        **request_kwargs,
+    )
     response.raise_for_status()
+    return _parse_api_response(response)
+
 
+def _parse_api_response(
+    response: Any,
+) -> JSONData:
+    """
+    Parse API responses into a consistent JSON payload.
+
+    Parameters
+    ----------
+    response : Any
+        HTTP response object exposing ``headers``, ``json()``, and ``text``.
+
+    Returns
+    -------
+    JSONData
+        Parsed JSON payload, or a fallback object with raw text.
+    """
     content_type = response.headers.get('content-type', '').lower()
     if 'application/json' in content_type:
         try:
@@ -99,6 +222,70 @@ def extract_from_api(
     return {'content': response.text, 'content_type': content_type}
 
 
+# SECTION: FUNCTIONS ======================================================== #
+
+
+def extract_from_api(
+    url: str,
+    method: HttpMethod | str = HttpMethod.GET,
+    **kwargs: Any,
+) -> JSONData:
+    """
+    Extract data from a REST API.
+
+    Parameters
+    ----------
+    url : str
+        API endpoint URL.
+    method : HttpMethod | str, optional
+        HTTP method to use. Defaults to ``GET``.
+    **kwargs : Any
+        Extra arguments forwarded to the underlying ``requests`` call
+        (for example, ``timeout``). To use a pre-configured
+        :class:`requests.Session`, provide it via ``session``.
+        When omitted, ``timeout`` defaults to 10 seconds.
+
+    Returns
+    -------
+    JSONData
+        Parsed JSON payload, or a fallback object with raw text.
+    """
+    env = {
+        'url': url,
+        'method': method,
+        'timeout': kwargs.pop('timeout', None),
+        'session': kwargs.pop('session', None),
+        'request_kwargs': kwargs,
+    }
+    return _extract_from_api_env(env, use_client=False)
+
+
+def extract_from_api_source(
+    cfg: Any,
+    source_obj: Any,
+    overrides: dict[str, Any],
+) -> JSONData:
+    """
+    Extract data from a REST API source connector.
+
+    Parameters
+    ----------
+    cfg : Any
+        Pipeline configuration.
+    source_obj : Any
+        Connector configuration.
+    overrides : dict[str, Any]
+        Extract-time overrides.
+
+    Returns
+    -------
+    JSONData
+        Extracted payload.
+    """
+    env = compose_api_request_env(cfg, source_obj, overrides)
+    return _extract_from_api_env(env, use_client=True)
+
+
 def extract_from_database(
     connection_string: str,
 ) -> JSONList: