etlplus 0.16.0__py3-none-any.whl → 0.16.2__py3-none-any.whl

etlplus/api/types.py

@@ -53,7 +53,31 @@ __all__ = [
 # SECTION: CONSTANTS ======================================================== #
 
 
-_UNSET = object()
+_UNSET: object = object()
+
+
+# SECTION: INTERNAL FUNCTIONS =============================================== #
+
+
+def _to_dict(
+    value: Mapping[str, Any] | object | None,
+) -> dict[str, Any] | None:
+    """
+    Return a defensive ``dict`` copy for mapping inputs.
+
+    Parameters
+    ----------
+    value : Mapping[str, Any] | object | None
+        Mapping to copy, or ``None``.
+
+    Returns
+    -------
+    dict[str, Any] | None
+        New ``dict`` instance or ``None`` when the input is ``None``.
+    """
+    if value is None:
+        return None
+    return cast(dict[str, Any], value)
 
 
 # SECTION: TYPED DICTS ====================================================== #
@@ -176,9 +200,9 @@ class RequestOptions:
 
     def __post_init__(self) -> None:
         if self.params is not None:
-            object.__setattr__(self, 'params', dict(self.params))
+            object.__setattr__(self, 'params', _to_dict(self.params))
         if self.headers is not None:
-            object.__setattr__(self, 'headers', dict(self.headers))
+            object.__setattr__(self, 'headers', _to_dict(self.headers))
 
     # -- Instance Methods -- #
 
@@ -224,23 +248,20 @@ class RequestOptions:
 
         Returns
         -------
-        RequestOptions
+        Self
             New snapshot reflecting the provided overrides.
         """
         if params is _UNSET:
             next_params = self.params
-        elif params is None:
-            next_params = None
         else:
-            next_params = cast(dict, params)
+            # next_params = _to_dict(params) if params is not None else None
+            next_params = _to_dict(params)
 
         if headers is _UNSET:
             next_headers = self.headers
-        elif headers is None:
-            next_headers = None
         else:
-            next_headers = cast(dict, headers)
-
+            # next_headers = _to_dict(headers) if headers is not None else None
+            next_headers = _to_dict(headers)
         if timeout is _UNSET:
             next_timeout = self.timeout
         else:

etlplus/ops/extract.py

@@ -6,11 +6,19 @@ Helpers to extract data from files, databases, and REST APIs.
 
 from __future__ import annotations
 
+from collections.abc import Mapping
 from pathlib import Path
 from typing import Any
 from typing import cast
+from urllib.parse import urlsplit
+from urllib.parse import urlunsplit
 
+from ..api import EndpointClient
 from ..api import HttpMethod
+from ..api import PaginationConfigMap
+from ..api import RequestOptions
+from ..api import compose_api_request_env
+from ..api import paginate_with_client
 from ..api.utils import resolve_request
 from ..connector import DataConnectorType
 from ..file import File
@@ -19,6 +27,7 @@ from ..types import JSONData
 from ..types import JSONDict
 from ..types import JSONList
 from ..types import StrPath
+from ..types import Timeout
 
 # SECTION: EXPORTS ========================================================== #
 
@@ -32,50 +41,164 @@ __all__ = [
 ]
 
 
-# SECTION: FUNCTIONS ======================================================== #
+# SECTION: INTERNAL FUNCTIONS =============================================== #
 
 
-def extract_from_api(
-    url: str,
-    method: HttpMethod | str = HttpMethod.GET,
-    **kwargs: Any,
+def _build_client(
+    *,
+    base_url: str,
+    base_path: str | None,
+    endpoints: dict[str, str],
+    retry: Any,
+    retry_network_errors: bool,
+    session: Any,
+) -> EndpointClient:
+    """
+    Construct an API client with shared defaults.
+
+    Parameters
+    ----------
+    base_url : str
+        API base URL.
+    base_path : str | None
+        Base path to prepend for endpoints.
+    endpoints : dict[str, str]
+        Endpoint name to path mappings.
+    retry : Any
+        Retry policy configuration.
+    retry_network_errors : bool
+        Whether to retry on network errors.
+    session : Any
+        Optional requests session.
+
+    Returns
+    -------
+    EndpointClient
+        Configured endpoint client instance.
+    """
+    ClientClass = EndpointClient  # noqa: N806
+    return ClientClass(
+        base_url=base_url,
+        base_path=base_path,
+        endpoints=endpoints,
+        retry=retry,
+        retry_network_errors=retry_network_errors,
+        session=session,
+    )
+
+
+def _extract_from_api_env(
+    env: Mapping[str, Any],
+    *,
+    use_client: bool,
 ) -> JSONData:
     """
-    Extract data from a REST API.
+    Extract API data from a normalized request environment.
 
     Parameters
     ----------
-    url : str
-        API endpoint URL.
-    method : HttpMethod | str, optional
-        HTTP method to use. Defaults to ``GET``.
-    **kwargs : Any
-        Extra arguments forwarded to the underlying ``requests`` call
-        (for example, ``timeout``). To use a pre-configured
-        :class:`requests.Session`, provide it via ``session``.
-        When omitted, ``timeout`` defaults to 10 seconds.
+    env : Mapping[str, Any]
+        Normalized environment describing API request parameters.
+    use_client : bool
+        Whether to use the endpoint client/pagination machinery.
 
     Returns
     -------
     JSONData
-        Parsed JSON payload, or a fallback object with raw text.
+        Extracted payload.
 
     Raises
     ------
-    TypeError
-        If a provided ``session`` does not expose the required HTTP
-        method (for example, ``get``).
+    ValueError
+        If required parameters are missing.
     """
-    timeout = kwargs.pop('timeout', None)
-    session = kwargs.pop('session', None)
+    if (
+        use_client
+        and env.get('use_endpoints')
+        and env.get('base_url')
+        and env.get('endpoints_map')
+        and env.get('endpoint_key')
+    ):
+        client = _build_client(
+            base_url=cast(str, env.get('base_url')),
+            base_path=cast(str | None, env.get('base_path')),
+            endpoints=cast(dict[str, str], env.get('endpoints_map', {})),
+            retry=env.get('retry'),
+            retry_network_errors=bool(env.get('retry_network_errors', False)),
+            session=env.get('session'),
+        )
+        return paginate_with_client(
+            client,
+            cast(str, env.get('endpoint_key')),
+            env.get('params'),
+            env.get('headers'),
+            env.get('timeout'),
+            env.get('pagination'),
+            cast(float | None, env.get('sleep_seconds')),
+        )
+
+    url = env.get('url')
+    if not url:
+        raise ValueError('API source missing URL')
+
+    if use_client:
+        parts = urlsplit(cast(str, url))
+        base = urlunsplit((parts.scheme, parts.netloc, '', '', ''))
+        client = _build_client(
+            base_url=base,
+            base_path=None,
+            endpoints={},
+            retry=env.get('retry'),
+            retry_network_errors=bool(env.get('retry_network_errors', False)),
+            session=env.get('session'),
+        )
+        request_options = RequestOptions(
+            params=cast(Mapping[str, Any] | None, env.get('params')),
+            headers=cast(Mapping[str, str] | None, env.get('headers')),
+            timeout=cast(Timeout | None, env.get('timeout')),
+        )
+
+        return client.paginate_url(
+            cast(str, url),
+            cast(PaginationConfigMap | None, env.get('pagination')),
+            request=request_options,
+            sleep_seconds=cast(float, env.get('sleep_seconds', 0.0)),
+        )
+
+    method = env.get('method', HttpMethod.GET)
+    timeout = env.get('timeout', None)
+    session = env.get('session', None)
+    request_kwargs = dict(env.get('request_kwargs') or {})
     request_callable, timeout, _ = resolve_request(
         method,
         session=session,
         timeout=timeout,
     )
-    response = request_callable(url, timeout=timeout, **kwargs)
+    response = request_callable(
+        cast(str, url),
+        timeout=timeout,
+        **request_kwargs,
+    )
     response.raise_for_status()
+    return _parse_api_response(response)
+
 
+def _parse_api_response(
+    response: Any,
+) -> JSONData:
+    """
+    Parse API responses into a consistent JSON payload.
+
+    Parameters
+    ----------
+    response : Any
+        HTTP response object exposing ``headers``, ``json()``, and ``text``.
+
+    Returns
+    -------
+    JSONData
+        Parsed JSON payload, or a fallback object with raw text.
+    """
     content_type = response.headers.get('content-type', '').lower()
     if 'application/json' in content_type:
         try:
@@ -99,6 +222,70 @@ def extract_from_api(
     return {'content': response.text, 'content_type': content_type}
 
 
+# SECTION: FUNCTIONS ======================================================== #
+
+
+def extract_from_api(
+    url: str,
+    method: HttpMethod | str = HttpMethod.GET,
+    **kwargs: Any,
+) -> JSONData:
+    """
+    Extract data from a REST API.
+
+    Parameters
+    ----------
+    url : str
+        API endpoint URL.
+    method : HttpMethod | str, optional
+        HTTP method to use. Defaults to ``GET``.
+    **kwargs : Any
+        Extra arguments forwarded to the underlying ``requests`` call
+        (for example, ``timeout``). To use a pre-configured
+        :class:`requests.Session`, provide it via ``session``.
+        When omitted, ``timeout`` defaults to 10 seconds.
+
+    Returns
+    -------
+    JSONData
+        Parsed JSON payload, or a fallback object with raw text.
+    """
+    env = {
+        'url': url,
+        'method': method,
+        'timeout': kwargs.pop('timeout', None),
+        'session': kwargs.pop('session', None),
+        'request_kwargs': kwargs,
+    }
+    return _extract_from_api_env(env, use_client=False)
+
+
+def extract_from_api_source(
+    cfg: Any,
+    source_obj: Any,
+    overrides: dict[str, Any],
+) -> JSONData:
+    """
+    Extract data from a REST API source connector.
+
+    Parameters
+    ----------
+    cfg : Any
+        Pipeline configuration.
+    source_obj : Any
+        Connector configuration.
+    overrides : dict[str, Any]
+        Extract-time overrides.
+
+    Returns
+    -------
+    JSONData
+        Extracted payload.
+    """
+    env = compose_api_request_env(cfg, source_obj, overrides)
+    return _extract_from_api_env(env, use_client=True)
+
+
 def extract_from_database(
     connection_string: str,
 ) -> JSONList:

etlplus/ops/load.py

@@ -8,11 +8,13 @@ from __future__ import annotations
 
 import json
 import sys
+from collections.abc import Mapping
 from pathlib import Path
 from typing import Any
 from typing import cast
 
 from ..api import HttpMethod
+from ..api import compose_api_target_env
 from ..api.utils import resolve_request
 from ..connector import DataConnectorType
 from ..file import File
@@ -39,6 +41,108 @@ __all__ = [
 # SECTION: INTERNAL FUNCTIONS ============================================== #
 
 
+def _load_data_from_str(
+    source: str,
+) -> JSONData:
+    """
+    Load JSON data from a string or file path.
+
+    Parameters
+    ----------
+    source : str
+        Input string representing a file path or JSON payload.
+
+    Returns
+    -------
+    JSONData
+        Parsed JSON payload.
+    """
+    # Special case: '-' means read JSON from STDIN (Unix convention).
+    if source == '-':
+        raw = sys.stdin.read()
+        return _parse_json_string(raw)
+
+    candidate = Path(source)
+    if candidate.exists():
+        try:
+            return File(candidate, FileFormat.JSON).read()
+        except (OSError, json.JSONDecodeError, ValueError):
+            # Fall back to treating the string as raw JSON content.
+            pass
+    return _parse_json_string(source)
+
+
+def _load_to_api_env(
+    data: JSONData,
+    env: Mapping[str, Any],
+) -> JSONDict:
+    """
+    Load data to an API target using a normalized environment.
+
+    Parameters
+    ----------
+    data : JSONData
+        Payload to load.
+    env : Mapping[str, Any]
+        Normalized request environment.
+
+    Returns
+    -------
+    JSONDict
+        Load result payload.
+
+    Raises
+    ------
+    ValueError
+        If required parameters are missing.
+    """
+    url = env.get('url')
+    if not url:
+        raise ValueError('API target missing "url"')
+    method = env.get('method') or 'post'
+    kwargs: dict[str, Any] = {}
+    headers = env.get('headers')
+    if headers:
+        kwargs['headers'] = cast(dict[str, str], headers)
+    if env.get('timeout') is not None:
+        kwargs['timeout'] = env.get('timeout')
+    session = env.get('session')
+    if session is not None:
+        kwargs['session'] = session
+    extra_kwargs = env.get('request_kwargs')
+    if isinstance(extra_kwargs, Mapping):
+        kwargs.update(extra_kwargs)
+    timeout = kwargs.pop('timeout', 10.0)
+    session = kwargs.pop('session', None)
+    request_callable, timeout, http_method = resolve_request(
+        method,
+        session=session,
+        timeout=timeout,
+    )
+    response = request_callable(
+        cast(str, url),
+        json=data,
+        timeout=timeout,
+        **kwargs,
+    )
+    response.raise_for_status()
+
+    # Try JSON first, fall back to text.
+    try:
+        payload: Any = response.json()
+    except ValueError:
+        payload = response.text
+
+    return {
+        'status': 'success',
+        'status_code': response.status_code,
+        'message': f'Data loaded to {url}',
+        'response': payload,
+        'records': count_records(data),
+        'method': http_method.value.upper(),
+    }
+
+
 def _parse_json_string(
     raw: str,
 ) -> JSONData:
@@ -113,18 +217,7 @@ def load_data(
         return File(source, FileFormat.JSON).read()
 
     if isinstance(source, str):
-        # Special case: '-' means read JSON from STDIN (Unix convention).
-        if source == '-':
-            raw = sys.stdin.read()
-            return _parse_json_string(raw)
-        candidate = Path(source)
-        if candidate.exists():
-            try:
-                return File(candidate, FileFormat.JSON).read()
-            except (OSError, json.JSONDecodeError, ValueError):
-                # Fall back to treating the string as raw JSON content.
-                pass
-        return _parse_json_string(source)
+        return _load_data_from_str(source)
 
     raise TypeError(
         'source must be a mapping, sequence of mappings, path, or JSON string',
@@ -158,30 +251,43 @@ def load_to_api(
         Result dictionary including response payload or text.
     """
     # Apply a conservative timeout to guard against hanging requests.
-    timeout = kwargs.pop('timeout', 10.0)
-    session = kwargs.pop('session', None)
-    request_callable, timeout, http_method = resolve_request(
-        method,
-        session=session,
-        timeout=timeout,
-    )
-    response = request_callable(url, json=data, timeout=timeout, **kwargs)
-    response.raise_for_status()
+    env = {
+        'url': url,
+        'method': method,
+        'timeout': kwargs.pop('timeout', 10.0),
+        'session': kwargs.pop('session', None),
+        'request_kwargs': kwargs,
+    }
+    return _load_to_api_env(data, env)
 
-    # Try JSON first, fall back to text.
-    try:
-        payload: Any = response.json()
-    except ValueError:
-        payload = response.text
 
-    return {
-        'status': 'success',
-        'status_code': response.status_code,
-        'message': f'Data loaded to {url}',
-        'response': payload,
-        'records': count_records(data),
-        'method': http_method.value.upper(),
-    }
+def load_to_api_target(
+    cfg: Any,
+    target_obj: Any,
+    overrides: dict[str, Any],
+    data: JSONData,
+) -> JSONDict:
+    """
+    Load data to an API target connector.
+
+    Parameters
+    ----------
+    cfg : Any
+        Pipeline configuration.
+    target_obj : Any
+        Connector configuration.
+    overrides : dict[str, Any]
+        Load-time overrides.
+    data : JSONData
+        Payload to load.
+
+    Returns
+    -------
+    JSONDict
+        Load result.
+    """
+    env = compose_api_target_env(cfg, target_obj, overrides)
+    return _load_to_api_env(data, env)
 
 
 def load_to_database(

etlplus/ops/run.py

@@ -6,31 +6,23 @@ A module for running ETL jobs defined in YAML configurations.
 
 from __future__ import annotations
 
-from collections.abc import Mapping
 from typing import Any
 from typing import Final
 from typing import cast
-from urllib.parse import urlsplit
-from urllib.parse import urlunsplit
 
-from ..api import EndpointClient  # noqa: F401 (re-exported for tests)
 from ..api import HttpMethod
-from ..api import PaginationConfigMap
-from ..api import RequestOptions
-from ..api import compose_api_request_env
-from ..api import compose_api_target_env
-from ..api import paginate_with_client
 from ..connector import DataConnectorType
 from ..file import FileFormat
 from ..types import JSONData
 from ..types import JSONDict
 from ..types import PipelineConfig
 from ..types import StrPath
-from ..types import Timeout
 from ..utils import print_json
 from ..workflow import load_pipeline_config
 from .extract import extract
+from .extract import extract_from_api_source
 from .load import load
+from .load import load_to_api_target
 from .transform import transform
 from .utils import maybe_validate
 from .validate import validate
@@ -54,6 +46,75 @@ DEFAULT_CONFIG_PATH: Final[str] = 'in/pipeline.yml'
 # SECTION: INTERNAL FUNCTIONS =============================================== #
 
 
+def _index_connectors(
+    connectors: list[Any],
+    *,
+    label: str,
+) -> dict[str, Any]:
+    """
+    Index connectors by name with a helpful error on duplicates.
+
+    Parameters
+    ----------
+    connectors : list[Any]
+        Connector objects to index.
+    label : str
+        Label used in error messages (e.g., ``"source"``).
+
+    Returns
+    -------
+    dict[str, Any]
+        Mapping of connector names to connector objects.
+
+    Raises
+    ------
+    ValueError
+        If duplicate connector names are found.
+    """
+    indexed: dict[str, Any] = {}
+    for connector in connectors:
+        name = getattr(connector, 'name', None)
+        if not isinstance(name, str) or not name:
+            continue
+        if name in indexed:
+            raise ValueError(f'Duplicate {label} connector name: {name}')
+        indexed[name] = connector
+    return indexed
+
+
+def _require_named_connector(
+    connectors: dict[str, Any],
+    name: str,
+    *,
+    label: str,
+) -> Any:
+    """
+    Return a connector by name or raise a helpful error.
+
+    Parameters
+    ----------
+    connectors : dict[str, Any]
+        Mapping of connector names to connector objects.
+    name : str
+        Connector name to retrieve.
+    label : str
+        Label used in error messages (e.g., ``"source"``).
+
+    Returns
+    -------
+    Any
+        Connector object.
+
+    Raises
+    ------
+    ValueError
+        If the connector name is not found.
+    """
+    if name not in connectors:
+        raise ValueError(f'Unknown {label}: {name}')
+    return connectors[name]
+
+
 def _resolve_validation_config(
     job_obj: Any,
     cfg: Any,
@@ -122,16 +183,18 @@ def run(
         raise ValueError(f'Job not found: {job}')
 
     # Index sources/targets by name
-    sources_by_name = {getattr(s, 'name', None): s for s in cfg.sources}
-    targets_by_name = {getattr(t, 'name', None): t for t in cfg.targets}
+    sources_by_name = _index_connectors(cfg.sources, label='source')
+    targets_by_name = _index_connectors(cfg.targets, label='target')
 
     # Extract.
     if not job_obj.extract:
         raise ValueError('Job missing "extract" section')
     source_name = job_obj.extract.source
-    if source_name not in sources_by_name:
-        raise ValueError(f'Unknown source: {source_name}')
-    source_obj = sources_by_name[source_name]
+    source_obj = _require_named_connector(
+        sources_by_name,
+        source_name,
+        label='source',
+    )
     ex_opts: dict[str, Any] = job_obj.extract.options or {}
 
     data: Any
@@ -151,68 +214,7 @@ def run(
             conn = getattr(source_obj, 'connection_string', '')
             data = extract('database', conn)
         case DataConnectorType.API:
-            env = compose_api_request_env(cfg, source_obj, ex_opts)
-            if (
-                env.get('use_endpoints')
-                and env.get('base_url')
-                and env.get('endpoints_map')
-                and env.get('endpoint_key')
-            ):
-                # Construct client using module-level EndpointClient so tests
-                # can monkeypatch this class on etlplus.ops.run.
-                ClientClass = EndpointClient  # noqa: N806
-                client = ClientClass(
-                    base_url=cast(str, env.get('base_url')),
-                    base_path=cast(str | None, env.get('base_path')),
-                    endpoints=cast(
-                        dict[str, str],
-                        env.get('endpoints_map', {}),
-                    ),
-                    retry=env.get('retry'),
-                    retry_network_errors=bool(
-                        env.get('retry_network_errors', False),
-                    ),
-                    session=env.get('session'),
-                )
-                data = paginate_with_client(
-                    client,
-                    cast(str, env.get('endpoint_key')),
-                    env.get('params'),
-                    env.get('headers'),
-                    env.get('timeout'),
-                    env.get('pagination'),
-                    cast(float | None, env.get('sleep_seconds')),
-                )
-            else:
-                url = env.get('url')
-                if not url:
-                    raise ValueError('API source missing URL')
-                parts = urlsplit(cast(str, url))
-                base = urlunsplit((parts.scheme, parts.netloc, '', '', ''))
-                ClientClass = EndpointClient  # noqa: N806
-                client = ClientClass(
-                    base_url=base,
-                    base_path=None,
-                    endpoints={},
-                    retry=env.get('retry'),
-                    retry_network_errors=bool(
-                        env.get('retry_network_errors', False),
-                    ),
-                    session=env.get('session'),
-                )
-
-                request_options = RequestOptions(
-                    params=cast(Mapping[str, Any] | None, env.get('params')),
-                    headers=cast(Mapping[str, str] | None, env.get('headers')),
-                    timeout=cast(Timeout | None, env.get('timeout')),
-                )
-
-                data = client.paginate_url(
-                    cast(str, url),
-                    cast(PaginationConfigMap | None, env.get('pagination')),
-                    request=request_options,
-                    sleep_seconds=cast(float, env.get('sleep_seconds', 0.0)),
-                )
+            data = extract_from_api_source(cfg, source_obj, ex_opts)
         case _:
             # :meth:`coerce` already raises for invalid connector types, but
             # keep explicit guard for defensive programming.
@@ -256,9 +258,11 @@ def run(
     if not job_obj.load:
         raise ValueError('Job missing "load" section')
     target_name = job_obj.load.target
-    if target_name not in targets_by_name:
-        raise ValueError(f'Unknown target: {target_name}')
-    target_obj = targets_by_name[target_name]
+    target_obj = _require_named_connector(
+        targets_by_name,
+        target_name,
+        label='target',
+    )
     overrides = job_obj.load.overrides or {}
 
     ttype_raw = getattr(target_obj, 'type', None)
@@ -274,26 +278,7 @@ def run(
                 raise ValueError('File target missing "path"')
             result = load(data, 'file', path, file_format=fmt)
         case DataConnectorType.API:
-            env_t = compose_api_target_env(cfg, target_obj, overrides)
-            url_t = env_t.get('url')
-            if not url_t:
-                raise ValueError('API target missing "url"')
-            kwargs_t: dict[str, Any] = {}
-            headers = env_t.get('headers')
-            if headers:
-                kwargs_t['headers'] = cast(dict[str, str], headers)
-            if env_t.get('timeout') is not None:
-                kwargs_t['timeout'] = env_t.get('timeout')
-            session = env_t.get('session')
-            if session is not None:
-                kwargs_t['session'] = session
-            result = load(
-                data,
-                'api',
-                cast(str, url_t),
-                method=cast(str | Any, env_t.get('method') or 'post'),
-                **kwargs_t,
-            )
+            result = load_to_api_target(cfg, target_obj, overrides, data)
         case DataConnectorType.DATABASE:
             conn = overrides.get('connection_string') or getattr(
                 target_obj,

etlplus/ops/transform.py

@@ -206,15 +206,12 @@ def _normalize_specs(
     """
     if config is None:
         return []
-    if isinstance(config, Sequence) and not isinstance(
-        config,
-        (str, bytes, bytearray),
-    ):
+    if _is_sequence_not_text(config):
         # Already a sequence of step specs; normalize to a list.
-        return list(config)  # type: ignore[list-item]
+        return list(cast(Sequence[StepSpec], config))
 
     # Single spec
-    return [config]
+    return [cast(StepSpec, config)]
 
 
 def _normalize_operation_keys(ops: Mapping[Any, Any]) -> dict[str, Any]:
@@ -702,7 +699,31 @@ def _apply_sort_step(
 # -- Helpers -- #
 
 
-def _is_plain_fields_list(obj: Any) -> bool:
+def _is_sequence_not_text(
+    obj: Any,
+) -> bool:
+    """
+    Return ``True`` for non-text sequences.
+
+    Parameters
+    ----------
+    obj : Any
+        The object to check.
+
+    Returns
+    -------
+    bool
+        ``True`` when *obj* is a non-text sequence.
+    """
+    return isinstance(obj, Sequence) and not isinstance(
+        obj,
+        (str, bytes, bytearray),
+    )
+
+
+def _is_plain_fields_list(
+    obj: Any,
+) -> bool:
     """
     Return True if obj is a non-text sequence of non-mapping items.
 
@@ -719,10 +740,8 @@ def _is_plain_fields_list(obj: Any) -> bool:
         True if obj is a non-text sequence of non-mapping items, False
         otherwise.
     """
-    return (
-        isinstance(obj, Sequence)
-        and not isinstance(obj, (str, bytes, bytearray))
-        and not any(isinstance(x, Mapping) for x in obj)
+    return _is_sequence_not_text(obj) and not any(
+        isinstance(x, Mapping) for x in obj
     )
 
 

etlplus/types.py

@@ -12,8 +12,7 @@ Notes
 See Also
 --------
 - :mod:`etlplus.api.types` for HTTP-specific aliases and data classes
-- :mod:`etlplus.connector.types` for connector-specific aliases and TypedDict
-    surfaces
+- :mod:`etlplus.connector.types` for connector-specific aliases
 
 Examples
 --------
@@ -81,6 +80,8 @@ __all__ = [
     # Type Aliases (Networking / Runtime)
     'Sleeper',
     'Timeout',
+    # Type Aliases (Templates)
+    'TemplateKey',
 ]
 
 

etlplus/workflow/__init__.py

@@ -14,6 +14,7 @@ from .jobs import TransformRef
 from .jobs import ValidationRef
 from .pipeline import PipelineConfig
 from .pipeline import load_pipeline_config
+from .profile import ProfileConfig
 
 # SECTION: EXPORTS ========================================================== #
 
@@ -24,6 +25,7 @@ __all__ = [
     'JobConfig',
     'LoadRef',
     'PipelineConfig',
+    'ProfileConfig',
     'TransformRef',
     'ValidationRef',
     # Functions

etlplus/workflow/dag.py

@@ -47,6 +47,28 @@ class DagError(ValueError):
         return self.message
 
 
+# SECTION: INTERNAL FUNCTIONS =============================================== #
+
+
+def _ready(
+    indegree: dict[str, int],
+) -> list[str]:
+    """
+    Return a sorted list of nodes with zero indegree.
+
+    Parameters
+    ----------
+    indegree : dict[str, int]
+        Mapping of node name to indegree.
+
+    Returns
+    -------
+    list[str]
+        Sorted list of node names ready to process.
+    """
+    return sorted(name for name, deg in indegree.items() if deg == 0)
+
+
 # SECTION: FUNCTIONS ======================================================== #
 
 
@@ -88,7 +110,7 @@ def topological_sort_jobs(
                 edges[dep].add(job.name)
                 indegree[job.name] += 1
 
-    queue = deque(sorted(name for name, deg in indegree.items() if deg == 0))
+    queue = deque(_ready(indegree))
     ordered: list[str] = []
 
     while queue:

etlplus/workflow/jobs.py

@@ -14,6 +14,7 @@ Notes
 
 from __future__ import annotations
 
+from collections.abc import Sequence
 from dataclasses import dataclass
 from dataclasses import field
 from typing import Any
@@ -76,13 +77,15 @@ def _parse_depends_on(
     """
     if isinstance(value, str):
         return [value]
-    if isinstance(value, list):
+    if isinstance(value, Sequence) and not isinstance(
+        value,
+        (str, bytes, bytearray),
+    ):
         return [entry for entry in value if isinstance(entry, str)]
     return []
 
 
 def _require_str(
-    # data: dict[str, Any],
     data: StrAnyMap,
     key: str,
 ) -> str | None:
@@ -149,13 +152,9 @@ class ExtractRef:
         data = maybe_mapping(obj)
         if not data:
             return None
-        source = _require_str(data, 'source')
-        if source is None:
+        if (source := _require_str(data, 'source')) is None:
             return None
-        return cls(
-            source=source,
-            options=coerce_dict(data.get('options')),
-        )
+        return cls(source=source, options=coerce_dict(data.get('options')))
 
 
 @dataclass(kw_only=True, slots=True)
@@ -214,18 +213,13 @@ class JobConfig:
         data = maybe_mapping(obj)
         if not data:
             return None
-        name = _require_str(data, 'name')
-        if name is None:
+        if (name := _require_str(data, 'name')) is None:
             return None
 
-        description = _coerce_optional_str(data.get('description'))
-
-        depends_on = _parse_depends_on(data.get('depends_on'))
-
         return cls(
             name=name,
-            description=description,
-            depends_on=depends_on,
+            description=_coerce_optional_str(data.get('description')),
+            depends_on=_parse_depends_on(data.get('depends_on')),
             extract=ExtractRef.from_obj(data.get('extract')),
             validate=ValidationRef.from_obj(data.get('validate')),
             transform=TransformRef.from_obj(data.get('transform')),
@@ -274,8 +268,7 @@ class LoadRef:
         data = maybe_mapping(obj)
         if not data:
             return None
-        target = _require_str(data, 'target')
-        if target is None:
+        if (target := _require_str(data, 'target')) is None:
             return None
         return cls(
             target=target,
@@ -321,8 +314,7 @@ class TransformRef:
         data = maybe_mapping(obj)
         if not data:
             return None
-        pipeline = _require_str(data, 'pipeline')
-        if pipeline is None:
+        if (pipeline := _require_str(data, 'pipeline')) is None:
             return None
         return cls(pipeline=pipeline)
 
@@ -372,13 +364,10 @@ class ValidationRef:
         data = maybe_mapping(obj)
         if not data:
             return None
-        ruleset = _require_str(data, 'ruleset')
-        if ruleset is None:
+        if (ruleset := _require_str(data, 'ruleset')) is None:
             return None
-        severity = _coerce_optional_str(data.get('severity'))
-        phase = _coerce_optional_str(data.get('phase'))
         return cls(
             ruleset=ruleset,
-            severity=severity,
-            phase=phase,
+            severity=_coerce_optional_str(data.get('severity')),
+            phase=_coerce_optional_str(data.get('phase')),
         )

etlplus/workflow/pipeline.py

@@ -50,20 +50,42 @@ __all__ = [
 # SECTION: INTERNAL FUNCTIONS =============================================== #
 
 
-def _collect_parsed[T](
+def _build_connectors(
     raw: StrAnyMap,
+    *,
     key: str,
-    parser: Callable[[Any], T | None],
-) -> list[T]:
+) -> list[Connector]:
     """
-    Collect parsed items from ``raw[key]`` using a tolerant parser.
+    Parse connector entries from a list under ``raw[key]``.
 
     Parameters
     ----------
     raw : StrAnyMap
         Raw pipeline mapping.
     key : str
-        Key pointing to a list-like payload.
+        Key pointing to connector entries (e.g., ``"sources"``).
+
+    Returns
+    -------
+    list[Connector]
+        Parsed connector instances.
+    """
+    return list(
+        _collect_parsed(raw.get(key, []) or [], _parse_connector_entry),
+    )
+
+
+def _collect_parsed[T](
+    items: Any,
+    parser: Callable[[Any], T | None],
+) -> list[T]:
+    """
+    Collect parsed items from ``raw[key]`` using a tolerant parser.
+
+    Parameters
+    ----------
+    items : Any
+        List-like payload to parse.
     parser : Callable[[Any], T | None]
         Parser that returns an instance or ``None`` for invalid entries.
 
@@ -72,12 +94,12 @@ def _collect_parsed[T](
     list[T]
         Parsed items, excluding invalid entries.
     """
-    items: list[T] = []
-    for entry in raw.get(key, []) or []:
+    parsed_items: list[T] = []
+    for entry in items or []:
         parsed = parser(entry)
         if parsed is not None:
-            items.append(parsed)
-    return items
+            parsed_items.append(parsed)
+    return parsed_items
 
 
 def _parse_connector_entry(
@@ -104,48 +126,6 @@ def _parse_connector_entry(
         return None
 
 
-def _build_sources(
-    raw: StrAnyMap,
-) -> list[Connector]:
-    """
-    Return a list of source connectors parsed from the mapping.
-
-    Parameters
-    ----------
-    raw : StrAnyMap
-        Raw pipeline mapping.
-
-    Returns
-    -------
-    list[Connector]
-        Parsed source connectors.
-    """
-    return list(
-        _collect_parsed(raw, 'sources', _parse_connector_entry),
-    )
-
-
-def _build_targets(
-    raw: StrAnyMap,
-) -> list[Connector]:
-    """
-    Return a list of target connectors parsed from the mapping.
-
-    Parameters
-    ----------
-    raw : StrAnyMap
-        Raw pipeline mapping.
-
-    Returns
-    -------
-    list[Connector]
-        Parsed target connectors.
-    """
-    return list(
-        _collect_parsed(raw, 'targets', _parse_connector_entry),
-    )
-
-
 # SECTION: FUNCTIONS ======================================================== #
 
 
@@ -311,17 +291,20 @@ class PipelineConfig:
         file_systems = coerce_dict(raw.get('file_systems'))
 
         # Sources
-        sources = _build_sources(raw)
+        sources = _build_connectors(raw, key='sources')
 
         # Validations/Transforms
         validations = coerce_dict(raw.get('validations'))
         transforms = coerce_dict(raw.get('transforms'))
 
         # Targets
-        targets = _build_targets(raw)
+        targets = _build_connectors(raw, key='targets')
 
         # Jobs
-        jobs = _collect_parsed(raw, 'jobs', JobConfig.from_obj)
+        jobs: list[JobConfig] = _collect_parsed(
+            raw.get('jobs', []) or [],
+            JobConfig.from_obj,
+        )
 
         # Table schemas (optional, tolerant pass-through structures).
         table_schemas: list[dict[str, Any]] = []

etlplus/workflow/profile.py

@@ -18,6 +18,7 @@ from typing import Self
 
 from ..types import StrAnyMap
 from ..utils import cast_str_dict
+from ..utils import maybe_mapping
 
 # SECTION: EXPORTS ========================================================== #
 
@@ -56,7 +57,8 @@ class ProfileConfig:
         cls,
         obj: StrAnyMap | None,
     ) -> Self:
-        """Parse a mapping into a :class:`ProfileConfig` instance.
+        """
+        Parse a mapping into a :class:`ProfileConfig` instance.
 
         Parameters
         ----------
@@ -73,7 +75,7 @@ class ProfileConfig:
             return cls()
 
         # Coerce all env values to strings using shared helper.
-        env = cast_str_dict(obj.get('env'))
+        env = cast_str_dict(maybe_mapping(obj.get('env')))
 
         return cls(
             default_target=obj.get('default_target'),

{etlplus-0.16.0.dist-info → etlplus-0.16.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: etlplus
-Version: 0.16.0
+Version: 0.16.2
 Summary: A Swiss Army knife for simple ETL operations
 Home-page: https://github.com/Dagitali/ETLPlus
 Author: ETLPlus Team

{etlplus-0.16.0.dist-info → etlplus-0.16.2.dist-info}/RECORD

@@ -5,7 +5,7 @@ etlplus/__version__.py,sha256=1E0GMK_yUWCMQFKxXjTvyMwofi0qT2k4CDNiHWiymWE,327
 etlplus/enums.py,sha256=8-uUOKe68cPzlmUg-e7gavkC95kbTJXRpRzvXehIsRk,6841
 etlplus/mixins.py,sha256=ifGpHwWv7U00yqGf-kN93vJax2IiK4jaGtTsPsO3Oak,1350
 etlplus/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-etlplus/types.py,sha256=vB1Sh_5771eVv8-RClJatBQ7PcsScY6C2MvJqnI8IG8,6216
+etlplus/types.py,sha256=J1ftMh0dmxe9ObFd3eXCbHiFvZI_5pV_hEHUpgeJQtY,6239
 etlplus/utils.py,sha256=X-k_Y8i6oDjlE5aQu9sw3gPw7O2ikiSn4uoheVv_ERc,17091
 etlplus/api/README.md,sha256=amxS_eIcsnNuVvD0x_w8nkyfedOTYbhlY0gGhaFg0DE,8705
 etlplus/api/__init__.py,sha256=PK2lQv1FbsE7ZZS_ejevFZQSuOUHGApBc22YfHAzMqA,4615
@@ -17,7 +17,7 @@ etlplus/api/errors.py,sha256=XjI2xW-sypMUNUbqfc2S57-IGyWnH3oCDFhCmKYYI_Q,4648
 etlplus/api/request_manager.py,sha256=fhzPV5x7DqpKqoLvfDR8GKhBX_QBMtvZsRXxVnQQElY,18674
 etlplus/api/retry_manager.py,sha256=aq9iNCxt-Puy4rAgKNtNucxw2eP1yqAKZ2lfgMkzbCk,11302
 etlplus/api/transport.py,sha256=abm-_WieBDSSbFanBwhmudBuVVm7LjYUb8vrlMXo7SA,9408
-etlplus/api/types.py,sha256=Ng1b83RaJSHn4jl-M1f1dsTgjXizQtrW4yOXAYjwk_4,7377
+etlplus/api/types.py,sha256=UcTrB347So12l8NplY-_HDf2T5IwZL_2r8CJDUSAm5Q,7975
 etlplus/api/utils.py,sha256=lNBfJKz3fJ4RhvnnX3uxVZC__6-WKksYMSGGYi0RRqM,26247
 etlplus/api/pagination/__init__.py,sha256=a4UX2J0AG8RMvmHt_CCofUm5vSmFo6GAfkb8XnSXypM,1395
 etlplus/api/pagination/client.py,sha256=yMEpWqRxTCD4zRc9OYtEyUtShpGH5atiHFEAt95v2FE,5394
@@ -117,10 +117,10 @@ etlplus/file/zip.py,sha256=8wnmnGW_pGTx65736CzAG67XIi5y98KxucRT8sNDeuQ,4195
 etlplus/file/zsav.py,sha256=5hMuBjYeHw--UL2ZCCDn6TzJkr_YNhdQhvKI6nr3WW0,1674
 etlplus/ops/README.md,sha256=8omi7DYZhelc26JKk8Cm8QR8I3OGwziysPj1ivx41iQ,1380
 etlplus/ops/__init__.py,sha256=NIIr2f-AZj5B0piBt6gjv46Yn0SzGYxEe6BPoopRh38,1702
-etlplus/ops/extract.py,sha256=xNHcKdx8LrnvIMWUFG-SmszpAlclz2jhvKFfX-sFGLI,5994
-etlplus/ops/load.py,sha256=sxADrZ0CE4NfJ7T4Chwa6ckTkK3gSjYRdJW4qL-B61k,8536
-etlplus/ops/run.py,sha256=x7bYrokpBE4XRfl6Rq7xeD3s-5A9P4HUVlufj_PxoTw,13526
-etlplus/ops/transform.py,sha256=H9DxAA-c9kwklAnmiZK21w7FMJnHfu0bLcguXfIGE-k,25417
+etlplus/ops/extract.py,sha256=LOYiPrALRMF7JDBabnRF24_HKnnIcfTdfXesWdS3QZM,11020
+etlplus/ops/load.py,sha256=yicciVwomUKkdbhuRqbavKBNpT2Hg813BnQzG6IgF4o,10811
+etlplus/ops/run.py,sha256=FYb2W5pi2PXx5E-l5etFMUcr7UmQWfMNHew9-otWIYE,11294
+etlplus/ops/transform.py,sha256=3qIJsy2lUSMPoTRWn8Yw4JocKV_ZTQx_fKRW0w73Cnc,25682
 etlplus/ops/utils.py,sha256=lJmrO1KDob-xZU8Gc2SvZvMgdYLsVoaz-fTV42KkLVo,10835
 etlplus/ops/validate.py,sha256=-OLAwQNNCmmDbmj0SB7zzYXDkJfcyBP_z9nTpqImLP0,13271
 etlplus/templates/README.md,sha256=IfPXlj1TGVA-uFWosHJhE2rabFW-znxOlOMazO9Z5cE,1361
@@ -128,14 +128,14 @@ etlplus/templates/__init__.py,sha256=tsniN7XJYs3NwYxJ6c2HD5upHP3CDkLx-bQCMt97UOM
 etlplus/templates/ddl.sql.j2,sha256=s8fMWvcb4eaJVXkifuib1aQPljtZ8buuyB_uA-ZdU3Q,4734
 etlplus/templates/view.sql.j2,sha256=Iy8DHfhq5yyvrUKDxqp_aHIEXY4Tm6j4wT7YDEFWAhk,2180
 etlplus/workflow/README.md,sha256=D1oloiJCOHiqpqgv3m3qpRSIUOMIQcWtIsOPv7KkNI0,1652
-etlplus/workflow/__init__.py,sha256=PWWAv4Bb94hls8YDZXU5DY5aXMbtSYOLYHqGRThxoA4,668
-etlplus/workflow/dag.py,sha256=kp31dORgk0GHbct_bipU5hu_0elwBtwLsXGjMWuhFHI,2503
-etlplus/workflow/jobs.py,sha256=onvzsZpTpZUWIivIGR4SjbUPSZ1X0nXpxAW0O0VumhQ,8945
-etlplus/workflow/pipeline.py,sha256=7bGvMywBu_H6XLsYjeNBS_NmVH7Bo7BmuzkWm-BAXMI,9633
-etlplus/workflow/profile.py,sha256=dZ6P50k_ZqXnrbgrbODUqgVkymbchcEqfZR-ExjTd3M,1935
-etlplus-0.16.0.dist-info/licenses/LICENSE,sha256=MuNO63i6kWmgnV2pbP2SLqP54mk1BGmu7CmbtxMmT-U,1069
-etlplus-0.16.0.dist-info/METADATA,sha256=soyAK0MDUoH-U0-WQt40XS6zH8L8Q5dywyHtEO3cUGo,28114
-etlplus-0.16.0.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
-etlplus-0.16.0.dist-info/entry_points.txt,sha256=6w-2-jzuPa55spzK34h-UKh2JTEShh38adFRONNP9QE,45
-etlplus-0.16.0.dist-info/top_level.txt,sha256=aWWF-udn_sLGuHTM6W6MLh99ArS9ROkUWO8Mi8y1_2U,8
-etlplus-0.16.0.dist-info/RECORD,,
+etlplus/workflow/__init__.py,sha256=ueothwpLruyLgr3-2hW8VT1unNyFJxdmT-l_3eB2ejc,724
+etlplus/workflow/dag.py,sha256=-f1x8N1eb-PUuiOwEvFLmJwfR7JaMDJihlCHlhrFhgE,2937
+etlplus/workflow/jobs.py,sha256=5DmAzmEZV6XXQ-xzowkLxFzplIh8Eno3wuCmjy79xHw,8818
+etlplus/workflow/pipeline.py,sha256=PA5zhcfrk--pAg3b3x4oBf29WMj5HqR8zOozz4oEmg8,9387
+etlplus/workflow/profile.py,sha256=FQU3bzBZ9_yjKC9kCXKN1FQDS9zjNUjtWB1r3UL95_Q,1993
+etlplus-0.16.2.dist-info/licenses/LICENSE,sha256=MuNO63i6kWmgnV2pbP2SLqP54mk1BGmu7CmbtxMmT-U,1069
+etlplus-0.16.2.dist-info/METADATA,sha256=QdFDSAYSrjZKyu5G8TWQSlC1Lobu8hA9qgXpnIOQ2eM,28114
+etlplus-0.16.2.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
+etlplus-0.16.2.dist-info/entry_points.txt,sha256=6w-2-jzuPa55spzK34h-UKh2JTEShh38adFRONNP9QE,45
+etlplus-0.16.2.dist-info/top_level.txt,sha256=aWWF-udn_sLGuHTM6W6MLh99ArS9ROkUWO8Mi8y1_2U,8
+etlplus-0.16.2.dist-info/RECORD,,