etlplus 0.16.0__py3-none-any.whl → 0.16.3__py3-none-any.whl

etlplus/ops/run.py

@@ -6,31 +6,23 @@ A module for running ETL jobs defined in YAML configurations.
 
 from __future__ import annotations
 
-from collections.abc import Mapping
 from typing import Any
 from typing import Final
 from typing import cast
-from urllib.parse import urlsplit
-from urllib.parse import urlunsplit
 
-from ..api import EndpointClient  # noqa: F401 (re-exported for tests)
 from ..api import HttpMethod
-from ..api import PaginationConfigMap
-from ..api import RequestOptions
-from ..api import compose_api_request_env
-from ..api import compose_api_target_env
-from ..api import paginate_with_client
 from ..connector import DataConnectorType
 from ..file import FileFormat
+from ..ops.types import PipelineConfig
 from ..types import JSONData
 from ..types import JSONDict
-from ..types import PipelineConfig
 from ..types import StrPath
-from ..types import Timeout
 from ..utils import print_json
 from ..workflow import load_pipeline_config
 from .extract import extract
+from .extract import extract_from_api_source
 from .load import load
+from .load import load_to_api_target
 from .transform import transform
 from .utils import maybe_validate
 from .validate import validate
@@ -54,6 +46,75 @@ DEFAULT_CONFIG_PATH: Final[str] = 'in/pipeline.yml'
 # SECTION: INTERNAL FUNCTIONS =============================================== #
 
 
+def _index_connectors(
+    connectors: list[Any],
+    *,
+    label: str,
+) -> dict[str, Any]:
+    """
+    Index connectors by name with a helpful error on duplicates.
+
+    Parameters
+    ----------
+    connectors : list[Any]
+        Connector objects to index.
+    label : str
+        Label used in error messages (e.g., ``"source"``).
+
+    Returns
+    -------
+    dict[str, Any]
+        Mapping of connector names to connector objects.
+
+    Raises
+    ------
+    ValueError
+        If duplicate connector names are found.
+    """
+    indexed: dict[str, Any] = {}
+    for connector in connectors:
+        name = getattr(connector, 'name', None)
+        if not isinstance(name, str) or not name:
+            continue
+        if name in indexed:
+            raise ValueError(f'Duplicate {label} connector name: {name}')
+        indexed[name] = connector
+    return indexed
+
+
+def _require_named_connector(
+    connectors: dict[str, Any],
+    name: str,
+    *,
+    label: str,
+) -> Any:
+    """
+    Return a connector by name or raise a helpful error.
+
+    Parameters
+    ----------
+    connectors : dict[str, Any]
+        Mapping of connector names to connector objects.
+    name : str
+        Connector name to retrieve.
+    label : str
+        Label used in error messages (e.g., ``"source"``).
+
+    Returns
+    -------
+    Any
+        Connector object.
+
+    Raises
+    ------
+    ValueError
+        If the connector name is not found.
+    """
+    if name not in connectors:
+        raise ValueError(f'Unknown {label}: {name}')
+    return connectors[name]
+
+
 def _resolve_validation_config(
     job_obj: Any,
     cfg: Any,
@@ -122,16 +183,18 @@ def run(
         raise ValueError(f'Job not found: {job}')
 
     # Index sources/targets by name
-    sources_by_name = {getattr(s, 'name', None): s for s in cfg.sources}
-    targets_by_name = {getattr(t, 'name', None): t for t in cfg.targets}
+    sources_by_name = _index_connectors(cfg.sources, label='source')
+    targets_by_name = _index_connectors(cfg.targets, label='target')
 
     # Extract.
     if not job_obj.extract:
         raise ValueError('Job missing "extract" section')
     source_name = job_obj.extract.source
-    if source_name not in sources_by_name:
-        raise ValueError(f'Unknown source: {source_name}')
-    source_obj = sources_by_name[source_name]
+    source_obj = _require_named_connector(
+        sources_by_name,
+        source_name,
+        label='source',
+    )
     ex_opts: dict[str, Any] = job_obj.extract.options or {}
 
     data: Any
@@ -151,68 +214,7 @@ def run(
             conn = getattr(source_obj, 'connection_string', '')
             data = extract('database', conn)
         case DataConnectorType.API:
-            env = compose_api_request_env(cfg, source_obj, ex_opts)
-            if (
-                env.get('use_endpoints')
-                and env.get('base_url')
-                and env.get('endpoints_map')
-                and env.get('endpoint_key')
-            ):
-                # Construct client using module-level EndpointClient so tests
-                # can monkeypatch this class on etlplus.ops.run.
-                ClientClass = EndpointClient  # noqa: N806
-                client = ClientClass(
-                    base_url=cast(str, env.get('base_url')),
-                    base_path=cast(str | None, env.get('base_path')),
-                    endpoints=cast(
-                        dict[str, str],
-                        env.get('endpoints_map', {}),
-                    ),
-                    retry=env.get('retry'),
-                    retry_network_errors=bool(
-                        env.get('retry_network_errors', False),
-                    ),
-                    session=env.get('session'),
-                )
-                data = paginate_with_client(
-                    client,
-                    cast(str, env.get('endpoint_key')),
-                    env.get('params'),
-                    env.get('headers'),
-                    env.get('timeout'),
-                    env.get('pagination'),
-                    cast(float | None, env.get('sleep_seconds')),
-                )
-            else:
-                url = env.get('url')
-                if not url:
-                    raise ValueError('API source missing URL')
-                parts = urlsplit(cast(str, url))
-                base = urlunsplit((parts.scheme, parts.netloc, '', '', ''))
-                ClientClass = EndpointClient  # noqa: N806
-                client = ClientClass(
-                    base_url=base,
-                    base_path=None,
-                    endpoints={},
-                    retry=env.get('retry'),
-                    retry_network_errors=bool(
-                        env.get('retry_network_errors', False),
-                    ),
-                    session=env.get('session'),
-                )
-
-                request_options = RequestOptions(
-                    params=cast(Mapping[str, Any] | None, env.get('params')),
-                    headers=cast(Mapping[str, str] | None, env.get('headers')),
-                    timeout=cast(Timeout | None, env.get('timeout')),
-                )
-
-                data = client.paginate_url(
-                    cast(str, url),
-                    cast(PaginationConfigMap | None, env.get('pagination')),
-                    request=request_options,
-                    sleep_seconds=cast(float, env.get('sleep_seconds', 0.0)),
-                )
+            data = extract_from_api_source(cfg, source_obj, ex_opts)
         case _:
             # :meth:`coerce` already raises for invalid connector types, but
             # keep explicit guard for defensive programming.
@@ -256,9 +258,11 @@ def run(
     if not job_obj.load:
         raise ValueError('Job missing "load" section')
     target_name = job_obj.load.target
-    if target_name not in targets_by_name:
-        raise ValueError(f'Unknown target: {target_name}')
-    target_obj = targets_by_name[target_name]
+    target_obj = _require_named_connector(
+        targets_by_name,
+        target_name,
+        label='target',
+    )
     overrides = job_obj.load.overrides or {}
 
     ttype_raw = getattr(target_obj, 'type', None)
@@ -274,26 +278,7 @@ def run(
                 raise ValueError('File target missing "path"')
             result = load(data, 'file', path, file_format=fmt)
         case DataConnectorType.API:
-            env_t = compose_api_target_env(cfg, target_obj, overrides)
-            url_t = env_t.get('url')
-            if not url_t:
-                raise ValueError('API target missing "url"')
-            kwargs_t: dict[str, Any] = {}
-            headers = env_t.get('headers')
-            if headers:
-                kwargs_t['headers'] = cast(dict[str, str], headers)
-            if env_t.get('timeout') is not None:
-                kwargs_t['timeout'] = env_t.get('timeout')
-            session = env_t.get('session')
-            if session is not None:
-                kwargs_t['session'] = session
-            result = load(
-                data,
-                'api',
-                cast(str, url_t),
-                method=cast(str | Any, env_t.get('method') or 'post'),
-                **kwargs_t,
-            )
+            result = load_to_api_target(cfg, target_obj, overrides, data)
         case DataConnectorType.DATABASE:
             conn = overrides.get('connection_string') or getattr(
                 target_obj,

etlplus/ops/transform.py

@@ -44,28 +44,28 @@ from collections.abc import Sequence
 from typing import Any
 from typing import cast
 
-from ..enums import AggregateName
-from ..enums import OperatorName
-from ..enums import PipelineStep
-from ..types import AggregateFunc
-from ..types import AggregateSpec
-from ..types import FieldName
-from ..types import Fields
-from ..types import FilterSpec
+from ..ops.types import PipelineConfig
 from ..types import JSONData
 from ..types import JSONDict
 from ..types import JSONList
-from ..types import MapSpec
-from ..types import OperatorFunc
-from ..types import PipelineConfig
-from ..types import PipelineStepName
-from ..types import SortKey
-from ..types import StepApplier
-from ..types import StepOrSteps
-from ..types import StepSpec
 from ..types import StrPath
 from ..utils import to_number
+from .enums import AggregateName
+from .enums import OperatorName
+from .enums import PipelineStep
 from .load import load_data
+from .types import AggregateFunc
+from .types import AggregateSpec
+from .types import FieldName
+from .types import Fields
+from .types import FilterSpec
+from .types import MapSpec
+from .types import OperatorFunc
+from .types import PipelineStepName
+from .types import SortKey
+from .types import StepApplier
+from .types import StepOrSteps
+from .types import StepSpec
 
 # SECTION: EXPORTS ========================================================== #
 
@@ -206,15 +206,12 @@ def _normalize_specs(
     """
     if config is None:
         return []
-    if isinstance(config, Sequence) and not isinstance(
-        config,
-        (str, bytes, bytearray),
-    ):
+    if _is_sequence_not_text(config):
         # Already a sequence of step specs; normalize to a list.
-        return list(config)  # type: ignore[list-item]
+        return list(cast(Sequence[StepSpec], config))
 
     # Single spec
-    return [config]
+    return [cast(StepSpec, config)]
 
 
 def _normalize_operation_keys(ops: Mapping[Any, Any]) -> dict[str, Any]:
@@ -702,7 +699,31 @@ def _apply_sort_step(
 # -- Helpers -- #
 
 
-def _is_plain_fields_list(obj: Any) -> bool:
+def _is_sequence_not_text(
+    obj: Any,
+) -> bool:
+    """
+    Return ``True`` for non-text sequences.
+
+    Parameters
+    ----------
+    obj : Any
+        The object to check.
+
+    Returns
+    -------
+    bool
+        ``True`` when *obj* is a non-text sequence.
+    """
+    return isinstance(obj, Sequence) and not isinstance(
+        obj,
+        (str, bytes, bytearray),
+    )
+
+
+def _is_plain_fields_list(
+    obj: Any,
+) -> bool:
     """
     Return True if obj is a non-text sequence of non-mapping items.
 
@@ -719,10 +740,8 @@ def _is_plain_fields_list(obj: Any) -> bool:
         True if obj is a non-text sequence of non-mapping items, False
         otherwise.
     """
-    return (
-        isinstance(obj, Sequence)
-        and not isinstance(obj, (str, bytes, bytearray))
-        and not any(isinstance(x, Mapping) for x in obj)
+    return _is_sequence_not_text(obj) and not any(
+        isinstance(x, Mapping) for x in obj
     )
 
 

etlplus/ops/types.py

@@ -0,0 +1,147 @@
+"""
+:mod:`etlplus.ops.types` module.
+
+Shared type aliases leveraged across :mod:`etlplus.ops` modules.
+
+Notes
+-----
+- Centralizes ops-focused aliases (functions, specs, and pipeline helpers).
+- Relies on Python 3.13 ``type`` statements for readability and IDE support.
+
+Examples
+--------
+>>> from etlplus.ops.types import AggregateFunc, OperatorFunc
+>>> def total(xs: list[float], _: int) -> float:
+...     return sum(xs)
+>>> agg: AggregateFunc = total
+>>> op: OperatorFunc = lambda a, b: a == b
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from collections.abc import Mapping
+from collections.abc import Sequence
+from typing import Any
+from typing import Literal
+
+from ..types import JSONList
+from ..types import StrAnyMap
+from ..types import StrSeqMap
+from ..types import StrStrMap
+
+# SECTION: EXPORTS ========================================================== #
+
+
+__all__ = [
+    # Type Aliases (Functions)
+    'AggregateFunc',
+    'OperatorFunc',
+    # Type Aliases (Records & Fields)
+    'FieldName',
+    'Fields',
+    # Type Aliases (Transform Specs)
+    'AggregateSpec',
+    'FilterSpec',
+    'MapSpec',
+    'SelectSpec',
+    'SortSpec',
+    # Type Aliases (Pipelines)
+    'StepOrSteps',
+    'StepSeq',
+    'StepSpec',
+    'PipelineConfig',
+    'PipelineStepName',
+    # Type Aliases (Helpers)
+    'StepApplier',
+    'SortKey',
+]
+
+
+# SECTION: TYPE ALIASES ===================================================== #
+
+
+# -- Functions -- #
+
+
+# TODO: Consider redefining to use `functools.reduce` signature.
+# TODO: Consider adding `**kwargs` to support richer aggregation functions.
+# TODO: Consider constraining first argument to `Sequence[float]`.
+# TODO: Consider constraining return type to `float | int | None`.
+# Callable reducing numeric collections into a summary value.
+type AggregateFunc = Callable[[list[float], int], Any]
+
+# Binary predicate consumed by filter operations.
+type OperatorFunc = Callable[[Any, Any], bool]
+
+# -- Records & Fields -- #
+
+# Individual field identifier referenced inside specs.
+type FieldName = str
+
+# Ordered list of :data:`FieldName` entries preserving projection order.
+type Fields = list[FieldName]
+
+# -- Transform Specs -- #
+
+# Filtering spec expecting ``field``, ``op``, and ``value`` keys.
+type FilterSpec = StrAnyMap
+
+# Field renaming instructions mapping old keys to new ones.
+type MapSpec = StrStrMap
+
+# Projection spec as a field list or mapping with metadata.
+#
+# Examples
+# --------
+# >>> from etlplus.ops.types import SelectSpec
+# >>> spec1: SelectSpec = ['a','b']
+# >>> spec2: SelectSpec = {'fields': [...]}
+type SelectSpec = Fields | StrSeqMap
+
+# Sort directive expressed as a field string or mapping with flags.
+#
+# Examples
+# --------
+# >>> from etlplus.ops.types import SortSpec
+# >>> spec1: SortSpec = 'field'
+# >>> spec2: SortSpec = {'field': 'x', 'reverse': True}
+type SortSpec = str | StrAnyMap
+
+# Aggregate instruction covering ``field``, ``func``, and optional alias.
+#
+# Supported functions: ``avg``, ``count``, ``max``, ``min``, and ``sum``.
+# Examples
+# --------
+# >>> from etlplus.ops.types import AggregateSpec
+# >>> spec: AggregateSpec = \
+# ...   {'field': 'x', 'func': 'sum' | 'avg' | ..., 'alias'?: '...'}
+type AggregateSpec = StrAnyMap
+
+# -- Pipelines-- #
+
+# Unified pipeline step spec consumed by :mod:`etlplus.ops.transform`.
+type StepSpec = AggregateSpec | FilterSpec | MapSpec | SelectSpec | SortSpec
+
+# Collections of steps
+
+# Ordered collection of :data:`StepSpec` entries.
+type StepSeq = Sequence[StepSpec]
+
+# Accepts either a single :data:`StepSpec` or a sequence of them.
+type StepOrSteps = StepSpec | StepSeq
+
+# Canonical literal names for supported transform stages.
+type PipelineStepName = Literal['aggregate', 'filter', 'map', 'select', 'sort']
+
+# Mapping from step name to its associated specification payload.
+# TODO: Consider replacing with etlplus.workflow.types.PipelineConfig.
+type PipelineConfig = Mapping[PipelineStepName, StepOrSteps]
+
+# -- Helpers -- #
+
+# Callable that applies step configuration to a batch of records.
+type StepApplier = Callable[[JSONList, Any], JSONList]
+
+# Tuple combining stable sort index and computed sort value.
+type SortKey = tuple[int, Any]

etlplus/types.py

@@ -12,8 +12,7 @@ Notes
 See Also
 --------
 - :mod:`etlplus.api.types` for HTTP-specific aliases and data classes
-- :mod:`etlplus.connector.types` for connector-specific aliases and TypedDict
-    surfaces
+- :mod:`etlplus.connector.types` for connector-specific aliases
 
 Examples
 --------
@@ -54,33 +53,15 @@ __all__ = [
     'JSONRecords',
     # Type Aliases (File System)
     'StrPath',
-    # Type Aliases (Functions)
-    'AggregateFunc',
-    'OperatorFunc',
-    # Type Aliases (Records & Fields)
-    'FieldName',
-    'Fields',
     # Type Aliases (Transform Specs)
     'StrAnyMap',
     'StrSeqMap',
     'StrStrMap',
-    'AggregateSpec',
-    'FilterSpec',
-    'MapSpec',
-    'SelectSpec',
-    'SortSpec',
-    # Type Aliases (Pipelines)
-    'StepOrSteps',
-    'StepSeq',
-    'StepSpec',
-    'PipelineStepName',
-    'PipelineConfig',
-    # Type Aliases (Helpers)
-    'StepApplier',
-    'SortKey',
     # Type Aliases (Networking / Runtime)
     'Sleeper',
     'Timeout',
+    # Type Aliases (Templates)
+    'TemplateKey',
 ]
 
 
@@ -125,22 +106,6 @@ type JSONRecords = list[JSONRecord]
 # Path-like inputs accepted by file helpers.
 type StrPath = str | Path | PathLike[str]
 
-# -- Functions -- #
-
-# Callable reducing numeric collections into a summary value.
-type AggregateFunc = Callable[[list[float], int], Any]
-
-# Binary predicate consumed by filter operations.
-type OperatorFunc = Callable[[Any, Any], bool]
-
-# -- Records & Fields -- #
-
-# Individual field identifier referenced inside specs.
-type FieldName = str
-
-# Ordered list of :data:`FieldName` entries preserving projection order.
-type Fields = list[FieldName]
-
 # -- Transform Specs -- #
 
 # Kept intentionally broad for runtime-friendly validation in transform.py.
@@ -156,69 +121,6 @@ type StrStrMap = Mapping[str, str]
 # Mapping whose values are homogeneous sequences.
 type StrSeqMap = Mapping[str, Sequence[Any]]
 
-# Transform step specifications
-
-# Filtering spec expecting ``field``, ``op``, and ``value`` keys.
-type FilterSpec = StrAnyMap
-
-# Field renaming instructions mapping old keys to new ones.
-type MapSpec = StrStrMap
-
-# Projection spec as a field list or mapping with metadata.
-#
-# Examples
-# --------
-# >>> from etlplus.types import SelectSpec
-# >>> spec1: SelectSpec = ['a','b']
-# >>> spec2: SelectSpec = {'fields': [...]}
-type SelectSpec = Fields | StrSeqMap
-
-# Sort directive expressed as a field string or mapping with flags.
-#
-# Examples
-# --------
-# >>> from etlplus.types import SortSpec
-# >>> spec1: SortSpec = 'field'
-# >>> spec2: SortSpec = {'field': 'x', 'reverse': True}
-type SortSpec = str | StrAnyMap
-
-# Aggregate instruction covering ``field``, ``func``, and optional alias.
-#
-# Supported functions: ``avg``, ``count``, ``max``, ``min``, and ``sum``.
-# Examples
-# --------
-# >>> from etlplus.types import AggregateSpec
-# >>> spec: AggregateSpec = \
-# ...   {'field': 'x', 'func': 'sum' | 'avg' | ..., 'alias'?: '...'}
-type AggregateSpec = StrAnyMap
-
-# -- Pipelines-- #
-
-# Unified pipeline step spec consumed by :mod:`etlplus.ops.transform`.
-type StepSpec = AggregateSpec | FilterSpec | MapSpec | SelectSpec | SortSpec
-
-# Collections of steps
-
-# Ordered collection of :data:`StepSpec` entries.
-type StepSeq = Sequence[StepSpec]
-
-# Accepts either a single :data:`StepSpec` or a sequence of them.
-type StepOrSteps = StepSpec | StepSeq
-
-# Canonical literal names for supported transform stages.
-type PipelineStepName = Literal['filter', 'map', 'select', 'sort', 'aggregate']
-
-# Mapping from step name to its associated specification payload.
-type PipelineConfig = Mapping[PipelineStepName, StepOrSteps]
-
-# -- Helpers -- #
-
-# Callable that applies step configuration to a batch of records.
-type StepApplier = Callable[[JSONList, Any], JSONList]
-
-# Tuple combining stable sort index and computed sort value.
-type SortKey = tuple[int, Any]
-
 # -- Networking / Runtime -- #
 
 # Sleep function used by retry helpers.

etlplus/workflow/__init__.py

@@ -14,6 +14,7 @@ from .jobs import TransformRef
 from .jobs import ValidationRef
 from .pipeline import PipelineConfig
 from .pipeline import load_pipeline_config
+from .profile import ProfileConfig
 
 # SECTION: EXPORTS ========================================================== #
 
@@ -24,6 +25,7 @@ __all__ = [
     'JobConfig',
     'LoadRef',
     'PipelineConfig',
+    'ProfileConfig',
     'TransformRef',
     'ValidationRef',
     # Functions

etlplus/workflow/dag.py

@@ -47,6 +47,28 @@ class DagError(ValueError):
         return self.message
 
 
+# SECTION: INTERNAL FUNCTIONS =============================================== #
+
+
+def _ready(
+    indegree: dict[str, int],
+) -> list[str]:
+    """
+    Return a sorted list of nodes with zero indegree.
+
+    Parameters
+    ----------
+    indegree : dict[str, int]
+        Mapping of node name to indegree.
+
+    Returns
+    -------
+    list[str]
+        Sorted list of node names ready to process.
+    """
+    return sorted(name for name, deg in indegree.items() if deg == 0)
+
+
 # SECTION: FUNCTIONS ======================================================== #
 
 
@@ -88,7 +110,7 @@ def topological_sort_jobs(
                 edges[dep].add(job.name)
                 indegree[job.name] += 1
 
-    queue = deque(sorted(name for name, deg in indegree.items() if deg == 0))
+    queue = deque(_ready(indegree))
     ordered: list[str] = []
 
     while queue: