etlplus 0.16.2__py3-none-any.whl → 0.16.4__py3-none-any.whl

etlplus/README.md

@@ -31,6 +31,28 @@ assert validate(filtered, {"age": {"type": "number", "min": 0}})["valid"]
 load(filtered, "file", "output.json", file_format="json")
 ```
 
+## Loading and Validating Configs
+
+Use the provided classes to load and validate configuration files:
+
+```python
+from etlplus.workflow import Config
+
+cfg = Config.from_yaml("pipeline.yml")
+```
+
+- Supports YAML and JSON formats
+- Validates against expected schema
+
+## Example: Loading a Pipeline Config
+
+```python
+from etlplus.workflow import Config
+
+config = Config.from_yaml("configs/pipeline.yml")
+print(config)
+```
+
 ## See Also
 
 - [Top-level project README](../README.md)

etlplus/__init__.py

@@ -5,6 +5,7 @@ Top-level facade for the ETLPlus toolkit.
 """
 
 from .__version__ import __version__
+from .config import Config
 
 __author__ = 'ETLPlus Team'
 
@@ -15,4 +16,5 @@ __author__ = 'ETLPlus Team'
 __all__ = [
     '__author__',
     '__version__',
+    'Config',
 ]

etlplus/cli/commands.py

@@ -62,6 +62,16 @@ __all__ = ['app']
 # SECTION: TYPE ALIASES ==================================================== #
 
 
+ConfigOption = Annotated[
+    str,
+    typer.Option(
+        ...,
+        '--config',
+        metavar='PATH',
+        help='Path to YAML-formatted configuration file.',
+    ),
+]
+
 JobOption = Annotated[
     str | None,
     typer.Option(
@@ -97,16 +107,6 @@ OutputOption = Annotated[
     ),
 ]
 
-PipelineConfigOption = Annotated[
-    str,
-    typer.Option(
-        ...,
-        '--config',
-        metavar='PATH',
-        help='Path to pipeline YAML configuration file.',
-    ),
-]
-
 PipelineOption = Annotated[
     str | None,
     typer.Option(
@@ -407,7 +407,7 @@ def _root(
 @app.command('check')
 def check_cmd(
     ctx: typer.Context,
-    config: PipelineConfigOption,
+    config: ConfigOption,
     jobs: JobsOption = False,
     pipelines: PipelinesOption = False,
     sources: SourcesOption = False,
@@ -422,20 +422,20 @@ def check_cmd(
     ----------
     ctx : typer.Context
         The Typer context.
-    config : PipelineConfigOption
+    config : ConfigOption
         Path to pipeline YAML configuration file.
-    jobs : bool, optional
+    jobs : JobsOption, optional
         List available job names and exit. Default is ``False``.
-    pipelines : bool, optional
+    pipelines : PipelinesOption, optional
         List ETL pipelines. Default is ``False``.
-    sources : bool, optional
+    sources : SourcesOption, optional
         List data sources. Default is ``False``.
-    summary : bool, optional
+    summary : SummaryOption, optional
         Show pipeline summary (name, version, sources, targets, jobs). Default
         is ``False``.
-    targets : bool, optional
+    targets : TargetsOption, optional
         List data targets. Default is ``False``.
-    transforms : bool, optional
+    transforms : TransformsOption, optional
         List data transforms. Default is ``False``.
 
     Returns
@@ -725,7 +725,7 @@ def render_cmd(
 @app.command('run')
 def run_cmd(
     ctx: typer.Context,
-    config: PipelineConfigOption,
+    config: ConfigOption,
     job: JobOption = None,
     pipeline: PipelineOption = None,
 ) -> int:
@@ -736,11 +736,11 @@ def run_cmd(
     ----------
     ctx : typer.Context
         The Typer context.
-    config : PipelineConfigOption
+    config : ConfigOption
         Path to pipeline YAML configuration file.
-    job : str | None, optional
+    job : JobOption, optional
         Name of the job to run. Default is ``None``.
-    pipeline : str | None, optional
+    pipeline : PipelineOption, optional
         Name of the pipeline to run. Default is ``None``.
 
     Returns

etlplus/cli/handlers.py

@@ -14,6 +14,7 @@ from typing import Any
 from typing import Literal
 from typing import cast
 
+from .. import Config
 from ..database import load_table_spec
 from ..database import render_tables
 from ..file import File
@@ -26,8 +27,6 @@ from ..ops import validate
 from ..ops.validate import FieldRules
 from ..types import JSONData
 from ..types import TemplateKey
-from ..workflow import PipelineConfig
-from ..workflow import load_pipeline_config
 from . import io as cli_io
 
 # SECTION: EXPORTS ========================================================== #
@@ -73,14 +72,14 @@ def _collect_table_specs(
         specs.append(dict(load_table_spec(Path(spec_path))))
 
     if config_path:
-        cfg = load_pipeline_config(config_path, substitute=True)
+        cfg = Config.from_yaml(config_path, substitute=True)
         specs.extend(getattr(cfg, 'table_schemas', []))
 
     return specs
 
 
 def _check_sections(
-    cfg: PipelineConfig,
+    cfg: Config,
     *,
     jobs: bool,
     pipelines: bool,
@@ -93,7 +92,7 @@ def _check_sections(
 
     Parameters
     ----------
-    cfg : PipelineConfig
+    cfg : Config
         The loaded pipeline configuration.
     jobs : bool
         Whether to include job metadata.
@@ -133,14 +132,14 @@ def _check_sections(
 
 
 def _pipeline_summary(
-    cfg: PipelineConfig,
+    cfg: Config,
 ) -> dict[str, Any]:
     """
     Return a human-friendly snapshot of a pipeline config.
 
     Parameters
     ----------
-    cfg : PipelineConfig
+    cfg : Config
         The loaded pipeline configuration.
 
     Returns
@@ -229,7 +228,7 @@ def check_handler(
         Zero on success.
 
     """
-    cfg = load_pipeline_config(config, substitute=substitute)
+    cfg = Config.from_yaml(config, substitute=substitute)
     if summary:
         cli_io.emit_json(_pipeline_summary(cfg), pretty=True)
         return 0
@@ -514,7 +513,7 @@ def run_handler(
     int
         Zero on success.
     """
-    cfg = load_pipeline_config(config, substitute=True)
+    cfg = Config.from_yaml(config, substitute=True)
 
     job_name = job or pipeline
     if job_name:

etlplus/{workflow/pipeline.py → config.py}

@@ -1,7 +1,7 @@
 """
-:mod:`etlplus.workflow.pipeline` module.
+:mod:`etlplus.config` module.
 
-Pipeline configuration model and helpers for job orchestration.
+Configuration model and helpers for job pipeline orchestration.
 
 Notes
 -----
@@ -24,26 +24,24 @@ from pathlib import Path
 from typing import Any
 from typing import Self
 
-from ..api import ApiConfig
-from ..connector import Connector
-from ..connector import parse_connector
-from ..file import File
-from ..file import FileFormat
-from ..types import StrAnyMap
-from ..utils import coerce_dict
-from ..utils import deep_substitute
-from ..utils import maybe_mapping
-from .jobs import JobConfig
-from .profile import ProfileConfig
+from .api import ApiConfig
+from .connector import Connector
+from .connector import parse_connector
+from .file import File
+from .file import FileFormat
+from .types import StrAnyMap
+from .utils import coerce_dict
+from .utils import deep_substitute
+from .utils import maybe_mapping
+from .workflow.jobs import JobConfig
+from .workflow.profile import ProfileConfig
 
 # SECTION: EXPORTS ========================================================== #
 
 
 __all__ = [
     # Data Classes
-    'PipelineConfig',
-    # Functions
-    'load_pipeline_config',
+    'Config',
 ]
 
 
@@ -126,29 +124,11 @@ def _parse_connector_entry(
         return None
 
 
-# SECTION: FUNCTIONS ======================================================== #
-
-
-def load_pipeline_config(
-    path: Path | str,
-    *,
-    substitute: bool = False,
-    env: Mapping[str, str] | None = None,
-) -> PipelineConfig:
-    """
-    Load a pipeline YAML file into a ``PipelineConfig`` instance.
-
-    Delegates to ``PipelineConfig.from_yaml`` for construction and optional
-    variable substitution.
-    """
-    return PipelineConfig.from_yaml(path, substitute=substitute, env=env)
-
-
 # SECTION: DATA CLASSES ===================================================== #
 
 
 @dataclass(kw_only=True, slots=True)
-class PipelineConfig:
+class Config:
     """
     Configuration for the data processing pipeline.
 
@@ -211,7 +191,7 @@ class PipelineConfig:
         env: Mapping[str, str] | None = None,
     ) -> Self:
         """
-        Parse a YAML file into a ``PipelineConfig`` instance.
+        Parse a YAML file into a ``Config`` instance.
 
         Parameters
         ----------
@@ -259,7 +239,7 @@ class PipelineConfig:
         raw: StrAnyMap,
     ) -> Self:
         """
-        Parse a mapping into a ``PipelineConfig`` instance.
+        Parse a mapping into a ``Config`` instance.
 
         Parameters
         ----------

etlplus/enums.py

@@ -1,18 +1,14 @@
 """
 :mod:`etlplus.enums` module.
 
-Shared enumeration types used across ETLPlus modules.
+Shared enumeration base class.
 """
 
 from __future__ import annotations
 
 import enum
-import operator as _op
-from statistics import fmean
 from typing import Self
 
-from .types import AggregateFunc
-from .types import OperatorFunc
 from .types import StrStrMap
 
 # SECTION: EXPORTS ========================================================== #
@@ -20,10 +16,7 @@ from .types import StrStrMap
 
 __all__ = [
     # Enums
-    'AggregateName',
     'CoercibleStrEnum',
-    'OperatorName',
-    'PipelineStep',
 ]
 
 
@@ -41,6 +34,7 @@ class CoercibleStrEnum(enum.StrEnum):
     Notes
     -----
     - Values are normalized via ``str(value).strip().casefold()``.
+    - If value matching fails, the raw string is tried as a member name.
     - Error messages enumerate allowed values for easier debugging.
     """
 
@@ -56,7 +50,13 @@ class CoercibleStrEnum(enum.StrEnum):
         Returns
         -------
         StrStrMap
-            A mapping of alias names to their corresponding enum member names.
+            A mapping of alias strings to their corresponding enum member
+            values or names.
+
+        Notes
+        -----
+        - Alias keys are normalized via ``str(key).strip().casefold()``.
+        - Alias values should be member values or member names.
         """
         return {}
 
@@ -80,7 +80,7 @@ class CoercibleStrEnum(enum.StrEnum):
         Parameters
         ----------
         value : Self | str | object
-            An existing enum member or a text value to normalize.
+            An existing enum member or a string-like value to normalize.
 
         Returns
         -------
@@ -95,10 +95,26 @@ class CoercibleStrEnum(enum.StrEnum):
         if isinstance(value, cls):
             return value
         try:
-            normalized = str(value).strip().casefold()
-            resolved = cls.aliases().get(normalized, normalized)
-            return cls(resolved)  # type: ignore[arg-type]
-        except (ValueError, TypeError) as e:
+            raw = str(value).strip()
+            normalized = raw.casefold()
+            aliases = {
+                str(key).strip().casefold(): alias
+                for key, alias in cls.aliases().items()
+            }
+            resolved = aliases.get(normalized)
+            if resolved is None:
+                try:
+                    return cls(normalized)  # type: ignore[arg-type]
+                except (ValueError, TypeError):
+                    return cls[raw]  # type: ignore[index]
+            if isinstance(resolved, cls):
+                return resolved
+            try:
+                return cls(resolved)  # type: ignore[arg-type]
+            except (ValueError, TypeError):
+                # Allow aliases to reference member names.
+                return cls[resolved]  # type: ignore[index]
+        except (ValueError, TypeError, KeyError) as e:
             allowed = ', '.join(cls.choices())
             raise ValueError(
                 f'Invalid {cls.__name__} value: {value!r}. Allowed: {allowed}',
@@ -107,15 +123,15 @@ class CoercibleStrEnum(enum.StrEnum):
     @classmethod
     def try_coerce(
         cls,
-        value: object,
+        value: Self | str | object,
     ) -> Self | None:
         """
-        Best-effort parse; return ``None`` on failure instead of raising.
+        Attempt to coerce a value into the enum; return ``None`` on failure.
 
         Parameters
         ----------
-        value : object
-            An existing enum member or a text value to normalize.
+        value : Self | str | object
+            An existing enum member or a string-like value to normalize.
 
         Returns
         -------
@@ -124,153 +140,5 @@ class CoercibleStrEnum(enum.StrEnum):
         """
         try:
             return cls.coerce(value)
-        except ValueError:
+        except (ValueError, TypeError, KeyError):
             return None
-
-
-# SECTION: ENUMS ============================================================ #
-
-
-class AggregateName(CoercibleStrEnum):
-    """Supported aggregations with helpers."""
-
-    # -- Constants -- #
-
-    AVG = 'avg'
-    COUNT = 'count'
-    MAX = 'max'
-    MIN = 'min'
-    SUM = 'sum'
-
-    # -- Class Methods -- #
-
-    @property
-    def func(self) -> AggregateFunc:
-        """
-        Get the aggregation function for this aggregation type.
-
-        Returns
-        -------
-        AggregateFunc
-            The aggregation function corresponding to this aggregation type.
-        """
-        if self is AggregateName.COUNT:
-            return lambda xs, n: n
-        if self is AggregateName.MAX:
-            return lambda xs, n: (max(xs) if xs else None)
-        if self is AggregateName.MIN:
-            return lambda xs, n: (min(xs) if xs else None)
-        if self is AggregateName.SUM:
-            return lambda xs, n: sum(xs)
-
-        # AVG
-        return lambda xs, n: (fmean(xs) if xs else 0.0)
-
-
-class OperatorName(CoercibleStrEnum):
-    """Supported comparison operators with helpers."""
-
-    # -- Constants -- #
-
-    EQ = 'eq'
-    NE = 'ne'
-    GT = 'gt'
-    GTE = 'gte'
-    LT = 'lt'
-    LTE = 'lte'
-    IN = 'in'
-    CONTAINS = 'contains'
-
-    # -- Getters -- #
-
-    @property
-    def func(self) -> OperatorFunc:
-        """
-        Get the comparison function for this operator.
-
-        Returns
-        -------
-        OperatorFunc
-            The comparison function corresponding to this operator.
-        """
-        match self:
-            case OperatorName.EQ:
-                return _op.eq
-            case OperatorName.NE:
-                return _op.ne
-            case OperatorName.GT:
-                return _op.gt
-            case OperatorName.GTE:
-                return _op.ge
-            case OperatorName.LT:
-                return _op.lt
-            case OperatorName.LTE:
-                return _op.le
-            case OperatorName.IN:
-                return lambda a, b: a in b
-            case OperatorName.CONTAINS:
-                return lambda a, b: b in a
-
-    # -- Class Methods -- #
-
-    @classmethod
-    def aliases(cls) -> StrStrMap:
-        """
-        Return a mapping of common aliases for each enum member.
-
-        Returns
-        -------
-        StrStrMap
-            A mapping of alias names to their corresponding enum member names.
-        """
-        return {
-            '==': 'eq',
-            '=': 'eq',
-            '!=': 'ne',
-            '<>': 'ne',
-            '>=': 'gte',
-            '≥': 'gte',
-            '<=': 'lte',
-            '≤': 'lte',
-            '>': 'gt',
-            '<': 'lt',
-        }
-
-
-class PipelineStep(CoercibleStrEnum):
-    """Pipeline step names as an enum for internal orchestration."""
-
-    # -- Constants -- #
-
-    FILTER = 'filter'
-    MAP = 'map'
-    SELECT = 'select'
-    SORT = 'sort'
-    AGGREGATE = 'aggregate'
-
-    # -- Getters -- #
-
-    @property
-    def order(self) -> int:
-        """
-        Get the execution order of this pipeline step.
-
-        Returns
-        -------
-        int
-            The execution order of this pipeline step.
-        """
-        return _PIPELINE_ORDER_INDEX[self]
-
-
-# SECTION: INTERNAL CONSTANTS ============================================== #
-
-
-# Precomputed order index for PipelineStep; avoids recomputing on each access.
-_PIPELINE_ORDER_INDEX: dict[PipelineStep, int] = {
-    PipelineStep.FILTER: 0,
-    PipelineStep.MAP: 1,
-    PipelineStep.SELECT: 2,
-    PipelineStep.SORT: 3,
-    PipelineStep.AGGREGATE: 4,
-}

etlplus/ops/__init__.py

@@ -52,6 +52,7 @@ from .validate import validate
 
 
 __all__ = [
+    # Functions
     'extract',
     'load',
     'run',

etlplus/ops/enums.py

@@ -0,0 +1,173 @@
+"""
+:mod:`etlplus.ops.enums` module.
+
+Operation-specific enums and helpers.
+"""
+
+from __future__ import annotations
+
+import operator as _op
+from statistics import fmean
+
+from ..enums import CoercibleStrEnum
+from ..types import StrStrMap
+from .types import AggregateFunc
+from .types import OperatorFunc
+
+# SECTION: EXPORTS ========================================================= #
+
+
+__all__ = [
+    # Enums
+    'AggregateName',
+    'OperatorName',
+    'PipelineStep',
+]
+
+
+# SECTION: ENUMS ============================================================ #
+
+
+class AggregateName(CoercibleStrEnum):
+    """Supported aggregations with helpers."""
+
+    # -- Constants -- #
+
+    AVG = 'avg'
+    COUNT = 'count'
+    MAX = 'max'
+    MIN = 'min'
+    SUM = 'sum'
+
+    # -- Class Methods -- #
+
+    @property
+    def func(self) -> AggregateFunc:
+        """
+        Get the aggregation function for this aggregation type.
+
+        Returns
+        -------
+        AggregateFunc
+            The aggregation function corresponding to this aggregation type.
+        """
+        if self is AggregateName.COUNT:
+            return lambda xs, n: n
+        if self is AggregateName.MAX:
+            return lambda xs, n: (max(xs) if xs else None)
+        if self is AggregateName.MIN:
+            return lambda xs, n: (min(xs) if xs else None)
+        if self is AggregateName.SUM:
+            return lambda xs, n: sum(xs)
+
+        # AVG
+        return lambda xs, n: (fmean(xs) if xs else 0.0)
+
+
+class OperatorName(CoercibleStrEnum):
+    """Supported comparison operators with helpers."""
+
+    # -- Constants -- #
+
+    EQ = 'eq'
+    NE = 'ne'
+    GT = 'gt'
+    GTE = 'gte'
+    LT = 'lt'
+    LTE = 'lte'
+    IN = 'in'
+    CONTAINS = 'contains'
+
+    # -- Getters -- #
+
+    @property
+    def func(self) -> OperatorFunc:
+        """
+        Get the comparison function for this operator.
+
+        Returns
+        -------
+        OperatorFunc
+            The comparison function corresponding to this operator.
+        """
+        match self:
+            case OperatorName.EQ:
+                return _op.eq
+            case OperatorName.NE:
+                return _op.ne
+            case OperatorName.GT:
+                return _op.gt
+            case OperatorName.GTE:
+                return _op.ge
+            case OperatorName.LT:
+                return _op.lt
+            case OperatorName.LTE:
+                return _op.le
+            case OperatorName.IN:
+                return lambda a, b: a in b
+            case OperatorName.CONTAINS:
+                return lambda a, b: b in a
+
+    # -- Class Methods -- #
+
+    @classmethod
+    def aliases(cls) -> StrStrMap:
+        """
+        Return a mapping of common aliases for each enum member.
+
+        Returns
+        -------
+        StrStrMap
+            A mapping of alias names to their corresponding enum member names.
+        """
+        return {
+            '==': 'eq',
+            '=': 'eq',
+            '!=': 'ne',
+            '<>': 'ne',
+            '>=': 'gte',
+            '≥': 'gte',
+            '<=': 'lte',
+            '≤': 'lte',
+            '>': 'gt',
+            '<': 'lt',
+        }
+
+
+class PipelineStep(CoercibleStrEnum):
+    """Pipeline step names as an enum for internal orchestration."""
+
+    # -- Constants -- #
+
+    FILTER = 'filter'
+    MAP = 'map'
+    SELECT = 'select'
+    SORT = 'sort'
+    AGGREGATE = 'aggregate'
+
+    # -- Getters -- #
+
+    @property
+    def order(self) -> int:
+        """
+        Get the execution order of this pipeline step.
+
+        Returns
+        -------
+        int
+            The execution order of this pipeline step.
+        """
+        return _PIPELINE_ORDER_INDEX[self]
+
+
+# SECTION: INTERNAL CONSTANTS ============================================== #
+
+
+# Precomputed order index for PipelineStep; avoids recomputing on each access.
+_PIPELINE_ORDER_INDEX: dict[PipelineStep, int] = {
+    PipelineStep.FILTER: 0,
+    PipelineStep.MAP: 1,
+    PipelineStep.SELECT: 2,
+    PipelineStep.SORT: 3,
+    PipelineStep.AGGREGATE: 4,
+}

etlplus/ops/run.py

@@ -11,14 +11,14 @@ from typing import Final
 from typing import cast
 
 from ..api import HttpMethod
+from ..config import Config
 from ..connector import DataConnectorType
 from ..file import FileFormat
+from ..ops.types import PipelineConfig
 from ..types import JSONData
 from ..types import JSONDict
-from ..types import PipelineConfig
 from ..types import StrPath
 from ..utils import print_json
-from ..workflow import load_pipeline_config
 from .extract import extract
 from .extract import extract_from_api_source
 from .load import load
@@ -176,7 +176,7 @@ def run(
         If the job is not found or if there are configuration issues.
     """
     cfg_path = config_path or DEFAULT_CONFIG_PATH
-    cfg = load_pipeline_config(cfg_path, substitute=True)
+    cfg = Config.from_yaml(cfg_path, substitute=True)
 
     # Lookup job by name
     if not (job_obj := next((j for j in cfg.jobs if j.name == job), None)):

etlplus/ops/transform.py

@@ -44,28 +44,28 @@ from collections.abc import Sequence
 from typing import Any
 from typing import cast
 
-from ..enums import AggregateName
-from ..enums import OperatorName
-from ..enums import PipelineStep
-from ..types import AggregateFunc
-from ..types import AggregateSpec
-from ..types import FieldName
-from ..types import Fields
-from ..types import FilterSpec
+from ..ops.types import PipelineConfig
 from ..types import JSONData
 from ..types import JSONDict
 from ..types import JSONList
-from ..types import MapSpec
-from ..types import OperatorFunc
-from ..types import PipelineConfig
-from ..types import PipelineStepName
-from ..types import SortKey
-from ..types import StepApplier
-from ..types import StepOrSteps
-from ..types import StepSpec
 from ..types import StrPath
 from ..utils import to_number
+from .enums import AggregateName
+from .enums import OperatorName
+from .enums import PipelineStep
 from .load import load_data
+from .types import AggregateFunc
+from .types import AggregateSpec
+from .types import FieldName
+from .types import Fields
+from .types import FilterSpec
+from .types import MapSpec
+from .types import OperatorFunc
+from .types import PipelineStepName
+from .types import SortKey
+from .types import StepApplier
+from .types import StepOrSteps
+from .types import StepSpec
 
 # SECTION: EXPORTS ========================================================== #
 

etlplus/ops/types.py

@@ -0,0 +1,147 @@
+"""
+:mod:`etlplus.ops.types` module.
+
+Shared type aliases leveraged across :mod:`etlplus.ops` modules.
+
+Notes
+-----
+- Centralizes ops-focused aliases (functions, specs, and pipeline helpers).
+- Relies on Python 3.13 ``type`` statements for readability and IDE support.
+
+Examples
+--------
+>>> from etlplus.ops.types import AggregateFunc, OperatorFunc
+>>> def total(xs: list[float], _: int) -> float:
+...     return sum(xs)
+>>> agg: AggregateFunc = total
+>>> op: OperatorFunc = lambda a, b: a == b
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from collections.abc import Mapping
+from collections.abc import Sequence
+from typing import Any
+from typing import Literal
+
+from ..types import JSONList
+from ..types import StrAnyMap
+from ..types import StrSeqMap
+from ..types import StrStrMap
+
+# SECTION: EXPORTS ========================================================== #
+
+
+__all__ = [
+    # Type Aliases (Functions)
+    'AggregateFunc',
+    'OperatorFunc',
+    # Type Aliases (Records & Fields)
+    'FieldName',
+    'Fields',
+    # Type Aliases (Transform Specs)
+    'AggregateSpec',
+    'FilterSpec',
+    'MapSpec',
+    'SelectSpec',
+    'SortSpec',
+    # Type Aliases (Pipelines)
+    'StepOrSteps',
+    'StepSeq',
+    'StepSpec',
+    'PipelineConfig',
+    'PipelineStepName',
+    # Type Aliases (Helpers)
+    'StepApplier',
+    'SortKey',
+]
+
+
+# SECTION: TYPE ALIASES ===================================================== #
+
+
+# -- Functions -- #
+
+
+# TODO: Consider redefining to use `functools.reduce` signature.
+# TODO: Consider adding `**kwargs` to support richer aggregation functions.
+# TODO: Consider constraining first argument to `Sequence[float]`.
+# TODO: Consider constraining return type to `float | int | None`.
+# Callable reducing numeric collections into a summary value.
+type AggregateFunc = Callable[[list[float], int], Any]
+
+# Binary predicate consumed by filter operations.
+type OperatorFunc = Callable[[Any, Any], bool]
+
+# -- Records & Fields -- #
+
+# Individual field identifier referenced inside specs.
+type FieldName = str
+
+# Ordered list of :data:`FieldName` entries preserving projection order.
+type Fields = list[FieldName]
+
+# -- Transform Specs -- #
+
+# Filtering spec expecting ``field``, ``op``, and ``value`` keys.
+type FilterSpec = StrAnyMap
+
+# Field renaming instructions mapping old keys to new ones.
+type MapSpec = StrStrMap
+
+# Projection spec as a field list or mapping with metadata.
+#
+# Examples
+# --------
+# >>> from etlplus.ops.types import SelectSpec
+# >>> spec1: SelectSpec = ['a','b']
+# >>> spec2: SelectSpec = {'fields': [...]}
+type SelectSpec = Fields | StrSeqMap
+
+# Sort directive expressed as a field string or mapping with flags.
+#
+# Examples
+# --------
+# >>> from etlplus.ops.types import SortSpec
+# >>> spec1: SortSpec = 'field'
+# >>> spec2: SortSpec = {'field': 'x', 'reverse': True}
+type SortSpec = str | StrAnyMap
+
+# Aggregate instruction covering ``field``, ``func``, and optional alias.
+#
+# Supported functions: ``avg``, ``count``, ``max``, ``min``, and ``sum``.
+# Examples
+# --------
+# >>> from etlplus.ops.types import AggregateSpec
+# >>> spec: AggregateSpec = \
+# ...   {'field': 'x', 'func': 'sum' | 'avg' | ..., 'alias'?: '...'}
+type AggregateSpec = StrAnyMap
+
+# -- Pipelines-- #
+
+# Unified pipeline step spec consumed by :mod:`etlplus.ops.transform`.
+type StepSpec = AggregateSpec | FilterSpec | MapSpec | SelectSpec | SortSpec
+
+# Collections of steps
+
+# Ordered collection of :data:`StepSpec` entries.
+type StepSeq = Sequence[StepSpec]
+
+# Accepts either a single :data:`StepSpec` or a sequence of them.
+type StepOrSteps = StepSpec | StepSeq
+
+# Canonical literal names for supported transform stages.
+type PipelineStepName = Literal['aggregate', 'filter', 'map', 'select', 'sort']
+
+# Mapping from step name to its associated specification payload.
+# TODO: Consider replacing with etlplus.workflow.types.PipelineConfig.
+type PipelineConfig = Mapping[PipelineStepName, StepOrSteps]
+
+# -- Helpers -- #
+
+# Callable that applies step configuration to a batch of records.
+type StepApplier = Callable[[JSONList, Any], JSONList]
+
+# Tuple combining stable sort index and computed sort value.
+type SortKey = tuple[int, Any]

etlplus/types.py

@@ -16,7 +16,8 @@ See Also
 
 Examples
 --------
->>> from etlplus.types import JSONDict, PipelineConfig
+>>> from etlplus.types import JSONDict
+>>> from etlplus.ops.types import PipelineConfig
 >>> payload: JSONDict = {'id': 1, 'name': 'Ada'}
 >>> isinstance(payload, dict)
 True
@@ -53,30 +54,10 @@ __all__ = [
     'JSONRecords',
     # Type Aliases (File System)
     'StrPath',
-    # Type Aliases (Functions)
-    'AggregateFunc',
-    'OperatorFunc',
-    # Type Aliases (Records & Fields)
-    'FieldName',
-    'Fields',
     # Type Aliases (Transform Specs)
     'StrAnyMap',
     'StrSeqMap',
     'StrStrMap',
-    'AggregateSpec',
-    'FilterSpec',
-    'MapSpec',
-    'SelectSpec',
-    'SortSpec',
-    # Type Aliases (Pipelines)
-    'StepOrSteps',
-    'StepSeq',
-    'StepSpec',
-    'PipelineStepName',
-    'PipelineConfig',
-    # Type Aliases (Helpers)
-    'StepApplier',
-    'SortKey',
     # Type Aliases (Networking / Runtime)
     'Sleeper',
     'Timeout',
@@ -126,22 +107,6 @@ type JSONRecords = list[JSONRecord]
 # Path-like inputs accepted by file helpers.
 type StrPath = str | Path | PathLike[str]
 
-# -- Functions -- #
-
-# Callable reducing numeric collections into a summary value.
-type AggregateFunc = Callable[[list[float], int], Any]
-
-# Binary predicate consumed by filter operations.
-type OperatorFunc = Callable[[Any, Any], bool]
-
-# -- Records & Fields -- #
-
-# Individual field identifier referenced inside specs.
-type FieldName = str
-
-# Ordered list of :data:`FieldName` entries preserving projection order.
-type Fields = list[FieldName]
-
 # -- Transform Specs -- #
 
 # Kept intentionally broad for runtime-friendly validation in transform.py.
@@ -157,69 +122,6 @@ type StrStrMap = Mapping[str, str]
 # Mapping whose values are homogeneous sequences.
 type StrSeqMap = Mapping[str, Sequence[Any]]
 
-# Transform step specifications
-
-# Filtering spec expecting ``field``, ``op``, and ``value`` keys.
-type FilterSpec = StrAnyMap
-
-# Field renaming instructions mapping old keys to new ones.
-type MapSpec = StrStrMap
-
-# Projection spec as a field list or mapping with metadata.
-#
-# Examples
-# --------
-# >>> from etlplus.types import SelectSpec
-# >>> spec1: SelectSpec = ['a','b']
-# >>> spec2: SelectSpec = {'fields': [...]}
-type SelectSpec = Fields | StrSeqMap
-
-# Sort directive expressed as a field string or mapping with flags.
-#
-# Examples
-# --------
-# >>> from etlplus.types import SortSpec
-# >>> spec1: SortSpec = 'field'
-# >>> spec2: SortSpec = {'field': 'x', 'reverse': True}
-type SortSpec = str | StrAnyMap
-
-# Aggregate instruction covering ``field``, ``func``, and optional alias.
-#
-# Supported functions: ``avg``, ``count``, ``max``, ``min``, and ``sum``.
-# Examples
-# --------
-# >>> from etlplus.types import AggregateSpec
-# >>> spec: AggregateSpec = \
-# ...   {'field': 'x', 'func': 'sum' | 'avg' | ..., 'alias'?: '...'}
-type AggregateSpec = StrAnyMap
-
-# -- Pipelines-- #
-
-# Unified pipeline step spec consumed by :mod:`etlplus.ops.transform`.
-type StepSpec = AggregateSpec | FilterSpec | MapSpec | SelectSpec | SortSpec
-
-# Collections of steps
-
-# Ordered collection of :data:`StepSpec` entries.
-type StepSeq = Sequence[StepSpec]
-
-# Accepts either a single :data:`StepSpec` or a sequence of them.
-type StepOrSteps = StepSpec | StepSeq
-
-# Canonical literal names for supported transform stages.
-type PipelineStepName = Literal['filter', 'map', 'select', 'sort', 'aggregate']
-
-# Mapping from step name to its associated specification payload.
-type PipelineConfig = Mapping[PipelineStepName, StepOrSteps]
-
-# -- Helpers -- #
-
-# Callable that applies step configuration to a batch of records.
-type StepApplier = Callable[[JSONList, Any], JSONList]
-
-# Tuple combining stable sort index and computed sort value.
-type SortKey = tuple[int, Any]
-
 # -- Networking / Runtime -- #
 
 # Sleep function used by retry helpers.

etlplus/workflow/README.md

@@ -12,8 +12,6 @@ Back to project overview: see the top-level [README](../../README.md).
 
 - [`etlplus.workflow` Subpackage](#etlplusworkflow-subpackage)
   - [Supported Configuration Types](#supported-configuration-types)
-  - [Loading and Validating Configs](#loading-and-validating-configs)
-  - [Example: Loading a Pipeline Config](#example-loading-a-pipeline-config)
   - [See Also](#see-also)
 
 ## Supported Configuration Types
@@ -23,28 +21,6 @@ Back to project overview: see the top-level [README](../../README.md).
 - **Pipeline**: End-to-end pipeline configuration
 - **Profile**: User or environment-specific settings
 
-## Loading and Validating Configs
-
-Use the provided classes to load and validate configuration files:
-
-```python
-from etlplus.workflow import PipelineConfig
-
-cfg = PipelineConfig.from_yaml("pipeline.yml")
-```
-
-- Supports YAML and JSON formats
-- Validates against expected schema
-
-## Example: Loading a Pipeline Config
-
-```python
-from etlplus.workflow import PipelineConfig
-
-pipeline = PipelineConfig.from_yaml("configs/pipeline.yml")
-print(pipeline)
-```
-
 ## See Also
 
 - Top-level CLI and library usage in the main [README](../../README.md)

etlplus/workflow/__init__.py

@@ -12,8 +12,6 @@ from .jobs import JobConfig
 from .jobs import LoadRef
 from .jobs import TransformRef
 from .jobs import ValidationRef
-from .pipeline import PipelineConfig
-from .pipeline import load_pipeline_config
 from .profile import ProfileConfig
 
 # SECTION: EXPORTS ========================================================== #
@@ -24,11 +22,9 @@ __all__ = [
     'ExtractRef',
     'JobConfig',
     'LoadRef',
-    'PipelineConfig',
     'ProfileConfig',
     'TransformRef',
     'ValidationRef',
     # Functions
-    'load_pipeline_config',
     'topological_sort_jobs',
 ]

etlplus/workflow/jobs.py

@@ -6,8 +6,6 @@ transform, load).
 
 Notes
 -----
-- Lightweight references used inside :class:`PipelineConfig` to avoid storing
-    large nested structures.
 - All attributes are simple and optional where appropriate, keeping parsing
     tolerant.
 """

{etlplus-0.16.2.dist-info → etlplus-0.16.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: etlplus
-Version: 0.16.2
+Version: 0.16.4
 Summary: A Swiss Army knife for simple ETL operations
 Home-page: https://github.com/Dagitali/ETLPlus
 Author: ETLPlus Team

{etlplus-0.16.2.dist-info → etlplus-0.16.4.dist-info}/RECORD

@@ -1,11 +1,12 @@
-etlplus/README.md,sha256=JaMSomnMsHrTruDnonHqe83Rv4K0-e7Wy46tMeVoleU,1468
-etlplus/__init__.py,sha256=mgTP4PJmRmsEjTCAizzzdtzAmhuHtarmPzphzdjvLgM,277
+etlplus/README.md,sha256=L3un9q7Q7Mstfh0dmSjpsZMZzsOz2tvlWw4_-Y_LZEs,1887
+etlplus/__init__.py,sha256=yHZt-sjjfPjB_CrNhcT9bcMlZCfwiul39ZhQ2VfxpRs,318
 etlplus/__main__.py,sha256=btoROneNiigyfBU7BSzPKZ1R9gzBMpxcpsbPwmuHwTM,479
 etlplus/__version__.py,sha256=1E0GMK_yUWCMQFKxXjTvyMwofi0qT2k4CDNiHWiymWE,327
-etlplus/enums.py,sha256=8-uUOKe68cPzlmUg-e7gavkC95kbTJXRpRzvXehIsRk,6841
+etlplus/config.py,sha256=6BCI9hC1yWYAy5WclIHJlg90FYeWn5vAVrT1NWUTwpE,8817
+etlplus/enums.py,sha256=MfQhy3XDpN7oqLrF7_WwZojl7n8cW3RAzsZGRnAbWgc,4073
 etlplus/mixins.py,sha256=ifGpHwWv7U00yqGf-kN93vJax2IiK4jaGtTsPsO3Oak,1350
 etlplus/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-etlplus/types.py,sha256=J1ftMh0dmxe9ObFd3eXCbHiFvZI_5pV_hEHUpgeJQtY,6239
+etlplus/types.py,sha256=Op2H1dcmv0Srm9prFnBZjt7f1S4Mqrus7XrdsjoZkIM,3461
 etlplus/utils.py,sha256=X-k_Y8i6oDjlE5aQu9sw3gPw7O2ikiSn4uoheVv_ERc,17091
 etlplus/api/README.md,sha256=amxS_eIcsnNuVvD0x_w8nkyfedOTYbhlY0gGhaFg0DE,8705
 etlplus/api/__init__.py,sha256=PK2lQv1FbsE7ZZS_ejevFZQSuOUHGApBc22YfHAzMqA,4615
@@ -28,9 +29,9 @@ etlplus/api/rate_limiting/config.py,sha256=Byc_kmnwFmjjfDEFIdc_sHc7Wnjde1NGgsjPH
 etlplus/api/rate_limiting/rate_limiter.py,sha256=uYxn-l2qwLUKVclDQ3vJIIP3fozJx2JlHhz7_zyXVbA,7033
 etlplus/cli/README.md,sha256=8H_G2d3HteYIU6ReX9K9DM485QjWDT5vHMQbGD_vv20,1237
 etlplus/cli/__init__.py,sha256=J97-Rv931IL1_b4AXnB7Fbbd7HKnHBpx18NQfC_kE6c,299
-etlplus/cli/commands.py,sha256=HFlg29tO6Jwv1NXWAHmvniLCyRSlboL55Arn9B8nZAM,25028
+etlplus/cli/commands.py,sha256=Mbnu_YYUrOumbDjkul9x5VjP8VXW5u08xNi4nLF9Yyo,25048
 etlplus/cli/constants.py,sha256=0F7dXIQKWUhhVu2Us527GJeknJIWpBqz7CK2e5OQgcE,1947
-etlplus/cli/handlers.py,sha256=uvbAyF6Ux8_5C-obCWZOrOP0QP0oiT-Km1hPhE8tDx0,18558
+etlplus/cli/handlers.py,sha256=r06JatAUewxtZYGuu7lLWlcy4hKGQ8U-Oo_YtAuIhzI,18454
 etlplus/cli/io.py,sha256=tGGNQ4ecezqj-mD285fgBVrYdphdeqApsyV9VojOj1I,7836
 etlplus/cli/main.py,sha256=68_uJwmWajhOC9o4R_ns8IQloC9BFmAKC_9GlQOxKWg,5239
 etlplus/cli/options.py,sha256=vfXT3YLh7wG1iC-aTdSg6ItMC8l6n0Lozmy53XjqLbA,1199
@@ -116,26 +117,27 @@ etlplus/file/yaml.py,sha256=b_SxDSEQPVXQv9a9Ih4wAcI940pE5Ksy5pQE6K6ckhw,2062
 etlplus/file/zip.py,sha256=8wnmnGW_pGTx65736CzAG67XIi5y98KxucRT8sNDeuQ,4195
 etlplus/file/zsav.py,sha256=5hMuBjYeHw--UL2ZCCDn6TzJkr_YNhdQhvKI6nr3WW0,1674
 etlplus/ops/README.md,sha256=8omi7DYZhelc26JKk8Cm8QR8I3OGwziysPj1ivx41iQ,1380
-etlplus/ops/__init__.py,sha256=NIIr2f-AZj5B0piBt6gjv46Yn0SzGYxEe6BPoopRh38,1702
+etlplus/ops/__init__.py,sha256=r5_-pPhSLCD1nq1EbN0rQrLOGpudueeIxCH_JvT2bt0,1718
+etlplus/ops/enums.py,sha256=dC_8CfaTiB2i83Az-oG-2hkjMuAfDADNbcMF2f94UeU,4014
 etlplus/ops/extract.py,sha256=LOYiPrALRMF7JDBabnRF24_HKnnIcfTdfXesWdS3QZM,11020
 etlplus/ops/load.py,sha256=yicciVwomUKkdbhuRqbavKBNpT2Hg813BnQzG6IgF4o,10811
-etlplus/ops/run.py,sha256=FYb2W5pi2PXx5E-l5etFMUcr7UmQWfMNHew9-otWIYE,11294
-etlplus/ops/transform.py,sha256=3qIJsy2lUSMPoTRWn8Yw4JocKV_ZTQx_fKRW0w73Cnc,25682
+etlplus/ops/run.py,sha256=4HWelMevW0pW_76lJkoMcbzeQMiThMbxzO09wx6yoHg,11278
+etlplus/ops/transform.py,sha256=-41uw_pwOGsMTUYxtXaeYOmTF_fTkN-L4Q9KT1OFe78,25671
+etlplus/ops/types.py,sha256=Cvp8AJzJhJ1iYjyHd7j9ZLioxE2NdK__3g6fOI0qq6Q,4198
 etlplus/ops/utils.py,sha256=lJmrO1KDob-xZU8Gc2SvZvMgdYLsVoaz-fTV42KkLVo,10835
 etlplus/ops/validate.py,sha256=-OLAwQNNCmmDbmj0SB7zzYXDkJfcyBP_z9nTpqImLP0,13271
 etlplus/templates/README.md,sha256=IfPXlj1TGVA-uFWosHJhE2rabFW-znxOlOMazO9Z5cE,1361
 etlplus/templates/__init__.py,sha256=tsniN7XJYs3NwYxJ6c2HD5upHP3CDkLx-bQCMt97UOM,106
 etlplus/templates/ddl.sql.j2,sha256=s8fMWvcb4eaJVXkifuib1aQPljtZ8buuyB_uA-ZdU3Q,4734
 etlplus/templates/view.sql.j2,sha256=Iy8DHfhq5yyvrUKDxqp_aHIEXY4Tm6j4wT7YDEFWAhk,2180
-etlplus/workflow/README.md,sha256=D1oloiJCOHiqpqgv3m3qpRSIUOMIQcWtIsOPv7KkNI0,1652
-etlplus/workflow/__init__.py,sha256=ueothwpLruyLgr3-2hW8VT1unNyFJxdmT-l_3eB2ejc,724
+etlplus/workflow/README.md,sha256=QelyVFGX-sZM9mx3v6BXkzX36hv6MI1yK4eCPSOKNwI,1050
+etlplus/workflow/__init__.py,sha256=XgCQr684om0rONrQZ61yQ0r4qqFQL0iLAAB2Mn2BRSE,594
 etlplus/workflow/dag.py,sha256=-f1x8N1eb-PUuiOwEvFLmJwfR7JaMDJihlCHlhrFhgE,2937
-etlplus/workflow/jobs.py,sha256=5DmAzmEZV6XXQ-xzowkLxFzplIh8Eno3wuCmjy79xHw,8818
-etlplus/workflow/pipeline.py,sha256=PA5zhcfrk--pAg3b3x4oBf29WMj5HqR8zOozz4oEmg8,9387
+etlplus/workflow/jobs.py,sha256=hLE9QJUzQaI0aOEon0P-xxxa6xHp997ANei4F310WRY,8711
 etlplus/workflow/profile.py,sha256=FQU3bzBZ9_yjKC9kCXKN1FQDS9zjNUjtWB1r3UL95_Q,1993
-etlplus-0.16.2.dist-info/licenses/LICENSE,sha256=MuNO63i6kWmgnV2pbP2SLqP54mk1BGmu7CmbtxMmT-U,1069
-etlplus-0.16.2.dist-info/METADATA,sha256=QdFDSAYSrjZKyu5G8TWQSlC1Lobu8hA9qgXpnIOQ2eM,28114
-etlplus-0.16.2.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
-etlplus-0.16.2.dist-info/entry_points.txt,sha256=6w-2-jzuPa55spzK34h-UKh2JTEShh38adFRONNP9QE,45
-etlplus-0.16.2.dist-info/top_level.txt,sha256=aWWF-udn_sLGuHTM6W6MLh99ArS9ROkUWO8Mi8y1_2U,8
-etlplus-0.16.2.dist-info/RECORD,,
+etlplus-0.16.4.dist-info/licenses/LICENSE,sha256=MuNO63i6kWmgnV2pbP2SLqP54mk1BGmu7CmbtxMmT-U,1069
+etlplus-0.16.4.dist-info/METADATA,sha256=Pe-3vIQcAfTasRzy2HPq0A4cin2I5PJChAmA23S9dgo,28114
+etlplus-0.16.4.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
+etlplus-0.16.4.dist-info/entry_points.txt,sha256=6w-2-jzuPa55spzK34h-UKh2JTEShh38adFRONNP9QE,45
+etlplus-0.16.4.dist-info/top_level.txt,sha256=aWWF-udn_sLGuHTM6W6MLh99ArS9ROkUWO8Mi8y1_2U,8
+etlplus-0.16.4.dist-info/RECORD,,