etlplus 0.16.0__py3-none-any.whl → 0.16.7__py3-none-any.whl

etlplus/ops/types.py

@@ -0,0 +1,147 @@
+"""
+:mod:`etlplus.ops.types` module.
+
+Shared type aliases leveraged across :mod:`etlplus.ops` modules.
+
+Notes
+-----
+- Centralizes ops-focused aliases (functions, specs, and pipeline helpers).
+- Relies on Python 3.13 ``type`` statements for readability and IDE support.
+
+Examples
+--------
+>>> from etlplus.ops.types import AggregateFunc, OperatorFunc
+>>> def total(xs: list[float], _: int) -> float:
+...     return sum(xs)
+>>> agg: AggregateFunc = total
+>>> op: OperatorFunc = lambda a, b: a == b
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from collections.abc import Mapping
+from collections.abc import Sequence
+from typing import Any
+from typing import Literal
+
+from ..types import JSONList
+from ..types import StrAnyMap
+from ..types import StrSeqMap
+from ..types import StrStrMap
+
+# SECTION: EXPORTS ========================================================== #
+
+
+__all__ = [
+    # Type Aliases (Functions)
+    'AggregateFunc',
+    'OperatorFunc',
+    # Type Aliases (Records & Fields)
+    'FieldName',
+    'Fields',
+    # Type Aliases (Transform Specs)
+    'AggregateSpec',
+    'FilterSpec',
+    'MapSpec',
+    'SelectSpec',
+    'SortSpec',
+    # Type Aliases (Pipelines)
+    'StepOrSteps',
+    'StepSeq',
+    'StepSpec',
+    'PipelineConfig',
+    'PipelineStepName',
+    # Type Aliases (Helpers)
+    'StepApplier',
+    'SortKey',
+]
+
+
+# SECTION: TYPE ALIASES ===================================================== #
+
+
+# -- Functions -- #
+
+
+# TODO: Consider redefining to use `functools.reduce` signature.
+# TODO: Consider adding `**kwargs` to support richer aggregation functions.
+# TODO: Consider constraining first argument to `Sequence[float]`.
+# TODO: Consider constraining return type to `float | int | None`.
+# Callable reducing numeric collections into a summary value.
+type AggregateFunc = Callable[[list[float], int], Any]
+
+# Binary predicate consumed by filter operations.
+type OperatorFunc = Callable[[Any, Any], bool]
+
+# -- Records & Fields -- #
+
+# Individual field identifier referenced inside specs.
+type FieldName = str
+
+# Ordered list of :data:`FieldName` entries preserving projection order.
+type Fields = list[FieldName]
+
+# -- Transform Specs -- #
+
+# Filtering spec expecting ``field``, ``op``, and ``value`` keys.
+type FilterSpec = StrAnyMap
+
+# Field renaming instructions mapping old keys to new ones.
+type MapSpec = StrStrMap
+
+# Projection spec as a field list or mapping with metadata.
+#
+# Examples
+# --------
+# >>> from etlplus.ops.types import SelectSpec
+# >>> spec1: SelectSpec = ['a','b']
+# >>> spec2: SelectSpec = {'fields': [...]}
+type SelectSpec = Fields | StrSeqMap
+
+# Sort directive expressed as a field string or mapping with flags.
+#
+# Examples
+# --------
+# >>> from etlplus.ops.types import SortSpec
+# >>> spec1: SortSpec = 'field'
+# >>> spec2: SortSpec = {'field': 'x', 'reverse': True}
+type SortSpec = str | StrAnyMap
+
+# Aggregate instruction covering ``field``, ``func``, and optional alias.
+#
+# Supported functions: ``avg``, ``count``, ``max``, ``min``, and ``sum``.
+# Examples
+# --------
+# >>> from etlplus.ops.types import AggregateSpec
+# >>> spec: AggregateSpec = \
+# ...   {'field': 'x', 'func': 'sum' | 'avg' | ..., 'alias'?: '...'}
+type AggregateSpec = StrAnyMap
+
+# -- Pipelines-- #
+
+# Unified pipeline step spec consumed by :mod:`etlplus.ops.transform`.
+type StepSpec = AggregateSpec | FilterSpec | MapSpec | SelectSpec | SortSpec
+
+# Collections of steps
+
+# Ordered collection of :data:`StepSpec` entries.
+type StepSeq = Sequence[StepSpec]
+
+# Accepts either a single :data:`StepSpec` or a sequence of them.
+type StepOrSteps = StepSpec | StepSeq
+
+# Canonical literal names for supported transform stages.
+type PipelineStepName = Literal['aggregate', 'filter', 'map', 'select', 'sort']
+
+# Mapping from step name to its associated specification payload.
+# TODO: Consider replacing with etlplus.workflow.types.PipelineConfig.
+type PipelineConfig = Mapping[PipelineStepName, StepOrSteps]
+
+# -- Helpers -- #
+
+# Callable that applies step configuration to a batch of records.
+type StepApplier = Callable[[JSONList, Any], JSONList]
+
+# Tuple combining stable sort index and computed sort value.
+type SortKey = tuple[int, Any]

etlplus/ops/utils.py

@@ -26,7 +26,7 @@ from ..utils import normalize_choice
 # SECTION: TYPED DICTIONARIES =============================================== #
 
 
-class ValidationResult(TypedDict, total=False):
+class ValidationResultDict(TypedDict, total=False):
     """Shape returned by ``validate_fn`` callables."""
 
     valid: bool
@@ -44,7 +44,7 @@ type ValidationPhase = Literal['before_transform', 'after_transform']
 type ValidationWindow = Literal['before_transform', 'after_transform', 'both']
 type ValidationSeverity = Literal['warn', 'error']
 
-type ValidateFn = Callable[[Any, Ruleset], ValidationResult]
+type ValidateFn = Callable[[Any, Ruleset], ValidationResultDict]
 type PrintFn = Callable[[Any], None]
 
 
@@ -198,7 +198,7 @@ def maybe_validate(
         Failure severity (``"warn"`` or ``"error"``).
     validate_fn : ValidateFn
         Engine that performs validation and returns a
-        :class:`ValidationResult` instance.
+        :class:`ValidationResultDict` instance.
     print_json_fn : PrintFn
         Structured logger invoked when validation fails.
 
@@ -270,7 +270,7 @@ def _log_failure(
     phase: ValidationPhase,
     window: ValidationWindow,
     ruleset_name: str | None,
-    result: ValidationResult,
+    result: ValidationResultDict,
 ) -> None:
     """
     Emit a structured message describing the failed validation.
@@ -285,7 +285,7 @@ def _log_failure(
         Configured validation window.
     ruleset_name : str | None
         Name of the validation ruleset.
-    result : ValidationResult
+    result : ValidationResultDict
         Result of the failed validation.
     """
     printer(

etlplus/ops/validate.py

@@ -44,9 +44,9 @@ from .load import load_data
 
 
 __all__ = [
-    'FieldRules',
-    'FieldValidation',
-    'Validation',
+    'FieldRulesDict',
+    'FieldValidationDict',
+    'ValidationDict',
     'validate_field',
     'validate',
 ]
@@ -69,7 +69,7 @@ TYPE_MAP: Final[dict[str, type | tuple[type, ...]]] = {
 # SECTION: TYPED DICTS ====================================================== #
 
 
-class FieldRules(TypedDict, total=False):
+class FieldRulesDict(TypedDict, total=False):
     """
     Validation rules for a single field.
 
@@ -93,7 +93,7 @@ class FieldRules(TypedDict, total=False):
     enum: list[Any]
 
 
-class FieldValidation(TypedDict):
+class FieldValidationDict(TypedDict):
     """
     Validation result for a single field.
 
@@ -109,7 +109,7 @@ class FieldValidation(TypedDict):
     errors: list[str]
 
 
-class Validation(TypedDict):
+class ValidationDict(TypedDict):
     """
     Validation result for a complete data structure.
 
@@ -134,7 +134,7 @@ class Validation(TypedDict):
 # SECTION: TYPE ALIASES ===================================================== #
 
 
-type RulesMap = Mapping[str, FieldRules]
+type RulesMap = Mapping[str, FieldRulesDict]
 
 
 # SECTION: INTERNAL FUNCTIONS ============================================== #
@@ -339,8 +339,8 @@ def _validate_record(
 
 def validate_field(
     value: Any,
-    rules: StrAnyMap | FieldRules,
-) -> FieldValidation:
+    rules: StrAnyMap | FieldRulesDict,
+) -> FieldValidationDict:
     """
     Validate a single value against field rules.
 
@@ -348,14 +348,14 @@ def validate_field(
     ----------
     value : Any
         The value to validate. ``None`` is treated as missing.
-    rules : StrAnyMap | FieldRules
+    rules : StrAnyMap | FieldRulesDict
         Rule dictionary. Supported keys include ``required``, ``type``,
         ``min``, ``max``, ``minLength``, ``maxLength``, ``pattern``, and
         ``enum``.
 
     Returns
     -------
-    FieldValidation
+    FieldValidationDict
         Result with ``valid`` and a list of ``errors``.
 
     Notes
@@ -438,7 +438,7 @@ def validate_field(
 def validate(
     source: StrPath | JSONData,
     rules: RulesMap | None = None,
-) -> Validation:
+) -> ValidationDict:
     """
     Validate data against rules.
 
@@ -452,7 +452,7 @@ def validate(
 
     Returns
     -------
-    Validation
+    ValidationDict
         Structured result with keys ``valid``, ``errors``, ``field_errors``,
         and ``data``. If loading fails, ``data`` is ``None`` and an error is
         reported in ``errors``.

etlplus/templates/README.md

@@ -1,10 +1,11 @@
 # `etlplus.templates` Subpackage
 
-Documentation for the `etlplus.templates` subpackage: SQL and DDL template helpers.
+Documentation for the `etlplus.templates` subpackage: bundled SQL/DDL templates used by the database
+helpers.
 
 - Provides Jinja2 templates for DDL and view generation
-- Supports templated SQL for multiple database backends
-- Includes helpers for rendering templates with schema metadata
+- Used by `etlplus.database.render_table_sql` and related helpers
+- Exposed as plain template files you can reuse with your own Jinja2 setup
 
 Back to project overview: see the top-level [README](../../README.md).
 
@@ -21,21 +22,22 @@ Back to project overview: see the top-level [README](../../README.md).
 
 ## Rendering Templates
 
-Use the helpers to render templates with your schema or table metadata:
+ETLPlus does not currently expose a `render_template` helper in this package. Use the database
+helpers instead:
 
 ```python
-from etlplus.templates import render_template
+from etlplus.database import render_table_sql, load_table_spec
 
-sql = render_template("ddl.sql.j2", schema=my_schema)
+spec = load_table_spec("schemas/users.yml")
+sql = render_table_sql(spec, template="ddl")
 ```
 
 ## Example: Rendering a DDL Template
 
 ```python
-from etlplus.templates import render_template
+from etlplus.database import render_tables_to_string
 
-schema = {"name": "users", "columns": [ ... ]}
-sql = render_template("ddl.sql.j2", schema=schema)
+sql = render_tables_to_string(["schemas/users.yml"], template="ddl")
 print(sql)
 ```
 

etlplus/types.py

@@ -12,12 +12,12 @@ Notes
 See Also
 --------
 - :mod:`etlplus.api.types` for HTTP-specific aliases and data classes
-- :mod:`etlplus.connector.types` for connector-specific aliases and TypedDict
-    surfaces
+- :mod:`etlplus.connector.types` for connector-specific aliases
 
 Examples
 --------
->>> from etlplus.types import JSONDict, PipelineConfig
+>>> from etlplus.types import JSONDict
+>>> from etlplus.ops.types import PipelineConfig
 >>> payload: JSONDict = {'id': 1, 'name': 'Ada'}
 >>> isinstance(payload, dict)
 True
@@ -54,33 +54,15 @@ __all__ = [
     'JSONRecords',
     # Type Aliases (File System)
     'StrPath',
-    # Type Aliases (Functions)
-    'AggregateFunc',
-    'OperatorFunc',
-    # Type Aliases (Records & Fields)
-    'FieldName',
-    'Fields',
     # Type Aliases (Transform Specs)
     'StrAnyMap',
     'StrSeqMap',
     'StrStrMap',
-    'AggregateSpec',
-    'FilterSpec',
-    'MapSpec',
-    'SelectSpec',
-    'SortSpec',
-    # Type Aliases (Pipelines)
-    'StepOrSteps',
-    'StepSeq',
-    'StepSpec',
-    'PipelineStepName',
-    'PipelineConfig',
-    # Type Aliases (Helpers)
-    'StepApplier',
-    'SortKey',
     # Type Aliases (Networking / Runtime)
     'Sleeper',
     'Timeout',
+    # Type Aliases (Templates)
+    'TemplateKey',
 ]
 
 
@@ -125,22 +107,6 @@ type JSONRecords = list[JSONRecord]
 # Path-like inputs accepted by file helpers.
 type StrPath = str | Path | PathLike[str]
 
-# -- Functions -- #
-
-# Callable reducing numeric collections into a summary value.
-type AggregateFunc = Callable[[list[float], int], Any]
-
-# Binary predicate consumed by filter operations.
-type OperatorFunc = Callable[[Any, Any], bool]
-
-# -- Records & Fields -- #
-
-# Individual field identifier referenced inside specs.
-type FieldName = str
-
-# Ordered list of :data:`FieldName` entries preserving projection order.
-type Fields = list[FieldName]
-
 # -- Transform Specs -- #
 
 # Kept intentionally broad for runtime-friendly validation in transform.py.
@@ -156,69 +122,6 @@ type StrStrMap = Mapping[str, str]
 # Mapping whose values are homogeneous sequences.
 type StrSeqMap = Mapping[str, Sequence[Any]]
 
-# Transform step specifications
-
-# Filtering spec expecting ``field``, ``op``, and ``value`` keys.
-type FilterSpec = StrAnyMap
-
-# Field renaming instructions mapping old keys to new ones.
-type MapSpec = StrStrMap
-
-# Projection spec as a field list or mapping with metadata.
-#
-# Examples
-# --------
-# >>> from etlplus.types import SelectSpec
-# >>> spec1: SelectSpec = ['a','b']
-# >>> spec2: SelectSpec = {'fields': [...]}
-type SelectSpec = Fields | StrSeqMap
-
-# Sort directive expressed as a field string or mapping with flags.
-#
-# Examples
-# --------
-# >>> from etlplus.types import SortSpec
-# >>> spec1: SortSpec = 'field'
-# >>> spec2: SortSpec = {'field': 'x', 'reverse': True}
-type SortSpec = str | StrAnyMap
-
-# Aggregate instruction covering ``field``, ``func``, and optional alias.
-#
-# Supported functions: ``avg``, ``count``, ``max``, ``min``, and ``sum``.
-# Examples
-# --------
-# >>> from etlplus.types import AggregateSpec
-# >>> spec: AggregateSpec = \
-# ...   {'field': 'x', 'func': 'sum' | 'avg' | ..., 'alias'?: '...'}
-type AggregateSpec = StrAnyMap
-
-# -- Pipelines-- #
-
-# Unified pipeline step spec consumed by :mod:`etlplus.ops.transform`.
-type StepSpec = AggregateSpec | FilterSpec | MapSpec | SelectSpec | SortSpec
-
-# Collections of steps
-
-# Ordered collection of :data:`StepSpec` entries.
-type StepSeq = Sequence[StepSpec]
-
-# Accepts either a single :data:`StepSpec` or a sequence of them.
-type StepOrSteps = StepSpec | StepSeq
-
-# Canonical literal names for supported transform stages.
-type PipelineStepName = Literal['filter', 'map', 'select', 'sort', 'aggregate']
-
-# Mapping from step name to its associated specification payload.
-type PipelineConfig = Mapping[PipelineStepName, StepOrSteps]
-
-# -- Helpers -- #
-
-# Callable that applies step configuration to a batch of records.
-type StepApplier = Callable[[JSONList, Any], JSONList]
-
-# Tuple combining stable sort index and computed sort value.
-type SortKey = tuple[int, Any]
-
 # -- Networking / Runtime -- #
 
 # Sleep function used by retry helpers.

etlplus/workflow/README.md

@@ -12,8 +12,6 @@ Back to project overview: see the top-level [README](../../README.md).
 
 - [`etlplus.workflow` Subpackage](#etlplusworkflow-subpackage)
   - [Supported Configuration Types](#supported-configuration-types)
-  - [Loading and Validating Configs](#loading-and-validating-configs)
-  - [Example: Loading a Pipeline Config](#example-loading-a-pipeline-config)
   - [See Also](#see-also)
 
 ## Supported Configuration Types
@@ -23,28 +21,6 @@ Back to project overview: see the top-level [README](../../README.md).
 - **Pipeline**: End-to-end pipeline configuration
 - **Profile**: User or environment-specific settings
 
-## Loading and Validating Configs
-
-Use the provided classes to load and validate configuration files:
-
-```python
-from etlplus.workflow import PipelineConfig
-
-cfg = PipelineConfig.from_yaml("pipeline.yml")
-```
-
-- Supports YAML and JSON formats
-- Validates against expected schema
-
-## Example: Loading a Pipeline Config
-
-```python
-from etlplus.workflow import PipelineConfig
-
-pipeline = PipelineConfig.from_yaml("configs/pipeline.yml")
-print(pipeline)
-```
-
 ## See Also
 
 - Top-level CLI and library usage in the main [README](../../README.md)

etlplus/workflow/__init__.py

@@ -12,8 +12,7 @@ from .jobs import JobConfig
 from .jobs import LoadRef
 from .jobs import TransformRef
 from .jobs import ValidationRef
-from .pipeline import PipelineConfig
-from .pipeline import load_pipeline_config
+from .profile import ProfileConfig
 
 # SECTION: EXPORTS ========================================================== #
 
@@ -23,10 +22,9 @@ __all__ = [
     'ExtractRef',
     'JobConfig',
     'LoadRef',
-    'PipelineConfig',
+    'ProfileConfig',
     'TransformRef',
     'ValidationRef',
     # Functions
-    'load_pipeline_config',
     'topological_sort_jobs',
 ]

etlplus/workflow/dag.py

@@ -47,6 +47,28 @@ class DagError(ValueError):
         return self.message
 
 
+# SECTION: INTERNAL FUNCTIONS =============================================== #
+
+
+def _ready(
+    indegree: dict[str, int],
+) -> list[str]:
+    """
+    Return a sorted list of nodes with zero indegree.
+
+    Parameters
+    ----------
+    indegree : dict[str, int]
+        Mapping of node name to indegree.
+
+    Returns
+    -------
+    list[str]
+        Sorted list of node names ready to process.
+    """
+    return sorted(name for name, deg in indegree.items() if deg == 0)
+
+
 # SECTION: FUNCTIONS ======================================================== #
 
 
@@ -88,7 +110,7 @@ def topological_sort_jobs(
                 edges[dep].add(job.name)
                 indegree[job.name] += 1
 
-    queue = deque(sorted(name for name, deg in indegree.items() if deg == 0))
+    queue = deque(_ready(indegree))
     ordered: list[str] = []
 
     while queue:

etlplus/workflow/jobs.py

@@ -6,14 +6,13 @@ transform, load).
 
 Notes
 -----
-- Lightweight references used inside :class:`PipelineConfig` to avoid storing
-    large nested structures.
 - All attributes are simple and optional where appropriate, keeping parsing
     tolerant.
 """
 
 from __future__ import annotations
 
+from collections.abc import Sequence
 from dataclasses import dataclass
 from dataclasses import field
 from typing import Any
@@ -76,13 +75,15 @@ def _parse_depends_on(
     """
     if isinstance(value, str):
         return [value]
-    if isinstance(value, list):
+    if isinstance(value, Sequence) and not isinstance(
+        value,
+        (str, bytes, bytearray),
+    ):
         return [entry for entry in value if isinstance(entry, str)]
     return []
 
 
 def _require_str(
-    # data: dict[str, Any],
     data: StrAnyMap,
     key: str,
 ) -> str | None:
@@ -149,13 +150,9 @@ class ExtractRef:
         data = maybe_mapping(obj)
         if not data:
             return None
-        source = _require_str(data, 'source')
-        if source is None:
+        if (source := _require_str(data, 'source')) is None:
             return None
-        return cls(
-            source=source,
-            options=coerce_dict(data.get('options')),
-        )
+        return cls(source=source, options=coerce_dict(data.get('options')))
 
 
 @dataclass(kw_only=True, slots=True)
@@ -214,18 +211,13 @@ class JobConfig:
         data = maybe_mapping(obj)
         if not data:
             return None
-        name = _require_str(data, 'name')
-        if name is None:
+        if (name := _require_str(data, 'name')) is None:
             return None
 
-        description = _coerce_optional_str(data.get('description'))
-
-        depends_on = _parse_depends_on(data.get('depends_on'))
-
         return cls(
             name=name,
-            description=description,
-            depends_on=depends_on,
+            description=_coerce_optional_str(data.get('description')),
+            depends_on=_parse_depends_on(data.get('depends_on')),
             extract=ExtractRef.from_obj(data.get('extract')),
             validate=ValidationRef.from_obj(data.get('validate')),
             transform=TransformRef.from_obj(data.get('transform')),
@@ -274,8 +266,7 @@ class LoadRef:
         data = maybe_mapping(obj)
         if not data:
             return None
-        target = _require_str(data, 'target')
-        if target is None:
+        if (target := _require_str(data, 'target')) is None:
             return None
         return cls(
             target=target,
@@ -321,8 +312,7 @@ class TransformRef:
         data = maybe_mapping(obj)
         if not data:
             return None
-        pipeline = _require_str(data, 'pipeline')
-        if pipeline is None:
+        if (pipeline := _require_str(data, 'pipeline')) is None:
             return None
         return cls(pipeline=pipeline)
 
@@ -372,13 +362,10 @@ class ValidationRef:
         data = maybe_mapping(obj)
         if not data:
             return None
-        ruleset = _require_str(data, 'ruleset')
-        if ruleset is None:
+        if (ruleset := _require_str(data, 'ruleset')) is None:
             return None
-        severity = _coerce_optional_str(data.get('severity'))
-        phase = _coerce_optional_str(data.get('phase'))
         return cls(
             ruleset=ruleset,
-            severity=severity,
-            phase=phase,
+            severity=_coerce_optional_str(data.get('severity')),
+            phase=_coerce_optional_str(data.get('phase')),
         )