etlplus 0.9.0__py3-none-any.whl → 0.9.2__py3-none-any.whl

etlplus/{validate.py → ops/validate.py}

@@ -1,5 +1,5 @@
 """
-:mod:`etlplus.validation` module.
+:mod:`etlplus.ops.validate` module.
 
 Validate dicts and lists of dicts using simple, schema-like rules.
 
@@ -11,8 +11,8 @@ Highlights
 ----------
 - Centralized type map and helpers for clarity and reuse.
 - Consistent error wording; field and item paths like ``[2].email``.
-- Small, focused public API with ``load_data``, ``validate_field``,
-  ``validate``.
+- Small, focused public API with :func:`load_data`, :func:`validate_field`,
+    :func:`validate`.
 
 Examples
 --------
@@ -34,11 +34,11 @@ from typing import Final
 from typing import Literal
 from typing import TypedDict
 
+from ..types import JSONData
+from ..types import Record
+from ..types import StrAnyMap
+from ..types import StrPath
 from .load import load_data
-from .types import JSONData
-from .types import Record
-from .types import StrAnyMap
-from .types import StrPath
 
 # SECTION: EXPORTS ========================================================== #
 
@@ -66,7 +66,7 @@ TYPE_MAP: Final[dict[str, type | tuple[type, ...]]] = {
 }
 
 
-# SECTION: CLASSES ========================================================== #
+# SECTION: TYPED DICTS ====================================================== #
 
 
 class FieldRules(TypedDict, total=False):
@@ -279,11 +279,15 @@ def _type_matches(
     bool
         ``True`` if the value matches the expected type; ``False`` if not.
     """
-    py_type = TYPE_MAP.get(expected)
-    if py_type:
-        return isinstance(value, py_type)
+    if expected == 'number':
+        return _is_number(value)
+    if expected == 'integer':
+        return isinstance(value, int) and not isinstance(value, bool)
+    if expected == 'boolean':
+        return isinstance(value, bool)
 
-    return False
+    py_type = TYPE_MAP.get(expected)
+    return isinstance(value, py_type) if py_type else False
 
 
 def _validate_record(
@@ -330,6 +334,9 @@ def _validate_record(
 # SECTION: FUNCTIONS ======================================================== #
 
 
+# -- Helpers -- #
+
+
 def validate_field(
     value: Any,
     rules: StrAnyMap | FieldRules,
@@ -425,6 +432,9 @@ def validate_field(
     return {'valid': len(errors) == 0, 'errors': errors}
 
 
+# -- Orchestration -- #
+
+
 def validate(
     source: StrPath | JSONData,
     rules: RulesMap | None = None,

etlplus/templates/README.md

@@ -0,0 +1,46 @@
+# `etlplus.templates` Subpackage
+
+Documentation for the `etlplus.templates` subpackage: SQL and DDL template helpers.
+
+- Provides Jinja2 templates for DDL and view generation
+- Supports templated SQL for multiple database backends
+- Includes helpers for rendering templates with schema metadata
+
+Back to project overview: see the top-level [README](../../README.md).
+
+- [`etlplus.templates` Subpackage](#etlplus-templates-subpackage)
+    - [Available Templates](#available-templates)
+    - [Rendering Templates](#rendering-templates)
+    - [Example: Rendering a DDL Template](#example-rendering-a-ddl-template)
+    - [See Also](#see-also)
+
+## Available Templates
+
+- `ddl.sql.j2`: Generic DDL (CREATE TABLE) template
+- `view.sql.j2`: Generic view creation template
+
+## Rendering Templates
+
+Use the helpers to render templates with your schema or table metadata:
+
+```python
+from etlplus.templates import render_template
+
+sql = render_template("ddl.sql.j2", schema=my_schema)
+```
+
+## Example: Rendering a DDL Template
+
+```python
+from etlplus.templates import render_template
+
+schema = {"name": "users", "columns": [ ... ]}
+sql = render_template("ddl.sql.j2", schema=schema)
+print(sql)
+```
+
+## See Also
+
+- Top-level CLI and library usage in the main [README](../../README.md)
+- DDL template in [ddl.sql.j2](ddl.sql.j2)
+- View template in [view.sql.j2](view.sql.j2)

etlplus/types.py

@@ -11,8 +11,9 @@ Notes
 
 See Also
 --------
-- :mod:`etlplus.api.types` for HTTP-specific aliases
-- :mod:`etlplus.config.types` for TypedDict surfaces
+- :mod:`etlplus.api.types` for HTTP-specific aliases and data classes
+- :mod:`etlplus.workflow.types` for workflow-specific aliases and TypedDict
+    surfaces
 
 Examples
 --------
@@ -193,8 +194,8 @@ type AggregateSpec = StrAnyMap
 
 # -- Pipelines-- #
 
-# Unified pipeline step spec consumed by :mod:`etlplus.transform`.
-type StepSpec = FilterSpec | MapSpec | SelectSpec | SortSpec | AggregateSpec
+# Unified pipeline step spec consumed by :mod:`etlplus.ops.transform`.
+type StepSpec = AggregateSpec | FilterSpec | MapSpec | SelectSpec | SortSpec
 
 # Collections of steps
 

etlplus/utils.py

@@ -8,6 +8,7 @@ from __future__ import annotations
 
 import json
 from collections.abc import Callable
+from collections.abc import Iterable
 from collections.abc import Mapping
 from typing import Any
 from typing import TypeVar
@@ -25,6 +26,7 @@ __all__ = [
     # Mapping utilities
     'cast_str_dict',
     'coerce_dict',
+    'deep_substitute',
     'maybe_mapping',
     # Float coercion
     'to_float',
@@ -39,7 +41,8 @@ __all__ = [
     # Generic number coercion
     'to_number',
     # Text processing
-    'normalized_str',
+    'normalize_choice',
+    'normalize_str',
 ]
 
 
@@ -56,6 +59,52 @@ Num = TypeVar('Num', int, float)
 # -- Data Utilities -- #
 
 
+def deep_substitute(
+    value: Any,
+    vars_map: StrAnyMap | None,
+    env_map: Mapping[str, str] | None,
+) -> Any:
+    """
+    Recursively substitute ``${VAR}`` tokens in nested structures.
+
+    Only strings are substituted; other types are returned as-is.
+
+    Parameters
+    ----------
+    value : Any
+        The value to perform substitutions on.
+    vars_map : StrAnyMap | None
+        Mapping of variable names to replacement values (lower precedence).
+    env_map : Mapping[str, str] | None
+        Mapping of environment variables overriding ``vars_map`` values
+        (higher precedence).
+
+    Returns
+    -------
+    Any
+        New structure with substitutions applied where tokens were found.
+    """
+    substitutions = _prepare_substitutions(vars_map, env_map)
+
+    def _apply(node: Any) -> Any:
+        match node:
+            case str():
+                return _replace_tokens(node, substitutions)
+            case Mapping():
+                return {k: _apply(v) for k, v in node.items()}
+            case list() | tuple() as seq:
+                apply = [_apply(item) for item in seq]
+                return apply if isinstance(seq, list) else tuple(apply)
+            case set():
+                return {_apply(item) for item in node}
+            case frozenset():
+                return frozenset(_apply(item) for item in node)
+            case _:
+                return node
+
+    return _apply(value)
+
+
 def cast_str_dict(
     mapping: StrAnyMap | None,
 ) -> dict[str, str]:
@@ -372,7 +421,7 @@ def to_number(
 # -- Text Processing -- #
 
 
-def normalized_str(
+def normalize_str(
     value: str | None,
 ) -> str:
     """
@@ -392,6 +441,36 @@ def normalized_str(
     return (value or '').strip().lower()
 
 
+def normalize_choice(
+    value: str | None,
+    *,
+    mapping: Mapping[str, str],
+    default: str,
+    normalize: Callable[[str | None], str] = normalize_str,
+) -> str:
+    """
+    Normalize a string choice using a mapping and fallback.
+
+    Parameters
+    ----------
+    value : str | None
+        Input value to normalize.
+    mapping : Mapping[str, str]
+        Mapping of acceptable normalized inputs to output values.
+    default : str
+        Default return value when input is missing or unrecognized.
+    normalize : Callable[[str | None], str], optional
+        Normalization function applied to *value*. Defaults to
+        :func:`normalize_str`.
+
+    Returns
+    -------
+    str
+        Normalized mapped value or ``default``.
+    """
+    return mapping.get(normalize(value), default)
+
+
 # SECTION: INTERNAL FUNCTIONS =============================================== #
 
 
@@ -425,6 +504,61 @@ def _clamp(
     return value
 
 
+def _prepare_substitutions(
+    vars_map: StrAnyMap | None,
+    env_map: Mapping[str, Any] | None,
+) -> tuple[tuple[str, Any], ...]:
+    """
+    Merge variable and environment maps into an ordered substitutions list.
+
+    Parameters
+    ----------
+    vars_map : StrAnyMap | None
+        Mapping of variable names to replacement values (lower precedence).
+    env_map : Mapping[str, Any] | None
+        Environment-backed values that override entries from ``vars_map``.
+
+    Returns
+    -------
+    tuple[tuple[str, Any], ...]
+        Immutable sequence of ``(name, value)`` pairs suitable for token
+        replacement.
+    """
+    if not vars_map and not env_map:
+        return ()
+    merged: dict[str, Any] = {**(vars_map or {}), **(env_map or {})}
+    return tuple(merged.items())
+
+
+def _replace_tokens(
+    text: str,
+    substitutions: Iterable[tuple[str, Any]],
+) -> str:
+    """
+    Replace ``${VAR}`` tokens in ``text`` using ``substitutions``.
+
+    Parameters
+    ----------
+    text : str
+        Input string that may contain ``${VAR}`` tokens.
+    substitutions : Iterable[tuple[str, Any]]
+        Sequence of ``(name, value)`` pairs used for token replacement.
+
+    Returns
+    -------
+    str
+        Updated text with replacements applied.
+    """
+    if not substitutions:
+        return text
+    out = text
+    for name, replacement in substitutions:
+        token = f'${{{name}}}'
+        if token in out:
+            out = out.replace(token, str(replacement))
+    return out
+
+
 def _coerce_float(
     value: object,
 ) -> float | None:

etlplus/workflow/README.md

@@ -0,0 +1,52 @@
+# `etlplus.workflow` Subpackage
+
+Documentation for the `etlplus.workflow` subpackage: configuration helpers for connectors,
+pipelines, jobs, and profiles.
+
+- Provides classes and utilities for managing ETL pipeline configuration
+- Supports YAML/JSON config loading and validation
+- Includes helpers for connectors, jobs, pipelines, and profiles
+- Exposes type definitions for config schemas
+
+Back to project overview: see the top-level [README](../../README.md).
+
+- [`etlplus.workflow` Subpackage](#etlplusworkflow-subpackage)
+  - [Supported Configuration Types](#supported-configuration-types)
+  - [Loading and Validating Configs](#loading-and-validating-configs)
+  - [Example: Loading a Pipeline Config](#example-loading-a-pipeline-config)
+  - [See Also](#see-also)
+
+## Supported Configuration Types
+
+- **Connector**: Connection details for databases, files, or APIs
+- **Job**: ETL job definitions and scheduling
+- **Pipeline**: End-to-end pipeline configuration
+- **Profile**: User or environment-specific settings
+
+## Loading and Validating Configs
+
+Use the provided classes to load and validate configuration files:
+
+```python
+from etlplus.workflow import PipelineConfig
+
+cfg = PipelineConfig.from_yaml("pipeline.yml")
+```
+
+- Supports YAML and JSON formats
+- Validates against expected schema
+
+## Example: Loading a Pipeline Config
+
+```python
+from etlplus.workflow import PipelineConfig
+
+pipeline = PipelineConfig.from_yaml("configs/pipeline.yml")
+print(pipeline)
+```
+
+## See Also
+
+- Top-level CLI and library usage in the main [README](../../README.md)
+- Config type definitions in [types.py](types.py)
+- Config utilities in [utils.py](utils.py)

etlplus/{config → workflow}/__init__.py

@@ -1,17 +1,7 @@
 """
-:mod:`etlplus.config` package.
+:mod:`etlplus.workflow` package.
 
-Configuration models and helpers for ETLPlus.
-
-This package defines models for data sources/targets ("connectors"), APIs,
-pagination/rate limits, pipeline orchestration, and related utilities. The
-parsers are permissive (accepting ``Mapping[str, Any]``) and normalize to
-concrete types without raising on unknown/optional fields.
-
-Notes
------
-- The models use ``@dataclass(slots=True)`` and avoid mutating inputs.
-- TypedDicts are editor/type-checking hints and are not enforced at runtime.
+Job workflow helpers.
 """
 
 from __future__ import annotations
@@ -21,6 +11,7 @@ from .connector import ConnectorApi
 from .connector import ConnectorDb
 from .connector import ConnectorFile
 from .connector import parse_connector
+from .dag import topological_sort_jobs
 from .jobs import ExtractRef
 from .jobs import JobConfig
 from .jobs import LoadRef
@@ -28,29 +19,25 @@ from .jobs import TransformRef
 from .jobs import ValidationRef
 from .pipeline import PipelineConfig
 from .pipeline import load_pipeline_config
-from .profile import ProfileConfig
-from .types import ConnectorType
 
 # SECTION: EXPORTS ========================================================== #
 
 
 __all__ = [
-    # Connectors
-    'Connector',
-    'ConnectorType',
+    # Data Classes
     'ConnectorApi',
     'ConnectorDb',
     'ConnectorFile',
-    'parse_connector',
-    # Jobs / Refs
     'ExtractRef',
     'JobConfig',
     'LoadRef',
+    'PipelineConfig',
     'TransformRef',
     'ValidationRef',
-    # Pipeline
-    'PipelineConfig',
+    # Functions
     'load_pipeline_config',
-    # Profile
-    'ProfileConfig',
+    'parse_connector',
+    'topological_sort_jobs',
+    # Type Aliases
+    'Connector',
 ]

etlplus/{config → workflow}/connector.py

@@ -1,5 +1,5 @@
 """
-:mod:`etlplus.config.connector` module.
+:mod:`etlplus.workflow.connector` module.
 
 A module defining configuration types for data source/target connectors in ETL
 pipelines. A "connector" is any I/O endpoint:
@@ -11,25 +11,26 @@ pipelines. A "connector" is any I/O endpoint:
 
 Examples
 --------
-- Use ``ConnectorApi``/``ConnectorFile``/``ConnectorDb`` when you want the
-  concrete dataclasses.
-- Use the ``Connector`` union for typing a value that can be any connector.
-- Use ``parse_connector(obj)`` to construct a connector instance from a generic
-  mapping that includes a ``type`` key.
+- Use :class:`ConnectorApi`/:class:`ConnectorFile`/:class:`ConnectorDb` when
+    you want the concrete dataclasses.
+- Use the :class:`Connector` union for typing a value that can be any
+    connector.
+- Use :func:`parse_connector(obj)` to construct a connector instance from a
+    generic mapping that includes a *type* key.
 
 Notes
 -----
 - TypedDict shapes are editor hints; runtime parsing remains permissive
-  (from_obj accepts Mapping[str, Any]).
-- TypedDicts referenced in :mod:`etlplus.config.types` remain editor hints.
-  Runtime parsing stays permissive and tolerant.
+    (from_obj accepts Mapping[str, Any]).
+- TypedDicts referenced in :mod:`etlplus.workflow.types` remain editor hints.
+    Runtime parsing stays permissive and tolerant.
 
 See Also
 --------
 - TypedDict shapes for editor hints (not enforced at runtime):
-    :mod:`etlplus.config.types.ConnectorApiConfigMap`,
-    :mod:`etlplus.config.types.ConnectorDbConfigMap`,
-    :mod:`etlplus.config.types.ConnectorFileConfigMap`.
+    :mod:`etlplus.workflow.types.ConnectorApiConfigMap`,
+    :mod:`etlplus.workflow.types.ConnectorDbConfigMap`,
+    :mod:`etlplus.workflow.types.ConnectorFileConfigMap`.
 """
 
 from __future__ import annotations
@@ -59,7 +60,7 @@ if TYPE_CHECKING:  # Editor-only typing hints to avoid runtime imports
 
 
 __all__ = [
-    # Classes
+    # Data Classes
     'ConnectorApi',
     'ConnectorDb',
     'ConnectorFile',
@@ -70,6 +71,40 @@ __all__ = [
 ]
 
 
+# SECTION: INTERNAL FUNCTIONS ============================================== #
+
+
+def _require_name(
+    obj: StrAnyMap,
+    *,
+    kind: str,
+) -> str:
+    """
+    Extract and validate the ``name`` field from connector mappings.
+
+    Parameters
+    ----------
+    obj : StrAnyMap
+        Connector mapping with a ``name`` entry.
+    kind : str
+        Connector kind used in the error message.
+
+    Returns
+    -------
+    str
+        Valid connector name.
+
+    Raises
+    ------
+    TypeError
+        If ``name`` is missing or not a string.
+    """
+    name = obj.get('name')
+    if not isinstance(name, str):
+        raise TypeError(f'Connector{kind} requires a "name" (str)')
+    return name
+
+
 # SECTION: DATA CLASSES ===================================================== #
 
 
@@ -83,12 +118,12 @@ class ConnectorApi:
     name : str
         Unique connector name.
     type : ConnectorType
-        Connector kind literal, always ``"api"``.
+        Connector kind literal, always ``'api'``.
     url : str | None
         Direct absolute URL (when not using ``service``/``endpoint`` refs).
     method : str | None
         Optional HTTP method; typically omitted for sources (defaults to
-        GET) and used for targets (e.g., ``"post"``).
+        GET) and used for targets (e.g., ``'post'``).
     headers : dict[str, str]
         Additional request headers.
     query_params : dict[str, Any]
@@ -111,7 +146,7 @@ class ConnectorApi:
 
     # Direct form
     url: str | None = None
-    # Optional HTTP method; typically omitted for sources (defaults to GET
+    # Optional HTTP method; typically omitted for sources (defaults to GET)
     # at runtime) and used for targets (e.g., 'post', 'put').
     method: str | None = None
     headers: dict[str, str] = field(default_factory=dict)
@@ -150,15 +185,8 @@ class ConnectorApi:
         -------
         Self
             Parsed connector instance.
-
-        Raises
-        ------
-        TypeError
-            If ``name`` is missing or invalid.
         """
-        name = obj.get('name')
-        if not isinstance(name, str):
-            raise TypeError('ConnectorApi requires a "name" (str)')
+        name = _require_name(obj, kind='Api')
         headers = cast_str_dict(obj.get('headers'))
 
         return cls(
@@ -185,7 +213,7 @@ class ConnectorDb:
     name : str
         Unique connector name.
     type : ConnectorType
-        Connector kind literal, always ``"database"``.
+        Connector kind literal, always ``'database'``.
     connection_string : str | None
         Connection string/DSN for the database.
     query : str | None
@@ -193,7 +221,7 @@ class ConnectorDb:
     table : str | None
         Target/source table name (optional).
     mode : str | None
-        Load mode hint (e.g., ``"append"``, ``"replace"``) — future use.
+        Load mode hint (e.g., ``'append'``, ``'replace'``) — future use.
     """
 
     # -- Attributes -- #
@@ -232,15 +260,8 @@ class ConnectorDb:
         -------
         Self
             Parsed connector instance.
-
-        Raises
-        ------
-        TypeError
-            If ``name`` is missing or invalid.
         """
-        name = obj.get('name')
-        if not isinstance(name, str):
-            raise TypeError('ConnectorDb requires a "name" (str)')
+        name = _require_name(obj, kind='Db')
 
         return cls(
             name=name,
@@ -262,9 +283,9 @@ class ConnectorFile:
     name : str
         Unique connector name.
     type : ConnectorType
-        Connector kind literal, always ``"file"``.
+        Connector kind literal, always ``'file'``.
     format : str | None
-        File format (e.g., ``"json"``, ``"csv"``).
+        File format (e.g., ``'json'``, ``'csv'``).
     path : str | None
         File path or URI.
     options : dict[str, Any]
@@ -306,15 +327,8 @@ class ConnectorFile:
         -------
         Self
             Parsed connector instance.
-
-        Raises
-        ------
-        TypeError
-            If ``name`` is missing or invalid.
         """
-        name = obj.get('name')
-        if not isinstance(name, str):
-            raise TypeError('ConnectorFile requires a "name" (str)')
+        name = _require_name(obj, kind='File')
 
         return cls(
             name=name,