etlplus 0.5.4__py3-none-any.whl

etlplus/config/utils.py

@@ -0,0 +1,120 @@
+"""
+:mod:`etlplus.config.utils` module.
+
+A module defining utility helpers for ETL pipeline configuration.
+
+Notes
+-----
+- Inputs to parsers favor ``Mapping[str, Any]`` to remain permissive and
+    avoid unnecessary copies; normalization returns concrete types.
+- Substitution is shallow for strings and recursive for containers.
+- Numeric coercion helpers are intentionally forgiving: invalid values
+    become ``None`` rather than raising.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Iterable
+from collections.abc import Mapping
+from typing import Any
+
+from ..types import StrAnyMap
+
+# SECTION: EXPORTS ========================================================== #
+
+
+__all__ = [
+    # Functions
+    'deep_substitute',
+]
+
+
+# SECTION: FUNCTIONS ======================================================== #
+
+
+def deep_substitute(
+    value: Any,
+    vars_map: StrAnyMap | None,
+    env_map: Mapping[str, str] | None,
+) -> Any:
+    """
+    Recursively substitute ``${VAR}`` tokens in nested structures.
+
+    Only strings are substituted; other types are returned as-is.
+
+    Parameters
+    ----------
+    value : Any
+        The value to perform substitutions on.
+    vars_map : StrAnyMap | None
+        Mapping of variable names to replacement values (lower precedence).
+    env_map : Mapping[str, str] | None
+        Mapping of environment variables overriding ``vars_map`` values (higher
+        precedence).
+
+    Returns
+    -------
+    Any
+        New structure with substitutions applied where tokens were found.
+    """
+    substitutions = _prepare_substitutions(vars_map, env_map)
+
+    def _apply(node: Any) -> Any:
+        match node:
+            case str():
+                return _replace_tokens(node, substitutions)
+            case Mapping():
+                return {k: _apply(v) for k, v in node.items()}
+            case list() | tuple() as seq:
+                apply = [_apply(item) for item in seq]
+                return apply if isinstance(seq, list) else tuple(apply)
+            case set():
+                return {_apply(item) for item in node}
+            case frozenset():
+                return frozenset(_apply(item) for item in node)
+            case _:
+                return node
+
+    return _apply(value)
+
+
+# SECTION: INTERNAL FUNCTIONS ============================================== #
+
+
+def _prepare_substitutions(
+    vars_map: StrAnyMap | None,
+    env_map: Mapping[str, Any] | None,
+) -> tuple[tuple[str, Any], ...]:
+    """Merge variable and environment maps into an ordered substitutions list.
+
+    Parameters
+    ----------
+    vars_map : StrAnyMap | None
+        Mapping of variable names to replacement values (lower precedence).
+    env_map : Mapping[str, Any] | None
+        Environment-backed values that override entries from ``vars_map``.
+
+    Returns
+    -------
+    tuple[tuple[str, Any], ...]
+        Immutable sequence of ``(name, value)`` pairs suitable for token
+        replacement.
+    """
+    if not vars_map and not env_map:
+        return ()
+    merged: dict[str, Any] = {**(vars_map or {}), **(env_map or {})}
+    return tuple(merged.items())
+
+
+def _replace_tokens(
+    text: str,
+    substitutions: Iterable[tuple[str, Any]],
+) -> str:
+    if not substitutions:
+        return text
+    out = text
+    for name, replacement in substitutions:
+        token = f'${{{name}}}'
+        if token in out:
+            out = out.replace(token, str(replacement))
+    return out

etlplus/ddl.py

@@ -0,0 +1,197 @@
+"""
+:mod:`etlplus.ddl` module.
+
+DDL rendering utilities for pipeline table schemas.
+
+Exposes helpers to load YAML/JSON table specs and render them into SQL via
+Jinja templates. Mirrors the behavior of ``tools/render_ddl.py`` so the CLI
+can emit DDLs without shelling out to that script.
+"""
+
+from __future__ import annotations
+
+import importlib.resources
+import json
+import os
+from collections.abc import Iterable
+from collections.abc import Mapping
+from pathlib import Path
+from typing import Any
+
+from jinja2 import DictLoader
+from jinja2 import Environment
+from jinja2 import FileSystemLoader
+from jinja2 import StrictUndefined
+
+# SECTION: EXPORTS ========================================================== #
+
+
+__all__ = [
+    'TEMPLATES',
+    'load_table_spec',
+    'render_table_sql',
+    'render_tables',
+]
+
+
+# SECTION: CONSTANTS ======================================================== #
+
+
+TEMPLATES = {
+    'ddl': 'ddl.sql.j2',
+    'view': 'view.sql.j2',
+}
+
+
+# SECTION: INTERNAL FUNCTIONS =============================================== #
+
+
+def _build_env(
+    *,
+    template_key: str | None,
+    template_path: str | None,
+) -> Environment:
+    """Return a Jinja2 environment using a built-in or file template."""
+    file_override = template_path or os.environ.get('TEMPLATE_NAME')
+    if file_override:
+        path = Path(file_override)
+        if not path.exists():
+            raise FileNotFoundError(f'Template file not found: {path}')
+        loader = FileSystemLoader(str(path.parent))
+        env = Environment(
+            loader=loader,
+            undefined=StrictUndefined,
+            trim_blocks=True,
+            lstrip_blocks=True,
+        )
+        env.globals['TEMPLATE_NAME'] = path.name
+        return env
+
+    key = (template_key or 'ddl').strip()
+    if key not in TEMPLATES:
+        choices = ', '.join(sorted(TEMPLATES))
+        raise ValueError(
+            f'Unknown template key "{key}". Choose from: {choices}',
+        )
+
+    # Load template from package data
+    template_filename = TEMPLATES[key]
+    template_source = _load_template_text(template_filename)
+
+    env = Environment(
+        loader=DictLoader({key: template_source}),
+        undefined=StrictUndefined,
+        trim_blocks=True,
+        lstrip_blocks=True,
+    )
+    env.globals['TEMPLATE_NAME'] = key
+    return env
+
+
+def _load_template_text(filename: str) -> str:
+    """Return the raw template text bundled with the package."""
+
+    try:
+        return (
+            importlib.resources.files(
+                'etlplus.templates',
+            )
+            .joinpath(filename)
+            .read_text(encoding='utf-8')
+        )
+    except FileNotFoundError as exc:  # pragma: no cover - deployment guard
+        raise FileNotFoundError(
+            f'Could not load template {filename} '
+            f'from etlplus.templates package data.',
+        ) from exc
+
+
+# SECTION: FUNCTIONS ======================================================== #
+
+
+def load_table_spec(path: Path | str) -> dict[str, Any]:
+    """Load a table spec from JSON or YAML."""
+
+    spec_path = Path(path)
+    text = spec_path.read_text(encoding='utf-8')
+    suffix = spec_path.suffix.lower()
+
+    if suffix == '.json':
+        return json.loads(text)
+
+    if suffix in {'.yml', '.yaml'}:
+        try:
+            import yaml  # type: ignore
+        except Exception as exc:  # pragma: no cover
+            raise RuntimeError(
+                'Missing dependency: pyyaml is required for YAML specs.',
+            ) from exc
+        return yaml.safe_load(text)
+
+    raise ValueError('Spec must be .json, .yml, or .yaml')
+
+
+def render_table_sql(
+    spec: Mapping[str, Any],
+    *,
+    template: str | None = 'ddl',
+    template_path: str | None = None,
+) -> str:
+    """
+    Render a single table spec into SQL text.
+
+    Parameters
+    ----------
+    spec : Mapping[str, Any]
+        Table specification mapping.
+    template : str | None, optional
+        Template key to use (default: 'ddl').
+    template_path : str | None, optional
+        Path to a custom template file (overrides ``template``).
+
+    Returns
+    -------
+    str
+        Rendered SQL string.
+
+    Raises
+    ------
+    TypeError
+        If the loaded template name is not a string.
+    """
+    env = _build_env(template_key=template, template_path=template_path)
+    template_name = env.globals.get('TEMPLATE_NAME')
+    if not isinstance(template_name, str):
+        raise TypeError('TEMPLATE_NAME must be a string.')
+    tmpl = env.get_template(template_name)
+    return tmpl.render(spec=spec).rstrip() + '\n'
+
+
+def render_tables(
+    specs: Iterable[Mapping[str, Any]],
+    *,
+    template: str | None = 'ddl',
+    template_path: str | None = None,
+) -> list[str]:
+    """
+    Render multiple table specs into a list of SQL payloads.
+
+    Parameters
+    ----------
+    specs : Iterable[Mapping[str, Any]]
+        Table specification mappings.
+    template : str | None, optional
+        Template key to use (default: 'ddl').
+    template_path : str | None, optional
+        Path to a custom template file (overrides ``template``).
+
+    Returns
+    -------
+    list[str]
+        Rendered SQL strings for each table spec.
+    """
+
+    return [
+        render_table_sql(spec, template=template, template_path=template_path)
+        for spec in specs
+    ]