etlplus 0.4.7__py3-none-any.whl → 0.7.0__py3-none-any.whl

etlplus/config/pipeline.py

@@ -190,6 +190,8 @@ class PipelineConfig:
         Target connectors, parsed tolerantly.
     jobs : list[JobConfig]
         Job orchestration definitions.
+    table_schemas : list[dict[str, Any]]
+        Optional DDL-style table specifications used by the render command.
     """
 
     # -- Attributes -- #
@@ -208,6 +210,7 @@ class PipelineConfig:
     transforms: dict[str, dict[str, Any]] = field(default_factory=dict)
     targets: list[Connector] = field(default_factory=list)
     jobs: list[JobConfig] = field(default_factory=list)
+    table_schemas: list[dict[str, Any]] = field(default_factory=list)
 
     # -- Class Methods -- #
 
@@ -312,6 +315,13 @@ class PipelineConfig:
         # Jobs
         jobs = _build_jobs(raw)
 
+        # Table schemas (optional, tolerant pass-through structures).
+        table_schemas: list[dict[str, Any]] = []
+        for entry in raw.get('table_schemas', []) or []:
+            spec = maybe_mapping(entry)
+            if spec is not None:
+                table_schemas.append(dict(spec))
+
         return cls(
             name=name,
             version=version,
@@ -325,4 +335,5 @@ class PipelineConfig:
             transforms=transforms,
             targets=targets,
             jobs=jobs,
+            table_schemas=table_schemas,
         )

etlplus/database/__init__.py

@@ -0,0 +1,42 @@
+"""
+:mod:`etlplus.database` package.
+
+Database utilities for:
+- DDL rendering and schema management.
+- Schema parsing from configuration files.
+- Dynamic ORM generation.
+- Database engine/session management.
+"""
+
+from __future__ import annotations
+
+from .ddl import load_table_spec
+from .ddl import render_table_sql
+from .ddl import render_tables
+from .ddl import render_tables_to_string
+from .engine import engine
+from .engine import load_database_url_from_config
+from .engine import make_engine
+from .engine import session
+from .orm import build_models
+from .orm import load_and_build_models
+from .schema import load_table_specs
+
+# SECTION: EXPORTS ========================================================== #
+
+
+__all__ = [
+    # Functions
+    'build_models',
+    'load_and_build_models',
+    'load_database_url_from_config',
+    'load_table_spec',
+    'load_table_specs',
+    'make_engine',
+    'render_table_sql',
+    'render_tables',
+    'render_tables_to_string',
+    # Singletons
+    'engine',
+    'session',
+]

etlplus/database/ddl.py

@@ -0,0 +1,311 @@
+"""
+:mod:`etlplus.database.ddl` module.
+
+DDL rendering utilities for pipeline table schemas.
+
+Exposes helpers to load YAML/JSON table specs and render them into SQL via
+Jinja templates. Mirrors the behavior of ``tools/render_ddl.py`` so the CLI
+can emit DDLs without shelling out to that script.
+"""
+
+from __future__ import annotations
+
+import importlib.resources
+import os
+from collections.abc import Iterable
+from collections.abc import Mapping
+from pathlib import Path
+from typing import Any
+from typing import Final
+
+from jinja2 import DictLoader
+from jinja2 import Environment
+from jinja2 import FileSystemLoader
+from jinja2 import StrictUndefined
+
+from ..file import File
+
+# SECTION: EXPORTS ========================================================== #
+
+
+__all__ = [
+    'TEMPLATES',
+    'load_table_spec',
+    'render_table_sql',
+    'render_tables',
+    'render_tables_to_string',
+]
+
+
+# SECTION: INTERNAL CONSTANTS =============================================== #
+
+
+_SUPPORTED_SPEC_SUFFIXES: Final[frozenset[str]] = frozenset(
+    {
+        '.json',
+        '.yml',
+        '.yaml',
+    },
+)
+
+
+# SECTION: CONSTANTS ======================================================== #
+
+
+TEMPLATES: Final[dict[str, str]] = {
+    'ddl': 'ddl.sql.j2',
+    'view': 'view.sql.j2',
+}
+
+
+# SECTION: INTERNAL FUNCTIONS =============================================== #
+
+
+def _load_template_text(
+    filename: str,
+) -> str:
+    """Return the bundled template text.
+
+    Parameters
+    ----------
+    filename : str
+        Template filename located inside the package data folder.
+
+    Returns
+    -------
+    str
+        Raw template contents.
+
+    Raises
+    ------
+    FileNotFoundError
+        If the template file cannot be located in package data.
+    """
+
+    try:
+        return (
+            importlib.resources.files(
+                'etlplus.templates',
+            )
+            .joinpath(filename)
+            .read_text(encoding='utf-8')
+        )
+    except FileNotFoundError as exc:  # pragma: no cover - deployment guard
+        raise FileNotFoundError(
+            f'Could not load template {filename} '
+            f'from etlplus.templates package data.',
+        ) from exc
+
+
+def _resolve_template(
+    *,
+    template_key: str | None,
+    template_path: str | None,
+) -> tuple[Environment, str]:
+    """Return environment and template name for rendering.
+
+    Parameters
+    ----------
+    template_key : str | None
+        Named template key bundled with the package.
+    template_path : str | None
+        Explicit template file override.
+
+    Returns
+    -------
+    tuple[Environment, str]
+        Pair of configured Jinja environment and the template identifier.
+
+    Raises
+    ------
+    FileNotFoundError
+        If the provided template path does not exist.
+    ValueError
+        If the template key is unknown.
+    """
+    file_override = template_path or os.environ.get('TEMPLATE_NAME')
+    if file_override:
+        path = Path(file_override)
+        if not path.exists():
+            raise FileNotFoundError(f'Template file not found: {path}')
+        loader = FileSystemLoader(str(path.parent))
+        env = Environment(
+            loader=loader,
+            undefined=StrictUndefined,
+            trim_blocks=True,
+            lstrip_blocks=True,
+        )
+        return env, path.name
+
+    key = (template_key or 'ddl').strip()
+    if key not in TEMPLATES:
+        choices = ', '.join(sorted(TEMPLATES))
+        raise ValueError(
+            f'Unknown template key "{key}". Choose from: {choices}',
+        )
+
+    # Load template from package data
+    template_filename = TEMPLATES[key]
+    template_source = _load_template_text(template_filename)
+
+    env = Environment(
+        loader=DictLoader({key: template_source}),
+        undefined=StrictUndefined,
+        trim_blocks=True,
+        lstrip_blocks=True,
+    )
+    return env, key
+
+
+# SECTION: FUNCTIONS ======================================================== #
+
+
+def load_table_spec(
+    path: Path | str,
+) -> dict[str, Any]:
+    """
+    Load a table specification from disk.
+
+    Parameters
+    ----------
+    path : Path | str
+        Path to the JSON or YAML specification file.
+
+    Returns
+    -------
+    dict[str, Any]
+        Parsed table specification mapping.
+
+    Raises
+    ------
+    ImportError
+        If the file cannot be read due to missing dependencies.
+    RuntimeError
+        If the YAML dependency is missing for YAML specs.
+    TypeError
+        If the loaded spec is not a mapping.
+    ValueError
+        If the file suffix is not supported.
+    """
+
+    spec_path = Path(path)
+    suffix = spec_path.suffix.lower()
+
+    if suffix not in _SUPPORTED_SPEC_SUFFIXES:
+        raise ValueError('Spec must be .json, .yml, or .yaml')
+
+    try:
+        spec = File.read_file(spec_path)
+    except ImportError as e:
+        if suffix in {'.yml', '.yaml'}:
+            raise RuntimeError(
+                'Missing dependency: pyyaml is required for YAML specs.',
+            ) from e
+        raise
+
+    if not isinstance(spec, Mapping):
+        raise TypeError('Table spec must be a mapping')
+
+    return dict(spec)
+
+
+def render_table_sql(
+    spec: Mapping[str, Any],
+    *,
+    template: str | None = 'ddl',
+    template_path: str | None = None,
+) -> str:
+    """
+    Render a single table spec into SQL text.
+
+    Parameters
+    ----------
+    spec : Mapping[str, Any]
+        Table specification mapping.
+    template : str | None, optional
+        Template key to use (default: 'ddl').
+    template_path : str | None, optional
+        Path to a custom template file (overrides ``template``).
+
+    Returns
+    -------
+    str
+        Rendered SQL string.
+    """
+    env, template_name = _resolve_template(
+        template_key=template,
+        template_path=template_path,
+    )
+    tmpl = env.get_template(template_name)
+    return tmpl.render(spec=spec).rstrip() + '\n'
+
+
+def render_tables(
+    specs: Iterable[Mapping[str, Any]],
+    *,
+    template: str | None = 'ddl',
+    template_path: str | None = None,
+) -> list[str]:
+    """
+    Render multiple table specs into a list of SQL payloads.
+
+    Parameters
+    ----------
+    specs : Iterable[Mapping[str, Any]]
+        Table specification mappings.
+    template : str | None, optional
+        Template key to use (default: 'ddl').
+    template_path : str | None, optional
+        Path to a custom template file (overrides ``template``).
+
+    Returns
+    -------
+    list[str]
+        Rendered SQL strings for each table spec.
+    """
+
+    return [
+        render_table_sql(spec, template=template, template_path=template_path)
+        for spec in specs
+    ]
+
+
+def render_tables_to_string(
+    spec_paths: Iterable[Path | str],
+    *,
+    template: str | None = 'ddl',
+    template_path: Path | str | None = None,
+) -> str:
+    """
+    Render one or more specs and concatenate the SQL payloads.
+
+    Parameters
+    ----------
+    spec_paths : Iterable[Path | str]
+        Paths to table specification files.
+    template : str | None, optional
+        Template key bundled with ETLPlus. Defaults to ``'ddl'``.
+    template_path : Path | str | None, optional
+        Custom Jinja template to override the bundled templates.
+
+    Returns
+    -------
+    str
+        Concatenated SQL payload suitable for writing to disk or stdout.
+    """
+
+    resolved_template_path = (
+        str(template_path) if template_path is not None else None
+    )
+    rendered_sql: list[str] = []
+    for spec_path in spec_paths:
+        spec = load_table_spec(spec_path)
+        rendered_sql.append(
+            render_table_sql(
+                spec,
+                template=template,
+                template_path=resolved_template_path,
+            ),
+        )
+
+    return ''.join(rendered_sql)

etlplus/database/engine.py

@@ -0,0 +1,146 @@
+"""
+:mod:`etlplus.database.engine` module.
+
+Lightweight engine/session factory with optional config-driven URL loading.
+"""
+
+from __future__ import annotations
+
+import os
+from collections.abc import Mapping
+from pathlib import Path
+from typing import Any
+
+from sqlalchemy import create_engine
+from sqlalchemy.engine import Engine
+from sqlalchemy.orm import sessionmaker
+
+from ..file import File
+
+# SECTION: EXPORTS ========================================================== #
+
+
+__all__ = [
+    # Functions
+    'load_database_url_from_config',
+    'make_engine',
+    # Singletons
+    'engine',
+    'session',
+]
+
+
+# SECTION: INTERNAL CONSTANTS =============================================== #
+
+
+DATABASE_URL: str = (
+    os.getenv('DATABASE_URL')
+    or os.getenv('DATABASE_DSN')
+    or 'sqlite+pysqlite:///:memory:'
+)
+
+
+# SECTION: INTERNAL FUNCTIONS =============================================== #
+
+
+def _resolve_url_from_mapping(cfg: Mapping[str, Any]) -> str | None:
+    """
+    Return a URL/DSN from a mapping if present.
+
+    Parameters
+    ----------
+    cfg : Mapping[str, Any]
+        Configuration mapping potentially containing connection fields.
+
+    Returns
+    -------
+    str | None
+        Resolved URL/DSN string, if present.
+    """
+    conn = cfg.get('connection_string') or cfg.get('url') or cfg.get('dsn')
+    if isinstance(conn, str) and conn.strip():
+        return conn.strip()
+
+    # Some configs nest defaults.
+    # E.g., databases: { mssql: { default: {...} } }
+    default_cfg = cfg.get('default')
+    if isinstance(default_cfg, Mapping):
+        return _resolve_url_from_mapping(default_cfg)
+
+    return None
+
+
+# SECTION: FUNCTIONS ======================================================== #
+
+
+def load_database_url_from_config(
+    path: str | Path,
+    *,
+    name: str | None = None,
+) -> str:
+    """
+    Extract a database URL/DSN from a YAML/JSON config file.
+
+    The loader is schema-tolerant: it looks for a top-level "databases" map
+    and then for a named entry (``name``). Each entry may contain either a
+    ``connection_string``/``url``/``dsn`` or a nested ``default`` block with
+    those fields.
+
+    Parameters
+    ----------
+    path : str | Path
+        Location of the configuration file.
+    name : str | None, optional
+        Named database entry under the ``databases`` map (default:
+        ``default``).
+
+    Returns
+    -------
+    str
+        Resolved database URL/DSN string.
+
+    Raises
+    ------
+    KeyError
+        If the specified database entry is not found.
+    TypeError
+        If the config structure is invalid.
+    ValueError
+        If no connection string/URL/DSN is found for the specified entry.
+    """
+    cfg = File.read_file(Path(path))
+    if not isinstance(cfg, Mapping):
+        raise TypeError('Database config must be a mapping')
+
+    databases = cfg.get('databases') if isinstance(cfg, Mapping) else None
+    if not isinstance(databases, Mapping):
+        raise KeyError('Config missing top-level "databases" mapping')
+
+    target = name or 'default'
+    entry = databases.get(target)
+    if entry is None:
+        raise KeyError(f'Database entry "{target}" not found in config')
+    if not isinstance(entry, Mapping):
+        raise TypeError(f'Database entry "{target}" must be a mapping')
+
+    url = _resolve_url_from_mapping(entry)
+    if not url:
+        raise ValueError(
+            f'Database entry "{target}" lacks connection_string/url/dsn',
+        )
+    return url
+
+
+def make_engine(url: str | None = None, **engine_kwargs: Any) -> Engine:
+    """Create a SQLAlchemy Engine, defaulting to env config if no URL given."""
+
+    resolved_url = url or DATABASE_URL
+    return create_engine(resolved_url, pool_pre_ping=True, **engine_kwargs)
+
+
+# SECTION: SINGLETONS ======================================================= #
+
+
+# Default engine/session for callers that rely on module-level singletons.
+engine = make_engine()
+session = sessionmaker(bind=engine, autoflush=False, autocommit=False)