etlplus 0.4.7__py3-none-any.whl → 0.8.3__py3-none-any.whl

etlplus/database/orm.py

@@ -0,0 +1,354 @@
+"""
+:mod:`etlplus.database.orm` module.
+
+Dynamic SQLAlchemy model generation from YAML table specs.
+
+Usage
+-----
+>>> from etlplus.database.orm import load_and_build_models
+>>> registry = load_and_build_models('examples/configs/ddl_spec.yml')
+>>> Player = registry['dbo.Customers']
+"""
+
+from __future__ import annotations
+
+import re
+from typing import Any
+from typing import Final
+
+from sqlalchemy import Boolean
+from sqlalchemy import CheckConstraint
+from sqlalchemy import Date
+from sqlalchemy import DateTime
+from sqlalchemy import Enum
+from sqlalchemy import Float
+from sqlalchemy import ForeignKey
+from sqlalchemy import ForeignKeyConstraint
+from sqlalchemy import Index
+from sqlalchemy import Integer
+from sqlalchemy import LargeBinary
+from sqlalchemy import Numeric
+from sqlalchemy import PrimaryKeyConstraint
+from sqlalchemy import String
+from sqlalchemy import Text
+from sqlalchemy import Time
+from sqlalchemy import UniqueConstraint
+from sqlalchemy import text
+from sqlalchemy.dialects.postgresql import JSONB
+from sqlalchemy.dialects.postgresql import UUID as PG_UUID
+from sqlalchemy.orm import DeclarativeBase
+from sqlalchemy.orm import mapped_column
+from sqlalchemy.types import TypeEngine
+
+from ..types import StrPath
+from .schema import ForeignKeySpec
+from .schema import TableSpec
+from .schema import load_table_specs
+from .types import ModelRegistry
+from .types import TypeFactory
+
+# SECTION: EXPORTS ========================================================== #
+
+
+__all__ = [
+    # Classes
+    'Base',
+    # Functions
+    'build_models',
+    'load_and_build_models',
+    'resolve_type',
+]
+
+
+# SECTION: INTERNAL CONSTANTS =============================================== #
+
+_TYPE_MAPPING: Final[dict[str, TypeFactory]] = {
+    'int': lambda _: Integer(),
+    'integer': lambda _: Integer(),
+    'bigint': lambda _: Integer(),
+    'smallint': lambda _: Integer(),
+    'bool': lambda _: Boolean(),
+    'boolean': lambda _: Boolean(),
+    'uuid': lambda _: PG_UUID(as_uuid=True),
+    'uniqueidentifier': lambda _: PG_UUID(as_uuid=True),
+    'rowversion': lambda _: LargeBinary(),
+    'varbinary': lambda _: LargeBinary(),
+    'blob': lambda _: LargeBinary(),
+    'text': lambda _: Text(),
+    'string': lambda _: Text(),
+    'varchar': lambda p: String(length=p[0]) if p else String(),
+    'nvarchar': lambda p: String(length=p[0]) if p else String(),
+    'char': lambda p: String(length=p[0] if p else 1),
+    'nchar': lambda p: String(length=p[0] if p else 1),
+    'numeric': lambda p: Numeric(
+        precision=p[0] if p else None,
+        scale=p[1] if len(p) > 1 else None,
+    ),
+    'decimal': lambda p: Numeric(
+        precision=p[0] if p else None,
+        scale=p[1] if len(p) > 1 else None,
+    ),
+    'float': lambda _: Float(),
+    'real': lambda _: Float(),
+    'double': lambda _: Float(),
+    'datetime': lambda _: DateTime(timezone=True),
+    'datetime2': lambda _: DateTime(timezone=True),
+    'timestamp': lambda _: DateTime(timezone=True),
+    'date': lambda _: Date(),
+    'time': lambda _: Time(),
+    'json': lambda _: JSONB(),
+    'jsonb': lambda _: JSONB(),
+}
+
+
+# SECTION: CLASSES ========================================================== #
+
+
+class Base(DeclarativeBase):
+    """Base class for all ORM models."""
+
+    __abstract__ = True
+
+
+# SECTION: INTERNAL FUNCTIONS =============================================== #
+
+
+def _class_name(
+    table: str,
+) -> str:
+    """
+    Convert table name to PascalCase class name.
+
+    Parameters
+    ----------
+    table : str
+        Table name.
+
+    Returns
+    -------
+    str
+        PascalCase class name.
+    """
+    parts = re.split(r'[^A-Za-z0-9]+', table)
+    return ''.join(p.capitalize() for p in parts if p)
+
+
+def _parse_type_decl(
+    type_str: str,
+) -> tuple[str, list[int]]:
+    """
+    Parse a type declaration string into its name and parameters.
+
+    Parameters
+    ----------
+    type_str : str
+        Type declaration string, e.g., "varchar(255)".
+
+    Returns
+    -------
+    tuple[str, list[int]]
+        A tuple containing the type name and a list of integer parameters.
+    """
+    m = re.match(
+        r'^(?P<name>[A-Za-z0-9_]+)(?:\((?P<params>[^)]*)\))?$',
+        type_str.strip(),
+    )
+    if not m:
+        return type_str.lower(), []
+    name = m.group('name').lower()
+    params_raw = m.group('params')
+    if not params_raw:
+        return name, []
+    params = [p.strip() for p in params_raw.split(',') if p.strip()]
+    parsed: list[int] = []
+    for p in params:
+        try:
+            parsed.append(int(p))
+        except ValueError:
+            continue
+    return name, parsed
+
+
+def _table_kwargs(
+    spec: TableSpec,
+) -> dict[str, str]:
+    """
+    Generate table keyword arguments based on the table specification.
+
+    Parameters
+    ----------
+    spec : TableSpec
+        Table specification.
+
+    Returns
+    -------
+    dict[str, str]
+        Dictionary of table keyword arguments.
+    """
+    kwargs: dict[str, str] = {}
+    if spec.schema_name:
+        kwargs['schema'] = spec.schema_name
+    return kwargs
+
+
+# SECTION: FUNCTIONS ======================================================== #
+
+
+def build_models(
+    specs: list[TableSpec],
+    *,
+    base: type[DeclarativeBase] = Base,
+) -> ModelRegistry:
+    """
+    Build SQLAlchemy ORM models from table specifications.
+    Parameters
+    ----------
+    specs : list[TableSpec]
+        List of table specifications.
+    base : type[DeclarativeBase], optional
+        Base class for the ORM models (default: :class:`Base`).
+    Returns
+    -------
+    ModelRegistry
+        Registry mapping fully qualified table names to ORM model classes.
+    """
+    registry: ModelRegistry = {}
+
+    for spec in specs:
+        table_args: list[object] = []
+        table_kwargs = _table_kwargs(spec)
+        pk_cols = set(spec.primary_key.columns) if spec.primary_key else set()
+
+        # Pre-handle multi-column constraints.
+        if spec.primary_key and len(spec.primary_key.columns) > 1:
+            table_args.append(
+                PrimaryKeyConstraint(
+                    *spec.primary_key.columns,
+                    name=spec.primary_key.name,
+                ),
+            )
+        for uc in spec.unique_constraints:
+            table_args.append(UniqueConstraint(*uc.columns, name=uc.name))
+        for idx in spec.indexes:
+            table_args.append(
+                Index(
+                    idx.name,
+                    *idx.columns,
+                    unique=idx.unique,
+                    postgresql_where=text(idx.where) if idx.where else None,
+                ),
+            )
+        composite_fks = [fk for fk in spec.foreign_keys if len(fk.columns) > 1]
+        for fk in composite_fks:
+            table_args.append(
+                ForeignKeyConstraint(
+                    fk.columns,
+                    [f'{fk.ref_table}.{c}' for c in fk.ref_columns],
+                    ondelete=fk.ondelete,
+                ),
+            )
+
+        fk_by_column = {
+            fk.columns[0]: fk
+            for fk in spec.foreign_keys
+            if len(fk.columns) == 1 and len(fk.ref_columns) == 1
+        }
+
+        attrs: dict[str, object] = {'__tablename__': spec.table}
+
+        for col in spec.columns:
+            col_fk: ForeignKeySpec | None = fk_by_column.get(col.name)
+            fk_arg = (
+                ForeignKey(
+                    f'{col_fk.ref_table}.{col_fk.ref_columns[0]}',
+                    ondelete=col_fk.ondelete,
+                )
+                if col_fk
+                else None
+            )
+            col_type: TypeEngine = (
+                Enum(*col.enum, name=f'{spec.table}_{col.name}_enum')
+                if col.enum
+                else resolve_type(col.type)
+            )
+            fk_args: list[ForeignKey] = []
+            if fk_arg:
+                fk_args.append(fk_arg)
+
+            kwargs: dict[str, Any] = {
+                'nullable': col.nullable,
+                'primary_key': col.name in pk_cols and len(pk_cols) == 1,
+                'unique': col.unique,
+            }
+            if col.default:
+                kwargs['server_default'] = text(col.default)
+            if col.identity:
+                kwargs['autoincrement'] = True
+
+            attrs[col.name] = mapped_column(*fk_args, type_=col_type, **kwargs)
+
+            if col.check:
+                table_args.append(
+                    CheckConstraint(
+                        col.check,
+                        name=f'ck_{spec.table}_{col.name}',
+                    ),
+                )
+
+        if table_args or table_kwargs:
+            args_tuple = tuple(table_args)
+            attrs['__table_args__'] = (
+                (*args_tuple, table_kwargs) if table_kwargs else args_tuple
+            )
+
+        cls_name = _class_name(spec.table)
+        model_cls = type(cls_name, (base,), attrs)
+        registry[spec.fq_name] = model_cls
+
+    return registry
+
+
+def load_and_build_models(
+    path: StrPath,
+    *,
+    base: type[DeclarativeBase] = Base,
+) -> ModelRegistry:
+    """
+    Load table specifications from a file and build SQLAlchemy models.
+
+    Parameters
+    ----------
+    path : StrPath
+        Path to the YAML file containing table specifications.
+    base : type[DeclarativeBase], optional
+        Base class for the ORM models (default: :class:`Base`).
+
+    Returns
+    -------
+    ModelRegistry
+        Registry mapping fully qualified table names to ORM model classes.
+    """
+    return build_models(load_table_specs(path), base=base)
+
+
+def resolve_type(
+    type_str: str,
+) -> TypeEngine:
+    """
+    Resolve a string type declaration to a SQLAlchemy :class:`TypeEngine`.
+
+    Parameters
+    ----------
+    type_str : str
+        String representation of the type declaration.
+
+    Returns
+    -------
+    TypeEngine
+        SQLAlchemy type engine instance corresponding to the type declaration.
+    """
+    name, params = _parse_type_decl(type_str)
+    factory = _TYPE_MAPPING.get(name)
+    if factory:
+        return factory(params)
+    return Text()

etlplus/database/schema.py

@@ -0,0 +1,274 @@
+"""
+:mod:`etlplus.database.schema` module.
+
+Helpers for loading and translating YAML definitions of database table schema
+specifications into Pydantic models for dynamic SQLAlchemy generation.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+from typing import ClassVar
+
+from pydantic import BaseModel
+from pydantic import ConfigDict
+from pydantic import Field
+
+from ..file import File
+from ..types import StrPath
+
+# SECTION: EXPORTS ========================================================== #
+
+
+__all__ = [
+    # Classes
+    'ColumnSpec',
+    'ForeignKeySpec',
+    'IdentitySpec',
+    'IndexSpec',
+    'PrimaryKeySpec',
+    'UniqueConstraintSpec',
+    'TableSpec',
+    # Functions
+    'load_table_specs',
+]
+
+
+# SECTION: CLASSES ========================================================== #
+
+
+class ColumnSpec(BaseModel):
+    """
+    Column specification suitable for ODBC / SQLite DDL.
+
+    Attributes
+    ----------
+    model_config : ClassVar[ConfigDict]
+        Pydantic model configuration.
+    name : str
+        Unquoted column name.
+    type : str
+        SQL type string, e.g., INT, NVARCHAR(100).
+    nullable : bool
+        True if NULL values are allowed.
+    default : str | None
+        Default value expression, or None if no default.
+    identity : IdentitySpec | None
+        Identity specification, or None if not an identity column.
+    check : str | None
+        Check constraint expression, or None if no check constraint.
+    enum : list[str] | None
+        List of allowed string values for enum-like columns, or None.
+    unique : bool
+        True if the column has a UNIQUE constraint.
+    """
+
+    model_config: ClassVar[ConfigDict] = ConfigDict(extra='forbid')
+
+    name: str
+    type: str = Field(description='SQL type string, e.g., INT, NVARCHAR(100)')
+    nullable: bool = True
+    default: str | None = None
+    identity: IdentitySpec | None = None
+    check: str | None = None
+    enum: list[str] | None = None
+    unique: bool = False
+
+
+class ForeignKeySpec(BaseModel):
+    """
+    Foreign key specification.
+
+    Attributes
+    ----------
+    model_config : ClassVar[ConfigDict]
+        Pydantic model configuration.
+    columns : list[str]
+        List of local column names.
+    ref_table : str
+        Referenced table name.
+    ref_columns : list[str]
+        List of referenced column names.
+    ondelete : str | None
+        ON DELETE action, or None.
+    """
+
+    model_config: ClassVar[ConfigDict] = ConfigDict(extra='forbid')
+
+    columns: list[str]
+    ref_table: str
+    ref_columns: list[str]
+    ondelete: str | None = None
+
+
+class IdentitySpec(BaseModel):
+    """
+    Identity specification.
+
+    Attributes
+    ----------
+    model_config : ClassVar[ConfigDict]
+        Pydantic model configuration.
+    seed : int | None
+        Identity seed value (default: 1).
+    increment : int | None
+        Identity increment value (default: 1).
+    """
+
+    model_config: ClassVar[ConfigDict] = ConfigDict(extra='forbid')
+
+    seed: int | None = Field(default=None, ge=1)
+    increment: int | None = Field(default=None, ge=1)
+
+
+class IndexSpec(BaseModel):
+    """
+    Index specification.
+
+    Attributes
+    ----------
+    model_config : ClassVar[ConfigDict]
+        Pydantic model configuration.
+    name : str
+        Index name.
+    columns : list[str]
+        List of column names included in the index.
+    unique : bool
+        True if the index is unique.
+    where : str | None
+        Optional WHERE clause for filtered indexes.
+    """
+
+    model_config: ClassVar[ConfigDict] = ConfigDict(extra='forbid')
+
+    name: str
+    columns: list[str]
+    unique: bool = False
+    where: str | None = None
+
+
+class PrimaryKeySpec(BaseModel):
+    """
+    Primary key specification.
+
+    Attributes
+    ----------
+    model_config : ClassVar[ConfigDict]
+        Pydantic model configuration.
+    name : str | None
+        Primary key constraint name, or None if unnamed.
+    columns : list[str]
+        List of column names included in the primary key.
+    """
+
+    model_config: ClassVar[ConfigDict] = ConfigDict(extra='forbid')
+
+    name: str | None = None
+    columns: list[str]
+
+
+class UniqueConstraintSpec(BaseModel):
+    """
+    Unique constraint specification.
+
+    Attributes
+    ----------
+    model_config : ClassVar[ConfigDict]
+        Pydantic model configuration.
+    name : str | None
+        Unique constraint name, or None if unnamed.
+    columns : list[str]
+        List of column names included in the unique constraint.
+    """
+
+    model_config: ClassVar[ConfigDict] = ConfigDict(extra='forbid')
+
+    name: str | None = None
+    columns: list[str]
+
+
+class TableSpec(BaseModel):
+    """
+    Table specification.
+
+    Attributes
+    ----------
+    model_config : ClassVar[ConfigDict]
+        Pydantic model configuration.
+    table : str
+        Table name.
+    schema_name : str | None
+        Schema name, or None if not specified.
+    create_schema : bool
+        Whether to create the schema if it does not exist.
+    columns : list[ColumnSpec]
+        List of column specifications.
+    primary_key : PrimaryKeySpec | None
+        Primary key specification, or None if no primary key.
+    unique_constraints : list[UniqueConstraintSpec]
+        List of unique constraint specifications.
+    indexes : list[IndexSpec]
+        List of index specifications.
+    foreign_keys : list[ForeignKeySpec]
+        List of foreign key specifications.
+    """
+
+    model_config: ClassVar[ConfigDict] = ConfigDict(extra='forbid')
+
+    table: str = Field(alias='name')
+    schema_name: str | None = Field(default=None, alias='schema')
+    create_schema: bool = False
+    columns: list[ColumnSpec]
+    primary_key: PrimaryKeySpec | None = None
+    unique_constraints: list[UniqueConstraintSpec] = Field(
+        default_factory=list,
+    )
+    indexes: list[IndexSpec] = Field(default_factory=list)
+    foreign_keys: list[ForeignKeySpec] = Field(default_factory=list)
+
+    # -- Properties -- #
+
+    @property
+    def fq_name(self) -> str:
+        """
+        Fully qualified table name, including schema if specified.
+        """
+        return (
+            f'{self.schema_name}.{self.table}'
+            if self.schema_name
+            else self.table
+        )
+
+
+# SECTION: FUNCTIONS ======================================================== #
+
+
+def load_table_specs(
+    path: StrPath,
+) -> list[TableSpec]:
+    """
+    Load table specifications from a YAML file.
+
+    Parameters
+    ----------
+    path : StrPath
+        Path to the YAML file containing table specifications.
+
+    Returns
+    -------
+    list[TableSpec]
+        A list of TableSpec instances parsed from the YAML file.
+    """
+    data = File.read_file(Path(path))
+    if not data:
+        return []
+
+    if isinstance(data, dict) and 'table_schemas' in data:
+        items: list[Any] = data['table_schemas'] or []
+    elif isinstance(data, list):
+        items = data
+    else:
+        items = [data]
+
+    return [TableSpec.model_validate(item) for item in items]

etlplus/database/types.py

@@ -0,0 +1,33 @@
+"""
+:mod:`etlplus.database.types` module.
+
+Shared type aliases leveraged across :mod:`etlplus.database` modules.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+
+from sqlalchemy.orm import DeclarativeBase
+from sqlalchemy.types import TypeEngine
+
+# SECTION: EXPORTS ========================================================== #
+
+
+__all__ = [
+    # Type Aliases
+    'ModelRegistry',
+    'TypeFactory',
+]
+
+
+# SECTION: TYPE ALIASES ===================================================== #
+
+
+# pylint: disable=invalid-name
+
+# Registry mapping fully qualified table names to declarative classes.
+type ModelRegistry = dict[str, type[DeclarativeBase]]
+
+# Callable producing a SQLAlchemy TypeEngine from parsed parameters.
+type TypeFactory = Callable[[list[int]], TypeEngine]

etlplus/run.py

@@ -142,10 +142,8 @@ def run(
     """
     Run a pipeline job defined in a YAML configuration.
 
-    This mirrors the run-mode logic from ``etlplus.cli.cmd_pipeline``
-    (without the list/summary modes). By default it reads the configuration
-    from ``in/pipeline.yml``, but callers can provide an explicit
-    ``config_path`` to override this.
+    By default it reads the configuration from ``in/pipeline.yml``, but callers
+    can provide an explicit ``config_path`` to override this.
 
     Parameters
     ----------

etlplus/templates/__init__.py

@@ -0,0 +1,5 @@
+"""
+:mod:`etlplus.templates` package.
+
+This package defines bundled Jinja2 templates for ``etlplus``.
+"""