lsst-felis 24.1.6rc1__py3-none-any.whl

felis/metadata.py

@@ -0,0 +1,383 @@
+"""Build SQLAlchemy metadata from a Felis schema."""
+
+# This file is part of felis.
+#
+# Developed for the LSST Data Management System.
+# This product includes software developed by the LSST Project
+# (https://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+from __future__ import annotations
+
+import logging
+from typing import Any, Literal
+
+from lsst.utils.iteration import ensure_iterable
+from sqlalchemy import (
+    CheckConstraint,
+    Column,
+    Constraint,
+    ForeignKeyConstraint,
+    Index,
+    MetaData,
+    PrimaryKeyConstraint,
+    Table,
+    TextClause,
+    UniqueConstraint,
+    text,
+)
+from sqlalchemy.dialects import mysql, postgresql
+from sqlalchemy.types import TypeEngine
+
+from felis.datamodel import Schema
+from felis.db.variants import make_variant_dict
+
+from . import datamodel
+from .db import sqltypes
+from .types import FelisType
+
+__all__ = ("MetaDataBuilder", "get_datatype_with_variants")
+
+logger = logging.getLogger(__name__)
+
+
+def _handle_timestamp_column(column_obj: datamodel.Column, variant_dict: dict[str, TypeEngine[Any]]) -> None:
+    """Handle columns with the timestamp datatype.
+
+    Parameters
+    ----------
+    column_obj
+        The column object representing the timestamp.
+    variant_dict
+        The dictionary of variant overrides for the datatype.
+
+    Notes
+    -----
+    This function updates the variant dictionary with the appropriate
+    timestamp type for the column object but only if the precision is set.
+    Otherwise, the default timestamp objects defined in the Felis type system
+    will be used instead.
+    """
+    if column_obj.precision is not None:
+        args: Any = [False, column_obj.precision]  # Turn off timezone.
+        variant_dict.update({"postgresql": postgresql.TIMESTAMP(*args), "mysql": mysql.DATETIME(*args)})
+
+
+def get_datatype_with_variants(column_obj: datamodel.Column) -> TypeEngine:
+    """Use the Felis type system to get a SQLAlchemy datatype with variant
+    overrides from the information in a Felis column object.
+
+    Parameters
+    ----------
+    column_obj
+        The column object from which to get the datatype.
+
+    Returns
+    -------
+    `~sqlalchemy.types.TypeEngine`
+        The SQLAlchemy datatype object.
+
+    Raises
+    ------
+    ValueError
+        Raised if the column has a sized type but no length or if the datatype
+        is invalid.
+    """
+    variant_dict = make_variant_dict(column_obj)
+    felis_type = FelisType.felis_type(column_obj.datatype.value)
+    datatype_fun = getattr(sqltypes, column_obj.datatype.value, None)
+    if datatype_fun is None:
+        raise ValueError(f"Unknown datatype: {column_obj.datatype.value}")
+    args = []
+    if felis_type.is_sized:
+        # Add length argument for size types.
+        if not column_obj.length:
+            raise ValueError(f"Column {column_obj.name} has sized type '{column_obj.datatype}' but no length")
+        args = [column_obj.length]
+    if felis_type.is_timestamp:
+        _handle_timestamp_column(column_obj, variant_dict)
+    return datatype_fun(*args, **variant_dict)
+
+
+_VALID_SERVER_DEFAULTS = ("CURRENT_TIMESTAMP", "NOW()", "LOCALTIMESTAMP", "NULL")
+
+
+class MetaDataBuilder:
+    """Build a SQLAlchemy metadata object from a Felis schema.
+
+    Parameters
+    ----------
+    schema
+        The schema object from which to build the SQLAlchemy metadata.
+    apply_schema_to_metadata
+        Whether to apply the schema name to the metadata object.
+    apply_schema_to_tables
+        Whether to apply the schema name to the tables.
+    ignore_constraints
+        Whether to ignore constraints when building the metadata.
+    """
+
+    def __init__(
+        self,
+        schema: Schema,
+        apply_schema_to_metadata: bool = True,
+        apply_schema_to_tables: bool = True,
+        ignore_constraints: bool = False,
+    ) -> None:
+        """Initialize the metadata builder."""
+        self.schema = schema
+        if not apply_schema_to_metadata:
+            logger.debug("Schema name will not be applied to metadata")
+        if not apply_schema_to_tables:
+            logger.debug("Schema name will not be applied to tables")
+        self.metadata = MetaData(schema=schema.name if apply_schema_to_metadata else None)
+        self._objects: dict[str, Any] = {}
+        self.apply_schema_to_tables = apply_schema_to_tables
+        self.ignore_constraints = ignore_constraints
+
+    def build(self) -> MetaData:
+        """Build the SQLAlchemy tables and constraints from the schema.
+
+        Notes
+        -----
+        This first builds the tables and then makes a second pass to build the
+        constraints. This is necessary because the constraints may reference
+        objects that are not yet created when the tables are built.
+
+        Returns
+        -------
+        `~sqlalchemy.sql.schema.MetaData`
+            The SQLAlchemy metadata object.
+        """
+        self.build_tables()
+        if not self.ignore_constraints:
+            self.build_constraints()
+        else:
+            logger.warning("Ignoring constraints")
+        return self.metadata
+
+    def build_tables(self) -> None:
+        """Build the SQLAlchemy tables from the schema."""
+        for table in self.schema.tables:
+            self.build_table(table)
+            if table.primary_key:
+                primary_key = self.build_primary_key(table.primary_key)
+                self._objects[table.id].append_constraint(primary_key)
+
+    def build_primary_key(self, primary_key_columns: str | list[str]) -> PrimaryKeyConstraint:
+        """Build a SQAlchemy ``PrimaryKeyConstraint`` from a single column ID
+        or a list of them.
+
+        Parameters
+        ----------
+        primary_key_columns
+            The column ID or list of column IDs from which to build the primary
+            key.
+
+        Returns
+        -------
+        `~sqlalchemy.sql.schema.PrimaryKeyConstraint`
+            The SQLAlchemy primary key constraint object.
+
+        Notes
+        -----
+        The ``primary_key_columns`` is a string or a list of strings
+        representing IDs which will be used to find the columnn objects in the
+        builder's internal ID map.
+        """
+        return PrimaryKeyConstraint(
+            *[self._objects[column_id] for column_id in ensure_iterable(primary_key_columns)]
+        )
+
+    def build_table(self, table_obj: datamodel.Table) -> None:
+        """Build a SQLAlchemy ``Table`` from a Felis table and add it to the
+        metadata.
+
+        Parameters
+        ----------
+        table_obj
+            The Felis table object from which to build the SQLAlchemy table.
+
+        Notes
+        -----
+        Several MySQL table options, including the engine and charset, are
+        handled by adding annotations to the table. This is not needed for
+        Postgres, as Felis does not support any table options for this dialect.
+        """
+        # Process mysql table options.
+        optargs = {}
+        if table_obj.mysql_engine:
+            optargs["mysql_engine"] = table_obj.mysql_engine
+        if table_obj.mysql_charset:
+            optargs["mysql_charset"] = table_obj.mysql_charset
+
+        # Create the SQLAlchemy table object and its columns.
+        name = table_obj.name
+        id = table_obj.id
+        description = table_obj.description
+        columns = [self.build_column(column) for column in table_obj.columns]
+        table = Table(
+            name,
+            self.metadata,
+            *columns,
+            comment=description,
+            schema=self.schema.name if self.apply_schema_to_tables else None,
+            **optargs,  # type: ignore[arg-type]
+        )
+
+        # Create the indexes and add them to the table.
+        indexes = [self.build_index(index) for index in table_obj.indexes]
+        for index in indexes:
+            index._set_parent(table)
+            table.indexes.add(index)
+
+        self._objects[id] = table
+
+    def build_column(self, column_obj: datamodel.Column) -> Column:
+        """Build a SQLAlchemy ``Column`` from a Felis column object.
+
+        Parameters
+        ----------
+        column_obj
+            The column object from which to build the SQLAlchemy column.
+
+        Returns
+        -------
+        `~sqlalchemy.sql.schema.Column`
+            The SQLAlchemy column object.
+        """
+        # Get basic column attributes.
+        name = column_obj.name
+        id = column_obj.id
+        description = column_obj.description
+        value = column_obj.value
+        nullable = column_obj.nullable
+
+        # Get datatype, handling variant overrides such as "mysql:datatype".
+        datatype = get_datatype_with_variants(column_obj)
+
+        # Set autoincrement, depending on if it was provided explicitly.
+        autoincrement: Literal["auto"] | bool = (
+            column_obj.autoincrement if column_obj.autoincrement is not None else "auto"
+        )
+
+        server_default: str | TextClause | None = None
+        if value is not None:
+            server_default = str(value)
+            if server_default in _VALID_SERVER_DEFAULTS or not isinstance(value, str):
+                # If the server default is a valid keyword or not a string,
+                # use it as is.
+                server_default = text(server_default)
+
+        if server_default is not None:
+            logger.debug(f"Column '{id}' has default value: {server_default}")
+
+        column: Column = Column(
+            name,
+            datatype,
+            comment=description,
+            autoincrement=autoincrement,
+            nullable=nullable,
+            server_default=server_default,
+        )
+
+        self._objects[id] = column
+
+        return column
+
+    def build_constraints(self) -> None:
+        """Build the SQLAlchemy constraints from the Felis schema and append
+        them to the associated table in the metadata.
+
+        Notes
+        -----
+        This is performed as a separate step after building the tables so that
+        all the referenced objects in the constraints will be present and can
+        be looked up by their ID.
+        """
+        for table_obj in self.schema.tables:
+            table = self._objects[table_obj.id]
+            for constraint_obj in table_obj.constraints:
+                constraint = self.build_constraint(constraint_obj)
+                table.append_constraint(constraint)
+
+    def build_constraint(self, constraint_obj: datamodel.Constraint) -> Constraint:
+        """Build a SQLAlchemy ``Constraint`` from a  Felis constraint.
+
+        Parameters
+        ----------
+        constraint_obj
+            The Felis object from which to build the constraint.
+
+        Returns
+        -------
+        `~sqlalchemy.sql.schema.Constraint`
+            The SQLAlchemy constraint object.
+
+        Raises
+        ------
+        ValueError
+            If the constraint type is not recognized.
+        TypeError
+            If the constraint object is not the expected type.
+        """
+        args: dict[str, Any] = {
+            "name": constraint_obj.name or None,
+            "comment": constraint_obj.description or None,
+            "deferrable": constraint_obj.deferrable or None,
+            "initially": constraint_obj.initially or None,
+        }
+        constraint: Constraint
+
+        if isinstance(constraint_obj, datamodel.ForeignKeyConstraint):
+            fk_obj: datamodel.ForeignKeyConstraint = constraint_obj
+            columns = [self._objects[column_id] for column_id in fk_obj.columns]
+            refcolumns = [self._objects[column_id] for column_id in fk_obj.referenced_columns]
+            constraint = ForeignKeyConstraint(columns, refcolumns, **args)
+        elif isinstance(constraint_obj, datamodel.CheckConstraint):
+            check_obj: datamodel.CheckConstraint = constraint_obj
+            expression = check_obj.expression
+            constraint = CheckConstraint(expression, **args)
+        elif isinstance(constraint_obj, datamodel.UniqueConstraint):
+            uniq_obj: datamodel.UniqueConstraint = constraint_obj
+            columns = [self._objects[column_id] for column_id in uniq_obj.columns]
+            constraint = UniqueConstraint(*columns, **args)
+        else:
+            raise ValueError(f"Unknown constraint type: {type(constraint_obj)}")
+
+        self._objects[constraint_obj.id] = constraint
+
+        return constraint
+
+    def build_index(self, index_obj: datamodel.Index) -> Index:
+        """Build a SQLAlchemy ``Index`` from a Felis `~felis.datamodel.Index`.
+
+        Parameters
+        ----------
+        index_obj
+            The Felis object from which to build the SQLAlchemy index.
+
+        Returns
+        -------
+        `~sqlalchemy.sql.schema.Index`
+            The SQLAlchemy index object.
+        """
+        columns = [self._objects[c_id] for c_id in (index_obj.columns if index_obj.columns else [])]
+        expressions = index_obj.expressions if index_obj.expressions else []
+        index = Index(index_obj.name, *columns, *expressions)
+        self._objects[index_obj.id] = index
+        return index

felis/schemas/tap_schema_std.yaml

@@ -0,0 +1,273 @@
+name: TAP_SCHEMA
+version: "1.1"
+description: A TAP-standard-mandated schema to describe tablesets in a TAP 1.1 service
+tables:
+- name: "schemas"
+  description: description of schemas in this tableset
+  primaryKey: "#schemas.schema_name"
+  tap:table_index: 100000
+  mysql:engine: "InnoDB"
+  columns:
+  - name: "schema_name"
+    datatype: "string"
+    description: schema name for reference to tap_schema.schemas
+    length: 64
+    nullable: false
+    tap:principal: 1
+    tap:std: 1
+    tap:column_index: 1
+  - name: "utype"
+    datatype: "string"
+    description: lists the utypes of schemas in the tableset
+    length: 512
+    tap:principal: 1
+    tap:std: 1
+    tap:column_index: 2
+  - name: "description"
+    datatype: "string"
+    description: describes schemas in the tableset
+    length: 512
+    tap:principal: 1
+    tap:std: 1
+    tap:column_index: 3
+  - name: "schema_index"
+    datatype: "int"
+    description: recommended sort order when listing schemas
+    tap:principal: 1
+    tap:std: 1
+    tap:column_index: 4
+- name: "tables"
+  description: description of tables in this tableset
+  primaryKey: "#tables.table_name"
+  tap:table_index: 101000
+  mysql:engine: "InnoDB"
+  columns:
+  - name: schema_name
+    datatype: string
+    description: the schema this table belongs to
+    length: 64
+    nullable: false
+    tap:principal: 1
+    tap:std: 1
+    tap:column_index: 1
+  - name: table_name
+    datatype: string
+    description: the fully qualified table name
+    length: 128
+    nullable: false
+    tap:principal: 1
+    tap:std: 1
+    tap:column_index: 2
+  - name: table_type
+    datatype: string
+    description: "one of: table view"
+    length: 8
+    nullable: false
+    tap:principal: 1
+    tap:std: 1
+    tap:column_index: 3
+  - name: utype
+    datatype: string
+    description: lists the utype of tables in the tableset
+    length: 512
+    tap:principal: 1
+    tap:std: 1
+    tap:column_index: 4
+  - name: description
+    datatype: string
+    description: describes tables in the tableset
+    length: 512
+    tap:principal: 1
+    tap:std: 1
+    tap:column_index: 5
+  - name: table_index
+    datatype: int
+    description: recommended sort order when listing tables
+    tap:principal: 1
+    tap:std: 1
+    tap:column_index: 6
+  constraints:
+  - name: "k1"
+    "@type": ForeignKey
+    columns: ["#tables.schema_name"]
+    referencedColumns: ["#schemas.schema_name"]
+- name: "columns"
+  description: description of columns in this tableset
+  primaryKey: ["#columns.table_name", "#columns.column_name"]
+  tap_table_index: 102000
+  mysql:engine: "InnoDB"
+  columns:
+  - name: table_name
+    datatype: string
+    description: the table this column belongs to
+    length: 128
+    nullable: false
+    tap:principal: 1
+    tap:std: 1
+    tap:column_index: 1
+  - name: column_name
+    datatype: string
+    description: the column name
+    length: 64
+    nullable: false
+    tap:principal: 1
+    tap:std: 1
+    tap:column_index: 2
+  - name: utype
+    datatype: string
+    description: lists the utypes of columns in the tableset
+    length: 512
+    tap:principal: 1
+    tap:std: 1
+    tap:column_index: 3
+  - name: ucd
+    datatype: string
+    description: lists the UCDs of columns in the tableset
+    length: 64
+    tap:principal: 1
+    tap:std: 1
+    tap:column_index: 4
+  - name: unit
+    datatype: string
+    description: lists the unit used for column values in the tableset
+    length: 64
+    tap:principal: 1
+    tap:std: 1
+    tap:column_index: 5
+  - name: description
+    datatype: string
+    description: describes the columns in the tableset
+    length: 512
+    tap:principal: 1
+    tap:std: 1
+    tap:column_index: 6
+  - name: datatype
+    datatype: string
+    description: lists the ADQL datatype of columns in the tableset
+    length: 64
+    nullable: false
+    tap:principal: 1
+    tap:std: 1
+    tap:column_index: 7
+  - name: arraysize
+    datatype: string
+    description: lists the size of variable-length columns in the tableset
+    length: 16
+    tap:principal: 1
+    tap:std: 1
+    tap:column_index: 8
+  - name: xtype
+    datatype: string
+    description: a DALI or custom extended type annotation
+    length: 64
+    tap:principal: 1
+    tap:std: 1
+    tap:column_index: 9
+  - name: size
+    datatype: int
+    description: "deprecated: use arraysize"
+    tap:principal: 1
+    tap:std: 1
+    tap:column_index: 10
+  - name: principal
+    datatype: int
+    description: a principal column; 1 means 1, 0 means 0
+    nullable: false
+    tap:principal: 1
+    tap:std: 1
+    tap:column_index: 11
+  - name: indexed
+    datatype: int
+    description: an indexed column; 1 means 1, 0 means 0
+    nullable: false
+    tap:principal: 1
+    tap:std: 1
+    tap:column_index: 12
+  - name: std
+    datatype: int
+    description: a standard column; 1 means 1, 0 means 0
+    nullable: false
+    tap:principal: 1
+    tap:std: 1
+    tap:column_index: 13
+  - name: column_index
+    datatype: int
+    description: recommended sort order when listing columns
+    tap:principal: 1
+    tap:std: 1
+    tap:column_index: 14
+  constraints:
+  - name: "k2"
+    "@type": ForeignKey
+    columns: ["#columns.table_name"]
+    referencedColumns: ["#tables.table_name"]
+- name: "keys"
+  description: description of foreign keys in this tableset
+  primaryKey: "#keys.key_id"
+  tap:table_index: 103000
+  mysql:engine: "InnoDB"
+  columns:
+  - name: key_id
+    datatype: string
+    description: unique key to join to tap_schema.key_columns
+    length: 64
+    nullable: false
+  - name: from_table
+    datatype: string
+    description: the table with the foreign key
+    length: 128
+    nullable: false
+  - name: target_table
+    datatype: string
+    description: the table with the primary key
+    length: 128
+    nullable: false
+  - name: utype
+    datatype: string
+    description: lists the utype of keys in the tableset
+    length: 512
+  - name: description
+    datatype: string
+    description: describes keys in the tableset
+    length: 512
+  constraints:
+  - name: "k3"
+    "@type": ForeignKey
+    columns: ["#keys.from_table"]
+    referencedColumns: ["#tables.table_name"]
+  - name: "k4"
+    "@type": ForeignKey
+    columns: ["#keys.target_table"]
+    referencedColumns: ["#tables.table_name"]
+- name: "key_columns"
+  description: description of foreign key columns in this tableset
+  tap:table_index: 104000
+  mysql:engine: "InnoDB"
+  columns:
+  - name: key_id
+    datatype: string
+    length: 64
+    nullable: false
+  - name: from_column
+    datatype: string
+    length: 64
+    nullable: false
+  - name: target_column
+    datatype: string
+    length: 64
+    nullable: false
+  constraints:
+  - name: "k5"
+    "@type": ForeignKey
+    columns: ["#key_columns.key_id"]
+    referencedColumns: ["#keys.key_id"]
+    # FIXME: These can't be defined as FK constraints, because they refer
+    # to non-unique columns, e.g., column_name from the columns table.
+    # - name: "k6"
+    #  "@type": ForeignKey
+    #  columns: ["#key_columns.from_column"]
+    #  referencedColumns: ["#columns.column_name"]
+    # - name: "k7"
+    #  "@type": ForeignKey
+    #  columns: ["#key_columns.target_column"]
+    #  referencedColumns: ["#columns.column_name"]