lsst-felis 27.2024.2300__py3-none-any.whl → 27.2024.2500__py3-none-any.whl

felis/cli.py

@@ -29,14 +29,14 @@ from typing import IO
 import click
 import yaml
 from pydantic import ValidationError
-from sqlalchemy.engine import Engine, create_engine, create_mock_engine, make_url
+from sqlalchemy.engine import Engine, create_engine, make_url
 from sqlalchemy.engine.mock import MockConnection
 
 from . import __version__
 from .datamodel import Schema
-from .metadata import DatabaseContext, InsertDump, MetaDataBuilder
+from .db.utils import DatabaseContext
+from .metadata import MetaDataBuilder
 from .tap import Tap11Base, TapLoadingVisitor, init_tables
-from .validation import get_schema
 
 logger = logging.getLogger("felis")
 
@@ -92,29 +92,27 @@ def create(
     """Create database objects from the Felis file."""
     yaml_data = yaml.safe_load(file)
     schema = Schema.model_validate(yaml_data)
-    url_obj = make_url(engine_url)
+    url = make_url(engine_url)
     if schema_name:
         logger.info(f"Overriding schema name with: {schema_name}")
         schema.name = schema_name
-    elif url_obj.drivername == "sqlite":
+    elif url.drivername == "sqlite":
         logger.info("Overriding schema name for sqlite with: main")
         schema.name = "main"
-    if not url_obj.host and not url_obj.drivername == "sqlite":
+    if not url.host and not url.drivername == "sqlite":
         dry_run = True
         logger.info("Forcing dry run for non-sqlite engine URL with no host")
 
-    builder = MetaDataBuilder(schema)
-    builder.build()
-    metadata = builder.metadata
+    metadata = MetaDataBuilder(schema).build()
     logger.debug(f"Created metadata with schema name: {metadata.schema}")
 
     engine: Engine | MockConnection
     if not dry_run and not output_file:
-        engine = create_engine(engine_url, echo=echo)
+        engine = create_engine(url, echo=echo)
     else:
         if dry_run:
             logger.info("Dry run will be executed")
-        engine = DatabaseContext.create_mock_engine(url_obj, output_file)
+        engine = DatabaseContext.create_mock_engine(url, output_file)
         if output_file:
             logger.info("Writing SQL output to: " + output_file.name)
 
@@ -229,10 +227,7 @@ def load_tap(
         )
         tap_visitor.visit_schema(schema)
     else:
-        _insert_dump = InsertDump()
-        conn = create_mock_engine(make_url(engine_url), executor=_insert_dump.dump, paramstyle="pyformat")
-        # After the engine is created, update the executor with the dialect
-        _insert_dump.dialect = conn.dialect
+        conn = DatabaseContext.create_mock_engine(engine_url)
 
         tap_visitor = TapLoadingVisitor.from_mock_connection(
             conn,
@@ -245,42 +240,44 @@ def load_tap(
 
 
 @cli.command("validate")
+@click.option("--check-description", is_flag=True, help="Require description for all objects", default=False)
 @click.option(
-    "-s",
-    "--schema-name",
-    help="Schema name for validation",
-    type=click.Choice(["RSP", "default"]),
-    default="default",
+    "--check-redundant-datatypes", is_flag=True, help="Check for redundant datatypes", default=False
 )
 @click.option(
-    "-d", "--require-description", is_flag=True, help="Require description for all objects", default=False
+    "--check-tap-table-indexes",
+    is_flag=True,
+    help="Check that every table has a unique TAP table index",
+    default=False,
 )
 @click.option(
-    "-t", "--check-redundant-datatypes", is_flag=True, help="Check for redundant datatypes", default=False
+    "--check-tap-principal",
+    is_flag=True,
+    help="Check that at least one column per table is flagged as TAP principal",
+    default=False,
 )
 @click.argument("files", nargs=-1, type=click.File())
 def validate(
-    schema_name: str,
-    require_description: bool,
+    check_description: bool,
     check_redundant_datatypes: bool,
+    check_tap_table_indexes: bool,
+    check_tap_principal: bool,
     files: Iterable[io.TextIOBase],
 ) -> None:
     """Validate one or more felis YAML files."""
-    schema_class = get_schema(schema_name)
-    if schema_name != "default":
-        logger.info(f"Using schema '{schema_class.__name__}'")
-
     rc = 0
     for file in files:
         file_name = getattr(file, "name", None)
         logger.info(f"Validating {file_name}")
         try:
             data = yaml.load(file, Loader=yaml.SafeLoader)
-            schema_class.model_validate(
+            Schema.model_validate(
                 data,
                 context={
+                    "check_description": check_description,
                     "check_redundant_datatypes": check_redundant_datatypes,
-                    "require_description": require_description,
+                    "check_tap_table_indexes": check_tap_table_indexes,
+                    "check_tap_principal": check_tap_principal,
                 },
             )
         except ValidationError as e:

felis/datamodel.py

@@ -22,7 +22,6 @@
 from __future__ import annotations
 
 import logging
-import re
 from collections.abc import Mapping, Sequence
 from enum import StrEnum, auto
 from typing import Annotated, Any, Literal, TypeAlias
@@ -30,13 +29,10 @@ from typing import Annotated, Any, Literal, TypeAlias
 from astropy import units as units  # type: ignore
 from astropy.io.votable import ucd  # type: ignore
 from pydantic import BaseModel, ConfigDict, Field, ValidationInfo, field_validator, model_validator
-from sqlalchemy import dialects
-from sqlalchemy import types as sqa_types
-from sqlalchemy.engine import create_mock_engine
-from sqlalchemy.engine.interfaces import Dialect
-from sqlalchemy.types import TypeEngine
 
+from .db.dialects import get_supported_dialects
 from .db.sqltypes import get_type_func
+from .db.utils import string_to_typeengine
 from .types import Boolean, Byte, Char, Double, FelisType, Float, Int, Long, Short, String, Text, Unicode
 
 logger = logging.getLogger(__name__)
@@ -100,7 +96,7 @@ class BaseObject(BaseModel):
     def check_description(self, info: ValidationInfo) -> BaseObject:
         """Check that the description is present if required."""
         context = info.context
-        if not context or not context.get("require_description", False):
+        if not context or not context.get("check_description", False):
             return self
         if self.description is None or self.description == "":
             raise ValueError("Description is required and must be non-empty")
@@ -127,51 +123,6 @@ class DataType(StrEnum):
     timestamp = auto()
 
 
-_DIALECTS = {
-    "mysql": create_mock_engine("mysql://", executor=None).dialect,
-    "postgresql": create_mock_engine("postgresql://", executor=None).dialect,
-}
-"""Dictionary of dialect names to SQLAlchemy dialects."""
-
-_DIALECT_MODULES = {"mysql": getattr(dialects, "mysql"), "postgresql": getattr(dialects, "postgresql")}
-"""Dictionary of dialect names to SQLAlchemy dialect modules."""
-
-_DATATYPE_REGEXP = re.compile(r"(\w+)(\((.*)\))?")
-"""Regular expression to match data types in the form "type(length)"""
-
-
-def string_to_typeengine(
-    type_string: str, dialect: Dialect | None = None, length: int | None = None
-) -> TypeEngine:
-    match = _DATATYPE_REGEXP.search(type_string)
-    if not match:
-        raise ValueError(f"Invalid type string: {type_string}")
-
-    type_name, _, params = match.groups()
-    if dialect is None:
-        type_class = getattr(sqa_types, type_name.upper(), None)
-    else:
-        try:
-            dialect_module = _DIALECT_MODULES[dialect.name]
-        except KeyError:
-            raise ValueError(f"Unsupported dialect: {dialect}")
-        type_class = getattr(dialect_module, type_name.upper(), None)
-
-    if not type_class:
-        raise ValueError(f"Unsupported type: {type_class}")
-
-    if params:
-        params = [int(param) if param.isdigit() else param for param in params.split(",")]
-        type_obj = type_class(*params)
-    else:
-        type_obj = type_class()
-
-    if hasattr(type_obj, "length") and getattr(type_obj, "length") is None and length is not None:
-        type_obj.length = length
-
-    return type_obj
-
-
 class Column(BaseObject):
     """A column in a table."""
 
@@ -257,12 +208,11 @@ class Column(BaseObject):
                 raise ValueError(f"Invalid IVOA UCD: {e}")
         return ivoa_ucd
 
-    @model_validator(mode="before")
-    @classmethod
-    def check_units(cls, values: dict[str, Any]) -> dict[str, Any]:
+    @model_validator(mode="after")
+    def check_units(self) -> Column:
         """Check that units are valid."""
-        fits_unit = values.get("fits:tunit")
-        ivoa_unit = values.get("ivoa:unit")
+        fits_unit = self.fits_tunit
+        ivoa_unit = self.ivoa_unit
 
         if fits_unit and ivoa_unit:
             raise ValueError("Column cannot have both FITS and IVOA units")
@@ -274,7 +224,7 @@ class Column(BaseObject):
             except ValueError as e:
                 raise ValueError(f"Invalid unit: {e}")
 
-        return values
+        return self
 
     @model_validator(mode="before")
     @classmethod
@@ -299,12 +249,15 @@ class Column(BaseObject):
         return values
 
     @model_validator(mode="after")
-    def check_datatypes(self, info: ValidationInfo) -> Column:
+    def check_redundant_datatypes(self, info: ValidationInfo) -> Column:
         """Check for redundant datatypes on columns."""
         context = info.context
         if not context or not context.get("check_redundant_datatypes", False):
             return self
-        if all(getattr(self, f"{dialect}:datatype", None) is not None for dialect in _DIALECTS.keys()):
+        if all(
+            getattr(self, f"{dialect}:datatype", None) is not None
+            for dialect in get_supported_dialects().keys()
+        ):
             return self
 
         datatype = self.datatype
@@ -317,7 +270,7 @@ class Column(BaseObject):
         else:
             datatype_obj = datatype_func()
 
-        for dialect_name, dialect in _DIALECTS.items():
+        for dialect_name, dialect in get_supported_dialects().items():
             db_annotation = f"{dialect_name}_datatype"
             if datatype_string := self.model_dump().get(db_annotation):
                 db_datatype_obj = string_to_typeengine(datatype_string, dialect, length)
@@ -465,6 +418,29 @@ class Table(BaseObject):
             raise ValueError("Column names must be unique")
         return columns
 
+    @model_validator(mode="after")
+    def check_tap_table_index(self, info: ValidationInfo) -> Table:
+        """Check that the table has a TAP table index."""
+        context = info.context
+        if not context or not context.get("check_tap_table_indexes", False):
+            return self
+        if self.tap_table_index is None:
+            raise ValueError("Table is missing a TAP table index")
+        return self
+
+    @model_validator(mode="after")
+    def check_tap_principal(self, info: ValidationInfo) -> Table:
+        """Check that at least one column is flagged as 'principal' for TAP
+        purposes.
+        """
+        context = info.context
+        if not context or not context.get("check_tap_principal", False):
+            return self
+        for col in self.columns:
+            if col.tap_principal == 1:
+                return self
+        raise ValueError(f"Table '{self.name}' is missing at least one column designated as 'tap:principal'")
+
 
 class SchemaVersion(BaseModel):
     """The version of the schema."""
@@ -554,6 +530,21 @@ class Schema(BaseObject):
             raise ValueError("Table names must be unique")
         return tables
 
+    @model_validator(mode="after")
+    def check_tap_table_indexes(self, info: ValidationInfo) -> Schema:
+        """Check that the TAP table indexes are unique."""
+        context = info.context
+        if not context or not context.get("check_tap_table_indexes", False):
+            return self
+        table_indicies = set()
+        for table in self.tables:
+            table_index = table.tap_table_index
+            if table_index is not None:
+                if table_index in table_indicies:
+                    raise ValueError(f"Duplicate 'tap:table_index' value {table_index} found in schema")
+                table_indicies.add(table_index)
+        return self
+
     def _create_id_map(self: Schema) -> Schema:
         """Create a map of IDs to objects.
 
@@ -566,7 +557,6 @@ class Schema(BaseObject):
             return self
         visitor: SchemaIdVisitor = SchemaIdVisitor()
         visitor.visit_schema(self)
-        logger.debug(f"Created schema ID map with {len(self.id_map.keys())} objects")
         if len(visitor.duplicates):
             raise ValueError(
                 "Duplicate IDs found in schema:\n    " + "\n    ".join(visitor.duplicates) + "\n"

felis/db/dialects.py

@@ -0,0 +1,63 @@
+# This file is part of felis.
+#
+# Developed for the LSST Data Management System.
+# This product includes software developed by the LSST Project
+# (https://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+import logging
+from types import ModuleType
+
+from sqlalchemy import dialects
+from sqlalchemy.engine import Dialect
+from sqlalchemy.engine.mock import create_mock_engine
+
+from .sqltypes import MYSQL, ORACLE, POSTGRES, SQLITE
+
+logger = logging.getLogger(__name__)
+
+_DIALECT_NAMES = [MYSQL, POSTGRES, SQLITE, ORACLE]
+
+
+def _dialect(dialect_name: str) -> Dialect:
+    """Create the SQLAlchemy dialect for the given name."""
+    return create_mock_engine(f"{dialect_name}://", executor=None).dialect
+
+
+_DIALECTS = {name: _dialect(name) for name in _DIALECT_NAMES}
+"""Dictionary of dialect names to SQLAlchemy dialects."""
+
+
+def get_supported_dialects() -> dict[str, Dialect]:
+    """Get a dictionary of the supported SQLAlchemy dialects."""
+    return _DIALECTS
+
+
+def _dialect_module(dialect_name: str) -> ModuleType:
+    """Get the SQLAlchemy dialect module for the given name."""
+    return getattr(dialects, dialect_name)
+
+
+_DIALECT_MODULES = {name: _dialect_module(name) for name in _DIALECT_NAMES}
+"""Dictionary of dialect names to SQLAlchemy modules for type instantiation."""
+
+
+def get_dialect_module(dialect_name: str) -> ModuleType:
+    """Get the SQLAlchemy dialect module for the given name."""
+    if dialect_name not in _DIALECT_MODULES:
+        raise ValueError(f"Unsupported dialect: {dialect_name}")
+    return _DIALECT_MODULES[dialect_name]

felis/db/utils.py

@@ -0,0 +1,248 @@
+# This file is part of felis.
+#
+# Developed for the LSST Data Management System.
+# This product includes software developed by the LSST Project
+# (https://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+from __future__ import annotations
+
+import logging
+import re
+from typing import IO, Any
+
+from sqlalchemy import MetaData, types
+from sqlalchemy.engine import Dialect, Engine, ResultProxy
+from sqlalchemy.engine.mock import MockConnection, create_mock_engine
+from sqlalchemy.engine.url import URL
+from sqlalchemy.exc import SQLAlchemyError
+from sqlalchemy.schema import CreateSchema, DropSchema
+from sqlalchemy.sql import text
+from sqlalchemy.types import TypeEngine
+
+from .dialects import get_dialect_module
+
+logger = logging.getLogger("felis")
+
+_DATATYPE_REGEXP = re.compile(r"(\w+)(\((.*)\))?")
+"""Regular expression to match data types in the form "type(length)"""
+
+
+def string_to_typeengine(
+    type_string: str, dialect: Dialect | None = None, length: int | None = None
+) -> TypeEngine:
+    """Convert a string representation of a data type to a SQLAlchemy
+    TypeEngine.
+    """
+    match = _DATATYPE_REGEXP.search(type_string)
+    if not match:
+        raise ValueError(f"Invalid type string: {type_string}")
+
+    type_name, _, params = match.groups()
+    if dialect is None:
+        type_class = getattr(types, type_name.upper(), None)
+    else:
+        try:
+            dialect_module = get_dialect_module(dialect.name)
+        except KeyError:
+            raise ValueError(f"Unsupported dialect: {dialect}")
+        type_class = getattr(dialect_module, type_name.upper(), None)
+
+    if not type_class:
+        raise ValueError(f"Unsupported type: {type_class}")
+
+    if params:
+        params = [int(param) if param.isdigit() else param for param in params.split(",")]
+        type_obj = type_class(*params)
+    else:
+        type_obj = type_class()
+
+    if hasattr(type_obj, "length") and getattr(type_obj, "length") is None and length is not None:
+        type_obj.length = length
+
+    return type_obj
+
+
+class SQLWriter:
+    """Writes SQL statements to stdout or a file."""
+
+    def __init__(self, file: IO[str] | None = None) -> None:
+        """Initialize the SQL writer.
+
+        Parameters
+        ----------
+        file : `io.TextIOBase` or `None`, optional
+            The file to write the SQL statements to. If None, the statements
+            will be written to stdout.
+        """
+        self.file = file
+        self.dialect: Dialect | None = None
+
+    def write(self, sql: Any, *multiparams: Any, **params: Any) -> None:
+        """Write the SQL statement to a file or stdout.
+
+        Statements with parameters will be formatted with the values
+        inserted into the resultant SQL output.
+
+        Parameters
+        ----------
+        sql : `typing.Any`
+            The SQL statement to write.
+        multiparams : `typing.Any`
+            The multiparams to use for the SQL statement.
+        params : `typing.Any`
+            The params to use for the SQL statement.
+        """
+        compiled = sql.compile(dialect=self.dialect)
+        sql_str = str(compiled) + ";"
+        params_list = [compiled.params]
+        for params in params_list:
+            if not params:
+                print(sql_str, file=self.file)
+                continue
+            new_params = {}
+            for key, value in params.items():
+                if isinstance(value, str):
+                    new_params[key] = f"'{value}'"
+                elif value is None:
+                    new_params[key] = "null"
+                else:
+                    new_params[key] = value
+            print(sql_str % new_params, file=self.file)
+
+
+class ConnectionWrapper:
+    """A wrapper for a SQLAlchemy engine or mock connection which provides a
+    consistent interface for executing SQL statements.
+    """
+
+    def __init__(self, engine: Engine | MockConnection):
+        """Initialize the connection wrapper.
+
+        Parameters
+        ----------
+        engine : `sqlalchemy.Engine` or `sqlalchemy.MockConnection`
+            The SQLAlchemy engine or mock connection to wrap.
+        """
+        self.engine = engine
+
+    def execute(self, statement: Any) -> ResultProxy:
+        """Execute a SQL statement on the engine and return the result."""
+        if isinstance(statement, str):
+            statement = text(statement)
+        if isinstance(self.engine, MockConnection):
+            return self.engine.connect().execute(statement)
+        else:
+            with self.engine.begin() as connection:
+                result = connection.execute(statement)
+                return result
+
+
+class DatabaseContext:
+    """A class for managing the schema and its database connection."""
+
+    def __init__(self, metadata: MetaData, engine: Engine | MockConnection):
+        """Initialize the database context.
+
+        Parameters
+        ----------
+        metadata : `sqlalchemy.MetaData`
+            The SQLAlchemy metadata object.
+
+        engine : `sqlalchemy.Engine` or `sqlalchemy.MockConnection`
+            The SQLAlchemy engine or mock connection object.
+        """
+        self.engine = engine
+        self.dialect_name = engine.dialect.name
+        self.metadata = metadata
+        self.conn = ConnectionWrapper(engine)
+
+    def create_if_not_exists(self) -> None:
+        """Create the schema in the database if it does not exist.
+
+        In MySQL, this will create a new database. In PostgreSQL, it will
+        create a new schema. For other variants, this is an unsupported
+        operation.
+
+        Parameters
+        ----------
+        engine: `sqlalchemy.Engine`
+            The SQLAlchemy engine object.
+        schema_name: `str`
+            The name of the schema (or database) to create.
+        """
+        schema_name = self.metadata.schema
+        try:
+            if self.dialect_name == "mysql":
+                logger.debug(f"Creating MySQL database: {schema_name}")
+                self.conn.execute(text(f"CREATE DATABASE IF NOT EXISTS {schema_name}"))
+            elif self.dialect_name == "postgresql":
+                logger.debug(f"Creating PG schema: {schema_name}")
+                self.conn.execute(CreateSchema(schema_name, if_not_exists=True))
+            else:
+                raise ValueError("Unsupported database type:" + self.dialect_name)
+        except SQLAlchemyError as e:
+            logger.error(f"Error creating schema: {e}")
+            raise
+
+    def drop_if_exists(self) -> None:
+        """Drop the schema in the database if it exists.
+
+        In MySQL, this will drop a database. In PostgreSQL, it will drop a
+        schema. For other variants, this is unsupported for now.
+
+        Parameters
+        ----------
+        engine: `sqlalchemy.Engine`
+            The SQLAlchemy engine object.
+        schema_name: `str`
+            The name of the schema (or database) to drop.
+        """
+        schema_name = self.metadata.schema
+        try:
+            if self.dialect_name == "mysql":
+                logger.debug(f"Dropping MySQL database if exists: {schema_name}")
+                self.conn.execute(text(f"DROP DATABASE IF EXISTS {schema_name}"))
+            elif self.dialect_name == "postgresql":
+                logger.debug(f"Dropping PostgreSQL schema if exists: {schema_name}")
+                self.conn.execute(DropSchema(schema_name, if_exists=True, cascade=True))
+            else:
+                raise ValueError(f"Unsupported database type: {self.dialect_name}")
+        except SQLAlchemyError as e:
+            logger.error(f"Error dropping schema: {e}")
+            raise
+
+    def create_all(self) -> None:
+        """Create all tables in the schema using the metadata object."""
+        self.metadata.create_all(self.engine)
+
+    @staticmethod
+    def create_mock_engine(engine_url: str | URL, output_file: IO[str] | None = None) -> MockConnection:
+        """Create a mock engine for testing or dumping DDL statements.
+
+        Parameters
+        ----------
+        engine_url : `sqlalchemy.engine.url.URL`
+            The SQLAlchemy engine URL.
+        output_file : `typing.IO` [ `str` ] or `None`, optional
+            The file to write the SQL statements to. If None, the statements
+            will be written to stdout.
+        """
+        writer = SQLWriter(output_file)
+        engine = create_mock_engine(engine_url, executor=writer.write)
+        writer.dialect = engine.dialect
+        return engine

felis/db/{_variants.py → variants.py}

@@ -23,38 +23,44 @@ import re
 from typing import Any
 
 from sqlalchemy import types
-from sqlalchemy.dialects import mysql, oracle, postgresql, sqlite
 from sqlalchemy.types import TypeEngine
 
 from ..datamodel import Column
+from .dialects import get_dialect_module, get_supported_dialects
 
-MYSQL = "mysql"
-ORACLE = "oracle"
-POSTGRES = "postgresql"
-SQLITE = "sqlite"
 
-TABLE_OPTS = {
-    "mysql:engine": "mysql_engine",
-    "mysql:charset": "mysql_charset",
-    "oracle:compress": "oracle_compress",
-}
+def _create_column_variant_overrides() -> dict[str, str]:
+    """Create a dictionary of column variant overrides."""
+    column_variant_overrides = {}
+    for dialect_name in get_supported_dialects().keys():
+        column_variant_overrides[f"{dialect_name}_datatype"] = dialect_name
+    return column_variant_overrides
 
-COLUMN_VARIANT_OVERRIDE = {
-    "mysql_datatype": "mysql",
-    "oracle_datatype": "oracle",
-    "postgresql_datatype": "postgresql",
-    "sqlite_datatype": "sqlite",
-}
 
-DIALECT_MODULES = {MYSQL: mysql, ORACLE: oracle, SQLITE: sqlite, POSTGRES: postgresql}
+_COLUMN_VARIANT_OVERRIDES = _create_column_variant_overrides()
+
+
+def _get_column_variant_overrides() -> dict[str, str]:
+    """Return a dictionary of column variant overrides."""
+    return _COLUMN_VARIANT_OVERRIDES
+
+
+def _get_column_variant_override(field_name: str) -> str:
+    """Return the dialect name from an override field name on the column like
+    ``mysql_datatype``.
+    """
+    if field_name not in _COLUMN_VARIANT_OVERRIDES:
+        raise ValueError(f"Field name {field_name} not found in column variant overrides")
+    return _COLUMN_VARIANT_OVERRIDES[field_name]
+
 
 _length_regex = re.compile(r"\((\d+)\)")
 """A regular expression that is looking for numbers within parentheses."""
 
 
-def process_variant_override(dialect_name: str, variant_override_str: str) -> types.TypeEngine:
+def _process_variant_override(dialect_name: str, variant_override_str: str) -> types.TypeEngine:
     """Return variant type for given dialect."""
-    dialect = DIALECT_MODULES[dialect_name]
+    dialect = get_dialect_module(dialect_name)
     variant_type_name = variant_override_str.split("(")[0]
 
     # Process Variant Type
@@ -86,9 +92,10 @@ def make_variant_dict(column_obj: Column) -> dict[str, TypeEngine[Any]]:
         variant datatype information (e.g., for mysql, postgresql, etc).
     """
     variant_dict = {}
+    variant_overrides = _get_column_variant_overrides()
     for field_name, value in iter(column_obj):
-        if field_name in COLUMN_VARIANT_OVERRIDE and value is not None:
-            dialect = COLUMN_VARIANT_OVERRIDE[field_name]
-            variant: TypeEngine = process_variant_override(dialect, value)
+        if field_name in variant_overrides and value is not None:
+            dialect = _get_column_variant_override(field_name)
+            variant: TypeEngine = _process_variant_override(dialect, value)
             variant_dict[dialect] = variant
     return variant_dict

felis/metadata.py

@@ -22,35 +22,26 @@
 from __future__ import annotations
 
 import logging
-from typing import IO, Any, Literal
+from typing import Any, Literal
 
-import sqlalchemy.schema as sqa_schema
 from lsst.utils.iteration import ensure_iterable
 from sqlalchemy import (
     CheckConstraint,
     Column,
     Constraint,
-    Engine,
     ForeignKeyConstraint,
     Index,
     MetaData,
     PrimaryKeyConstraint,
-    ResultProxy,
     Table,
     TextClause,
     UniqueConstraint,
-    create_mock_engine,
-    make_url,
     text,
 )
-from sqlalchemy.engine.interfaces import Dialect
-from sqlalchemy.engine.mock import MockConnection
-from sqlalchemy.engine.url import URL
-from sqlalchemy.exc import SQLAlchemyError
 from sqlalchemy.types import TypeEngine
 
 from felis.datamodel import Schema
-from felis.db._variants import make_variant_dict
+from felis.db.variants import make_variant_dict
 
 from . import datamodel
 from .db import sqltypes
@@ -59,56 +50,6 @@ from .types import FelisType
 logger = logging.getLogger(__name__)
 
 
-class InsertDump:
-    """An Insert Dumper for SQL statements which supports writing messages
-    to stdout or a file.
-    """
-
-    def __init__(self, file: IO[str] | None = None) -> None:
-        """Initialize the insert dumper.
-
-        Parameters
-        ----------
-        file : `io.TextIOBase` or `None`, optional
-            The file to write the SQL statements to. If None, the statements
-            will be written to stdout.
-        """
-        self.file = file
-        self.dialect: Dialect | None = None
-
-    def dump(self, sql: Any, *multiparams: Any, **params: Any) -> None:
-        """Dump the SQL statement to a file or stdout.
-
-        Statements with parameters will be formatted with the values
-        inserted into the resultant SQL output.
-
-        Parameters
-        ----------
-        sql : `typing.Any`
-            The SQL statement to dump.
-        multiparams : `typing.Any`
-            The multiparams to use for the SQL statement.
-        params : `typing.Any`
-            The params to use for the SQL statement.
-        """
-        compiled = sql.compile(dialect=self.dialect)
-        sql_str = str(compiled) + ";"
-        params_list = [compiled.params]
-        for params in params_list:
-            if not params:
-                print(sql_str, file=self.file)
-                continue
-            new_params = {}
-            for key, value in params.items():
-                if isinstance(value, str):
-                    new_params[key] = f"'{value}'"
-                elif value is None:
-                    new_params[key] = "null"
-                else:
-                    new_params[key] = value
-            print(sql_str % new_params, file=self.file)
-
-
 def get_datatype_with_variants(column_obj: datamodel.Column) -> TypeEngine:
     """Use the Felis type system to get a SQLAlchemy datatype with variant
     overrides from the information in a `Column` object.
@@ -387,127 +328,3 @@ class MetaDataBuilder:
         index = Index(index_obj.name, *columns, *expressions)
         self._objects[index_obj.id] = index
         return index
-
-
-class ConnectionWrapper:
-    """A wrapper for a SQLAlchemy engine or mock connection which provides a
-    consistent interface for executing SQL statements.
-    """
-
-    def __init__(self, engine: Engine | MockConnection):
-        """Initialize the connection wrapper.
-
-        Parameters
-        ----------
-        engine : `sqlalchemy.Engine` or `sqlalchemy.MockConnection`
-            The SQLAlchemy engine or mock connection to wrap.
-        """
-        self.engine = engine
-
-    def execute(self, statement: Any) -> ResultProxy:
-        """Execute a SQL statement on the engine and return the result."""
-        if isinstance(statement, str):
-            statement = text(statement)
-        if isinstance(self.engine, MockConnection):
-            return self.engine.connect().execute(statement)
-        else:
-            with self.engine.begin() as connection:
-                result = connection.execute(statement)
-                return result
-
-
-class DatabaseContext:
-    """A class for managing the schema and its database connection."""
-
-    def __init__(self, metadata: MetaData, engine: Engine | MockConnection):
-        """Initialize the database context.
-
-        Parameters
-        ----------
-        metadata : `sqlalchemy.MetaData`
-            The SQLAlchemy metadata object.
-
-        engine : `sqlalchemy.Engine` or `sqlalchemy.MockConnection`
-            The SQLAlchemy engine or mock connection object.
-        """
-        self.engine = engine
-        self.metadata = metadata
-        self.connection = ConnectionWrapper(engine)
-
-    def create_if_not_exists(self) -> None:
-        """Create the schema in the database if it does not exist.
-
-        In MySQL, this will create a new database. In PostgreSQL, it will
-        create a new schema. For other variants, this is an unsupported
-        operation.
-
-        Parameters
-        ----------
-        engine: `sqlalchemy.Engine`
-            The SQLAlchemy engine object.
-        schema_name: `str`
-            The name of the schema (or database) to create.
-        """
-        db_type = self.engine.dialect.name
-        schema_name = self.metadata.schema
-        try:
-            if db_type == "mysql":
-                logger.info(f"Creating MySQL database: {schema_name}")
-                self.connection.execute(text(f"CREATE DATABASE IF NOT EXISTS {schema_name}"))
-            elif db_type == "postgresql":
-                logger.info(f"Creating PG schema: {schema_name}")
-                self.connection.execute(sqa_schema.CreateSchema(schema_name, if_not_exists=True))
-            else:
-                raise ValueError("Unsupported database type:" + db_type)
-        except SQLAlchemyError as e:
-            logger.error(f"Error creating schema: {e}")
-            raise
-
-    def drop_if_exists(self) -> None:
-        """Drop the schema in the database if it exists.
-
-        In MySQL, this will drop a database. In PostgreSQL, it will drop a
-        schema. For other variants, this is unsupported for now.
-
-        Parameters
-        ----------
-        engine: `sqlalchemy.Engine`
-            The SQLAlchemy engine object.
-        schema_name: `str`
-            The name of the schema (or database) to drop.
-        """
-        db_type = self.engine.dialect.name
-        schema_name = self.metadata.schema
-        try:
-            if db_type == "mysql":
-                logger.info(f"Dropping MySQL database if exists: {schema_name}")
-                self.connection.execute(text(f"DROP DATABASE IF EXISTS {schema_name}"))
-            elif db_type == "postgresql":
-                logger.info(f"Dropping PostgreSQL schema if exists: {schema_name}")
-                self.connection.execute(sqa_schema.DropSchema(schema_name, if_exists=True, cascade=True))
-            else:
-                raise ValueError(f"Unsupported database type: {db_type}")
-        except SQLAlchemyError as e:
-            logger.error(f"Error dropping schema: {e}")
-            raise
-
-    def create_all(self) -> None:
-        """Create all tables in the schema using the metadata object."""
-        self.metadata.create_all(self.engine)
-
-    @staticmethod
-    def create_mock_engine(engine_url: URL, output_file: IO[str] | None = None) -> MockConnection:
-        """Create a mock engine for testing or dumping DDL statements.
-
-        Parameters
-        ----------
-        engine_url : `sqlalchemy.engine.url.URL`
-            The SQLAlchemy engine URL.
-        output_file : `typing.IO` [ `str` ] or `None`, optional
-            The file to write the SQL statements to. If None, the statements
-            will be written to stdout.
-        """
-        dumper = InsertDump(output_file)
-        engine = create_mock_engine(make_url(engine_url), executor=dumper.dump)
-        dumper.dialect = engine.dialect
-        return engine

felis/version.py

@@ -1,2 +1,2 @@
 __all__ = ["__version__"]
-__version__ = "27.2024.2300"
+__version__ = "27.2024.2500"

{lsst_felis-27.2024.2300.dist-info → lsst_felis-27.2024.2500.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lsst-felis
-Version: 27.2024.2300
+Version: 27.2024.2500
 Summary: A vocabulary for describing catalogs and acting on those descriptions
 Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
 License: GNU General Public License v3 or later (GPLv3+)

lsst_felis-27.2024.2500.dist-info/RECORD

@@ -0,0 +1,21 @@
+felis/__init__.py,sha256=THmRg3ylB4E73XhFjJX7YlnV_CM3lr_gZO_HqQFzIQ4,937
+felis/cli.py,sha256=7acvMUku17wDgS1cxGUhKMo5agkofHFN5NUdIdZyJ3o,10132
+felis/datamodel.py,sha256=psSj2IBI7Va_2RvSnKrlZTDIwA__g5YeHh0Q3tweNWo,21208
+felis/metadata.py,sha256=lSgKy9CksmyJp9zUkzkwOAzjk2Hzivb9VVbYfTPNmvY,12140
+felis/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+felis/tap.py,sha256=fVYvvIFk_vccXqbcFYdgK2yOfp4P5E4guvsSGktsNxs,16795
+felis/types.py,sha256=z_ECfSxpqiFSGppjxKwCO4fPP7TLBaIN3Qo1AGF16Go,4418
+felis/version.py,sha256=DpIrT4nE40bHhTZ12_Zx-z9xNkBs4IQms4j7HZqcdCA,55
+felis/db/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+felis/db/dialects.py,sha256=mSYgS8gqUJQGgZw3IA-zBfKlrJ2r4nEUR94wKZNRKZg,2267
+felis/db/sqltypes.py,sha256=yFlautQ1hv21MHF4AIfBp7_2m1-exKBfc76xYsMHBgk,5735
+felis/db/utils.py,sha256=oYPGOt5K_82GnScJOg8WarOyvval-Cu1nX7CduFxKss,9122
+felis/db/variants.py,sha256=Ti2oZf7nFTe8aFyG-GeFSW4bIb5ClNikm9xOJtRcxLY,3862
+lsst_felis-27.2024.2500.dist-info/COPYRIGHT,sha256=bUmNy19uUxqITMpjeHFe69q3IzQpjxvvBw6oV7kR7ho,129
+lsst_felis-27.2024.2500.dist-info/LICENSE,sha256=jOtLnuWt7d5Hsx6XXB2QxzrSe2sWWh3NgMfFRetluQM,35147
+lsst_felis-27.2024.2500.dist-info/METADATA,sha256=co1oyBz9lU3XhIYJmN86G1Qt2iMW9LW66ShIkXrA2E4,1191
+lsst_felis-27.2024.2500.dist-info/WHEEL,sha256=cpQTJ5IWu9CdaPViMhC9YzF8gZuS5-vlfoFihTBC86A,91
+lsst_felis-27.2024.2500.dist-info/entry_points.txt,sha256=Gk2XFujA_Gp52VBk45g5kim8TDoMDJFPctsMqiq72EM,40
+lsst_felis-27.2024.2500.dist-info/top_level.txt,sha256=F4SvPip3iZRVyISi50CHhwTIAokAhSxjWiVcn4IVWRI,6
+lsst_felis-27.2024.2500.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
+lsst_felis-27.2024.2500.dist-info/RECORD,,

{lsst_felis-27.2024.2300.dist-info → lsst_felis-27.2024.2500.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.43.0)
+Generator: setuptools (70.1.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

felis/validation.py

@@ -1,103 +0,0 @@
-# This file is part of felis.
-#
-# Developed for the LSST Data Management System.
-# This product includes software developed by the LSST Project
-# (https://www.lsst.org).
-# See the COPYRIGHT file at the top-level directory of this distribution
-# for details of code ownership.
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <https://www.gnu.org/licenses/>.
-
-from __future__ import annotations
-
-import logging
-from collections.abc import Sequence
-from typing import Any
-
-from pydantic import Field, model_validator
-
-from .datamodel import Column, DescriptionStr, Schema, Table
-
-logger = logging.getLogger(__name__)
-
-__all__ = ["RspColumn", "RspSchema", "RspTable", "get_schema"]
-
-
-class RspColumn(Column):
-    """Column for RSP data validation."""
-
-    description: DescriptionStr
-    """Redefine description to make it required."""
-
-
-class RspTable(Table):
-    """Table for RSP data validation.
-
-    The list of columns is overridden to use RspColumn instead of Column.
-
-    Tables for the RSP must have a TAP table index and a valid description.
-    """
-
-    description: DescriptionStr
-    """Redefine description to make it required."""
-
-    tap_table_index: int = Field(..., alias="tap:table_index")
-    """Redefine the TAP_SCHEMA table index so that it is required."""
-
-    columns: Sequence[RspColumn]
-    """Redefine columns to include RSP validation."""
-
-    @model_validator(mode="after")  # type: ignore[arg-type]
-    @classmethod
-    def check_tap_principal(cls: Any, tbl: RspTable) -> RspTable:
-        """Check that at least one column is flagged as 'principal' for
-        TAP purposes.
-        """
-        for col in tbl.columns:
-            if col.tap_principal == 1:
-                return tbl
-        raise ValueError(f"Table '{tbl.name}' is missing at least one column designated as 'tap:principal'")
-
-
-class RspSchema(Schema):
-    """Schema for RSP data validation.
-
-    TAP table indexes must be unique across all tables.
-    """
-
-    tables: Sequence[RspTable]
-    """Redefine tables to include RSP validation."""
-
-    @model_validator(mode="after")  # type: ignore[arg-type]
-    @classmethod
-    def check_tap_table_indexes(cls: Any, sch: RspSchema) -> RspSchema:
-        """Check that the TAP table indexes are unique."""
-        table_indicies = set()
-        for table in sch.tables:
-            table_index = table.tap_table_index
-            if table_index is not None:
-                if table_index in table_indicies:
-                    raise ValueError(f"Duplicate 'tap:table_index' value {table_index} found in schema")
-                table_indicies.add(table_index)
-        return sch
-
-
-def get_schema(schema_name: str) -> type[Schema]:
-    """Get the schema class for the given name."""
-    if schema_name == "default":
-        return Schema
-    elif schema_name == "RSP":
-        return RspSchema
-    else:
-        raise ValueError(f"Unknown schema name '{schema_name}'")

lsst_felis-27.2024.2300.dist-info/RECORD

@@ -1,20 +0,0 @@
-felis/__init__.py,sha256=THmRg3ylB4E73XhFjJX7YlnV_CM3lr_gZO_HqQFzIQ4,937
-felis/cli.py,sha256=l_4srdXPghBLAVuOvfJVdyIVhq45kTV5KYskmYSsIUA,10279
-felis/datamodel.py,sha256=0LWiqjQgsgn0do4YjVWcf-_5JyJVT6UoFDuVMAULhPI,21351
-felis/metadata.py,sha256=df64ep8F7nY7wiO-Myo1jUJNaNOq12qctSWmyIjGN5k,18910
-felis/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-felis/tap.py,sha256=fVYvvIFk_vccXqbcFYdgK2yOfp4P5E4guvsSGktsNxs,16795
-felis/types.py,sha256=z_ECfSxpqiFSGppjxKwCO4fPP7TLBaIN3Qo1AGF16Go,4418
-felis/validation.py,sha256=Zq0gyCvPCwRlhfQ-w_p6ccDTkjcyhxSA1-Gr5plXiZI,3465
-felis/version.py,sha256=udxtsGErEX4XE4c2P4cd332clbtDeA7Ok9WBuyZT92E,55
-felis/db/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-felis/db/_variants.py,sha256=zCuXDgU_x_pTZcWkBLgqQCiOhlA6y2tBt-PUQfafwmM,3368
-felis/db/sqltypes.py,sha256=yFlautQ1hv21MHF4AIfBp7_2m1-exKBfc76xYsMHBgk,5735
-lsst_felis-27.2024.2300.dist-info/COPYRIGHT,sha256=bUmNy19uUxqITMpjeHFe69q3IzQpjxvvBw6oV7kR7ho,129
-lsst_felis-27.2024.2300.dist-info/LICENSE,sha256=jOtLnuWt7d5Hsx6XXB2QxzrSe2sWWh3NgMfFRetluQM,35147
-lsst_felis-27.2024.2300.dist-info/METADATA,sha256=2cg_4wdGV39QnRdDiJxGi3mUs9EBw02Dn0YbkCveB7k,1191
-lsst_felis-27.2024.2300.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-lsst_felis-27.2024.2300.dist-info/entry_points.txt,sha256=Gk2XFujA_Gp52VBk45g5kim8TDoMDJFPctsMqiq72EM,40
-lsst_felis-27.2024.2300.dist-info/top_level.txt,sha256=F4SvPip3iZRVyISi50CHhwTIAokAhSxjWiVcn4IVWRI,6
-lsst_felis-27.2024.2300.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-lsst_felis-27.2024.2300.dist-info/RECORD,,