lsst-felis 26.2024.1500__py3-none-any.whl → 26.2024.1700__py3-none-any.whl

felis/cli.py

@@ -183,6 +183,7 @@ def init_tap(
 @click.option("--tap-columns-table", help="Alt Table Name for TAP_SCHEMA.columns")
 @click.option("--tap-keys-table", help="Alt Table Name for TAP_SCHEMA.keys")
 @click.option("--tap-key-columns-table", help="Alt Table Name for TAP_SCHEMA.key_columns")
+@click.option("--tap-schema-index", type=int, help="TAP_SCHEMA index of the schema")
 @click.argument("file", type=click.File())
 def load_tap(
     engine_url: str,
@@ -196,6 +197,7 @@ def load_tap(
     tap_columns_table: str,
     tap_keys_table: str,
     tap_key_columns_table: str,
+    tap_schema_index: int,
     file: io.TextIOBase,
 ) -> None:
     """Load TAP metadata from a Felis FILE.
@@ -203,28 +205,8 @@ def load_tap(
     This command loads the associated TAP metadata from a Felis FILE
     to the TAP_SCHEMA tables.
     """
-    top_level_object = yaml.load(file, Loader=yaml.SafeLoader)
-    schema_obj: dict
-    if isinstance(top_level_object, dict):
-        schema_obj = top_level_object
-        if "@graph" not in schema_obj:
-            schema_obj["@type"] = "felis:Schema"
-        schema_obj["@context"] = DEFAULT_CONTEXT
-    elif isinstance(top_level_object, list):
-        schema_obj = {"@context": DEFAULT_CONTEXT, "@graph": top_level_object}
-    else:
-        logger.error("Schema object not of recognizable type")
-        raise click.exceptions.Exit(1)
-
-    normalized = _normalize(schema_obj, embed="@always")
-    if len(normalized["@graph"]) > 1 and (schema_name or catalog_name):
-        logger.error("--schema-name and --catalog-name incompatible with multiple schemas")
-        raise click.exceptions.Exit(1)
-
-    # Force normalized["@graph"] to a list, which is what happens when there's
-    # multiple schemas
-    if isinstance(normalized["@graph"], dict):
-        normalized["@graph"] = [normalized["@graph"]]
+    yaml_data = yaml.load(file, Loader=yaml.SafeLoader)
+    schema = Schema.model_validate(yaml_data)
 
     tap_tables = init_tables(
         tap_schema_name,
@@ -243,28 +225,28 @@ def load_tap(
             # In Memory SQLite - Mostly used to test
             Tap11Base.metadata.create_all(engine)
 
-        for schema in normalized["@graph"]:
-            tap_visitor = TapLoadingVisitor(
-                engine,
-                catalog_name=catalog_name,
-                schema_name=schema_name,
-                tap_tables=tap_tables,
-            )
-            tap_visitor.visit_schema(schema)
+        tap_visitor = TapLoadingVisitor(
+            engine,
+            catalog_name=catalog_name,
+            schema_name=schema_name,
+            tap_tables=tap_tables,
+            tap_schema_index=tap_schema_index,
+        )
+        tap_visitor.visit_schema(schema)
     else:
         _insert_dump = InsertDump()
         conn = create_mock_engine(make_url(engine_url), executor=_insert_dump.dump, paramstyle="pyformat")
         # After the engine is created, update the executor with the dialect
         _insert_dump.dialect = conn.dialect
 
-        for schema in normalized["@graph"]:
-            tap_visitor = TapLoadingVisitor.from_mock_connection(
-                conn,
-                catalog_name=catalog_name,
-                schema_name=schema_name,
-                tap_tables=tap_tables,
-            )
-            tap_visitor.visit_schema(schema)
+        tap_visitor = TapLoadingVisitor.from_mock_connection(
+            conn,
+            catalog_name=catalog_name,
+            schema_name=schema_name,
+            tap_tables=tap_tables,
+            tap_schema_index=tap_schema_index,
+        )
+        tap_visitor.visit_schema(schema)
 
 
 @cli.command("modify-tap")
@@ -373,22 +355,37 @@ def merge(files: Iterable[io.TextIOBase]) -> None:
     type=click.Choice(["RSP", "default"]),
     default="default",
 )
-@click.option("-d", "--require-description", is_flag=True, help="Require description for all objects")
+@click.option(
+    "-d", "--require-description", is_flag=True, help="Require description for all objects", default=False
+)
+@click.option(
+    "-t", "--check-redundant-datatypes", is_flag=True, help="Check for redundant datatypes", default=False
+)
 @click.argument("files", nargs=-1, type=click.File())
-def validate(schema_name: str, require_description: bool, files: Iterable[io.TextIOBase]) -> None:
+def validate(
+    schema_name: str,
+    require_description: bool,
+    check_redundant_datatypes: bool,
+    files: Iterable[io.TextIOBase],
+) -> None:
     """Validate one or more felis YAML files."""
     schema_class = get_schema(schema_name)
-    logger.info(f"Using schema '{schema_class.__name__}'")
-
-    if require_description:
-        Schema.require_description(True)
+    if schema_name != "default":
+        logger.info(f"Using schema '{schema_class.__name__}'")
 
     rc = 0
     for file in files:
         file_name = getattr(file, "name", None)
         logger.info(f"Validating {file_name}")
         try:
-            schema_class.model_validate(yaml.load(file, Loader=yaml.SafeLoader))
+            data = yaml.load(file, Loader=yaml.SafeLoader)
+            schema_class.model_validate(
+                data,
+                context={
+                    "check_redundant_datatypes": check_redundant_datatypes,
+                    "require_description": require_description,
+                },
+            )
         except ValidationError as e:
             logger.error(e)
             rc = 1

felis/datamodel.py

@@ -22,13 +22,22 @@
 from __future__ import annotations
 
 import logging
+import re
 from collections.abc import Mapping, Sequence
-from enum import Enum
+from enum import StrEnum, auto
 from typing import Annotated, Any, Literal, TypeAlias
 
 from astropy import units as units  # type: ignore
 from astropy.io.votable import ucd  # type: ignore
-from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
+from pydantic import BaseModel, ConfigDict, Field, ValidationInfo, field_validator, model_validator
+from sqlalchemy import dialects
+from sqlalchemy import types as sqa_types
+from sqlalchemy.engine import create_mock_engine
+from sqlalchemy.engine.interfaces import Dialect
+from sqlalchemy.types import TypeEngine
+
+from .db.sqltypes import get_type_func
+from .types import FelisType
 
 logger = logging.getLogger(__name__)
 
@@ -49,7 +58,6 @@ __all__ = (
 CONFIG = ConfigDict(
     populate_by_name=True,  # Populate attributes by name.
     extra="forbid",  # Do not allow extra fields.
-    validate_assignment=True,  # Validate assignments after model is created.
     str_strip_whitespace=True,  # Strip whitespace from string fields.
 )
 """Pydantic model configuration as described in:
@@ -83,40 +91,85 @@ class BaseObject(BaseModel):
     """
 
     description: DescriptionStr | None = None
-    """A description of the database object.
+    """A description of the database object."""
 
-    By default, the description is optional but will be required if
-    `BaseObject.Config.require_description` is set to `True` by the user.
-    """
+    votable_utype: str | None = Field(None, alias="votable:utype")
+    """The VOTable utype (usage-specific or unique type) of the object."""
 
-    @model_validator(mode="before")
-    @classmethod
-    def check_description(cls, values: dict[str, Any]) -> dict[str, Any]:
+    @model_validator(mode="after")
+    def check_description(self, info: ValidationInfo) -> BaseObject:
         """Check that the description is present if required."""
-        if Schema.is_description_required():
-            if "description" not in values or not values["description"]:
-                raise ValueError("Description is required and must be non-empty")
-            if len(values["description"].strip()) < DESCR_MIN_LENGTH:
-                raise ValueError(f"Description must be at least {DESCR_MIN_LENGTH} characters long")
-        return values
+        context = info.context
+        if not context or not context.get("require_description", False):
+            return self
+        if self.description is None or self.description == "":
+            raise ValueError("Description is required and must be non-empty")
+        if len(self.description) < DESCR_MIN_LENGTH:
+            raise ValueError(f"Description must be at least {DESCR_MIN_LENGTH} characters long")
+        return self
 
 
-class DataType(Enum):
+class DataType(StrEnum):
     """`Enum` representing the data types supported by Felis."""
 
-    BOOLEAN = "boolean"
-    BYTE = "byte"
-    SHORT = "short"
-    INT = "int"
-    LONG = "long"
-    FLOAT = "float"
-    DOUBLE = "double"
-    CHAR = "char"
-    STRING = "string"
-    UNICODE = "unicode"
-    TEXT = "text"
-    BINARY = "binary"
-    TIMESTAMP = "timestamp"
+    boolean = auto()
+    byte = auto()
+    short = auto()
+    int = auto()
+    long = auto()
+    float = auto()
+    double = auto()
+    char = auto()
+    string = auto()
+    unicode = auto()
+    text = auto()
+    binary = auto()
+    timestamp = auto()
+
+
+_DIALECTS = {
+    "mysql": create_mock_engine("mysql://", executor=None).dialect,
+    "postgresql": create_mock_engine("postgresql://", executor=None).dialect,
+}
+"""Dictionary of dialect names to SQLAlchemy dialects."""
+
+_DIALECT_MODULES = {"mysql": getattr(dialects, "mysql"), "postgresql": getattr(dialects, "postgresql")}
+"""Dictionary of dialect names to SQLAlchemy dialect modules."""
+
+_DATATYPE_REGEXP = re.compile(r"(\w+)(\((.*)\))?")
+"""Regular expression to match data types in the form "type(length)"""
+
+
+def string_to_typeengine(
+    type_string: str, dialect: Dialect | None = None, length: int | None = None
+) -> TypeEngine:
+    match = _DATATYPE_REGEXP.search(type_string)
+    if not match:
+        raise ValueError(f"Invalid type string: {type_string}")
+
+    type_name, _, params = match.groups()
+    if dialect is None:
+        type_class = getattr(sqa_types, type_name.upper(), None)
+    else:
+        try:
+            dialect_module = _DIALECT_MODULES[dialect.name]
+        except KeyError:
+            raise ValueError(f"Unsupported dialect: {dialect}")
+        type_class = getattr(dialect_module, type_name.upper(), None)
+
+    if not type_class:
+        raise ValueError(f"Unsupported type: {type_class}")
+
+    if params:
+        params = [int(param) if param.isdigit() else param for param in params.split(",")]
+        type_obj = type_class(*params)
+    else:
+        type_obj = type_class()
+
+    if hasattr(type_obj, "length") and getattr(type_obj, "length") is None and length is not None:
+        type_obj.length = length
+
+    return type_obj
 
 
 class Column(BaseObject):
@@ -128,13 +181,8 @@ class Column(BaseObject):
     length: int | None = None
     """The length of the column."""
 
-    nullable: bool | None = None
-    """Whether the column can be ``NULL``.
-
-    If `None`, this value was not set explicitly in the YAML data. In this
-    case, it will be set to `False` for columns with numeric types and `True`
-    otherwise.
-    """
+    nullable: bool = True
+    """Whether the column can be ``NULL``."""
 
     value: Any = None
     """The default value of the column."""
@@ -171,12 +219,12 @@ class Column(BaseObject):
     """TAP_SCHEMA indication that this column is defined by an IVOA standard.
     """
 
-    votable_utype: str | None = Field(None, alias="votable:utype")
-    """The VOTable utype (usage-specific or unique type) of the column."""
-
     votable_xtype: str | None = Field(None, alias="votable:xtype")
     """The VOTable xtype (extended type) of the column."""
 
+    votable_datatype: str | None = Field(None, alias="votable:datatype")
+    """The VOTable datatype of the column."""
+
     @field_validator("ivoa_ucd")
     @classmethod
     def check_ivoa_ucd(cls, ivoa_ucd: str) -> str:
@@ -207,6 +255,57 @@ class Column(BaseObject):
 
         return values
 
+    @model_validator(mode="after")  # type: ignore[arg-type]
+    @classmethod
+    def validate_datatypes(cls, col: Column, info: ValidationInfo) -> Column:
+        """Check for redundant datatypes on columns."""
+        context = info.context
+        if not context or not context.get("check_redundant_datatypes", False):
+            return col
+        if all(getattr(col, f"{dialect}:datatype", None) is not None for dialect in _DIALECTS.keys()):
+            return col
+
+        datatype = col.datatype
+        length: int | None = col.length or None
+
+        datatype_func = get_type_func(datatype)
+        felis_type = FelisType.felis_type(datatype)
+        if felis_type.is_sized:
+            if length is not None:
+                datatype_obj = datatype_func(length)
+            else:
+                raise ValueError(f"Length must be provided for sized type '{datatype}' in column '{col.id}'")
+        else:
+            datatype_obj = datatype_func()
+
+        for dialect_name, dialect in _DIALECTS.items():
+            db_annotation = f"{dialect_name}_datatype"
+            if datatype_string := col.model_dump().get(db_annotation):
+                db_datatype_obj = string_to_typeengine(datatype_string, dialect, length)
+                if datatype_obj.compile(dialect) == db_datatype_obj.compile(dialect):
+                    raise ValueError(
+                        "'{}: {}' is a redundant override of 'datatype: {}' in column '{}'{}".format(
+                            db_annotation,
+                            datatype_string,
+                            col.datatype,
+                            col.id,
+                            "" if length is None else f" with length {length}",
+                        )
+                    )
+                else:
+                    logger.debug(
+                        "Type override of 'datatype: {}' with '{}: {}' in column '{}' "
+                        "compiled to '{}' and '{}'".format(
+                            col.datatype,
+                            db_annotation,
+                            datatype_string,
+                            col.id,
+                            datatype_obj.compile(dialect),
+                            db_datatype_obj.compile(dialect),
+                        )
+                    )
+        return col
+
 
 class Constraint(BaseObject):
     """A database table constraint."""
@@ -404,15 +503,6 @@ class SchemaIdVisitor:
 class Schema(BaseObject):
     """The database schema containing the tables."""
 
-    class ValidationConfig:
-        """Validation configuration which is specific to Felis."""
-
-        _require_description = False
-        """Flag to require a description for all objects.
-
-        This is set by the `require_description` class method.
-        """
-
     version: SchemaVersion | str | None = None
     """The version of the schema."""
 
@@ -430,21 +520,29 @@ class Schema(BaseObject):
             raise ValueError("Table names must be unique")
         return tables
 
-    @model_validator(mode="after")
-    def create_id_map(self: Schema) -> Schema:
-        """Create a map of IDs to objects."""
+    def _create_id_map(self: Schema) -> Schema:
+        """Create a map of IDs to objects.
+
+        This method should not be called by users. It is called automatically
+        by the ``model_post_init()`` method. If the ID map is already
+        populated, this method will return immediately.
+        """
         if len(self.id_map):
-            logger.debug("ID map was already populated")
+            logger.debug("Ignoring call to create_id_map() - ID map was already populated")
             return self
         visitor: SchemaIdVisitor = SchemaIdVisitor()
         visitor.visit_schema(self)
-        logger.debug(f"ID map contains {len(self.id_map.keys())} objects")
+        logger.debug(f"Created schema ID map with {len(self.id_map.keys())} objects")
         if len(visitor.duplicates):
             raise ValueError(
                 "Duplicate IDs found in schema:\n    " + "\n    ".join(visitor.duplicates) + "\n"
             )
         return self
 
+    def model_post_init(self, ctx: Any) -> None:
+        """Post-initialization hook for the model."""
+        self._create_id_map()
+
     def __getitem__(self, id: str) -> BaseObject:
         """Get an object by its ID."""
         if id not in self:
@@ -454,20 +552,3 @@ class Schema(BaseObject):
     def __contains__(self, id: str) -> bool:
         """Check if an object with the given ID is in the schema."""
         return id in self.id_map
-
-    @classmethod
-    def require_description(cls, rd: bool = True) -> None:
-        """Set whether a description is required for all objects.
-
-        This includes the schema, tables, columns, and constraints.
-
-        Users should call this method to set the requirement for a description
-        when validating schemas, rather than change the flag value directly.
-        """
-        logger.debug(f"Setting description requirement to '{rd}'")
-        cls.ValidationConfig._require_description = rd
-
-    @classmethod
-    def is_description_required(cls) -> bool:
-        """Return whether a description is required for all objects."""
-        return cls.ValidationConfig._require_description

felis/db/_variants.py

@@ -40,10 +40,10 @@ TABLE_OPTS = {
 }
 
 COLUMN_VARIANT_OVERRIDE = {
-    "mysql:datatype": "mysql",
-    "oracle:datatype": "oracle",
-    "postgresql:datatype": "postgresql",
-    "sqlite:datatype": "sqlite",
+    "mysql_datatype": "mysql",
+    "oracle_datatype": "oracle",
+    "postgresql_datatype": "postgresql",
+    "sqlite_datatype": "sqlite",
 }
 
 DIALECT_MODULES = {MYSQL: mysql, ORACLE: oracle, SQLITE: sqlite, POSTGRES: postgresql}
@@ -87,7 +87,7 @@ def make_variant_dict(column_obj: Column) -> dict[str, TypeEngine[Any]]:
     """
     variant_dict = {}
     for field_name, value in iter(column_obj):
-        if field_name in COLUMN_VARIANT_OVERRIDE:
+        if field_name in COLUMN_VARIANT_OVERRIDE and value is not None:
             dialect = COLUMN_VARIANT_OVERRIDE[field_name]
             variant: TypeEngine = process_variant_override(dialect, value)
             variant_dict[dialect] = variant

felis/db/sqltypes.py

@@ -21,9 +21,9 @@
 
 import builtins
 from collections.abc import Mapping
-from typing import Any
+from typing import Any, Callable
 
-from sqlalchemy import Float, SmallInteger, types
+from sqlalchemy import SmallInteger, types
 from sqlalchemy.dialects import mysql, oracle, postgresql
 from sqlalchemy.ext.compiler import compiles
 
@@ -39,27 +39,15 @@ class TINYINT(SmallInteger):
     __visit_name__ = "TINYINT"
 
 
-class DOUBLE(Float):
-    """The non-standard DOUBLE type."""
-
-    __visit_name__ = "DOUBLE"
-
-
 @compiles(TINYINT)
 def compile_tinyint(type_: Any, compiler: Any, **kw: Any) -> str:
     """Return type name for TINYINT."""
     return "TINYINT"
 
 
-@compiles(DOUBLE)
-def compile_double(type_: Any, compiler: Any, **kw: Any) -> str:
-    """Return type name for double precision type."""
-    return "DOUBLE"
-
-
 _TypeMap = Mapping[str, types.TypeEngine | type[types.TypeEngine]]
 
-boolean_map: _TypeMap = {MYSQL: mysql.BIT(1), ORACLE: oracle.NUMBER(1), POSTGRES: postgresql.BOOLEAN()}
+boolean_map: _TypeMap = {MYSQL: mysql.BOOLEAN, ORACLE: oracle.NUMBER(1), POSTGRES: postgresql.BOOLEAN()}
 
 byte_map: _TypeMap = {
     MYSQL: mysql.TINYINT(),
@@ -160,7 +148,7 @@ def float(**kwargs: Any) -> types.TypeEngine:
 
 def double(**kwargs: Any) -> types.TypeEngine:
     """Return SQLAlchemy type for double precision float."""
-    return _vary(DOUBLE(), double_map, kwargs)
+    return _vary(types.DOUBLE(), double_map, kwargs)
 
 
 def char(length: builtins.int, **kwargs: Any) -> types.TypeEngine:
@@ -178,9 +166,9 @@ def unicode(length: builtins.int, **kwargs: Any) -> types.TypeEngine:
     return _vary(types.NVARCHAR(length), unicode_map, kwargs, length)
 
 
-def text(length: builtins.int, **kwargs: Any) -> types.TypeEngine:
+def text(**kwargs: Any) -> types.TypeEngine:
     """Return SQLAlchemy type for text."""
-    return _vary(types.CLOB(length), text_map, kwargs, length)
+    return _vary(types.TEXT(), text_map, kwargs)
 
 
 def binary(length: builtins.int, **kwargs: Any) -> types.TypeEngine:
@@ -193,6 +181,13 @@ def timestamp(**kwargs: Any) -> types.TypeEngine:
     return types.TIMESTAMP()
 
 
+def get_type_func(type_name: str) -> Callable:
+    """Return the function for the type with the given name."""
+    if type_name not in globals():
+        raise ValueError(f"Unknown type: {type_name}")
+    return globals()[type_name]
+
+
 def _vary(
     type_: types.TypeEngine,
     variant_map: _TypeMap,
@@ -203,7 +198,7 @@ def _vary(
     variants.update(overrides)
     for dialect, variant in variants.items():
         # If this is a class and not an instance, instantiate
-        if isinstance(variant, type):
+        if callable(variant):
             variant = variant(*args)
         type_ = type_.with_variant(variant, dialect)
     return type_

felis/metadata.py

@@ -34,7 +34,6 @@ from sqlalchemy import (
     ForeignKeyConstraint,
     Index,
     MetaData,
-    Numeric,
     PrimaryKeyConstraint,
     ResultProxy,
     Table,
@@ -265,17 +264,12 @@ class MetaDataBuilder:
         id = column_obj.id
         description = column_obj.description
         default = column_obj.value
+        nullable = column_obj.nullable
 
-        # Handle variant overrides for the column (e.g., "mysql:datatype").
+        # Get datatype, handling variant overrides such as "mysql:datatype".
         datatype = get_datatype_with_variants(column_obj)
 
-        # Set default value of nullable based on column type and then whether
-        # it was explicitly provided in the schema data.
-        nullable = column_obj.nullable
-        if nullable is None:
-            nullable = False if isinstance(datatype, Numeric) else True
-
-        # Set autoincrement depending on if it was provided explicitly.
+        # Set autoincrement, depending on if it was provided explicitly.
         autoincrement: Literal["auto"] | bool = (
             column_obj.autoincrement if column_obj.autoincrement is not None else "auto"
         )

felis/tap.py

@@ -24,7 +24,7 @@ from __future__ import annotations
 __all__ = ["Tap11Base", "TapLoadingVisitor", "init_tables"]
 
 import logging
-from collections.abc import Iterable, Mapping, MutableMapping
+from collections.abc import Iterable, MutableMapping
 from typing import Any
 
 from sqlalchemy import Column, Integer, String
@@ -34,14 +34,13 @@ from sqlalchemy.orm import Session, declarative_base, sessionmaker
 from sqlalchemy.schema import MetaData
 from sqlalchemy.sql.expression import Insert, insert
 
-from .check import FelisValidator
-from .types import FelisType
-from .visitor import Visitor
+from felis import datamodel
 
-_Mapping = Mapping[str, Any]
+from .datamodel import Constraint, Index, Schema, Table
+from .types import FelisType
 
 Tap11Base: Any = declarative_base()  # Any to avoid mypy mess with SA 2
-logger = logging.getLogger("felis")
+logger = logging.getLogger(__name__)
 
 IDENTIFIER_LENGTH = 128
 SMALL_FIELD_LENGTH = 32
@@ -133,7 +132,7 @@ def init_tables(
     )
 
 
-class TapLoadingVisitor(Visitor[None, tuple, Tap11Base, None, tuple, None, None]):
+class TapLoadingVisitor:
     """Felis schema visitor for generating TAP schema.
 
     Parameters
@@ -154,6 +153,7 @@ class TapLoadingVisitor(Visitor[None, tuple, Tap11Base, None, tuple, None, None]
         catalog_name: str | None = None,
         schema_name: str | None = None,
         tap_tables: MutableMapping[str, Any] | None = None,
+        tap_schema_index: int | None = None,
     ):
         self.graph_index: MutableMapping[str, Any] = {}
         self.catalog_name = catalog_name
@@ -161,7 +161,7 @@ class TapLoadingVisitor(Visitor[None, tuple, Tap11Base, None, tuple, None, None]
         self.engine = engine
         self._mock_connection: MockConnection | None = None
         self.tables = tap_tables or init_tables()
-        self.checker = FelisValidator()
+        self.tap_schema_index = tap_schema_index
 
     @classmethod
     def from_mock_connection(
@@ -170,30 +170,30 @@ class TapLoadingVisitor(Visitor[None, tuple, Tap11Base, None, tuple, None, None]
         catalog_name: str | None = None,
         schema_name: str | None = None,
         tap_tables: MutableMapping[str, Any] | None = None,
+        tap_schema_index: int | None = None,
     ) -> TapLoadingVisitor:
         visitor = cls(engine=None, catalog_name=catalog_name, schema_name=schema_name, tap_tables=tap_tables)
         visitor._mock_connection = mock_connection
+        visitor.tap_schema_index = tap_schema_index
         return visitor
 
-    def visit_schema(self, schema_obj: _Mapping) -> None:
-        self.checker.check_schema(schema_obj)
-        if (version_obj := schema_obj.get("version")) is not None:
-            self.visit_schema_version(version_obj, schema_obj)
+    def visit_schema(self, schema_obj: Schema) -> None:
         schema = self.tables["schemas"]()
         # Override with default
-        self.schema_name = self.schema_name or schema_obj["name"]
+        self.schema_name = self.schema_name or schema_obj.name
 
         schema.schema_name = self._schema_name()
-        schema.description = schema_obj.get("description")
-        schema.utype = schema_obj.get("votable:utype")
-        schema.schema_index = int(schema_obj.get("tap:schema_index", 0))
+        schema.description = schema_obj.description
+        schema.utype = schema_obj.votable_utype
+        schema.schema_index = self.tap_schema_index
+        logger.debug("Set TAP_SCHEMA index: {}".format(self.tap_schema_index))
 
         if self.engine is not None:
             session: Session = sessionmaker(self.engine)()
 
             session.add(schema)
 
-            for table_obj in schema_obj["tables"]:
+            for table_obj in schema_obj.tables:
                 table, columns = self.visit_table(table_obj, schema_obj)
                 session.add(table)
                 session.add_all(columns)
@@ -202,6 +202,8 @@ class TapLoadingVisitor(Visitor[None, tuple, Tap11Base, None, tuple, None, None]
             session.add_all(keys)
             session.add_all(key_columns)
 
+            logger.debug("Committing TAP schema: %s", schema_obj.name)
+            logger.debug("TAP tables: %s", len(self.tables))
             session.commit()
         else:
             logger.info("Dry run, not inserting into database")
@@ -211,7 +213,7 @@ class TapLoadingVisitor(Visitor[None, tuple, Tap11Base, None, tuple, None, None]
             conn = self._mock_connection
             conn.execute(_insert(self.tables["schemas"], schema))
 
-            for table_obj in schema_obj["tables"]:
+            for table_obj in schema_obj.tables:
                 table, columns = self.visit_table(table_obj, schema_obj)
                 conn.execute(_insert(self.tables["tables"], table))
                 for column in columns:
@@ -223,56 +225,45 @@ class TapLoadingVisitor(Visitor[None, tuple, Tap11Base, None, tuple, None, None]
             for key_column in key_columns:
                 conn.execute(_insert(self.tables["key_columns"], key_column))
 
-    def visit_constraints(self, schema_obj: _Mapping) -> tuple:
+    def visit_constraints(self, schema_obj: Schema) -> tuple:
         all_keys = []
         all_key_columns = []
-        for table_obj in schema_obj["tables"]:
-            for c in table_obj.get("constraints", []):
-                key, key_columns = self.visit_constraint(c, table_obj)
+        for table_obj in schema_obj.tables:
+            for c in table_obj.constraints:
+                key, key_columns = self.visit_constraint(c)
                 if not key:
                     continue
                 all_keys.append(key)
                 all_key_columns += key_columns
         return all_keys, all_key_columns
 
-    def visit_schema_version(
-        self, version_obj: str | Mapping[str, Any], schema_obj: Mapping[str, Any]
-    ) -> None:
-        # Docstring is inherited.
-
-        # For now we ignore schema versioning completely, still do some checks.
-        self.checker.check_schema_version(version_obj, schema_obj)
-
-    def visit_table(self, table_obj: _Mapping, schema_obj: _Mapping) -> tuple:
-        self.checker.check_table(table_obj, schema_obj)
-        table_id = table_obj["@id"]
+    def visit_table(self, table_obj: Table, schema_obj: Schema) -> tuple:
+        table_id = table_obj.id
         table = self.tables["tables"]()
         table.schema_name = self._schema_name()
-        table.table_name = self._table_name(table_obj["name"])
+        table.table_name = self._table_name(table_obj.name)
         table.table_type = "table"
-        table.utype = table_obj.get("votable:utype")
-        table.description = table_obj.get("description")
-        table.table_index = int(table_obj.get("tap:table_index", 0))
+        table.utype = table_obj.votable_utype
+        table.description = table_obj.description
+        table.table_index = 0 if table_obj.tap_table_index is None else table_obj.tap_table_index
 
-        columns = [self.visit_column(c, table_obj) for c in table_obj["columns"]]
-        self.visit_primary_key(table_obj.get("primaryKey", []), table_obj)
+        columns = [self.visit_column(c, table_obj) for c in table_obj.columns]
+        self.visit_primary_key(table_obj.primary_key, table_obj)
 
-        for i in table_obj.get("indexes", []):
+        for i in table_obj.indexes:
             self.visit_index(i, table)
 
         self.graph_index[table_id] = table
         return table, columns
 
-    def check_column(self, column_obj: _Mapping, table_obj: _Mapping) -> None:
-        self.checker.check_column(column_obj, table_obj)
-        _id = column_obj["@id"]
-        # Guaranteed to exist at this point, for mypy use "" as default
-        datatype_name = column_obj.get("datatype", "")
-        felis_type = FelisType.felis_type(datatype_name)
+    def check_column(self, column_obj: datamodel.Column) -> None:
+        _id = column_obj.id
+        datatype_name = column_obj.datatype
+        felis_type = FelisType.felis_type(datatype_name.value)
         if felis_type.is_sized:
             # It is expected that both arraysize and length are fine for
             # length types.
-            arraysize = column_obj.get("votable:arraysize", column_obj.get("length"))
+            arraysize = column_obj.votable_arraysize or column_obj.length
             if arraysize is None:
                 logger.warning(
                     f"votable:arraysize and length for {_id} are None for type {datatype_name}. "
@@ -283,7 +274,7 @@ class TapLoadingVisitor(Visitor[None, tuple, Tap11Base, None, tuple, None, None]
             # datetime types really should have a votable:arraysize, because
             # they are converted to strings and the `length` is loosely to the
             # string size
-            if "votable:arraysize" not in column_obj:
+            if not column_obj.votable_arraysize:
                 logger.warning(
                     f"votable:arraysize for {_id} is None for type {datatype_name}. "
                     f'Using length "*". '
@@ -291,47 +282,45 @@ class TapLoadingVisitor(Visitor[None, tuple, Tap11Base, None, tuple, None, None]
                     "materialized datetime/timestamp strings."
                 )
 
-    def visit_column(self, column_obj: _Mapping, table_obj: _Mapping) -> Tap11Base:
-        self.check_column(column_obj, table_obj)
-        column_id = column_obj["@id"]
-        table_name = self._table_name(table_obj["name"])
+    def visit_column(self, column_obj: datamodel.Column, table_obj: Table) -> Tap11Base:
+        self.check_column(column_obj)
+        column_id = column_obj.id
+        table_name = self._table_name(table_obj.name)
 
         column = self.tables["columns"]()
         column.table_name = table_name
-        column.column_name = column_obj["name"]
+        column.column_name = column_obj.name
 
-        felis_datatype = column_obj["datatype"]
-        felis_type = FelisType.felis_type(felis_datatype)
-        column.datatype = column_obj.get("votable:datatype", felis_type.votable_name)
+        felis_datatype = column_obj.datatype
+        felis_type = FelisType.felis_type(felis_datatype.value)
+        column.datatype = column_obj.votable_datatype or felis_type.votable_name
 
         arraysize = None
         if felis_type.is_sized:
-            # prefer votable:arraysize to length, fall back to `*`
-            arraysize = column_obj.get("votable:arraysize", column_obj.get("length", "*"))
+            arraysize = column_obj.votable_arraysize or column_obj.length or "*"
         if felis_type.is_timestamp:
-            arraysize = column_obj.get("votable:arraysize", "*")
+            arraysize = column_obj.votable_arraysize or "*"
         column.arraysize = arraysize
 
-        column.xtype = column_obj.get("votable:xtype")
-        column.description = column_obj.get("description")
-        column.utype = column_obj.get("votable:utype")
+        column.xtype = column_obj.votable_xtype
+        column.description = column_obj.description
+        column.utype = column_obj.votable_utype
 
-        unit = column_obj.get("ivoa:unit") or column_obj.get("fits:tunit")
+        unit = column_obj.ivoa_unit or column_obj.fits_tunit
         column.unit = unit
-        column.ucd = column_obj.get("ivoa:ucd")
+        column.ucd = column_obj.ivoa_ucd
 
         # We modify this after we process columns
         column.indexed = 0
 
-        column.principal = column_obj.get("tap:principal", 0)
-        column.std = column_obj.get("tap:std", 0)
-        column.column_index = column_obj.get("tap:column_index")
+        column.principal = column_obj.tap_principal
+        column.std = column_obj.tap_std
+        column.column_index = column_obj.tap_column_index
 
         self.graph_index[column_id] = column
         return column
 
-    def visit_primary_key(self, primary_key_obj: str | Iterable[str], table_obj: _Mapping) -> None:
-        self.checker.check_primary_key(primary_key_obj, table_obj)
+    def visit_primary_key(self, primary_key_obj: str | Iterable[str] | None, table_obj: Table) -> None:
         if primary_key_obj:
             if isinstance(primary_key_obj, str):
                 primary_key_obj = [primary_key_obj]
@@ -341,19 +330,18 @@ class TapLoadingVisitor(Visitor[None, tuple, Tap11Base, None, tuple, None, None]
                 columns[0].indexed = 1
         return None
 
-    def visit_constraint(self, constraint_obj: _Mapping, table_obj: _Mapping) -> tuple:
-        self.checker.check_constraint(constraint_obj, table_obj)
-        constraint_type = constraint_obj["@type"]
+    def visit_constraint(self, constraint_obj: Constraint) -> tuple:
+        constraint_type = constraint_obj.type
         key = None
         key_columns = []
         if constraint_type == "ForeignKey":
-            constraint_name = constraint_obj["name"]
-            description = constraint_obj.get("description")
-            utype = constraint_obj.get("votable:utype")
+            constraint_name = constraint_obj.name
+            description = constraint_obj.description
+            utype = constraint_obj.votable_utype
 
-            columns = [self.graph_index[col["@id"]] for col in constraint_obj.get("columns", [])]
+            columns = [self.graph_index[col_id] for col_id in getattr(constraint_obj, "columns", [])]
             refcolumns = [
-                self.graph_index[refcol["@id"]] for refcol in constraint_obj.get("referencedColumns", [])
+                self.graph_index[refcol_id] for refcol_id in getattr(constraint_obj, "referenced_columns", [])
             ]
 
             table_name = None
@@ -386,9 +374,8 @@ class TapLoadingVisitor(Visitor[None, tuple, Tap11Base, None, tuple, None, None]
                 key_columns.append(key_column)
         return key, key_columns
 
-    def visit_index(self, index_obj: _Mapping, table_obj: _Mapping) -> None:
-        self.checker.check_index(index_obj, table_obj)
-        columns = [self.graph_index[col["@id"]] for col in index_obj.get("columns", [])]
+    def visit_index(self, index_obj: Index, table_obj: Table) -> None:
+        columns = [self.graph_index[col_id] for col_id in getattr(index_obj, "columns", [])]
         # if just one column and it's indexed, update the object
         if len(columns) == 1:
             columns[0].indexed = 1

felis/types.py

@@ -125,7 +125,7 @@ class Unicode(FelisType, felis_name="unicode", votable_name="unicodeChar", is_si
     """Felis definition of unicode string type."""
 
 
-class Text(FelisType, felis_name="text", votable_name="unicodeChar", is_sized=True):
+class Text(FelisType, felis_name="text", votable_name="char"):
     """Felis definition of text type."""
 
 

felis/version.py

@@ -1,2 +1,2 @@
 __all__ = ["__version__"]
-__version__ = "26.2024.1500"
+__version__ = "26.2024.1700"

{lsst_felis-26.2024.1500.dist-info → lsst_felis-26.2024.1700.dist-info}/METADATA

@@ -1,10 +1,11 @@
 Metadata-Version: 2.1
 Name: lsst-felis
-Version: 26.2024.1500
+Version: 26.2024.1700
 Summary: A vocabulary for describing catalogs and acting on those descriptions
 Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
 License: GNU General Public License v3 or later (GPLv3+)
-Project-URL: Homepage, https://github.com/lsst/felis
+Project-URL: Homepage, https://felis.lsst.io
+Project-URL: Source, https://github.com/lsst/felis
 Keywords: lsst
 Classifier: Intended Audience :: Science/Research
 Classifier: License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)
@@ -24,6 +25,8 @@ Requires-Dist: pyyaml >=6
 Requires-Dist: pyld >=2
 Requires-Dist: pydantic <3,>=2
 Requires-Dist: lsst-utils
+Provides-Extra: dev
+Requires-Dist: documenteer[guide] ; extra == 'dev'
 Provides-Extra: test
 Requires-Dist: pytest >=3.2 ; extra == 'test'
 

lsst_felis-26.2024.1700.dist-info/RECORD

@@ -0,0 +1,24 @@
+felis/__init__.py,sha256=_Pw-QKMYj0WRgE8fW2N2pBXJUj-Pjv8dSKJBzykjyZU,1842
+felis/check.py,sha256=RBxXq7XwPGIucrs1PPgPtgk8MrWAJlOmoxCNySEz9-I,13892
+felis/cli.py,sha256=VBlEoo65Y5lPf7wzokuZeW1HL-ObfIrrVzo155RZK0Q,16202
+felis/datamodel.py,sha256=VEymalSBkVmTPWL-xm1DPp6fAR2ze0KvFRGwVf2YR08,19315
+felis/metadata.py,sha256=gcBjpB_JpEfKXpKc7hMwG11PizqeDR8IJaFfhFtBIEw,18294
+felis/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+felis/simple.py,sha256=yzv_aoZrZhfakd1Xm7gLDeVKyJjCDZ7wAyYYp-l_Sxs,14414
+felis/tap.py,sha256=MOqe9_K6KK9oUqEuKyKtSLFrzTluohbcvQ59DeoRiLY,16803
+felis/types.py,sha256=z_ECfSxpqiFSGppjxKwCO4fPP7TLBaIN3Qo1AGF16Go,4418
+felis/utils.py,sha256=tYxr0xFdPN4gDHibeAD9d5DFgU8hKlSZVKmZoDzi4e8,4164
+felis/validation.py,sha256=f9VKvp7q-cnim2D5voTKwCdt0NRsYBpTwom1Z_3OKkc,3469
+felis/version.py,sha256=PH-FIbvh4hvlcfNHl8Bt-EVFDlE5_dasGcUm02k5uxI,55
+felis/visitor.py,sha256=EazU4nYbkKBj3mCZYvsTCBTNmh0qRaUNZIzCcM3dqOQ,6439
+felis/db/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+felis/db/_variants.py,sha256=zCuXDgU_x_pTZcWkBLgqQCiOhlA6y2tBt-PUQfafwmM,3368
+felis/db/sqltypes.py,sha256=n6E1K-Hvdt62GVBQedefzTqBpNS7ks92lchRM5BF_Oo,5735
+lsst_felis-26.2024.1700.dist-info/COPYRIGHT,sha256=bUmNy19uUxqITMpjeHFe69q3IzQpjxvvBw6oV7kR7ho,129
+lsst_felis-26.2024.1700.dist-info/LICENSE,sha256=jOtLnuWt7d5Hsx6XXB2QxzrSe2sWWh3NgMfFRetluQM,35147
+lsst_felis-26.2024.1700.dist-info/METADATA,sha256=e4j5d28w8iR-ZzYbhvD_tSKHAIC7Gx07rSW4llYDaa8,1215
+lsst_felis-26.2024.1700.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+lsst_felis-26.2024.1700.dist-info/entry_points.txt,sha256=Gk2XFujA_Gp52VBk45g5kim8TDoMDJFPctsMqiq72EM,40
+lsst_felis-26.2024.1700.dist-info/top_level.txt,sha256=F4SvPip3iZRVyISi50CHhwTIAokAhSxjWiVcn4IVWRI,6
+lsst_felis-26.2024.1700.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
+lsst_felis-26.2024.1700.dist-info/RECORD,,

lsst_felis-26.2024.1500.dist-info/RECORD

@@ -1,24 +0,0 @@
-felis/__init__.py,sha256=_Pw-QKMYj0WRgE8fW2N2pBXJUj-Pjv8dSKJBzykjyZU,1842
-felis/check.py,sha256=RBxXq7XwPGIucrs1PPgPtgk8MrWAJlOmoxCNySEz9-I,13892
-felis/cli.py,sha256=YeGSiA3ywPVMMdB1YxH1_Gdac1kl4oPJvJtajfCs5VU,16637
-felis/datamodel.py,sha256=ooNSg68OuNk89EVu1MtxupLUWgSyzmb50wjza1joDO4,16002
-felis/metadata.py,sha256=5DE2YMnu6YuhwntBSe-OheCD7C2-vA4yb64BpjTC68A,18542
-felis/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-felis/simple.py,sha256=yzv_aoZrZhfakd1Xm7gLDeVKyJjCDZ7wAyYYp-l_Sxs,14414
-felis/tap.py,sha256=RBwEKyU3S0oXSNIMoI2WRAuC9WB5eai9BdQQUYN5Qdc,17704
-felis/types.py,sha256=1GL6IkHcIsIydiyw1eX98REh-lWCVIRO-9qjmaZfqvw,4440
-felis/utils.py,sha256=tYxr0xFdPN4gDHibeAD9d5DFgU8hKlSZVKmZoDzi4e8,4164
-felis/validation.py,sha256=f9VKvp7q-cnim2D5voTKwCdt0NRsYBpTwom1Z_3OKkc,3469
-felis/version.py,sha256=v0ZRn4V9ZMSKzM31IPNIMtcYAuVRbtzceeMS49b6RIE,55
-felis/visitor.py,sha256=EazU4nYbkKBj3mCZYvsTCBTNmh0qRaUNZIzCcM3dqOQ,6439
-felis/db/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-felis/db/_variants.py,sha256=aW0Q7R4KEtxLR7VMashQjDLWdzDNrMVAH521MSvMey0,3346
-felis/db/sqltypes.py,sha256=0HOEqvL0OailGP-j6Jj5tnOSu_Pt7Hi29PPof4Q5d2c,5787
-lsst_felis-26.2024.1500.dist-info/COPYRIGHT,sha256=bUmNy19uUxqITMpjeHFe69q3IzQpjxvvBw6oV7kR7ho,129
-lsst_felis-26.2024.1500.dist-info/LICENSE,sha256=jOtLnuWt7d5Hsx6XXB2QxzrSe2sWWh3NgMfFRetluQM,35147
-lsst_felis-26.2024.1500.dist-info/METADATA,sha256=o3cHiTSh5w6RHPvPAIG1sXMQZ6t1xW4lkNrgbMzMVlk,1101
-lsst_felis-26.2024.1500.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-lsst_felis-26.2024.1500.dist-info/entry_points.txt,sha256=Gk2XFujA_Gp52VBk45g5kim8TDoMDJFPctsMqiq72EM,40
-lsst_felis-26.2024.1500.dist-info/top_level.txt,sha256=F4SvPip3iZRVyISi50CHhwTIAokAhSxjWiVcn4IVWRI,6
-lsst_felis-26.2024.1500.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-lsst_felis-26.2024.1500.dist-info/RECORD,,