lsst-felis 29.2025.2400__tar.gz → 29.2025.2600__tar.gz

{lsst_felis-29.2025.2400/python/lsst_felis.egg-info → lsst_felis-29.2025.2600}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lsst-felis
-Version: 29.2025.2400
+Version: 29.2025.2600
 Summary: A vocabulary for describing catalogs and acting on those descriptions
 Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
 License: GNU General Public License v3 or later (GPLv3+)

{lsst_felis-29.2025.2400 → lsst_felis-29.2025.2600}/pyproject.toml

@@ -60,7 +60,7 @@ zip-safe = true
 license-files = ["COPYRIGHT", "LICENSE"]
 
 [tool.setuptools.package-data]
-"felis" = ["py.typed", "schemas/*.yaml"]
+"felis" = ["py.typed", "config/tap_schema/*.yaml", "config/tap_schema/*.csv"]
 
 [tool.setuptools.dynamic]
 version = { attr = "lsst_versions.get_lsst_version" }

{lsst_felis-29.2025.2400 → lsst_felis-29.2025.2600}/python/felis/cli.py

@@ -38,7 +38,7 @@ from .db.schema import create_database
 from .db.utils import DatabaseContext, is_mock_url
 from .diff import DatabaseDiff, FormattedSchemaDiff, SchemaDiff
 from .metadata import MetaDataBuilder
-from .tap_schema import DataLoader, TableManager
+from .tap_schema import DataLoader, MetadataInserter, TableManager
 
 __all__ = ["cli"]
 
@@ -284,14 +284,20 @@ def load_tap_schema(
 
 
 @cli.command("init-tap-schema", help="Initialize a standard TAP_SCHEMA database")
-@click.option("--engine-url", envvar="FELIS_ENGINE_URL", help="SQLAlchemy Engine URL")
+@click.option("--engine-url", envvar="FELIS_ENGINE_URL", help="SQLAlchemy Engine URL", required=True)
 @click.option("--tap-schema-name", help="Name of the TAP_SCHEMA schema in the database")
 @click.option(
     "--tap-tables-postfix", help="Postfix which is applied to standard TAP_SCHEMA table names", default=""
 )
+@click.option(
+    "--insert-metadata/--no-insert-metadata",
+    is_flag=True,
+    help="Insert metadata describing TAP_SCHEMA itself",
+    default=True,
+)
 @click.pass_context
 def init_tap_schema(
-    ctx: click.Context, engine_url: str, tap_schema_name: str, tap_tables_postfix: str
+    ctx: click.Context, engine_url: str, tap_schema_name: str, tap_tables_postfix: str, insert_metadata: bool
 ) -> None:
     """Initialize a standard TAP_SCHEMA database.
 
@@ -303,6 +309,10 @@ def init_tap_schema(
         Name of the TAP_SCHEMA schema in the database.
     tap_tables_postfix
         Postfix which is applied to standard TAP_SCHEMA table names.
+    insert_metadata
+        Insert metadata describing TAP_SCHEMA itself.
+        If set to False, only the TAP_SCHEMA tables will be created, but no
+        metadata will be inserted.
     """
     url = make_url(engine_url)
     engine: Engine | MockConnection
@@ -315,6 +325,9 @@ def init_tap_schema(
         table_name_postfix=tap_tables_postfix,
     )
     mgr.initialize_database(engine)
+    if insert_metadata:
+        inserter = MetadataInserter(mgr, engine)
+        inserter.insert_metadata()
 
 
 @cli.command("validate", help="Validate one or more Felis YAML files")
@@ -465,12 +478,21 @@ def diff(
       felis dump schema.yaml schema_dump.yaml
     """,
 )
+@click.option(
+    "--strip-ids/--no-strip-ids",
+    is_flag=True,
+    help="Strip IDs from the output schema",
+    default=False,
+)
 @click.argument("files", nargs=2, type=click.Path())
 @click.pass_context
 def dump(
     ctx: click.Context,
+    strip_ids: bool,
     files: list[str],
 ) -> None:
+    if strip_ids:
+        logger.info("Stripping IDs from the output schema")
     if files[1].endswith(".json"):
         format = "json"
     elif files[1].endswith(".yaml"):
@@ -480,9 +502,9 @@ def dump(
     schema = Schema.from_uri(files[0], context={"id_generation": ctx.obj["id_generation"]})
     with open(files[1], "w") as f:
         if format == "yaml":
-            schema.dump_yaml(f)
+            schema.dump_yaml(f, strip_ids=strip_ids)
         elif format == "json":
-            schema.dump_json(f)
+            schema.dump_json(f, strip_ids=strip_ids)
 
 
 if __name__ == "__main__":

lsst_felis-29.2025.2600/python/felis/config/tap_schema/columns.csv

@@ -0,0 +1,33 @@
+table_name,column_name,utype,ucd,unit,description,datatype,arraysize,xtype,size,principal,indexed,std,column_index
+tap_schema.columns,"""size""",\N,\N,\N,deprecated: use arraysize,int,\N,\N,\N,1,0,1,9
+tap_schema.columns,arraysize,\N,\N,\N,lists the size of variable-length columns in the tableset,char,16*,\N,16,1,0,1,8
+tap_schema.columns,column_index,\N,\N,\N,recommended sort order when listing columns of a table,int,\N,\N,\N,1,0,1,13
+tap_schema.columns,column_name,\N,\N,\N,the column name,char,64*,\N,64,1,0,1,2
+tap_schema.columns,datatype,\N,\N,\N,lists the ADQL datatype of columns in the tableset,char,64*,\N,64,1,0,1,7
+tap_schema.columns,description,\N,\N,\N,describes the columns in the tableset,char,512*,\N,512,1,0,1,6
+tap_schema.columns,indexed,\N,\N,\N,"an indexed column; 1 means 1, 0 means 0",int,\N,\N,\N,1,0,1,11
+tap_schema.columns,principal,\N,\N,\N,"a principal column; 1 means 1, 0 means 0",int,\N,\N,\N,1,0,1,10
+tap_schema.columns,std,\N,\N,\N,"a standard column; 1 means 1, 0 means 0",int,\N,\N,\N,1,0,1,12
+tap_schema.columns,table_name,\N,\N,\N,the table this column belongs to,char,64*,\N,64,1,0,1,1
+tap_schema.columns,ucd,\N,\N,\N,lists the UCDs of columns in the tableset,char,64*,\N,64,1,0,1,4
+tap_schema.columns,unit,\N,\N,\N,lists the unit used for column values in the tableset,char,64*,\N,64,1,0,1,5
+tap_schema.columns,utype,\N,\N,\N,lists the utypes of columns in the tableset,char,512*,\N,512,1,0,1,3
+tap_schema.columns,xtype,\N,\N,\N,a DALI or custom extended type annotation,char,64*,\N,64,1,0,1,7
+tap_schema.key_columns,from_column,\N,\N,\N,column in the from_table,char,64*,\N,64,1,0,1,2
+tap_schema.key_columns,key_id,\N,\N,\N,key to join to tap_schema.keys,char,64*,\N,64,1,0,1,1
+tap_schema.key_columns,target_column,\N,\N,\N,column in the target_table,char,64*,\N,64,1,0,1,3
+tap_schema.keys,description,\N,\N,\N,describes keys in the tableset,char,512*,\N,512,1,0,1,5
+tap_schema.keys,from_table,\N,\N,\N,the table with the foreign key,char,64*,\N,64,1,0,1,2
+tap_schema.keys,key_id,\N,\N,\N,unique key to join to tap_schema.key_columns,char,64*,\N,64,1,0,1,1
+tap_schema.keys,target_table,\N,\N,\N,the table with the primary key,char,64*,\N,64,1,0,1,3
+tap_schema.keys,utype,\N,\N,\N,lists the utype of keys in the tableset,char,512*,\N,512,1,0,1,4
+tap_schema.schemas,description,\N,\N,\N,describes schemas in the tableset,char,512*,\N,512,1,0,1,3
+tap_schema.schemas,schema_index,\N,\N,\N,recommended sort order when listing schemas,int,\N,\N,\N,1,0,1,4
+tap_schema.schemas,schema_name,\N,\N,\N,schema name for reference to tap_schema.schemas,char,64*,\N,64,1,0,1,1
+tap_schema.schemas,utype,\N,\N,\N,lists the utypes of schemas in the tableset,char,512*,\N,512,1,0,1,2
+tap_schema.tables,description,\N,\N,\N,describes tables in the tableset,char,512*,\N,512,1,0,1,5
+tap_schema.tables,schema_name,\N,\N,\N,the schema this table belongs to,char,512*,\N,512,1,0,1,1
+tap_schema.tables,table_index,\N,\N,\N,recommended sort order when listing tables,int,\N,\N,\N,1,0,1,6
+tap_schema.tables,table_name,\N,\N,\N,the fully qualified table name,char,64*,\N,64,1,0,1,2
+tap_schema.tables,table_type,\N,\N,\N,one of: table view,char,8*,\N,8,1,0,1,3
+tap_schema.tables,utype,\N,\N,\N,lists the utype of tables in the tableset,char,512*,\N,512,1,0,1,4

lsst_felis-29.2025.2600/python/felis/config/tap_schema/key_columns.csv

@@ -0,0 +1,8 @@
+key_id,from_column,target_column
+k1,schema_name,schema_name
+k2,table_name,table_name
+k3,from_table,table_name
+k4,target_table,table_name
+k5,key_id,key_id
+k6,from_column,column_name
+k7,target_column,column_name

lsst_felis-29.2025.2600/python/felis/config/tap_schema/keys.csv

@@ -0,0 +1,8 @@
+key_id,from_table,target_table,utype,description
+k1,tap_schema.tables,tap_schema.schemas,\N,\N
+k2,tap_schema.columns,tap_schema.tables,\N,\N
+k3,tap_schema.keys,tap_schema.tables,\N,\N
+k4,tap_schema.keys,tap_schema.tables,\N,\N
+k5,tap_schema.key_columns,tap_schema.keys,\N,\N
+k6,tap_schema.key_columns,tap_schema.columns,\N,\N
+k7,tap_schema.key_columns,tap_schema.columns,\N,\N

lsst_felis-29.2025.2600/python/felis/config/tap_schema/schemas.csv

@@ -0,0 +1,2 @@
+schema_name,utype,description,schema_index
+tap_schema,\N,A TAP-standard-mandated schema to describe tablesets in a TAP 1.1 service,100000

lsst_felis-29.2025.2600/python/felis/config/tap_schema/tables.csv

@@ -0,0 +1,6 @@
+schema_name,table_name,table_type,utype,description,table_index
+tap_schema,tap_schema.columns,table,\N,description of columns in this tableset,102000
+tap_schema,tap_schema.key_columns,table,\N,description of foreign key columns in this tableset,104000
+tap_schema,tap_schema.keys,table,\N,description of foreign keys in this tableset,103000
+tap_schema,tap_schema.schemas,table,\N,description of schemas in this tableset,100000
+tap_schema,tap_schema.tables,table,\N,description of tables in this tableset,101000

{lsst_felis-29.2025.2400 → lsst_felis-29.2025.2600}/python/felis/datamodel.py

@@ -39,11 +39,13 @@ from pydantic import (
     ConfigDict,
     Field,
     PrivateAttr,
+    ValidationError,
     ValidationInfo,
     field_serializer,
     field_validator,
     model_validator,
 )
+from pydantic_core import InitErrorDetails
 
 from .db.dialects import get_supported_dialects
 from .db.sqltypes import get_type_func
@@ -633,6 +635,12 @@ class ForeignKeyConstraint(Constraint):
     referenced_columns: list[str] = Field(alias="referencedColumns")
     """The columns referenced by the foreign key."""
 
+    on_delete: Literal["CASCADE", "SET NULL", "SET DEFAULT", "RESTRICT", "NO ACTION"] | None = None
+    """Action to take when the referenced row is deleted."""
+
+    on_update: Literal["CASCADE", "SET NULL", "SET DEFAULT", "RESTRICT", "NO ACTION"] | None = None
+    """Action to take when the referenced row is updated."""
+
     @field_serializer("type")
     def serialize_type(self, value: str) -> str:
         """Ensure '@type' is included in serialized output.
@@ -752,7 +760,10 @@ class ColumnGroup(BaseObject):
         for col in self.columns:
             if isinstance(col, str):
                 # Dereference ColumnRef to Column object
-                col_obj = self.table._find_column_by_id(col)
+                try:
+                    col_obj = self.table._find_column_by_id(col)
+                except KeyError as e:
+                    raise ValueError(f"Column '{col}' not found in table '{self.table.name}'") from e
                 dereferenced_columns.append(col_obj)
             else:
                 dereferenced_columns.append(col)
@@ -902,7 +913,7 @@ class Table(BaseObject):
         for column in self.columns:
             if column.id == id:
                 return column
-        raise ValueError(f"Column '{id}' not found in table '{self.name}'")
+        raise KeyError(f"Column '{id}' not found in table '{self.name}'")
 
     @model_validator(mode="after")
     def dereference_column_groups(self: Table) -> Table:
@@ -1022,6 +1033,56 @@ class SchemaIdVisitor:
 T = TypeVar("T", bound=BaseObject)
 
 
+def _strip_ids(data: Any) -> Any:
+    """Recursively strip '@id' fields from a dictionary or list.
+
+    Parameters
+    ----------
+    data
+        The data to strip IDs from, which can be a dictionary, list, or any
+        other type. Other types will be returned unchanged.
+    """
+    if isinstance(data, dict):
+        data.pop("@id", None)
+        for k, v in data.items():
+            data[k] = _strip_ids(v)
+        return data
+    elif isinstance(data, list):
+        return [_strip_ids(item) for item in data]
+    else:
+        return data
+
+
+def _append_error(
+    errors: list[InitErrorDetails],
+    loc: tuple,
+    input_value: Any,
+    error_message: str,
+    error_type: str = "value_error",
+) -> None:
+    """Append an error to the errors list.
+
+    Parameters
+    ----------
+    errors : list[InitErrorDetails]
+        The list of errors to append to.
+    loc : tuple
+        The location of the error in the schema.
+    input_value : Any
+        The input value that caused the error.
+    error_message : str
+        The error message to include in the context.
+    """
+    errors.append(
+        {
+            "type": error_type,
+            "loc": loc,
+            "input": input_value,
+            "ctx": {"error": error_message},
+        }
+    )
+
+
 class Schema(BaseObject, Generic[T]):
     """Database schema model.
 
@@ -1204,18 +1265,19 @@ class Schema(BaseObject, Generic[T]):
 
         return self
 
-    def _create_id_map(self: Schema) -> Schema:
+    @model_validator(mode="after")
+    def create_id_map(self: Schema) -> Schema:
         """Create a map of IDs to objects.
 
+        Returns
+        -------
+        `Schema`
+            The schema with the ID map created.
+
         Raises
         ------
         ValueError
             Raised if duplicate identifiers are found in the schema.
-
-        Notes
-        -----
-        This is called automatically by the `model_post_init` method. If the
-        ID map is already populated, this method will return immediately.
         """
         if self._id_map:
             logger.debug("Ignoring call to create_id_map() - ID map was already populated")
@@ -1226,25 +1288,152 @@ class Schema(BaseObject, Generic[T]):
             raise ValueError(
                 "Duplicate IDs found in schema:\n    " + "\n    ".join(visitor.duplicates) + "\n"
             )
+        logger.debug("Created ID map with %d entries", len(self._id_map))
         return self
 
-    def model_post_init(self, ctx: Any) -> None:
-        """Post-initialization hook for the model.
+    def _validate_column_id(
+        self: Schema,
+        column_id: str,
+        loc: tuple,
+        errors: list[InitErrorDetails],
+    ) -> None:
+        """Validate a column ID from a constraint and append errors if invalid.
 
         Parameters
         ----------
-        ctx
-            The context object which was passed to the model.
+        schema : Schema
+            The schema being validated.
+        column_id : str
+            The column ID to validate.
+        loc : tuple
+            The location of the error in the schema.
+        errors : list[InitErrorDetails]
+            The list of errors to append to.
+        """
+        if column_id not in self:
+            _append_error(
+                errors,
+                loc,
+                column_id,
+                f"Column ID '{column_id}' not found in schema",
+            )
+        elif not isinstance(self[column_id], Column):
+            _append_error(
+                errors,
+                loc,
+                column_id,
+                f"ID '{column_id}' does not refer to a Column object",
+            )
 
-        Notes
-        -----
-        This method is called automatically by Pydantic after the model is
-        initialized. It is used to create the ID map for the schema.
+    def _validate_foreign_key_column(
+        self: Schema,
+        column_id: str,
+        table: Table,
+        loc: tuple,
+        errors: list[InitErrorDetails],
+    ) -> None:
+        """Validate a foreign key column ID from a constraint and append errors
+        if invalid.
 
-        The ``ctx`` argument has the type `Any` because this is the function
-        signature in Pydantic itself.
+        Parameters
+        ----------
+        schema : Schema
+            The schema being validated.
+        column_id : str
+            The foreign key column ID to validate.
+        loc : tuple
+            The location of the error in the schema.
+        errors : list[InitErrorDetails]
+            The list of errors to append to.
         """
-        self._create_id_map()
+        try:
+            table._find_column_by_id(column_id)
+        except KeyError:
+            _append_error(
+                errors,
+                loc,
+                column_id,
+                f"Column '{column_id}' not found in table '{table.name}'",
+            )
+
+    @model_validator(mode="after")
+    def check_constraints(self: Schema) -> Schema:
+        """Check constraint objects for validity. This needs to be deferred
+        until after the schema is fully loaded and the ID map is created.
+
+        Raises
+        ------
+        pydantic.ValidationError
+            Raised if any constraints are invalid.
+
+        Returns
+        -------
+        `Schema`
+            The schema being validated.
+        """
+        errors: list[InitErrorDetails] = []
+
+        for table_index, table in enumerate(self.tables):
+            for constraint_index, constraint in enumerate(table.constraints):
+                column_ids: list[str] = []
+                referenced_column_ids: list[str] = []
+
+                if isinstance(constraint, ForeignKeyConstraint):
+                    column_ids += constraint.columns
+                    referenced_column_ids += constraint.referenced_columns
+                elif isinstance(constraint, UniqueConstraint):
+                    column_ids += constraint.columns
+                # No extra checks are required on CheckConstraint objects.
+
+                # Validate the foreign key columns
+                for column_id in column_ids:
+                    self._validate_column_id(
+                        column_id,
+                        (
+                            "tables",
+                            table_index,
+                            "constraints",
+                            constraint_index,
+                            "columns",
+                            column_id,
+                        ),
+                        errors,
+                    )
+                    # Check that the foreign key column is within the source
+                    # table.
+                    self._validate_foreign_key_column(
+                        column_id,
+                        table,
+                        (
+                            "tables",
+                            table_index,
+                            "constraints",
+                            constraint_index,
+                            "columns",
+                            column_id,
+                        ),
+                        errors,
+                    )
+
+                # Validate the primary key (reference) columns
+                for referenced_column_id in referenced_column_ids:
+                    self._validate_column_id(
+                        referenced_column_id,
+                        (
+                            "tables",
+                            table_index,
+                            "constraints",
+                            constraint_index,
+                            "referenced_columns",
+                            referenced_column_id,
+                        ),
+                        errors,
+                    )
+
+        if errors:
+            raise ValidationError.from_exception_data("Schema validation failed", errors)
+
+        return self
 
     def __getitem__(self, id: str) -> BaseObject:
         """Get an object by its ID.
@@ -1388,31 +1577,56 @@ class Schema(BaseObject, Generic[T]):
         yaml_data = yaml.safe_load(source)
         return Schema.model_validate(yaml_data, context=context)
 
-    def dump_yaml(self, stream: IO[str] = sys.stdout) -> None:
+    def _model_dump(self, strip_ids: bool = False) -> dict[str, Any]:
+        """Dump the schema as a dictionary with some default arguments
+        applied.
+
+        Parameters
+        ----------
+        strip_ids
+            Whether to strip the IDs from the dumped data. Defaults to `False`.
+
+        Returns
+        -------
+        `dict` [ `str`, `Any` ]
+            The dumped schema data as a dictionary.
+        """
+        data = self.model_dump(by_alias=True, exclude_none=True, exclude_defaults=True)
+        if strip_ids:
+            data = _strip_ids(data)
+        return data
+
+    def dump_yaml(self, stream: IO[str] = sys.stdout, strip_ids: bool = False) -> None:
         """Pretty print the schema as YAML.
 
         Parameters
         ----------
         stream
             The stream to write the YAML data to.
+        strip_ids
+            Whether to strip the IDs from the dumped data. Defaults to `False`.
         """
+        data = self._model_dump(strip_ids=strip_ids)
         yaml.safe_dump(
-            self.model_dump(by_alias=True, exclude_none=True, exclude_defaults=True),
+            data,
             stream,
             default_flow_style=False,
             sort_keys=False,
         )
 
-    def dump_json(self, stream: IO[str] = sys.stdout) -> None:
+    def dump_json(self, stream: IO[str] = sys.stdout, strip_ids: bool = False) -> None:
         """Pretty print the schema as JSON.
 
         Parameters
         ----------
         stream
             The stream to write the JSON data to.
+        strip_ids
+            Whether to strip the IDs from the dumped data. Defaults to `False`.
         """
+        data = self._model_dump(strip_ids=strip_ids)
         json.dump(
-            self.model_dump(by_alias=True, exclude_none=True, exclude_defaults=True),
+            data,
             stream,
             indent=4,
             sort_keys=False,

{lsst_felis-29.2025.2400 → lsst_felis-29.2025.2600}/python/felis/metadata.py

@@ -338,12 +338,17 @@ class MetaDataBuilder:
             "deferrable": constraint_obj.deferrable or None,
             "initially": constraint_obj.initially or None,
         }
+
         constraint: Constraint
 
         if isinstance(constraint_obj, datamodel.ForeignKeyConstraint):
             fk_obj: datamodel.ForeignKeyConstraint = constraint_obj
             columns = [self._objects[column_id] for column_id in fk_obj.columns]
             refcolumns = [self._objects[column_id] for column_id in fk_obj.referenced_columns]
+            if constraint_obj.on_delete is not None:
+                args["ondelete"] = constraint_obj.on_delete
+            if constraint_obj.on_update is not None:
+                args["onupdate"] = constraint_obj.on_update
             constraint = ForeignKeyConstraint(columns, refcolumns, **args)
         elif isinstance(constraint_obj, datamodel.CheckConstraint):
             check_obj: datamodel.CheckConstraint = constraint_obj

{lsst_felis-29.2025.2400 → lsst_felis-29.2025.2600}/python/felis/tap_schema.py

@@ -21,6 +21,8 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
+import csv
+import io
 import logging
 import os
 import re
@@ -208,7 +210,7 @@ class TableManager:
         str
             The path to the standard TAP_SCHEMA schema resource.
         """
-        return os.path.join(os.path.dirname(__file__), "schemas", "tap_schema_std.yaml")
+        return os.path.join(os.path.dirname(__file__), "config", "tap_schema", "tap_schema_std.yaml")
 
     @classmethod
     def get_tap_schema_std_resource(cls) -> ResourcePath:
@@ -219,7 +221,7 @@ class TableManager:
         `~lsst.resources.ResourcePath`
             The standard TAP_SCHEMA schema resource.
         """
-        return ResourcePath("resource://felis/schemas/tap_schema_std.yaml")
+        return ResourcePath("resource://felis/config/tap_schema/tap_schema_std.yaml")
 
     @classmethod
     def get_table_names_std(cls) -> list[str]:
@@ -708,3 +710,48 @@ class DataLoader:
             if index.columns and len(index.columns) == 1 and index.columns[0] == column.id:
                 return 1
         return 0
+
+
+class MetadataInserter:
+    """Insert TAP_SCHEMA self-description rows into the database.
+
+    Parameters
+    ----------
+    mgr
+        The table manager that contains the TAP_SCHEMA tables.
+    engine
+        The engine for connecting to the TAP_SCHEMA database.
+    """
+
+    def __init__(self, mgr: TableManager, engine: Engine):
+        """Initialize the metadata inserter.
+
+        Parameters
+        ----------
+        mgr
+            The table manager representing the TAP_SCHEMA tables.
+        engine
+            The SQLAlchemy engine for connecting to the database.
+        """
+        self._mgr = mgr
+        self._engine = engine
+
+    def insert_metadata(self) -> None:
+        """Insert the TAP_SCHEMA metadata into the database."""
+        for table_name in self._mgr.get_table_names_std():
+            table = self._mgr[table_name]
+            csv_bytes = ResourcePath(f"resource://felis/config/tap_schema/{table_name}.csv").read()
+            text_stream = io.TextIOWrapper(io.BytesIO(csv_bytes), encoding="utf-8")
+            reader = csv.reader(text_stream)
+            headers = next(reader)
+            rows = [
+                {key: None if value == "\\N" else value for key, value in zip(headers, row)} for row in reader
+            ]
+            logger.debug(
+                "Inserting %d rows into table '%s' with headers: %s",
+                len(rows),
+                table_name,
+                headers,
+            )
+            with self._engine.begin() as conn:
+                conn.execute(table.insert(), rows)

{lsst_felis-29.2025.2400 → lsst_felis-29.2025.2600/python/lsst_felis.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lsst-felis
-Version: 29.2025.2400
+Version: 29.2025.2600
 Summary: A vocabulary for describing catalogs and acting on those descriptions
 Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
 License: GNU General Public License v3 or later (GPLv3+)

{lsst_felis-29.2025.2400 → lsst_felis-29.2025.2600}/python/lsst_felis.egg-info/SOURCES.txt

@@ -11,13 +11,18 @@ python/felis/metadata.py
 python/felis/py.typed
 python/felis/tap_schema.py
 python/felis/types.py
+python/felis/config/tap_schema/columns.csv
+python/felis/config/tap_schema/key_columns.csv
+python/felis/config/tap_schema/keys.csv
+python/felis/config/tap_schema/schemas.csv
+python/felis/config/tap_schema/tables.csv
+python/felis/config/tap_schema/tap_schema_std.yaml
 python/felis/db/__init__.py
 python/felis/db/dialects.py
 python/felis/db/schema.py
 python/felis/db/sqltypes.py
 python/felis/db/utils.py
 python/felis/db/variants.py
-python/felis/schemas/tap_schema_std.yaml
 python/felis/tests/__init__.py
 python/felis/tests/postgresql.py
 python/felis/tests/run_cli.py

{lsst_felis-29.2025.2400 → lsst_felis-29.2025.2600}/tests/test_cli.py

@@ -23,7 +23,9 @@ import os
 import shutil
 import tempfile
 import unittest
+from typing import Any
 
+import yaml
 from sqlalchemy import create_engine
 
 import felis.tap_schema as tap_schema
@@ -170,11 +172,51 @@ class CliTestCase(unittest.TestCase):
         with tempfile.NamedTemporaryFile(delete=True, suffix=".yaml") as temp_file:
             run_cli(["dump", TEST_YAML, temp_file.name], print_output=True)
 
+    @classmethod
+    def _check_strip_ids(cls, obj: Any) -> None:
+        """
+        Recursively check that a dict/list structure has no attributes with key
+        '@id'. Raises a ValueError if any '@id' key is found. This is used to
+        check the output of the `--strip-ids` option in the `dump` command.
+        """
+        if isinstance(obj, dict):
+            for k, v in obj.items():
+                if k == "@id":
+                    raise ValueError("Found forbidden key '@id'")
+                cls._check_strip_ids(v)
+        elif isinstance(obj, list):
+            for item in obj:
+                cls._check_strip_ids(item)
+
+    def test_dump_yaml_with_strip_ids(self) -> None:
+        """Test for ``dump`` command with YAML output and stripped IDs."""
+        with tempfile.NamedTemporaryFile(delete=True, suffix=".yaml") as temp_file:
+            run_cli(["dump", "--strip-ids", TEST_YAML, temp_file.name], print_output=True)
+            dumped_data = temp_file.read().decode("utf-8")
+            try:
+                # Load the dumped YAML data to check for '@id' keys.
+                data = yaml.safe_load(dumped_data)
+                self._check_strip_ids(data)
+            except ValueError:
+                self.fail("Dumped YAML contains forbidden key '@id'")
+
     def test_dump_json(self) -> None:
         """Test for ``dump`` command with JSON output."""
         with tempfile.NamedTemporaryFile(delete=True, suffix=".json") as temp_file:
             run_cli(["dump", TEST_YAML, temp_file.name], print_output=True)
 
+    def test_dump_json_with_strip_ids(self) -> None:
+        """Test for ``dump`` command with JSON output."""
+        with tempfile.NamedTemporaryFile(delete=True, suffix=".json") as temp_file:
+            run_cli(["dump", "--strip-ids", TEST_YAML, temp_file.name], print_output=True)
+            dumped_data = temp_file.read().decode("utf-8")
+            try:
+                # Load the dumped YAML data to check for '@id' keys.
+                data = yaml.safe_load(dumped_data)
+                self._check_strip_ids(data)
+            except ValueError:
+                self.fail("Dumped YAML contains forbidden key '@id'")
+
     def test_dump_with_invalid_file_extension_error(self) -> None:
         """Test for ``dump`` command with JSON output."""
         run_cli(["dump", TEST_YAML, "out.bad"], expect_error=True)

{lsst_felis-29.2025.2400 → lsst_felis-29.2025.2600}/tests/test_datamodel.py

@@ -451,6 +451,105 @@ class ConstraintTestCase(unittest.TestCase):
         with self.assertRaises(ValidationError):
             UniqueConstraint(name="uniq_test", id="#uniq_test", columns=["test_column"], type="BAD_TYPE")
 
+    def test_constraint_column_checks(self) -> None:
+        """Test the extra validation in the ``Schema`` that checks the
+        constraint column references.
+        """
+
+        def _create_test_schema(constraint: Constraint) -> None:
+            """Create a test schema with the given constraint."""
+            test_col = Column(name="testColumn", id="#test_col_id", datatype="int")
+            test_col2 = Column(name="testColumn2", id="#test_col_id2", datatype="int")
+            test_tbl = Table(
+                name="testTable", id="#test_tbl_id", columns=[test_col, test_col2], constraints=[constraint]
+            )
+            test_col = Column(name="testColumn", id="#test_col2_id", datatype="int")
+            test_col2 = Column(name="testColumn2", id="#test_col2_id2", datatype="int")
+            test_tbl2 = Table(name="testTable2", id="#test_tbl2_id", columns=[test_col, test_col2])
+            Schema(name="testSchema", id="#test_schema_id", tables=[test_tbl, test_tbl2])
+
+        # Creating a unique constraint on a bad column should raise an
+        # exception.
+        with self.assertRaises(ValidationError):
+            _create_test_schema(
+                UniqueConstraint(name="testConstraint", id="#test_constraint_id", columns=["bad_column"])
+            )
+
+        # Creating a foreign key constraint with a bad column should raise an
+        # exception.
+        with self.assertRaises(ValidationError):
+            _create_test_schema(
+                ForeignKeyConstraint(
+                    name="testForeignKey",
+                    id="#test_fk_id",
+                    columns=["bad_column"],
+                    referenced_columns=["#test_col2_id"],
+                )
+            )
+
+        # Creating a foreign key constraint with a bad referenced column should
+        # raise an exception.
+        with self.assertRaises(ValidationError):
+            _create_test_schema(
+                ForeignKeyConstraint(
+                    name="testForeignKey",
+                    id="#test_fk_id",
+                    columns=["#test_col_id"],
+                    referenced_columns=["bad_column"],
+                )
+            )
+
+        # Creating a foreign key constraint where the source column is not in
+        # the same table as the constraint should raise an exception.
+        with self.assertRaises(ValidationError):
+            _create_test_schema(
+                ForeignKeyConstraint(
+                    name="testForeignKey",
+                    id="#test_fk_id",
+                    columns=["#test_col2_id"],  # This column is in test_tbl2, not test_tbl
+                    referenced_columns=["#test_col_id"],
+                )
+            )
+
+        # Creating a foreign key constraint where the referenced column is not
+        # a column object should raise an exception.
+        with self.assertRaises(ValidationError):
+            _create_test_schema(
+                ForeignKeyConstraint(
+                    name="testForeignKey",
+                    id="#test_fk_id",
+                    columns=["#test_col_id"],
+                    referenced_columns=["#test_schema_id"],
+                )
+            )
+
+        # Creating a valid unique constraint should not raise an exception.
+        _create_test_schema(
+            UniqueConstraint(name="testConstraint", id="#test_constraint_id", columns=["#test_col_id"])
+        )
+
+        # Creating a valid foreign key constraint should not raise an
+        # exception.
+        _create_test_schema(
+            ForeignKeyConstraint(
+                name="testForeignKey",
+                id="#test_fk_id",
+                columns=["#test_col_id"],
+                referenced_columns=["#test_col2_id"],
+            )
+        )
+
+        # Creating a foreign key constraint with a composite key should not
+        # raise an exception.
+        _create_test_schema(
+            ForeignKeyConstraint(
+                name="testCompositeForeignKey",
+                id="#test_composite_fk_id",
+                columns=["#test_col_id", "#test_col_id2"],
+                referenced_columns=["#test_col2_id", "#test_col2_id2"],
+            )
+        )
+
 
 class IndexTestCase(unittest.TestCase):
     """Test Pydantic validation of the ``Index`` class."""
@@ -575,8 +674,8 @@ class SchemaTestCase(unittest.TestCase):
 
     def test_check_unique_index_names(self) -> None:
         """Test that index names are unique."""
-        test_col = Column(name="test_column1", id="#test_table#test_column1", datatype="int")
-        test_col2 = Column(name="test_column2", id="##test_table#test_column2", datatype="string", length=256)
+        test_col = Column(name="test_column1", id="#test_table.test_column1", datatype="int")
+        test_col2 = Column(name="test_column2", id="#test_table.test_column2", datatype="string", length=256)
         test_tbl = Table(name="test_table", id="#test_table", columns=[test_col, test_col2])
         test_idx = Index(name="idx_test", id="#idx_test", columns=[test_col.id])
         test_idx2 = Index(name="idx_test", id="#idx_test2", columns=[test_col2.id])
@@ -646,13 +745,14 @@ class SchemaTestCase(unittest.TestCase):
         """Test loading a schema from a resource."""
         # Test loading a schema from a resource string.
         schema = Schema.from_uri(
-            "resource://felis/schemas/tap_schema_std.yaml", context={"id_generation": True}
+            "resource://felis/config/tap_schema/tap_schema_std.yaml", context={"id_generation": True}
         )
         self.assertIsInstance(schema, Schema)
 
         # Test loading a schema from a ResourcePath.
         schema = Schema.from_uri(
-            ResourcePath("resource://felis/schemas/tap_schema_std.yaml"), context={"id_generation": True}
+            ResourcePath("resource://felis/config/tap_schema/tap_schema_std.yaml"),
+            context={"id_generation": True},
         )
         self.assertIsInstance(schema, Schema)
 
@@ -662,7 +762,7 @@ class SchemaTestCase(unittest.TestCase):
 
         # Without ID generation enabled, this schema should fail validation.
         with self.assertRaises(ValidationError):
-            Schema.from_uri("resource://felis/schemas/tap_schema_std.yaml")
+            Schema.from_uri("resource://felis/config/tap_schema/tap_schema_std.yaml")
 
 
 class SchemaVersionTest(unittest.TestCase):

{lsst_felis-29.2025.2400 → lsst_felis-29.2025.2600}/tests/test_metadata.py

@@ -118,6 +118,8 @@ class MetaDataTestCase(unittest.TestCase):
                         md_db_fk: ForeignKeyConstraint = md_db_constraint
                         self.assertEqual(md_fk.referred_table.name, md_db_fk.referred_table.name)
                         self.assertEqual(md_fk.column_keys, md_db_fk.column_keys)
+                        self.assertEqual(md_fk.ondelete, md_db_fk.ondelete)
+                        self.assertEqual(md_fk.onupdate, md_db_fk.onupdate)
                     elif isinstance(md_constraint, UniqueConstraint) and isinstance(
                         md_db_constraint, UniqueConstraint
                     ):
@@ -240,6 +242,21 @@ class MetaDataTestCase(unittest.TestCase):
         for table in md.tables.values():
             self.assertTrue(table.name.endswith("_test"))
 
+    def test_fk_actions(self) -> None:
+        """Test that foreign key constraints with on delete and on update
+        actions are created correctly.
+        """
+        schema = Schema.model_validate(self.yaml_data)
+        schema.name = "main"
+        builder = MetaDataBuilder(schema)
+        md = builder.build()
+
+        for table in md.tables.values():
+            for constraint in table.constraints:
+                if isinstance(constraint, ForeignKeyConstraint):
+                    self.assertIn(constraint.ondelete, ["SET NULL"])
+                    self.assertIn(constraint.onupdate, ["CASCADE"])
+
 
 if __name__ == "__main__":
     unittest.main()