lsst-felis 28.2025.402__tar.gz → 28.2025.600__tar.gz

{lsst_felis-28.2025.402/python/lsst_felis.egg-info → lsst_felis-28.2025.600}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: lsst-felis
-Version: 28.2025.402
+Version: 28.2025.600
 Summary: A vocabulary for describing catalogs and acting on those descriptions
 Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
 License: GNU General Public License v3 or later (GPLv3+)

{lsst_felis-28.2025.402 → lsst_felis-28.2025.600}/python/felis/cli.py

@@ -38,7 +38,6 @@ from .db.schema import create_database
 from .db.utils import DatabaseContext, is_mock_url
 from .diff import DatabaseDiff, FormattedSchemaDiff, SchemaDiff
 from .metadata import MetaDataBuilder
-from .tap import Tap11Base, TapLoadingVisitor, init_tables
 from .tap_schema import DataLoader, TableManager
 
 __all__ = ["cli"]
@@ -179,174 +178,9 @@ def create(
         raise click.ClickException(str(e))
 
 
-@cli.command("init-tap", help="Initialize TAP_SCHEMA objects in the database")
-@click.option("--tap-schema-name", help="Alternate database schema name for 'TAP_SCHEMA'")
-@click.option("--tap-schemas-table", help="Alternate table name for 'schemas'")
-@click.option("--tap-tables-table", help="Alternate table name for 'tables'")
-@click.option("--tap-columns-table", help="Alternate table name for 'columns'")
-@click.option("--tap-keys-table", help="Alternate table name for 'keys'")
-@click.option("--tap-key-columns-table", help="Alternate table name for 'key_columns'")
-@click.argument("engine-url")
-def init_tap(
-    engine_url: str,
-    tap_schema_name: str,
-    tap_schemas_table: str,
-    tap_tables_table: str,
-    tap_columns_table: str,
-    tap_keys_table: str,
-    tap_key_columns_table: str,
-) -> None:
-    """Initialize TAP_SCHEMA objects in the database.
-
-    Parameters
-    ----------
-    engine_url
-        SQLAlchemy Engine URL. The target PostgreSQL schema or MySQL database
-        must already exist and be referenced in the URL.
-    tap_schema_name
-        Alterate name for the database schema ``TAP_SCHEMA``.
-    tap_schemas_table
-        Alterate table name for ``schemas``.
-    tap_tables_table
-        Alterate table name for ``tables``.
-    tap_columns_table
-        Alterate table name for ``columns``.
-    tap_keys_table
-        Alterate table name for ``keys``.
-    tap_key_columns_table
-        Alterate table name for ``key_columns``.
-
-    Notes
-    -----
-    The supported version of TAP_SCHEMA in the SQLAlchemy metadata is 1.1. The
-    tables are created in the database schema specified by the engine URL,
-    which must be a PostgreSQL schema or MySQL database that already exists.
-    """
-    engine = create_engine(engine_url)
-    init_tables(
-        tap_schema_name,
-        tap_schemas_table,
-        tap_tables_table,
-        tap_columns_table,
-        tap_keys_table,
-        tap_key_columns_table,
-    )
-    Tap11Base.metadata.create_all(engine)
-
-
-@cli.command("load-tap", help="Load metadata from a Felis file into a TAP_SCHEMA database")
-@click.option("--engine-url", envvar="FELIS_ENGINE_URL", help="SQLAlchemy Engine URL")
-@click.option("--schema-name", help="Alternate Schema Name for Felis file")
-@click.option("--catalog-name", help="Catalog Name for Schema")
-@click.option("--dry-run", is_flag=True, help="Dry Run Only. Prints out the DDL that would be executed")
-@click.option("--tap-schema-name", help="Alternate schema name for 'TAP_SCHEMA'")
-@click.option("--tap-tables-postfix", help="Postfix for TAP_SCHEMA table names")
-@click.option("--tap-schemas-table", help="Alternate table name for 'schemas'")
-@click.option("--tap-tables-table", help="Alternate table name for 'tables'")
-@click.option("--tap-columns-table", help="Alternate table name for 'columns'")
-@click.option("--tap-keys-table", help="Alternate table name for 'keys'")
-@click.option("--tap-key-columns-table", help="Alternate table name for 'key_columns'")
-@click.option("--tap-schema-index", type=int, help="TAP_SCHEMA index of the schema in this environment")
-@click.argument("file", type=click.File())
-def load_tap(
-    engine_url: str,
-    schema_name: str,
-    catalog_name: str,
-    dry_run: bool,
-    tap_schema_name: str,
-    tap_tables_postfix: str,
-    tap_schemas_table: str,
-    tap_tables_table: str,
-    tap_columns_table: str,
-    tap_keys_table: str,
-    tap_key_columns_table: str,
-    tap_schema_index: int,
-    file: IO[str],
-) -> None:
-    """Load TAP metadata from a Felis file.
-
-    This command loads the associated TAP metadata from a Felis YAML file
-    into the TAP_SCHEMA tables.
-
-    Parameters
-    ----------
-    engine_url
-        SQLAlchemy Engine URL to catalog.
-    schema_name
-        Alternate schema name. This overrides the schema name in the
-        ``catalog`` field of the Felis file.
-    catalog_name
-        Catalog name for the schema. This possibly duplicates the
-        ``tap_schema_name`` argument (DM-44870).
-    dry_run
-        Dry run only to print out commands instead of executing.
-    tap_schema_name
-        Alternate name for the schema of TAP_SCHEMA in the database.
-    tap_tables_postfix
-        Postfix for TAP table names that will be automatically appended.
-    tap_schemas_table
-        Alternate table name for ``schemas``.
-    tap_tables_table
-        Alternate table name for ``tables``.
-    tap_columns_table
-        Alternate table name for ``columns``.
-    tap_keys_table
-        Alternate table name for ``keys``.
-    tap_key_columns_table
-        Alternate table name for ``key_columns``.
-    tap_schema_index
-        TAP_SCHEMA index of the schema in this TAP environment.
-    file
-        Felis file to read.
-
-    Notes
-    -----
-    The data will be loaded into the TAP_SCHEMA from the engine URL. The
-    tables must have already been initialized or an error will occur.
-    """
-    schema = Schema.from_stream(file)
-
-    tap_tables = init_tables(
-        tap_schema_name,
-        tap_tables_postfix,
-        tap_schemas_table,
-        tap_tables_table,
-        tap_columns_table,
-        tap_keys_table,
-        tap_key_columns_table,
-    )
-
-    if not dry_run:
-        engine = create_engine(engine_url)
-
-        if engine_url == "sqlite://" and not dry_run:
-            # In Memory SQLite - Mostly used to test
-            Tap11Base.metadata.create_all(engine)
-
-        tap_visitor = TapLoadingVisitor(
-            engine,
-            catalog_name=catalog_name,
-            schema_name=schema_name,
-            tap_tables=tap_tables,
-            tap_schema_index=tap_schema_index,
-        )
-        tap_visitor.visit_schema(schema)
-    else:
-        conn = DatabaseContext.create_mock_engine(engine_url)
-
-        tap_visitor = TapLoadingVisitor.from_mock_connection(
-            conn,
-            catalog_name=catalog_name,
-            schema_name=schema_name,
-            tap_tables=tap_tables,
-            tap_schema_index=tap_schema_index,
-        )
-        tap_visitor.visit_schema(schema)
-
-
 @cli.command("load-tap-schema", help="Load metadata from a Felis file into a TAP_SCHEMA database")
 @click.option("--engine-url", envvar="FELIS_ENGINE_URL", help="SQLAlchemy Engine URL")
-@click.option("--tap-schema-name", help="Name of the TAP_SCHEMA schema in the database")
+@click.option("--tap-schema-name", help="Name of the TAP_SCHEMA schema in the database (default: TAP_SCHEMA)")
 @click.option(
     "--tap-tables-postfix", help="Postfix which is applied to standard TAP_SCHEMA table names", default=""
 )
@@ -417,6 +251,40 @@ def load_tap_schema(
     ).load()
 
 
+@cli.command("init-tap-schema", help="Initialize a standard TAP_SCHEMA database")
+@click.option("--engine-url", envvar="FELIS_ENGINE_URL", help="SQLAlchemy Engine URL")
+@click.option("--tap-schema-name", help="Name of the TAP_SCHEMA schema in the database")
+@click.option(
+    "--tap-tables-postfix", help="Postfix which is applied to standard TAP_SCHEMA table names", default=""
+)
+@click.pass_context
+def init_tap_schema(
+    ctx: click.Context, engine_url: str, tap_schema_name: str, tap_tables_postfix: str
+) -> None:
+    """Initialize a standard TAP_SCHEMA database.
+
+    Parameters
+    ----------
+    engine_url
+        SQLAlchemy Engine URL.
+    tap_schema_name
+        Name of the TAP_SCHEMA schema in the database.
+    tap_tables_postfix
+        Postfix which is applied to standard TAP_SCHEMA table names.
+    """
+    url = make_url(engine_url)
+    engine: Engine | MockConnection
+    if is_mock_url(url):
+        raise click.ClickException("Mock engine URL is not supported for this command")
+    engine = create_engine(engine_url)
+    mgr = TableManager(
+        apply_schema_to_metadata=False if engine.dialect.name == "sqlite" else True,
+        schema_name=tap_schema_name,
+        table_name_postfix=tap_tables_postfix,
+    )
+    mgr.initialize_database(engine)
+
+
 @cli.command("validate", help="Validate one or more Felis YAML files")
 @click.option(
     "--check-description", is_flag=True, help="Check that all objects have a description", default=False

{lsst_felis-28.2025.402 → lsst_felis-28.2025.600}/python/felis/datamodel.py

@@ -134,6 +134,32 @@ class DataType(StrEnum):
     timestamp = auto()
 
 
+def validate_ivoa_ucd(ivoa_ucd: str) -> str:
+    """Validate IVOA UCD values.
+
+    Parameters
+    ----------
+    ivoa_ucd
+        IVOA UCD value to check.
+
+    Returns
+    -------
+    `str`
+        The IVOA UCD value if it is valid.
+
+    Raises
+    ------
+    ValueError
+        If the IVOA UCD value is invalid.
+    """
+    if ivoa_ucd is not None:
+        try:
+            ucd.parse_ucd(ivoa_ucd, check_controlled_vocabulary=True, has_colon=";" in ivoa_ucd)
+        except ValueError as e:
+            raise ValueError(f"Invalid IVOA UCD: {e}")
+    return ivoa_ucd
+
+
 class Column(BaseObject):
     """Column model."""
 
@@ -235,12 +261,7 @@ class Column(BaseObject):
         `str`
             The IVOA UCD value if it is valid.
         """
-        if ivoa_ucd is not None:
-            try:
-                ucd.parse_ucd(ivoa_ucd, check_controlled_vocabulary=True, has_colon=";" in ivoa_ucd)
-            except ValueError as e:
-                raise ValueError(f"Invalid IVOA UCD: {e}")
-        return ivoa_ucd
+        return validate_ivoa_ucd(ivoa_ucd)
 
     @model_validator(mode="after")
     def check_units(self) -> Column:
@@ -551,6 +572,70 @@ _ConstraintType = Annotated[
 """Type alias for a constraint type."""
 
 
+ColumnRef: TypeAlias = str
+"""Type alias for a column reference."""
+
+
+class ColumnGroup(BaseObject):
+    """Column group model."""
+
+    columns: list[ColumnRef | Column] = Field(..., min_length=1)
+    """Columns in the group."""
+
+    ivoa_ucd: str | None = Field(None, alias="ivoa:ucd")
+    """IVOA UCD of the column."""
+
+    table: Table | None = None
+    """Reference to the parent table."""
+
+    @field_validator("ivoa_ucd")
+    @classmethod
+    def check_ivoa_ucd(cls, ivoa_ucd: str) -> str:
+        """Check that IVOA UCD values are valid.
+
+        Parameters
+        ----------
+        ivoa_ucd
+            IVOA UCD value to check.
+
+        Returns
+        -------
+        `str`
+            The IVOA UCD value if it is valid.
+        """
+        return validate_ivoa_ucd(ivoa_ucd)
+
+    @model_validator(mode="after")
+    def check_unique_columns(self) -> ColumnGroup:
+        """Check that the columns list contains unique items.
+
+        Returns
+        -------
+        `ColumnGroup`
+            The column group being validated.
+        """
+        column_ids = [col if isinstance(col, str) else col.id for col in self.columns]
+        if len(column_ids) != len(set(column_ids)):
+            raise ValueError("Columns in the group must be unique")
+        return self
+
+    def _dereference_columns(self) -> None:
+        """Dereference ColumnRef to Column objects."""
+        if self.table is None:
+            raise ValueError("ColumnGroup must have a reference to its parent table")
+
+        dereferenced_columns: list[ColumnRef | Column] = []
+        for col in self.columns:
+            if isinstance(col, str):
+                # Dereference ColumnRef to Column object
+                col_obj = self.table._find_column_by_id(col)
+                dereferenced_columns.append(col_obj)
+            else:
+                dereferenced_columns.append(col)
+
+        self.columns = dereferenced_columns
+
+
 class Table(BaseObject):
     """Table model."""
 
@@ -563,6 +648,9 @@ class Table(BaseObject):
     indexes: list[Index] = Field(default_factory=list)
     """Indexes on the table."""
 
+    column_groups: list[ColumnGroup] = Field(default_factory=list, alias="columnGroups")
+    """Column groups in the table."""
+
     primary_key: str | list[str] | None = Field(None, alias="primaryKey")
     """Primary key of the table."""
 
@@ -653,6 +741,43 @@ class Table(BaseObject):
                 return self
         raise ValueError(f"Table '{self.name}' is missing at least one column designated as 'tap:principal'")
 
+    def _find_column_by_id(self, id: str) -> Column:
+        """Find a column by ID.
+
+        Parameters
+        ----------
+        id
+            The ID of the column to find.
+
+        Returns
+        -------
+        `Column`
+            The column with the given ID.
+
+        Raises
+        ------
+        ValueError
+            Raised if the column is not found.
+        """
+        for column in self.columns:
+            if column.id == id:
+                return column
+        raise ValueError(f"Column '{id}' not found in table '{self.name}'")
+
+    @model_validator(mode="after")
+    def dereference_column_groups(self: Table) -> Table:
+        """Dereference columns in column groups.
+
+        Returns
+        -------
+        `Table`
+            The table with dereferenced column groups.
+        """
+        for group in self.column_groups:
+            group.table = self
+            group._dereference_columns()
+        return self
+
 
 class SchemaVersion(BaseModel):
     """Schema version model."""

{lsst_felis-28.2025.402 → lsst_felis-28.2025.600}/python/felis/metadata.py

@@ -127,6 +127,8 @@ class MetaDataBuilder:
         Whether to apply the schema name to the metadata object.
     ignore_constraints
         Whether to ignore constraints when building the metadata.
+    table_name_postfix
+        A string to append to the table names when building the metadata.
     """
 
     def __init__(
@@ -134,6 +136,7 @@ class MetaDataBuilder:
         schema: Schema,
         apply_schema_to_metadata: bool = True,
         ignore_constraints: bool = False,
+        table_name_postfix: str = "",
     ) -> None:
         """Initialize the metadata builder."""
         self.schema = schema
@@ -142,6 +145,7 @@ class MetaDataBuilder:
         self.metadata = MetaData(schema=schema.name if apply_schema_to_metadata else None)
         self._objects: dict[str, Any] = {}
         self.ignore_constraints = ignore_constraints
+        self.table_name_postfix = table_name_postfix
 
     def build(self) -> MetaData:
         """Build the SQLAlchemy tables and constraints from the schema.
@@ -225,7 +229,7 @@ class MetaDataBuilder:
         description = table_obj.description
         columns = [self.build_column(column) for column in table_obj.columns]
         table = Table(
-            name,
+            name + self.table_name_postfix,
             self.metadata,
             *columns,
             comment=description,

{lsst_felis-28.2025.402 → lsst_felis-28.2025.600}/python/felis/tap_schema.py

@@ -91,9 +91,15 @@ class TableManager:
         self.table_name_postfix = table_name_postfix
         self.apply_schema_to_metadata = apply_schema_to_metadata
         self.schema_name = schema_name or TableManager._SCHEMA_NAME_STD
+        self.table_name_postfix = table_name_postfix
 
         if is_valid_engine(engine):
             assert isinstance(engine, Engine)
+            if table_name_postfix != "":
+                logger.warning(
+                    "Table name postfix '%s' will be ignored when reflecting TAP_SCHEMA database",
+                    table_name_postfix,
+                )
             logger.debug(
                 "Reflecting TAP_SCHEMA database from existing database at %s",
                 engine.url._replace(password="***"),
@@ -131,7 +137,9 @@ class TableManager:
             self.schema_name = self.schema.name
 
         self._metadata = MetaDataBuilder(
-            self.schema, apply_schema_to_metadata=self.apply_schema_to_metadata
+            self.schema,
+            apply_schema_to_metadata=self.apply_schema_to_metadata,
+            table_name_postfix=self.table_name_postfix,
         ).build()
 
         logger.debug("Loaded TAP_SCHEMA '%s' from YAML resource", self.schema_name)
@@ -353,7 +361,7 @@ class TableManager:
         engine
             The SQLAlchemy engine to use to create the tables.
         """
-        logger.info("Creating TAP_SCHEMA database '%s'", self.metadata.schema)
+        logger.info("Creating TAP_SCHEMA database '%s'", self.schema_name)
         self._create_schema(engine)
         self.metadata.create_all(engine)
 
@@ -424,7 +432,7 @@ class DataLoader:
             # Execute the inserts if not in dry run mode.
             self._execute_inserts()
         else:
-            logger.info("Dry run: not loading data into database")
+            logger.info("Dry run - not loading data into database")
 
     def _insert_schemas(self) -> None:
         """Insert the schema data into the schemas table."""
@@ -565,7 +573,7 @@ class DataLoader:
     def _print_sql(self) -> None:
         """Print the generated inserts to stdout."""
         for insert_str in self._compiled_inserts():
-            print(insert_str)
+            print(insert_str + ";")
 
     def _write_sql_to_file(self) -> None:
         """Write the generated insert statements to a file."""
@@ -573,7 +581,7 @@ class DataLoader:
             raise ValueError("No output path specified")
         with open(self.output_path, "w") as outfile:
             for insert_str in self._compiled_inserts():
-                outfile.write(insert_str + "\n")
+                outfile.write(insert_str + ";" + "\n")
 
     def _insert(self, table_name: str, record: list[Any] | dict[str, Any]) -> None:
         """Generate an insert statement for a record.

lsst_felis-28.2025.600/python/felis/version.py

@@ -0,0 +1,2 @@
+__all__ = ["__version__"]
+__version__ = "28.2025.600"

{lsst_felis-28.2025.402 → lsst_felis-28.2025.600/python/lsst_felis.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: lsst-felis
-Version: 28.2025.402
+Version: 28.2025.600
 Summary: A vocabulary for describing catalogs and acting on those descriptions
 Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
 License: GNU General Public License v3 or later (GPLv3+)

{lsst_felis-28.2025.402 → lsst_felis-28.2025.600}/python/lsst_felis.egg-info/SOURCES.txt

@@ -9,7 +9,6 @@ python/felis/datamodel.py
 python/felis/diff.py
 python/felis/metadata.py
 python/felis/py.typed
-python/felis/tap.py
 python/felis/tap_schema.py
 python/felis/types.py
 python/felis/version.py
@@ -35,6 +34,5 @@ tests/test_db.py
 tests/test_diff.py
 tests/test_metadata.py
 tests/test_postgres.py
-tests/test_tap.py
 tests/test_tap_schema.py
 tests/test_tap_schema_postgres.py

{lsst_felis-28.2025.402 → lsst_felis-28.2025.600}/tests/test_cli.py

@@ -30,7 +30,6 @@ from sqlalchemy import create_engine
 import felis.tap_schema as tap_schema
 from felis.cli import cli
 from felis.datamodel import Schema
-from felis.db.dialects import get_supported_dialects
 from felis.metadata import MetaDataBuilder
 
 TESTDIR = os.path.abspath(os.path.dirname(__file__))
@@ -91,38 +90,6 @@ class CliTestCase(unittest.TestCase):
         )
         self.assertEqual(result.exit_code, 0)
 
-    def test_init_tap(self) -> None:
-        """Test for ``init-tap`` command."""
-        url = f"sqlite:///{self.tmpdir}/tap.sqlite3"
-        runner = CliRunner()
-        result = runner.invoke(cli, ["init-tap", url], catch_exceptions=False)
-        self.assertEqual(result.exit_code, 0)
-
-    def test_load_tap(self) -> None:
-        """Test for ``load-tap`` command."""
-        # Cannot use the same url for both init-tap and load-tap in the same
-        # process.
-        url = f"sqlite:///{self.tmpdir}/tap.sqlite3"
-
-        # Need to run init-tap first.
-        runner = CliRunner()
-        result = runner.invoke(cli, ["init-tap", url])
-        self.assertEqual(result.exit_code, 0)
-
-        result = runner.invoke(cli, ["load-tap", f"--engine-url={url}", TEST_YAML], catch_exceptions=False)
-        self.assertEqual(result.exit_code, 0)
-
-    def test_load_tap_mock(self) -> None:
-        """Test ``load-tap --dry-run`` command on supported dialects."""
-        urls = [f"{dialect_name}://" for dialect_name in get_supported_dialects().keys()]
-
-        for url in urls:
-            runner = CliRunner()
-            result = runner.invoke(
-                cli, ["load-tap", f"--engine-url={url}", "--dry-run", TEST_YAML], catch_exceptions=False
-            )
-            self.assertEqual(result.exit_code, 0)
-
     def test_validate_default(self) -> None:
         """Test validate command."""
         runner = CliRunner()
@@ -172,9 +139,9 @@ class CliTestCase(unittest.TestCase):
         self.assertTrue(result.exit_code != 0)
 
     def test_load_tap_schema(self) -> None:
-        """Test for ``load-tap-schema`` command."""
+        """Test load-tap-schema command."""
         # Create the TAP_SCHEMA database.
-        url = f"sqlite:///{self.tmpdir}/tap_schema.sqlite3"
+        url = f"sqlite:///{self.tmpdir}/load_tap_schema.sqlite3"
         runner = CliRunner()
         tap_schema_path = tap_schema.TableManager.get_tap_schema_std_path()
         result = runner.invoke(
@@ -191,6 +158,21 @@ class CliTestCase(unittest.TestCase):
         )
         self.assertEqual(result.exit_code, 0)
 
+    def test_init_tap_schema(self) -> None:
+        """Test init-tap-schema command."""
+        url = f"sqlite:///{self.tmpdir}/init_tap_schema.sqlite3"
+        runner = CliRunner()
+        result = runner.invoke(cli, ["init-tap-schema", f"--engine-url={url}"], catch_exceptions=False)
+        self.assertEqual(result.exit_code, 0)
+
+    def test_init_tap_schema_mock(self) -> None:
+        """Test init-tap-schema command with a mock URL, which should throw
+        an error, as this is not supported.
+        """
+        runner = CliRunner()
+        result = runner.invoke(cli, ["init-tap-schema", "sqlite://"], catch_exceptions=False)
+        self.assertNotEqual(result.exit_code, 0)
+
     def test_diff(self) -> None:
         """Test for ``diff`` command."""
         test_diff1 = os.path.join(TESTDIR, "data", "test_diff1.yaml")