lsst-felis 28.2024.4500__py3-none-any.whl → 29.2025.4500__py3-none-any.whl

felis/tap_schema.py

@@ -21,13 +21,15 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
+import csv
+import io
 import logging
 import os
 import re
 from typing import Any
 
 from lsst.resources import ResourcePath
-from sqlalchemy import MetaData, Table, text
+from sqlalchemy import MetaData, Table, select, text
 from sqlalchemy.engine import Connection, Engine
 from sqlalchemy.engine.mock import MockConnection
 from sqlalchemy.exc import SQLAlchemyError
@@ -35,13 +37,13 @@ from sqlalchemy.schema import CreateSchema
 from sqlalchemy.sql.dml import Insert
 
 from felis import datamodel
-from felis.datamodel import Schema
+from felis.datamodel import Constraint, Schema
 from felis.db.utils import is_valid_engine
 from felis.metadata import MetaDataBuilder
 
 from .types import FelisType
 
-__all__ = ["TableManager", "DataLoader"]
+__all__ = ["DataLoader", "TableManager"]
 
 logger = logging.getLogger(__name__)
 
@@ -91,9 +93,15 @@ class TableManager:
         self.table_name_postfix = table_name_postfix
         self.apply_schema_to_metadata = apply_schema_to_metadata
         self.schema_name = schema_name or TableManager._SCHEMA_NAME_STD
+        self.table_name_postfix = table_name_postfix
 
         if is_valid_engine(engine):
             assert isinstance(engine, Engine)
+            if table_name_postfix != "":
+                logger.warning(
+                    "Table name postfix '%s' will be ignored when reflecting TAP_SCHEMA database",
+                    table_name_postfix,
+                )
             logger.debug(
                 "Reflecting TAP_SCHEMA database from existing database at %s",
                 engine.url._replace(password="***"),
@@ -133,7 +141,7 @@ class TableManager:
         self._metadata = MetaDataBuilder(
             self.schema,
             apply_schema_to_metadata=self.apply_schema_to_metadata,
-            apply_schema_to_tables=self.apply_schema_to_metadata,
+            table_name_postfix=self.table_name_postfix,
         ).build()
 
         logger.debug("Loaded TAP_SCHEMA '%s' from YAML resource", self.schema_name)
@@ -157,7 +165,7 @@ class TableManager:
         tables to be accessed by their standard TAP_SCHEMA names.
         """
         if table_name not in self._table_map:
-            raise KeyError(f"Table '{table_name}' not found in table map")
+            raise KeyError(f"Table '{table_name}' not found in TAP_SCHEMA")
         return self.metadata.tables[self._table_map[table_name]]
 
     @property
@@ -202,7 +210,7 @@ class TableManager:
         str
             The path to the standard TAP_SCHEMA schema resource.
         """
-        return os.path.join(os.path.dirname(__file__), "schemas", "tap_schema_std.yaml")
+        return os.path.join(os.path.dirname(__file__), "config", "tap_schema", "tap_schema_std.yaml")
 
     @classmethod
     def get_tap_schema_std_resource(cls) -> ResourcePath:
@@ -213,7 +221,7 @@ class TableManager:
         `~lsst.resources.ResourcePath`
             The standard TAP_SCHEMA schema resource.
         """
-        return ResourcePath("resource://felis/schemas/tap_schema_std.yaml")
+        return ResourcePath("resource://felis/config/tap_schema/tap_schema_std.yaml")
 
     @classmethod
     def get_table_names_std(cls) -> list[str]:
@@ -355,10 +363,38 @@ class TableManager:
         engine
             The SQLAlchemy engine to use to create the tables.
         """
-        logger.info("Creating TAP_SCHEMA database '%s'", self.metadata.schema)
+        logger.info("Creating TAP_SCHEMA database '%s'", self.schema_name)
         self._create_schema(engine)
         self.metadata.create_all(engine)
 
+    def select(self, engine: Engine, table_name: str, filter_condition: str = "") -> list[dict[str, Any]]:
+        """Select all rows from a TAP_SCHEMA table with an optional filter
+        condition.
+
+        Parameters
+        ----------
+        engine
+            The SQLAlchemy engine to use to connect to the database.
+        table_name
+            The name of the table to select from.
+        filter_condition
+            The filter condition as a string. If empty, no filter will be
+            applied.
+
+        Returns
+        -------
+        list
+            A list of dictionaries containing the rows from the table.
+        """
+        table = self[table_name]
+        query = select(table)
+        if filter_condition:
+            query = query.where(text(filter_condition))
+        with engine.connect() as connection:
+            result = connection.execute(query)
+            rows = [dict(row._mapping) for row in result]
+        return rows
+
 
 class DataLoader:
     """Load data into the TAP_SCHEMA tables.
@@ -380,6 +416,9 @@ class DataLoader:
         If True, print the SQL statements that will be executed.
     dry_run
         If True, the data will not be loaded into the database.
+    unique_keys
+        If True, prepend the schema name to the key name to make it unique
+        when loading data into the keys and key_columns tables.
     """
 
     def __init__(
@@ -391,6 +430,7 @@ class DataLoader:
         output_path: str | None = None,
         print_sql: bool = False,
         dry_run: bool = False,
+        unique_keys: bool = False,
     ):
         self.schema = schema
         self.mgr = mgr
@@ -400,6 +440,7 @@ class DataLoader:
         self.output_path = output_path
         self.print_sql = print_sql
         self.dry_run = dry_run
+        self.unique_keys = unique_keys
 
     def load(self) -> None:
         """Load the schema data into the TAP_SCHEMA tables.
@@ -426,10 +467,10 @@ class DataLoader:
             # Execute the inserts if not in dry run mode.
             self._execute_inserts()
         else:
-            logger.info("Dry run: not loading data into database")
+            logger.info("Dry run - not loading data into database")
 
     def _insert_schemas(self) -> None:
-        """Insert the schema data into the schemas table."""
+        """Insert the schema data into the ``schemas`` table."""
         schema_record = {
             "schema_name": self.schema.name,
             "utype": self.schema.votable_utype,
@@ -454,7 +495,7 @@ class DataLoader:
         return f"{self.schema.name}.{table.name}"
 
     def _insert_tables(self) -> None:
-        """Insert the table data into the tables table."""
+        """Insert the table data into the ``tables`` table."""
         for table in self.schema.tables:
             table_record = {
                 "schema_name": self.schema.name,
@@ -467,7 +508,7 @@ class DataLoader:
             self._insert("tables", table_record)
 
     def _insert_columns(self) -> None:
-        """Insert the column data into the columns table."""
+        """Insert the column data into the ``columns`` table."""
         for table in self.schema.tables:
             for column in table.columns:
                 felis_type = FelisType.felis_type(column.datatype.value)
@@ -495,18 +536,49 @@ class DataLoader:
                 }
                 self._insert("columns", column_record)
 
+    def _get_key(self, constraint: Constraint) -> str:
+        """Get the key name for a constraint.
+
+        Parameters
+        ----------
+        constraint
+            The constraint to get the key name for.
+
+        Returns
+        -------
+        str
+            The key name for the constraint.
+
+        Notes
+        -----
+        This will prepend the name of the schema to the key name if the
+        `unique_keys` attribute is set to True. Otherwise, it will just return
+        the name of the constraint.
+        """
+        if self.unique_keys:
+            key_id = f"{self.schema.name}_{constraint.name}"
+            logger.debug("Generated unique key_id: %s -> %s", constraint.name, key_id)
+        else:
+            key_id = constraint.name
+        return key_id
+
     def _insert_keys(self) -> None:
-        """Insert the foreign keys into the keys and key_columns tables."""
+        """Insert the foreign keys into the ``keys`` and ``key_columns``
+        tables.
+        """
         for table in self.schema.tables:
             for constraint in table.constraints:
                 if isinstance(constraint, datamodel.ForeignKeyConstraint):
+                    ###########################################################
                     # Handle keys table
+                    ###########################################################
                     referenced_column = self.schema.find_object_by_id(
                         constraint.referenced_columns[0], datamodel.Column
                     )
                     referenced_table = self.schema.get_table_by_column(referenced_column)
+                    key_id = self._get_key(constraint)
                     key_record = {
-                        "key_id": constraint.name,
+                        "key_id": key_id,
                         "from_table": self._get_table_name(table),
                         "target_table": self._get_table_name(referenced_table),
                         "description": constraint.description,
@@ -514,17 +586,23 @@ class DataLoader:
                     }
                     self._insert("keys", key_record)
 
+                    ###########################################################
                     # Handle key_columns table
-                    from_column = self.schema.find_object_by_id(constraint.columns[0], datamodel.Column)
-                    target_column = self.schema.find_object_by_id(
-                        constraint.referenced_columns[0], datamodel.Column
-                    )
-                    key_columns_record = {
-                        "key_id": constraint.name,
-                        "from_column": from_column.name,
-                        "target_column": target_column.name,
-                    }
-                    self._insert("key_columns", key_columns_record)
+                    ###########################################################
+                    # Loop over the corresponding columns and referenced
+                    # columns and insert a record for each pair. This is
+                    # necessary for proper handling of composite keys.
+                    for from_column_id, target_column_id in zip(
+                        constraint.columns, constraint.referenced_columns
+                    ):
+                        from_column = self.schema.find_object_by_id(from_column_id, datamodel.Column)
+                        target_column = self.schema.find_object_by_id(target_column_id, datamodel.Column)
+                        key_columns_record = {
+                            "key_id": key_id,
+                            "from_column": from_column.name,
+                            "target_column": target_column.name,
+                        }
+                        self._insert("key_columns", key_columns_record)
 
     def _generate_all_inserts(self) -> None:
         """Generate the inserts for all the data."""
@@ -567,7 +645,7 @@ class DataLoader:
     def _print_sql(self) -> None:
         """Print the generated inserts to stdout."""
         for insert_str in self._compiled_inserts():
-            print(insert_str)
+            print(insert_str + ";")
 
     def _write_sql_to_file(self) -> None:
         """Write the generated insert statements to a file."""
@@ -575,7 +653,7 @@ class DataLoader:
             raise ValueError("No output path specified")
         with open(self.output_path, "w") as outfile:
             for insert_str in self._compiled_inserts():
-                outfile.write(insert_str + "\n")
+                outfile.write(insert_str + ";" + "\n")
 
     def _insert(self, table_name: str, record: list[Any] | dict[str, Any]) -> None:
         """Generate an insert statement for a record.
@@ -642,3 +720,48 @@ class DataLoader:
             if index.columns and len(index.columns) == 1 and index.columns[0] == column.id:
                 return 1
         return 0
+
+
+class MetadataInserter:
+    """Insert TAP_SCHEMA self-description rows into the database.
+
+    Parameters
+    ----------
+    mgr
+        The table manager that contains the TAP_SCHEMA tables.
+    engine
+        The engine for connecting to the TAP_SCHEMA database.
+    """
+
+    def __init__(self, mgr: TableManager, engine: Engine):
+        """Initialize the metadata inserter.
+
+        Parameters
+        ----------
+        mgr
+            The table manager representing the TAP_SCHEMA tables.
+        engine
+            The SQLAlchemy engine for connecting to the database.
+        """
+        self._mgr = mgr
+        self._engine = engine
+
+    def insert_metadata(self) -> None:
+        """Insert the TAP_SCHEMA metadata into the database."""
+        for table_name in self._mgr.get_table_names_std():
+            table = self._mgr[table_name]
+            csv_bytes = ResourcePath(f"resource://felis/config/tap_schema/{table_name}.csv").read()
+            text_stream = io.TextIOWrapper(io.BytesIO(csv_bytes), encoding="utf-8")
+            reader = csv.reader(text_stream)
+            headers = next(reader)
+            rows = [
+                {key: None if value == "\\N" else value for key, value in zip(headers, row)} for row in reader
+            ]
+            logger.debug(
+                "Inserting %d rows into table '%s' with headers: %s",
+                len(rows),
+                table_name,
+                headers,
+            )
+            with self._engine.begin() as conn:
+                conn.execute(table.insert(), rows)

felis/tests/run_cli.py

@@ -0,0 +1,79 @@
+"""Test utility for running cli commands."""
+
+# This file is part of felis.
+#
+# Developed for the LSST Data Management System.
+# This product includes software developed by the LSST Project
+# (https://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+import logging
+
+from click.testing import CliRunner
+
+from felis.cli import cli
+
+__all__ = ["run_cli"]
+
+
+def run_cli(
+    cmd: list[str],
+    log_level: int = logging.WARNING,
+    catch_exceptions: bool = False,
+    expect_error: bool = False,
+    print_cmd: bool = False,
+    print_output: bool = False,
+    id_generation: bool = False,
+) -> None:
+    """Run a CLI command and check the exit code.
+
+    Parameters
+    ----------
+    cmd : list[str]
+        The command to run.
+    log_level : int
+        The logging level to use, by default logging.WARNING.
+    catch_exceptions : bool
+        Whether to catch exceptions, by default False.
+    expect_error : bool
+        Whether to expect an error, by default False.
+    print_cmd : bool
+        Whether to print the command, by default False.
+    print_output : bool
+        Whether to print the output, by default False.
+    id_generation : bool
+        Whether to enable id generation, by default False.
+    """
+    if not cmd:
+        raise ValueError("No command provided.")
+    full_cmd = ["--log-level", logging.getLevelName(log_level)] + cmd
+    if id_generation:
+        full_cmd = ["--id-generation"] + full_cmd
+    if print_cmd:
+        print(f"Running command: felis {' '.join(full_cmd)}")
+    runner = CliRunner()
+    result = runner.invoke(
+        cli,
+        full_cmd,
+        catch_exceptions=catch_exceptions,
+    )
+    if print_output:
+        print(result.output)
+    if expect_error:
+        assert result.exit_code != 0
+    else:
+        assert result.exit_code == 0

felis/types.py

@@ -26,20 +26,20 @@ from __future__ import annotations
 from typing import Any
 
 __all__ = [
-    "FelisType",
+    "Binary",
     "Boolean",
     "Byte",
-    "Short",
+    "Char",
+    "Double",
+    "FelisType",
+    "Float",
     "Int",
     "Long",
-    "Float",
-    "Double",
-    "Char",
+    "Short",
     "String",
-    "Unicode",
     "Text",
-    "Binary",
     "Timestamp",
+    "Unicode",
 ]
 
 

{lsst_felis-28.2024.4500.dist-info → lsst_felis-29.2025.4500.dist-info}/METADATA

@@ -1,34 +1,38 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: lsst-felis
-Version: 28.2024.4500
+Version: 29.2025.4500
 Summary: A vocabulary for describing catalogs and acting on those descriptions
 Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
-License: GNU General Public License v3 or later (GPLv3+)
+License-Expression: GPL-3.0-or-later
 Project-URL: Homepage, https://felis.lsst.io
 Project-URL: Source, https://github.com/lsst/felis
 Keywords: lsst
 Classifier: Intended Audience :: Science/Research
-Classifier: License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
 Classifier: Topic :: Scientific/Engineering :: Astronomy
 Requires-Python: >=3.11.0
 Description-Content-Type: text/markdown
 License-File: COPYRIGHT
 License-File: LICENSE
-Requires-Dist: astropy >=4
-Requires-Dist: sqlalchemy >=1.4
-Requires-Dist: click >=7
-Requires-Dist: pyyaml >=6
-Requires-Dist: pydantic <3,>=2
-Requires-Dist: lsst-utils
+Requires-Dist: alembic
+Requires-Dist: astropy
+Requires-Dist: click
+Requires-Dist: deepdiff
 Requires-Dist: lsst-resources
-Provides-Extra: dev
-Requires-Dist: documenteer[guide] <2 ; extra == 'dev'
-Requires-Dist: autodoc-pydantic ; extra == 'dev'
-Requires-Dist: sphinx-click ; extra == 'dev'
+Requires-Dist: lsst-utils
+Requires-Dist: numpy
+Requires-Dist: pydantic<3,>=2
+Requires-Dist: pyyaml
+Requires-Dist: sqlalchemy
 Provides-Extra: test
-Requires-Dist: pytest >=3.2 ; extra == 'test'
-
+Requires-Dist: pytest>=3.2; extra == "test"
+Provides-Extra: dev
+Requires-Dist: documenteer[guide]<2; extra == "dev"
+Requires-Dist: autodoc_pydantic; extra == "dev"
+Requires-Dist: sphinx-click; extra == "dev"
+Dynamic: license-file

lsst_felis-29.2025.4500.dist-info/RECORD

@@ -0,0 +1,31 @@
+felis/__init__.py,sha256=HnwWzLaPOSnPzAoppSIHzTrGfixEgvkzJdBxa8-03cw,1294
+felis/cli.py,sha256=g6OrBrIylNLiflSvrLlef86BjoiehV3L5eAvVPrxPog,16911
+felis/datamodel.py,sha256=VAJ9DqOBtfu3fWtpDOcpg4Ca1jfq8NMG6MHH8GbHpl0,52135
+felis/diff.py,sha256=ZzjOJ57p5ZwFn6eem7CYoPjSnxti5OZY33B6Ds5Q-Rg,7797
+felis/metadata.py,sha256=79YcaIqeFP-pj9zhWpqXlvw_piUTUwuLrV5_8eVYalQ,13763
+felis/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+felis/tap_schema.py,sha256=Kfpnv6AO_ni_B-DEgi3PRSkGKkTEybuYfWlA9gOMM_I,27071
+felis/types.py,sha256=ifZQjc-Uw5CM3L7hmFUb7wcHY1O_HgJCw6HPqyUkHvk,5510
+felis/config/tap_schema/columns.csv,sha256=9RsyuPObUQ_6myux9vKtlQ-aJgs7rvvxoLf6yYSRWqc,3272
+felis/config/tap_schema/key_columns.csv,sha256=dRezco5ltcM1mG--2DvPsbOxB6cwVaBwczwi3It2vag,210
+felis/config/tap_schema/keys.csv,sha256=6zTXyo-1GNfu5sBWpX-7ZJFAtHrxOys78AViCcdPgu8,377
+felis/config/tap_schema/schemas.csv,sha256=z5g1bW1Y9H8nKLZyH4e5xiBBoK9JezR2Xf8L79K2TZk,138
+felis/config/tap_schema/tables.csv,sha256=o0KioOiL7hw9ntCyKWili-iFMjAaGRMUOE-nM30LBD0,510
+felis/config/tap_schema/tap_schema_std.yaml,sha256=sPW-Vk72nY0PFpCvP5d8L8fWvhkif-x32sGtcfDZ8bU,7131
+felis/db/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+felis/db/dialects.py,sha256=XTZEbTnfy96GJDNRLCQMbAV6irerC87vhO_HyTIXLbs,3517
+felis/db/schema.py,sha256=NOFXzBoBQcgpoRlgT3LoC70FKp7pCSmFEJ7rU8FIT-c,2101
+felis/db/sqltypes.py,sha256=Q2p3Af3O5-B1ZxQ4M2j_w8SH1o_kp6ezg8h7LmSlfww,11060
+felis/db/utils.py,sha256=jiKQ_SirKRdQITHe8gSiT_i3ckRHZbkAnwUlEHk2u4Y,14116
+felis/db/variants.py,sha256=eahthrbVeV8ZdGamWQccNmWgx6CCscGrU0vQRs5HZK8,5260
+felis/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+felis/tests/postgresql.py,sha256=B_xk4fLual5-viGDqP20r94okuc0pbSvytRH_L0fvMs,4035
+felis/tests/run_cli.py,sha256=Gg8loUIGj9t6KlkRKrEc9Z9b5dtlkpJy94ORuj4BrxU,2503
+lsst_felis-29.2025.4500.dist-info/licenses/COPYRIGHT,sha256=vJAFLFTSF1mhy9eIuA3P6R-3yxTWKQgpig88P-1IzRw,129
+lsst_felis-29.2025.4500.dist-info/licenses/LICENSE,sha256=jOtLnuWt7d5Hsx6XXB2QxzrSe2sWWh3NgMfFRetluQM,35147
+lsst_felis-29.2025.4500.dist-info/METADATA,sha256=_brJSKPh9-Izo-XR_3FTFQb_zfsnECQyL2t6gjLW6p8,1377
+lsst_felis-29.2025.4500.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+lsst_felis-29.2025.4500.dist-info/entry_points.txt,sha256=Gk2XFujA_Gp52VBk45g5kim8TDoMDJFPctsMqiq72EM,40
+lsst_felis-29.2025.4500.dist-info/top_level.txt,sha256=F4SvPip3iZRVyISi50CHhwTIAokAhSxjWiVcn4IVWRI,6
+lsst_felis-29.2025.4500.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
+lsst_felis-29.2025.4500.dist-info/RECORD,,

{lsst_felis-28.2024.4500.dist-info → lsst_felis-29.2025.4500.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (75.3.0)
+Generator: setuptools (80.9.0)
 Root-Is-Purelib: true
 Tag: py3-none-any