lsst-felis 28.2024.4800__py3-none-any.whl → 28.2025.500__py3-none-any.whl

felis/diff.py

@@ -0,0 +1,229 @@
+"""Compare schemas and print the differences."""
+
+# This file is part of felis.
+#
+# Developed for the LSST Data Management System.
+# This product includes software developed by the LSST Project
+# (https://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+import logging
+import pprint
+import re
+from collections.abc import Callable
+from typing import Any
+
+from alembic.autogenerate import compare_metadata
+from alembic.migration import MigrationContext
+from deepdiff.diff import DeepDiff
+from sqlalchemy import Engine, MetaData
+
+from .datamodel import Schema
+from .metadata import MetaDataBuilder
+
+__all__ = ["SchemaDiff", "DatabaseDiff"]
+
+logger = logging.getLogger(__name__)
+
+# Change alembic log level to avoid unnecessary output
+logging.getLogger("alembic").setLevel(logging.WARNING)
+
+
+class SchemaDiff:
+    """
+    Compare two schemas using DeepDiff and print the differences.
+
+    Parameters
+    ----------
+    schema1
+        The first schema to compare.
+    schema2
+        The second schema to compare.
+    """
+
+    def __init__(self, schema1: Schema, schema2: Schema):
+        self.dict1 = schema1.model_dump(exclude_none=True)
+        self.dict2 = schema2.model_dump(exclude_none=True)
+        self.diff = DeepDiff(self.dict1, self.dict2, ignore_order=True)
+
+    def print(self) -> None:
+        """
+        Print the differences between the two schemas.
+        """
+        pprint.pprint(self.diff)
+
+    @property
+    def has_changes(self) -> bool:
+        """
+        Check if there are any differences between the two schemas.
+
+        Returns
+        -------
+        bool
+            True if there are differences, False otherwise.
+        """
+        return len(self.diff) > 0
+
+
+class FormattedSchemaDiff(SchemaDiff):
+    """
+    Compare two schemas using DeepDiff and print the differences using a
+    customized output format.
+
+    Parameters
+    ----------
+    schema1
+        The first schema to compare.
+    schema2
+        The second schema to compare.
+    """
+
+    def __init__(self, schema1: Schema, schema2: Schema):
+        super().__init__(schema1, schema2)
+
+    def print(self) -> None:
+        """
+        Print the differences between the two schemas using a custom format.
+        """
+        handlers: dict[str, Callable[[dict[str, Any]], None]] = {
+            "values_changed": self._handle_values_changed,
+            "iterable_item_added": self._handle_iterable_item_added,
+            "iterable_item_removed": self._handle_iterable_item_removed,
+            "dictionary_item_added": self._handle_dictionary_item_added,
+            "dictionary_item_removed": self._handle_dictionary_item_removed,
+        }
+
+        for change_type, handler in handlers.items():
+            if change_type in self.diff:
+                handler(self.diff[change_type])
+
+    def _print_header(self, id_dict: dict[str, Any], keys: list[int | str]) -> None:
+        id = self._get_id(id_dict, keys)
+        print(f"{id} @ {self._get_key_display(keys)}")
+
+    def _handle_values_changed(self, changes: dict[str, Any]) -> None:
+        for key in changes:
+            keys = self._parse_deepdiff_path(key)
+            value1 = self._get_value_from_keys(self.dict1, keys)
+            value2 = self._get_value_from_keys(self.dict2, keys)
+            self._print_header(self.dict1, keys)
+            print(f"- {value1}")
+            print(f"+ {value2}")
+
+    def _handle_iterable_item_added(self, changes: dict[str, Any]) -> None:
+        for key in changes:
+            keys = self._parse_deepdiff_path(key)
+            value = self._get_value_from_keys(self.dict2, keys)
+            self._print_header(self.dict2, keys)
+            print(f"+ {value}")
+
+    def _handle_iterable_item_removed(self, changes: dict[str, Any]) -> None:
+        for key in changes:
+            keys = self._parse_deepdiff_path(key)
+            value = self._get_value_from_keys(self.dict1, keys)
+            self._print_header(self.dict1, keys)
+            print(f"- {value}")
+
+    def _handle_dictionary_item_added(self, changes: dict[str, Any]) -> None:
+        for key in changes:
+            keys = self._parse_deepdiff_path(key)
+            value = self._get_value_from_keys(self.dict2, keys)
+            self._print_header(self.dict2, keys)
+            print(f"+ {value}")
+
+    def _handle_dictionary_item_removed(self, changes: dict[str, Any]) -> None:
+        for key in changes:
+            keys = self._parse_deepdiff_path(key)
+            value = self._get_value_from_keys(self.dict1, keys)
+            self._print_header(self.dict1, keys)
+            print(f"- {value}")
+
+    @staticmethod
+    def _get_id(values: dict, keys: list[str | int]) -> str:
+        value = values
+        last_id = None
+
+        for key in keys:
+            if isinstance(value, dict) and "id" in value:
+                last_id = value["id"]
+            value = value[key]
+
+        if isinstance(value, dict) and "id" in value:
+            last_id = value["id"]
+
+        if last_id is not None:
+            return last_id
+        else:
+            raise ValueError("No 'id' found in the specified path")
+
+    @staticmethod
+    def _get_key_display(keys: list[str | int]) -> str:
+        return ".".join(str(k) for k in keys)
+
+    @staticmethod
+    def _parse_deepdiff_path(path: str) -> list[str | int]:
+        if path.startswith("root"):
+            path = path[4:]
+
+        pattern = re.compile(r"\['([^']+)'\]|\[(\d+)\]")
+        matches = pattern.findall(path)
+
+        keys = []
+        for match in matches:
+            if match[0]:  # String key
+                keys.append(match[0])
+            elif match[1]:  # Integer index
+                keys.append(int(match[1]))
+
+        return keys
+
+    @staticmethod
+    def _get_value_from_keys(data: dict, keys: list[str | int]) -> Any:
+        value = data
+        for key in keys:
+            value = value[key]
+        return value
+
+
+class DatabaseDiff(SchemaDiff):
+    """
+    Compare a schema with a database and print the differences.
+
+    Parameters
+    ----------
+    schema
+        The schema to compare.
+    engine
+        The database engine to compare with.
+    """
+
+    def __init__(self, schema: Schema, engine: Engine):
+        db_metadata = MetaData()
+        with engine.connect() as connection:
+            db_metadata.reflect(bind=connection)
+            mc = MigrationContext.configure(
+                connection, opts={"compare_type": True, "target_metadata": db_metadata}
+            )
+            schema_metadata = MetaDataBuilder(schema, apply_schema_to_metadata=False).build()
+            self.diff = compare_metadata(mc, schema_metadata)
+
+    def print(self) -> None:
+        """
+        Print the differences between the schema and the database.
+        """
+        if self.has_changes:
+            pprint.pprint(self.diff)

felis/metadata.py

@@ -125,29 +125,27 @@ class MetaDataBuilder:
         The schema object from which to build the SQLAlchemy metadata.
     apply_schema_to_metadata
         Whether to apply the schema name to the metadata object.
-    apply_schema_to_tables
-        Whether to apply the schema name to the tables.
     ignore_constraints
         Whether to ignore constraints when building the metadata.
+    table_name_postfix
+        A string to append to the table names when building the metadata.
     """
 
     def __init__(
         self,
         schema: Schema,
         apply_schema_to_metadata: bool = True,
-        apply_schema_to_tables: bool = True,
         ignore_constraints: bool = False,
+        table_name_postfix: str = "",
     ) -> None:
         """Initialize the metadata builder."""
         self.schema = schema
         if not apply_schema_to_metadata:
             logger.debug("Schema name will not be applied to metadata")
-        if not apply_schema_to_tables:
-            logger.debug("Schema name will not be applied to tables")
         self.metadata = MetaData(schema=schema.name if apply_schema_to_metadata else None)
         self._objects: dict[str, Any] = {}
-        self.apply_schema_to_tables = apply_schema_to_tables
         self.ignore_constraints = ignore_constraints
+        self.table_name_postfix = table_name_postfix
 
     def build(self) -> MetaData:
         """Build the SQLAlchemy tables and constraints from the schema.
@@ -231,11 +229,10 @@ class MetaDataBuilder:
         description = table_obj.description
         columns = [self.build_column(column) for column in table_obj.columns]
         table = Table(
-            name,
+            name + self.table_name_postfix,
             self.metadata,
             *columns,
             comment=description,
-            schema=self.schema.name if self.apply_schema_to_tables else None,
             **optargs,  # type: ignore[arg-type]
         )
 

felis/tap_schema.py

@@ -91,9 +91,15 @@ class TableManager:
         self.table_name_postfix = table_name_postfix
         self.apply_schema_to_metadata = apply_schema_to_metadata
         self.schema_name = schema_name or TableManager._SCHEMA_NAME_STD
+        self.table_name_postfix = table_name_postfix
 
         if is_valid_engine(engine):
             assert isinstance(engine, Engine)
+            if table_name_postfix != "":
+                logger.warning(
+                    "Table name postfix '%s' will be ignored when reflecting TAP_SCHEMA database",
+                    table_name_postfix,
+                )
             logger.debug(
                 "Reflecting TAP_SCHEMA database from existing database at %s",
                 engine.url._replace(password="***"),
@@ -133,7 +139,7 @@ class TableManager:
         self._metadata = MetaDataBuilder(
             self.schema,
             apply_schema_to_metadata=self.apply_schema_to_metadata,
-            apply_schema_to_tables=self.apply_schema_to_metadata,
+            table_name_postfix=self.table_name_postfix,
         ).build()
 
         logger.debug("Loaded TAP_SCHEMA '%s' from YAML resource", self.schema_name)
@@ -355,7 +361,7 @@ class TableManager:
         engine
             The SQLAlchemy engine to use to create the tables.
         """
-        logger.info("Creating TAP_SCHEMA database '%s'", self.metadata.schema)
+        logger.info("Creating TAP_SCHEMA database '%s'", self.schema_name)
         self._create_schema(engine)
         self.metadata.create_all(engine)
 
@@ -426,7 +432,7 @@ class DataLoader:
             # Execute the inserts if not in dry run mode.
             self._execute_inserts()
         else:
-            logger.info("Dry run: not loading data into database")
+            logger.info("Dry run - not loading data into database")
 
     def _insert_schemas(self) -> None:
         """Insert the schema data into the schemas table."""
@@ -567,7 +573,7 @@ class DataLoader:
     def _print_sql(self) -> None:
         """Print the generated inserts to stdout."""
         for insert_str in self._compiled_inserts():
-            print(insert_str)
+            print(insert_str + ";")
 
     def _write_sql_to_file(self) -> None:
         """Write the generated insert statements to a file."""
@@ -575,7 +581,7 @@ class DataLoader:
             raise ValueError("No output path specified")
         with open(self.output_path, "w") as outfile:
             for insert_str in self._compiled_inserts():
-                outfile.write(insert_str + "\n")
+                outfile.write(insert_str + ";" + "\n")
 
     def _insert(self, table_name: str, record: list[Any] | dict[str, Any]) -> None:
         """Generate an insert statement for a record.

felis/version.py

@@ -1,2 +1,2 @@
 __all__ = ["__version__"]
-__version__ = "28.2024.4800"
+__version__ = "28.2025.500"

{lsst_felis-28.2024.4800.dist-info → lsst_felis-28.2025.500.dist-info}/METADATA

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: lsst-felis
-Version: 28.2024.4800
+Version: 28.2025.500
 Summary: A vocabulary for describing catalogs and acting on those descriptions
 Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
 License: GNU General Public License v3 or later (GPLv3+)
@@ -13,18 +13,22 @@ Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
 Classifier: Topic :: Scientific/Engineering :: Astronomy
 Requires-Python: >=3.11.0
 Description-Content-Type: text/markdown
 License-File: COPYRIGHT
 License-File: LICENSE
-Requires-Dist: astropy>=4
-Requires-Dist: sqlalchemy>=1.4
-Requires-Dist: click>=7
-Requires-Dist: pyyaml>=6
-Requires-Dist: pydantic<3,>=2
-Requires-Dist: lsst-utils
+Requires-Dist: alembic
+Requires-Dist: astropy
+Requires-Dist: click
+Requires-Dist: deepdiff
 Requires-Dist: lsst-resources
+Requires-Dist: lsst-utils
+Requires-Dist: numpy
+Requires-Dist: pydantic<3,>=2
+Requires-Dist: pyyaml
+Requires-Dist: sqlalchemy
 Provides-Extra: test
 Requires-Dist: pytest>=3.2; extra == "test"
 Provides-Extra: dev

lsst_felis-28.2025.500.dist-info/RECORD

@@ -0,0 +1,26 @@
+felis/__init__.py,sha256=r1KFSnc55gziwUuYb9s2EfwrI_85aa3LpaKwk6rUvvs,1108
+felis/cli.py,sha256=Wf-sEUZ-B9zzn4M1huY2ruV1nkgVmpzX8f8iuFfyxZc,14469
+felis/datamodel.py,sha256=NczAA4HBBC4-uxPNsrKAFX-hdlgvCT2qqEJCEqDy4yg,39265
+felis/diff.py,sha256=0N4OcBCzbL9DW_XGAeuvGsQ0zIhq8fY-Kx2QdvLv-Ds,7492
+felis/metadata.py,sha256=cYx_qizkLBqcoxWV46h4TbwTi1KVJAkuA2OuUmD-K5k,13536
+felis/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+felis/tap_schema.py,sha256=DgHH4hBf4q_F540TAR9GTKcALwUkk8iTw5pzQlmv1DA,22753
+felis/types.py,sha256=m80GSGfNHQ3-NzRuTzKOyRXLJboPxdk9kzpp1SO8XdY,5510
+felis/version.py,sha256=S4DyUuQLyaEbonnBItCg8kyZ-_mgVePitRnB-HX_q2Y,54
+felis/db/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+felis/db/dialects.py,sha256=n5La-shu-8fHLIyf8rrazHDyrzATmMCdELtKV_0ymxI,3517
+felis/db/schema.py,sha256=NOFXzBoBQcgpoRlgT3LoC70FKp7pCSmFEJ7rU8FIT-c,2101
+felis/db/sqltypes.py,sha256=JJy97U8KzAOg5pFi2xZgSjvU8CXXgrzkvCsmo6FLRG4,11060
+felis/db/utils.py,sha256=SIl2ryOT2Zn5n0BqdNDxC1HcOoxh0doaKk_hMUGvwAc,14116
+felis/db/variants.py,sha256=eahthrbVeV8ZdGamWQccNmWgx6CCscGrU0vQRs5HZK8,5260
+felis/schemas/tap_schema_std.yaml,sha256=sPW-Vk72nY0PFpCvP5d8L8fWvhkif-x32sGtcfDZ8bU,7131
+felis/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+felis/tests/postgresql.py,sha256=B_xk4fLual5-viGDqP20r94okuc0pbSvytRH_L0fvMs,4035
+lsst_felis-28.2025.500.dist-info/COPYRIGHT,sha256=vJAFLFTSF1mhy9eIuA3P6R-3yxTWKQgpig88P-1IzRw,129
+lsst_felis-28.2025.500.dist-info/LICENSE,sha256=jOtLnuWt7d5Hsx6XXB2QxzrSe2sWWh3NgMfFRetluQM,35147
+lsst_felis-28.2025.500.dist-info/METADATA,sha256=47DRp1MC63ZQA0CbZR8KWLxoh-dnv6V7iN-gtBO2Xms,1410
+lsst_felis-28.2025.500.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+lsst_felis-28.2025.500.dist-info/entry_points.txt,sha256=Gk2XFujA_Gp52VBk45g5kim8TDoMDJFPctsMqiq72EM,40
+lsst_felis-28.2025.500.dist-info/top_level.txt,sha256=F4SvPip3iZRVyISi50CHhwTIAokAhSxjWiVcn4IVWRI,6
+lsst_felis-28.2025.500.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
+lsst_felis-28.2025.500.dist-info/RECORD,,

{lsst_felis-28.2024.4800.dist-info → lsst_felis-28.2025.500.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (75.6.0)
+Generator: setuptools (75.8.0)
 Root-Is-Purelib: true
 Tag: py3-none-any