lsst-daf-butler 29.0.0rc1__tar.gz → 29.2025.1000__tar.gz

{lsst_daf_butler-29.0.0rc1/python/lsst_daf_butler.egg-info → lsst_daf_butler-29.2025.1000}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: lsst-daf-butler
-Version: 29.0.0rc1
+Version: 29.2025.1000
 Summary: An abstraction layer for reading and writing astronomical data to datastores.
 Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
 License: BSD 3-Clause License

{lsst_daf_butler-29.0.0rc1 → lsst_daf_butler-29.2025.1000}/python/lsst/daf/butler/datastores/fileDatastore.py

@@ -2649,6 +2649,7 @@ class FileDatastore(GenericBaseDatastore[StoredFileInfo]):
         artifact_existence: dict[ResourcePath, bool] | None = None,
         dry_run: bool = False,
     ) -> tuple[set[DatasetRef], set[DatasetRef]]:
+        log.verbose("transfer_from %s to %s", source_datastore.name, self.name)
         # Docstring inherited
         if type(self) is not type(source_datastore):
             raise TypeError(
@@ -2693,6 +2694,7 @@ class FileDatastore(GenericBaseDatastore[StoredFileInfo]):
         # What we really want is all the records in the source datastore
         # associated with these refs. Or derived ones if they don't exist
         # in the source.
+        log.verbose("Looking up source datastore records in %s", source_datastore.name)
         source_records = source_datastore._get_stored_records_associated_with_refs(
             refs, ignore_datastore_records=True
         )
@@ -2726,6 +2728,9 @@ class FileDatastore(GenericBaseDatastore[StoredFileInfo]):
             source_records.update(found_records)
 
         # See if we already have these records
+        log.verbose(
+            "Looking up existing datastore records in target %s for %d refs", self.name, len(requested_ids)
+        )
         target_records = self._get_stored_records_associated_with_refs(refs, ignore_datastore_records=True)
 
         # The artifacts to register
@@ -2744,6 +2749,7 @@ class FileDatastore(GenericBaseDatastore[StoredFileInfo]):
         direct_transfers = []
 
         # Now can transfer the artifacts
+        log.verbose("Transferring artifacts")
         for ref in refs:
             if not self.constraints.isAcceptable(ref):
                 # This datastore should not be accepting this dataset.
@@ -2818,6 +2824,7 @@ class FileDatastore(GenericBaseDatastore[StoredFileInfo]):
         # to difficulties if the dataset has previously been ingested
         # disassembled and is somehow now assembled, or vice versa.
         if not dry_run:
+            log.verbose("Registering datastore records in database")
             self._register_datasets(artifacts, insert_mode=DatabaseInsertMode.REPLACE)
 
         if already_present:
@@ -2828,6 +2835,13 @@ class FileDatastore(GenericBaseDatastore[StoredFileInfo]):
                 "" if n_skipped == 1 else "s",
             )
 
+        log.verbose(
+            "Finished transfer_from %s to %s with %d accepted, %d rejected",
+            source_datastore.name,
+            self.name,
+            len(accepted),
+            len(rejected),
+        )
         return accepted, rejected
 
     @transactional

{lsst_daf_butler-29.0.0rc1 → lsst_daf_butler-29.2025.1000}/python/lsst/daf/butler/registry/datasets/byDimensions/_manager.py

@@ -317,6 +317,16 @@ class ByDimensionsDatasetRecordStorageManagerUUID(DatasetRecordStorageManager):
             raise ValueError(
                 f"Component dataset types can not be stored in registry. Rejecting {dataset_type.name}"
             )
+
+        # If database universe and dimension group universe are different it
+        # can cause unexpected effects.
+        if dataset_type.dimensions.universe is not self._dimensions.universe:
+            raise ValueError(
+                "Incompatible dimension universe versions - "
+                f"database universe: {self._dimensions.universe}, "
+                f"dataset type universe: {dataset_type.dimensions.universe}."
+            )
+
         record = self._fetch_dataset_type_record(dataset_type.name)
         if record is None:
             if (dynamic_tables := self._cache.get_by_dimensions(dataset_type.dimensions)) is None:

{lsst_daf_butler-29.0.0rc1 → lsst_daf_butler-29.2025.1000}/python/lsst/daf/butler/registry/tests/_registry.py

@@ -75,7 +75,7 @@ from ..._exceptions import (
 from ..._exceptions_legacy import DatasetTypeError
 from ..._storage_class import StorageClass
 from ..._timespan import Timespan
-from ...dimensions import DataCoordinate, DataCoordinateSet, SkyPixDimension
+from ...dimensions import DataCoordinate, DataCoordinateSet, DimensionUniverse, SkyPixDimension
 from .._collection_summary import CollectionSummary
 from .._config import RegistryConfig
 from .._exceptions import (
@@ -326,6 +326,17 @@ class RegistryTests(ABC):
         self.assertCountEqual([dt.name for dt in types], ["test", "testNoneTemplate"])
         self.assertEqual(missing, ["notarealdatasettype"])
 
+        # Trying to register a dataset type with different universe version or
+        # namespace will raise.
+        wrong_universes = (DimensionUniverse(version=-1), DimensionUniverse(namespace="🔭"))
+        for universe in wrong_universes:
+            storageClass = StorageClass("testDatasetType")
+            dataset_type = DatasetType(
+                "wrong_universe", ("instrument", "visit"), storageClass, universe=universe
+            )
+            with self.assertRaisesRegex(ValueError, "Incompatible dimension universe versions"):
+                registry.registerDatasetType(dataset_type)
+
     def testDatasetTypeCache(self):
         """Test for dataset type cache update logic after a cache miss."""
         butler1 = self.make_butler()

{lsst_daf_butler-29.0.0rc1 → lsst_daf_butler-29.2025.1000}/python/lsst/daf/butler/tests/hybrid_butler_registry.py

@@ -144,6 +144,16 @@ class HybridButlerRegistry(Registry):
         return self._remote.getCollectionSummary(collection)
 
     def registerDatasetType(self, datasetType: DatasetType) -> bool:
+        # We need to make sure that dataset type universe is the same as
+        # direct registry universe.
+        if datasetType.dimensions.universe is self._remote.dimensions:
+            datasetType = DatasetType(
+                datasetType.name,
+                datasetType.dimensions.names,
+                datasetType.storageClass,
+                universe=self._direct.dimensions,
+                isCalibration=datasetType.isCalibration(),
+            )
         return self._direct.registerDatasetType(datasetType)
 
     def removeDatasetType(self, name: str | tuple[str, ...]) -> None:

lsst_daf_butler-29.2025.1000/python/lsst/daf/butler/version.py

@@ -0,0 +1,2 @@
+__all__ = ["__version__"]
+__version__ = "29.2025.1000"

{lsst_daf_butler-29.0.0rc1 → lsst_daf_butler-29.2025.1000/python/lsst_daf_butler.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: lsst-daf-butler
-Version: 29.0.0rc1
+Version: 29.2025.1000
 Summary: An abstraction layer for reading and writing astronomical data to datastores.
 Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
 License: BSD 3-Clause License

{lsst_daf_butler-29.0.0rc1 → lsst_daf_butler-29.2025.1000}/tests/test_simpleButler.py

@@ -847,6 +847,16 @@ class SimpleButlerTests(TestCaseMixin):
         self.assertEqual(butler1.get_dataset_type("b"), b)
         # Register them in the opposite order in a new repo.
         butler2 = self.makeButler(writeable=True)
+        # Dataset types have to use correct universe and with RemoteButler
+        # each butler instance has its own universe instance.
+        a = DatasetType("a", ["instrument"], universe=butler2.dimensions, storageClass="StructuredDataDict")
+        b = DatasetType(
+            "b",
+            ["instrument"],
+            universe=butler2.dimensions,
+            storageClass="StructuredDataDict",
+            isCalibration=True,
+        )
         butler2.registry.registerDatasetType(b)
         butler2.registry.registerDatasetType(a)
         self.assertEqual(butler2.get_dataset_type("a"), a)

lsst_daf_butler-29.0.0rc1/python/lsst/daf/butler/version.py

@@ -1,2 +0,0 @@
-__all__ = ["__version__"]
-__version__ = "29.0.0rc1"