lsst-daf-butler 30.2025.5000__tar.gz → 30.2025.5100__tar.gz

{lsst_daf_butler-30.2025.5000/python/lsst_daf_butler.egg-info → lsst_daf_butler-30.2025.5100}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lsst-daf-butler
-Version: 30.2025.5000
+Version: 30.2025.5100
 Summary: An abstraction layer for reading and writing astronomical data to datastores.
 Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
 License-Expression: BSD-3-Clause OR GPL-3.0-or-later

{lsst_daf_butler-30.2025.5000 → lsst_daf_butler-30.2025.5100}/python/lsst/daf/butler/datastore/record_data.py

@@ -49,7 +49,7 @@ if TYPE_CHECKING:
 # Pydantic requires the possible value types to be explicitly enumerated in
 # order for `uuid.UUID` in particular to work.  `typing.Any` does not work
 # here.
-_Record: TypeAlias = dict[str, int | str | uuid.UUID | None]
+_Record: TypeAlias = dict[str, int | str | None]
 
 
 class SerializedDatastoreRecordData(pydantic.BaseModel):

{lsst_daf_butler-30.2025.5000 → lsst_daf_butler-30.2025.5100}/python/lsst/daf/butler/datastores/fileDatastore.py

@@ -3166,6 +3166,20 @@ class FileDatastore(GenericBaseDatastore[StoredFileInfo]):
 
     def export_records(self, refs: Iterable[DatasetIdRef]) -> Mapping[str, DatastoreRecordData]:
         # Docstring inherited from the base class.
+
+        # This call to 'bridge.check' filters out "partially deleted" datasets.
+        # Specifically, ones in the unusual edge state that:
+        # 1. They have an entry in the registry dataset tables
+        # 2. They were "trashed" from the datastore, so they are not
+        # present in the "dataset_location" table.)
+        # 3. But the trash has not been "emptied", so there are still entries
+        #  in the "opaque" datastore records table.
+        #
+        # As far as I can tell, this can only occur in the case of a concurrent
+        # or aborted call to `Butler.pruneDatasets(unstore=True, purge=False)`.
+        # Datasets (with or without files existing on disk) can persist in
+        # this zombie state indefinitely, until someone manually empties
+        # the trash.
         exported_refs = list(self._bridge.check(refs))
         ids = {ref.id for ref in exported_refs}
         records: dict[DatasetId, dict[str, list[StoredDatastoreItemInfo]]] = {id: {} for id in ids}

{lsst_daf_butler-30.2025.5000 → lsst_daf_butler-30.2025.5100}/python/lsst/daf/butler/registry/bridge/monolithic.py

@@ -215,20 +215,24 @@ class MonolithicDatastoreRegistryBridge(DatastoreRegistryBridge):
     def check(self, refs: Iterable[DatasetIdRef]) -> Iterable[DatasetIdRef]:
         # Docstring inherited from DatastoreRegistryBridge
         byId = {ref.id: ref for ref in refs}
-        sql = (
-            sqlalchemy.sql.select(self._tables.dataset_location.columns.dataset_id)
-            .select_from(self._tables.dataset_location)
-            .where(
-                sqlalchemy.sql.and_(
-                    self._tables.dataset_location.columns.datastore_name == self.datastoreName,
-                    self._tables.dataset_location.columns.dataset_id.in_(byId.keys()),
+        found: list[DatasetIdRef] = []
+        with self._db.session():
+            for batch in chunk_iterable(byId.keys(), 50000):
+                sql = (
+                    sqlalchemy.sql.select(self._tables.dataset_location.columns.dataset_id)
+                    .select_from(self._tables.dataset_location)
+                    .where(
+                        sqlalchemy.sql.and_(
+                            self._tables.dataset_location.columns.datastore_name == self.datastoreName,
+                            self._tables.dataset_location.columns.dataset_id.in_(batch),
+                        )
+                    )
                 )
-            )
-        )
-        with self._db.query(sql) as sql_result:
-            sql_rows = sql_result.fetchall()
-        for row in sql_rows:
-            yield byId[row.dataset_id]
+                with self._db.query(sql) as sql_result:
+                    sql_ids = sql_result.scalars().all()
+                found.extend(byId[id] for id in sql_ids)
+
+        return found
 
     @contextmanager
     def emptyTrash(

{lsst_daf_butler-30.2025.5000 → lsst_daf_butler-30.2025.5100}/python/lsst/daf/butler/registry/datasets/byDimensions/_manager.py

@@ -12,6 +12,8 @@ from typing import TYPE_CHECKING, Any, ClassVar
 import astropy.time
 import sqlalchemy
 
+from lsst.utils.iteration import chunk_iterable
+
 from .... import ddl
 from ...._collection_type import CollectionType
 from ...._dataset_ref import DatasetId, DatasetIdFactory, DatasetIdGenEnum, DatasetRef
@@ -424,17 +426,18 @@ class ByDimensionsDatasetRecordStorageManagerUUID(DatasetRecordStorageManager):
         return result
 
     def get_dataset_refs(self, ids: list[DatasetId]) -> list[DatasetRef]:
-        # Look up the dataset types corresponding to the given Dataset IDs.
-        id_col = self._static.dataset.columns["id"]
-        sql = sqlalchemy.sql.select(
-            id_col,
-            self._static.dataset.columns["dataset_type_id"],
-        ).where(id_col.in_(ids))
-        with self._db.query(sql) as sql_result:
-            dataset_rows = sql_result.mappings().all()
-        dataset_type_map: dict[DatasetId, DatasetType] = {
-            row["id"]: self._get_dataset_type_by_id(row["dataset_type_id"]) for row in dataset_rows
-        }
+        dataset_type_map: dict[DatasetId, DatasetType] = {}
+        for batch in chunk_iterable(set(ids), 50000):
+            # Look up the dataset types corresponding to the given Dataset IDs.
+            id_col = self._static.dataset.columns["id"]
+            sql = sqlalchemy.sql.select(
+                id_col,
+                self._static.dataset.columns["dataset_type_id"],
+            ).where(id_col.in_(batch))
+            with self._db.query(sql) as sql_result:
+                dataset_rows = sql_result.mappings().all()
+            for row in dataset_rows:
+                dataset_type_map[row["id"]] = self._get_dataset_type_by_id(row["dataset_type_id"])
 
         # Group the given dataset IDs by the DimensionGroup of their dataset
         # types -- there is a separate tags table for each DimensionGroup.
@@ -448,40 +451,41 @@ class ByDimensionsDatasetRecordStorageManagerUUID(DatasetRecordStorageManager):
             # data IDs corresponding to the UUIDs found from the dataset table.
             dynamic_tables = self._get_dynamic_tables(dimension_group)
             tags_table = self._get_tags_table(dynamic_tables)
-            tags_sql = tags_table.select().where(tags_table.columns["dataset_id"].in_(datasets))
-            # Join in the collection table to fetch the run name.
-            collection_column = tags_table.columns[self._collections.getCollectionForeignKeyName()]
-            joined_collections = self._collections.join_collections_sql(collection_column, tags_sql)
-            tags_sql = joined_collections.joined_sql
-            run_name_column = joined_collections.name_column
-            tags_sql = tags_sql.add_columns(run_name_column)
-            # Tags table includes run collections and tagged
-            # collections.
-            # In theory the data ID for a given dataset should be the
-            # same in both, but nothing actually guarantees this.
-            # So skip any tagged collections, using the run collection
-            # as the definitive definition.
-            tags_sql = tags_sql.where(joined_collections.type_column == int(CollectionType.RUN))
-
-            with self._db.query(tags_sql) as sql_result:
-                data_id_rows = sql_result.mappings().all()
-
-            assert run_name_column.key is not None
-            for data_id_row in data_id_rows:
-                id = data_id_row["dataset_id"]
-                dataset_type = dataset_type_map[id]
-                run_name = data_id_row[run_name_column.key]
-                data_id = DataCoordinate.from_required_values(
-                    dimension_group,
-                    tuple(data_id_row[dimension] for dimension in dimension_group.required),
-                )
-                ref = DatasetRef(
-                    datasetType=dataset_type,
-                    dataId=data_id,
-                    id=id,
-                    run=run_name,
-                )
-                output_refs.append(ref)
+            for batch in chunk_iterable(datasets, 50000):
+                tags_sql = tags_table.select().where(tags_table.columns["dataset_id"].in_(batch))
+                # Join in the collection table to fetch the run name.
+                collection_column = tags_table.columns[self._collections.getCollectionForeignKeyName()]
+                joined_collections = self._collections.join_collections_sql(collection_column, tags_sql)
+                tags_sql = joined_collections.joined_sql
+                run_name_column = joined_collections.name_column
+                tags_sql = tags_sql.add_columns(run_name_column)
+                # Tags table includes run collections and tagged
+                # collections.
+                # In theory the data ID for a given dataset should be the
+                # same in both, but nothing actually guarantees this.
+                # So skip any tagged collections, using the run collection
+                # as the definitive definition.
+                tags_sql = tags_sql.where(joined_collections.type_column == int(CollectionType.RUN))
+
+                with self._db.query(tags_sql) as sql_result:
+                    data_id_rows = sql_result.mappings().all()
+
+                assert run_name_column.key is not None
+                for data_id_row in data_id_rows:
+                    id = data_id_row["dataset_id"]
+                    dataset_type = dataset_type_map[id]
+                    run_name = data_id_row[run_name_column.key]
+                    data_id = DataCoordinate.from_required_values(
+                        dimension_group,
+                        tuple(data_id_row[dimension] for dimension in dimension_group.required),
+                    )
+                    ref = DatasetRef(
+                        datasetType=dataset_type,
+                        dataId=data_id,
+                        id=id,
+                        run=run_name,
+                    )
+                    output_refs.append(ref)
 
         return output_refs
 

{lsst_daf_butler-30.2025.5000 → lsst_daf_butler-30.2025.5100}/python/lsst/daf/butler/registry/interfaces/_database.py

@@ -1562,7 +1562,12 @@ class Database(ABC):
                 return None
             else:
                 sql = table.insert()
-                return [connection.execute(sql, row).inserted_primary_key[0] for row in rows]
+                ids = []
+                for row in rows:
+                    key = connection.execute(sql, row).inserted_primary_key
+                    assert key is not None
+                    ids.append(key[0])
+                return ids
 
     @abstractmethod
     def replace(self, table: sqlalchemy.schema.Table, *rows: dict) -> None:

lsst_daf_butler-30.2025.5100/python/lsst/daf/butler/version.py

@@ -0,0 +1,2 @@
+__all__ = ["__version__"]
+__version__ = "30.2025.5100"

{lsst_daf_butler-30.2025.5000 → lsst_daf_butler-30.2025.5100/python/lsst_daf_butler.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lsst-daf-butler
-Version: 30.2025.5000
+Version: 30.2025.5100
 Summary: An abstraction layer for reading and writing astronomical data to datastores.
 Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
 License-Expression: BSD-3-Clause OR GPL-3.0-or-later

{lsst_daf_butler-30.2025.5000 → lsst_daf_butler-30.2025.5100}/tests/test_datastore.py

@@ -61,6 +61,7 @@ from lsst.daf.butler.datastore.cache_manager import (
     DatastoreCacheManagerConfig,
     DatastoreDisabledCacheManager,
 )
+from lsst.daf.butler.datastore.record_data import DatastoreRecordData, SerializedDatastoreRecordData
 from lsst.daf.butler.datastore.stored_file_info import StoredFileInfo, make_datastore_path_relative
 from lsst.daf.butler.formatters.yaml import YamlFormatter
 from lsst.daf.butler.tests import (
@@ -77,6 +78,7 @@ from lsst.daf.butler.tests.dict_convertible_model import DictConvertibleModel
 from lsst.daf.butler.tests.utils import TestCaseMixin
 from lsst.resources import ResourcePath
 from lsst.utils import doImport
+from lsst.utils.introspection import get_full_type_name
 
 TESTDIR = os.path.dirname(__file__)
 
@@ -2252,6 +2254,44 @@ class StoredFileInfoTestCase(DatasetTestHelper, unittest.TestCase):
         self.assertEqual(make_datastore_path_relative("http://server.com/some/path"), "some/path")
         self.assertEqual(make_datastore_path_relative("http://server.com/some/path#frag"), "some/path#frag")
 
+    def test_datastore_record_data_json_types(self):
+        """Test that we don't round-trip checksums to UUIDs when deserializing
+        datastore record data.
+        """
+        test_json = """
+            {
+                "dataset_ids": [
+                    "74478304-abf1-4a9c-9eb2-926090a84446"
+                ],
+                "records": {
+                    "lsst.daf.butler.datastore.stored_file_info.StoredFileInfo": {
+                    "74478304abf14a9c9eb2926090a84446": {
+                        "file_datastore_records": [
+                        {
+                            "formatter": "lsst.daf.butler.formatters.yaml.YamlFormatter",
+                            "path": "gain_factors/base-2025-158/gain_factors_spx_base-2025-158.yaml",
+                            "storage_class": "GainFactors",
+                            "component": "__NULL_STRING__",
+                            "checksum": "cab515f6-ab67-0484-393f-aaa525dd526f",
+                            "file_size": 5412
+                        }
+                        ]
+                    }
+                    }
+                }
+            }
+        """
+        id_str = "74478304abf14a9c9eb2926090a84446"
+        s = SerializedDatastoreRecordData.model_validate_json(test_json)
+        self.assertIsInstance(
+            s.records[get_full_type_name(StoredFileInfo)][id_str]["file_datastore_records"][0]["checksum"],
+            str,
+        )
+        id = uuid.UUID(id_str)
+        d = DatastoreRecordData.from_simple(s)
+        self.assertIsInstance(d.records[id]["file_datastore_records"][0], StoredFileInfo)
+        self.assertIsInstance(d.records[id]["file_datastore_records"][0].checksum, str)
+
 
 @contextlib.contextmanager
 def _temp_yaml_file(data: Any) -> Iterator[str]:

lsst_daf_butler-30.2025.5000/python/lsst/daf/butler/version.py

@@ -1,2 +0,0 @@
-__all__ = ["__version__"]
-__version__ = "30.2025.5000"