lsst-daf-butler 30.2025.5000__py3-none-any.whl → 30.2025.5100__py3-none-any.whl

lsst/daf/butler/datastore/record_data.py

@@ -49,7 +49,7 @@ if TYPE_CHECKING:
 # Pydantic requires the possible value types to be explicitly enumerated in
 # order for `uuid.UUID` in particular to work.  `typing.Any` does not work
 # here.
-_Record: TypeAlias = dict[str, int | str | uuid.UUID | None]
+_Record: TypeAlias = dict[str, int | str | None]
 
 
 class SerializedDatastoreRecordData(pydantic.BaseModel):

lsst/daf/butler/datastores/fileDatastore.py

@@ -3166,6 +3166,20 @@ class FileDatastore(GenericBaseDatastore[StoredFileInfo]):
 
     def export_records(self, refs: Iterable[DatasetIdRef]) -> Mapping[str, DatastoreRecordData]:
         # Docstring inherited from the base class.
+
+        # This call to 'bridge.check' filters out "partially deleted" datasets.
+        # Specifically, ones in the unusual edge state that:
+        # 1. They have an entry in the registry dataset tables
+        # 2. They were "trashed" from the datastore, so they are not
+        # present in the "dataset_location" table.)
+        # 3. But the trash has not been "emptied", so there are still entries
+        #  in the "opaque" datastore records table.
+        #
+        # As far as I can tell, this can only occur in the case of a concurrent
+        # or aborted call to `Butler.pruneDatasets(unstore=True, purge=False)`.
+        # Datasets (with or without files existing on disk) can persist in
+        # this zombie state indefinitely, until someone manually empties
+        # the trash.
         exported_refs = list(self._bridge.check(refs))
         ids = {ref.id for ref in exported_refs}
         records: dict[DatasetId, dict[str, list[StoredDatastoreItemInfo]]] = {id: {} for id in ids}

lsst/daf/butler/registry/bridge/monolithic.py

@@ -215,20 +215,24 @@ class MonolithicDatastoreRegistryBridge(DatastoreRegistryBridge):
     def check(self, refs: Iterable[DatasetIdRef]) -> Iterable[DatasetIdRef]:
         # Docstring inherited from DatastoreRegistryBridge
         byId = {ref.id: ref for ref in refs}
-        sql = (
-            sqlalchemy.sql.select(self._tables.dataset_location.columns.dataset_id)
-            .select_from(self._tables.dataset_location)
-            .where(
-                sqlalchemy.sql.and_(
-                    self._tables.dataset_location.columns.datastore_name == self.datastoreName,
-                    self._tables.dataset_location.columns.dataset_id.in_(byId.keys()),
+        found: list[DatasetIdRef] = []
+        with self._db.session():
+            for batch in chunk_iterable(byId.keys(), 50000):
+                sql = (
+                    sqlalchemy.sql.select(self._tables.dataset_location.columns.dataset_id)
+                    .select_from(self._tables.dataset_location)
+                    .where(
+                        sqlalchemy.sql.and_(
+                            self._tables.dataset_location.columns.datastore_name == self.datastoreName,
+                            self._tables.dataset_location.columns.dataset_id.in_(batch),
+                        )
+                    )
                 )
-            )
-        )
-        with self._db.query(sql) as sql_result:
-            sql_rows = sql_result.fetchall()
-        for row in sql_rows:
-            yield byId[row.dataset_id]
+                with self._db.query(sql) as sql_result:
+                    sql_ids = sql_result.scalars().all()
+                found.extend(byId[id] for id in sql_ids)
+
+        return found
 
     @contextmanager
     def emptyTrash(

lsst/daf/butler/registry/datasets/byDimensions/_manager.py

@@ -12,6 +12,8 @@ from typing import TYPE_CHECKING, Any, ClassVar
 import astropy.time
 import sqlalchemy
 
+from lsst.utils.iteration import chunk_iterable
+
 from .... import ddl
 from ...._collection_type import CollectionType
 from ...._dataset_ref import DatasetId, DatasetIdFactory, DatasetIdGenEnum, DatasetRef
@@ -424,17 +426,18 @@ class ByDimensionsDatasetRecordStorageManagerUUID(DatasetRecordStorageManager):
         return result
 
     def get_dataset_refs(self, ids: list[DatasetId]) -> list[DatasetRef]:
-        # Look up the dataset types corresponding to the given Dataset IDs.
-        id_col = self._static.dataset.columns["id"]
-        sql = sqlalchemy.sql.select(
-            id_col,
-            self._static.dataset.columns["dataset_type_id"],
-        ).where(id_col.in_(ids))
-        with self._db.query(sql) as sql_result:
-            dataset_rows = sql_result.mappings().all()
-        dataset_type_map: dict[DatasetId, DatasetType] = {
-            row["id"]: self._get_dataset_type_by_id(row["dataset_type_id"]) for row in dataset_rows
-        }
+        dataset_type_map: dict[DatasetId, DatasetType] = {}
+        for batch in chunk_iterable(set(ids), 50000):
+            # Look up the dataset types corresponding to the given Dataset IDs.
+            id_col = self._static.dataset.columns["id"]
+            sql = sqlalchemy.sql.select(
+                id_col,
+                self._static.dataset.columns["dataset_type_id"],
+            ).where(id_col.in_(batch))
+            with self._db.query(sql) as sql_result:
+                dataset_rows = sql_result.mappings().all()
+            for row in dataset_rows:
+                dataset_type_map[row["id"]] = self._get_dataset_type_by_id(row["dataset_type_id"])
 
         # Group the given dataset IDs by the DimensionGroup of their dataset
         # types -- there is a separate tags table for each DimensionGroup.
@@ -448,40 +451,41 @@ class ByDimensionsDatasetRecordStorageManagerUUID(DatasetRecordStorageManager):
             # data IDs corresponding to the UUIDs found from the dataset table.
             dynamic_tables = self._get_dynamic_tables(dimension_group)
             tags_table = self._get_tags_table(dynamic_tables)
-            tags_sql = tags_table.select().where(tags_table.columns["dataset_id"].in_(datasets))
-            # Join in the collection table to fetch the run name.
-            collection_column = tags_table.columns[self._collections.getCollectionForeignKeyName()]
-            joined_collections = self._collections.join_collections_sql(collection_column, tags_sql)
-            tags_sql = joined_collections.joined_sql
-            run_name_column = joined_collections.name_column
-            tags_sql = tags_sql.add_columns(run_name_column)
-            # Tags table includes run collections and tagged
-            # collections.
-            # In theory the data ID for a given dataset should be the
-            # same in both, but nothing actually guarantees this.
-            # So skip any tagged collections, using the run collection
-            # as the definitive definition.
-            tags_sql = tags_sql.where(joined_collections.type_column == int(CollectionType.RUN))
-
-            with self._db.query(tags_sql) as sql_result:
-                data_id_rows = sql_result.mappings().all()
-
-            assert run_name_column.key is not None
-            for data_id_row in data_id_rows:
-                id = data_id_row["dataset_id"]
-                dataset_type = dataset_type_map[id]
-                run_name = data_id_row[run_name_column.key]
-                data_id = DataCoordinate.from_required_values(
-                    dimension_group,
-                    tuple(data_id_row[dimension] for dimension in dimension_group.required),
-                )
-                ref = DatasetRef(
-                    datasetType=dataset_type,
-                    dataId=data_id,
-                    id=id,
-                    run=run_name,
-                )
-                output_refs.append(ref)
+            for batch in chunk_iterable(datasets, 50000):
+                tags_sql = tags_table.select().where(tags_table.columns["dataset_id"].in_(batch))
+                # Join in the collection table to fetch the run name.
+                collection_column = tags_table.columns[self._collections.getCollectionForeignKeyName()]
+                joined_collections = self._collections.join_collections_sql(collection_column, tags_sql)
+                tags_sql = joined_collections.joined_sql
+                run_name_column = joined_collections.name_column
+                tags_sql = tags_sql.add_columns(run_name_column)
+                # Tags table includes run collections and tagged
+                # collections.
+                # In theory the data ID for a given dataset should be the
+                # same in both, but nothing actually guarantees this.
+                # So skip any tagged collections, using the run collection
+                # as the definitive definition.
+                tags_sql = tags_sql.where(joined_collections.type_column == int(CollectionType.RUN))
+
+                with self._db.query(tags_sql) as sql_result:
+                    data_id_rows = sql_result.mappings().all()
+
+                assert run_name_column.key is not None
+                for data_id_row in data_id_rows:
+                    id = data_id_row["dataset_id"]
+                    dataset_type = dataset_type_map[id]
+                    run_name = data_id_row[run_name_column.key]
+                    data_id = DataCoordinate.from_required_values(
+                        dimension_group,
+                        tuple(data_id_row[dimension] for dimension in dimension_group.required),
+                    )
+                    ref = DatasetRef(
+                        datasetType=dataset_type,
+                        dataId=data_id,
+                        id=id,
+                        run=run_name,
+                    )
+                    output_refs.append(ref)
 
         return output_refs
 

lsst/daf/butler/registry/interfaces/_database.py

@@ -1562,7 +1562,12 @@ class Database(ABC):
                 return None
             else:
                 sql = table.insert()
-                return [connection.execute(sql, row).inserted_primary_key[0] for row in rows]
+                ids = []
+                for row in rows:
+                    key = connection.execute(sql, row).inserted_primary_key
+                    assert key is not None
+                    ids.append(key[0])
+                return ids
 
     @abstractmethod
     def replace(self, table: sqlalchemy.schema.Table, *rows: dict) -> None:

lsst/daf/butler/version.py

@@ -1,2 +1,2 @@
 __all__ = ["__version__"]
-__version__ = "30.2025.5000"
+__version__ = "30.2025.5100"

{lsst_daf_butler-30.2025.5000.dist-info → lsst_daf_butler-30.2025.5100.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lsst-daf-butler
-Version: 30.2025.5000
+Version: 30.2025.5100
 Summary: An abstraction layer for reading and writing astronomical data to datastores.
 Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
 License-Expression: BSD-3-Clause OR GPL-3.0-or-later

{lsst_daf_butler-30.2025.5000.dist-info → lsst_daf_butler-30.2025.5100.dist-info}/RECORD

@@ -51,7 +51,7 @@ lsst/daf/butler/repo_relocation.py,sha256=Ivhx2xU4slc53Z6RExhNnquMr2Hx-S8h62emml
 lsst/daf/butler/time_utils.py,sha256=MVTfOFI2xt3IeA46pa-fWY2kJRwSzaQyq1uzeUABcTM,11805
 lsst/daf/butler/timespan_database_representation.py,sha256=rYeQ_vp6gneRjboqV-gvNW0DWhm1QJM-KnVzFTDVZ0I,24550
 lsst/daf/butler/utils.py,sha256=5u50COK5z4u31grOhmQF7mFz55biNLOvSMRdQjEdsjo,5140
-lsst/daf/butler/version.py,sha256=arbNW1FaLmuy0rG2fl72Eu7RoD_LnOedd06IvO-wmLM,55
+lsst/daf/butler/version.py,sha256=T_ii-AmyfLEzX_XiWTNw5GEFMy94NcXrbg2rkzKZg7g,55
 lsst/daf/butler/_rubin/__init__.py,sha256=9z5kmc6LJ3C_iPFV46cvdlQ2qOGJbZh-2Ft5Z-rbE28,1569
 lsst/daf/butler/_rubin/file_datasets.py,sha256=P5_BIhxpVj9qfLuLiI2_dClMHsjO5Qm5oDXVr3WntNU,3607
 lsst/daf/butler/_utilities/__init__.py,sha256=vLzPZYAJ-9r1cnqsP64MVpFgSw2166yOpq0iPMSdAvw,1298
@@ -96,11 +96,11 @@ lsst/daf/butler/datastore/composites.py,sha256=NZ7rBK5yH-hrtpqZxC8d49UwwQqWZlEmm
 lsst/daf/butler/datastore/constraints.py,sha256=OcUXuXZq1UBnuQqq8U7Hp3Ezqu0RBN8pIo93BEq7lyI,5921
 lsst/daf/butler/datastore/file_templates.py,sha256=xnb4ZheW6NqeCE__vkIvLF91d57nhfcX3ynGWI0rX_0,35095
 lsst/daf/butler/datastore/generic_base.py,sha256=C15FN1fDVxs-XjeDc1hw5un3MMMVIaZN4QdFbjqQ168,5176
-lsst/daf/butler/datastore/record_data.py,sha256=dhku1kCG0WCeYeWOq9m0Euq02pa96XDF5W9nD_ZEe8k,10428
+lsst/daf/butler/datastore/record_data.py,sha256=kY-fqXrhdffpIHgczgBDvGYb4VnDoFtIeNfqUpteR1w,10416
 lsst/daf/butler/datastore/stored_file_info.py,sha256=s_9LsLZgIF7TwjiXEXgNPz3OaaRiUyzmf5SOdJ1elhk,15838
 lsst/daf/butler/datastores/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 lsst/daf/butler/datastores/chainedDatastore.py,sha256=3DUKoGLVbmqyqm2K4-De3GKswE4WALFp0P9ul6OF-Qk,56220
-lsst/daf/butler/datastores/fileDatastore.py,sha256=jEfKlhKRSgR5jzmGu7VWFQXLOHsLt566GufLFfQ32yE,136081
+lsst/daf/butler/datastores/fileDatastore.py,sha256=AN4IKz5hcz2jcfKQUhEnT4DxiPDs1nf5_WF5P_t-u68,136876
 lsst/daf/butler/datastores/inMemoryDatastore.py,sha256=ZLgJuPnKJPsnaZQ_8rTZn02YHl2MxsQ_kcxCXhcc5dk,30213
 lsst/daf/butler/datastores/file_datastore/__init__.py,sha256=vLzPZYAJ-9r1cnqsP64MVpFgSw2166yOpq0iPMSdAvw,1298
 lsst/daf/butler/datastores/file_datastore/get.py,sha256=w-5tuXxdthsSiOlI7MBFrFO6zWM_f1Uqqoy8VGAdvcQ,16964
@@ -203,7 +203,7 @@ lsst/daf/butler/registry/versions.py,sha256=egvrctt_1wBzZgh8iSfySaQJQ9bkx_9bUJWk
 lsst/daf/butler/registry/wildcards.py,sha256=akMGgqDkVM0mQ9RAFENv0IrnoUyMP3mhODYXDaWIQ8o,20277
 lsst/daf/butler/registry/bridge/__init__.py,sha256=vLzPZYAJ-9r1cnqsP64MVpFgSw2166yOpq0iPMSdAvw,1298
 lsst/daf/butler/registry/bridge/ephemeral.py,sha256=QSKMRwQTAHnMOwdigH5tSNrQaM2dK-IXymb7x87bY-w,5845
-lsst/daf/butler/registry/bridge/monolithic.py,sha256=JvVhMH0K9lfzOTaVe9k34JaTAXAHKLP8e2XvRANWEZg,18148
+lsst/daf/butler/registry/bridge/monolithic.py,sha256=NLMLPh9ORChrtHT0be8PtmIlnnDQlmeAsM2MPmTeZZM,18386
 lsst/daf/butler/registry/collections/__init__.py,sha256=vLzPZYAJ-9r1cnqsP64MVpFgSw2166yOpq0iPMSdAvw,1298
 lsst/daf/butler/registry/collections/_base.py,sha256=MxrvTh81aHUULXtf5hHzK6iAjwkes5NckouHMhTFdPM,37480
 lsst/daf/butler/registry/collections/nameKey.py,sha256=UZCwiY0hOyB1NXA_1ZzjD4tuKnQ_jvrMmPQlUN6tWuk,12890
@@ -214,7 +214,7 @@ lsst/daf/butler/registry/databases/sqlite.py,sha256=xW82mdbOoOOTBeuSYDQ9DiDPhnMG
 lsst/daf/butler/registry/datasets/__init__.py,sha256=vLzPZYAJ-9r1cnqsP64MVpFgSw2166yOpq0iPMSdAvw,1298
 lsst/daf/butler/registry/datasets/byDimensions/__init__.py,sha256=BG4C7mhKFbCzvfQSI31CIV_iTMc1gYL_LT4Plyu6LdE,1323
 lsst/daf/butler/registry/datasets/byDimensions/_dataset_type_cache.py,sha256=VmpKsw7PfHTu54cChAUH8qcOC2jizBLNJX_SE_DQ1yA,8315
-lsst/daf/butler/registry/datasets/byDimensions/_manager.py,sha256=f7DWQ_77tgOti6C_0wq-v1orf4YCW_mhmzhi_FMcbq0,71373
+lsst/daf/butler/registry/datasets/byDimensions/_manager.py,sha256=DeJhLIc7hqw19j_HYh-boQ8jow3NMurbhUegrQ8GO8s,71720
 lsst/daf/butler/registry/datasets/byDimensions/summaries.py,sha256=MuRk2p6fAKhvjId3jWnuFzhqMnKNF31ugExD7a2g48k,18534
 lsst/daf/butler/registry/datasets/byDimensions/tables.py,sha256=9MXcpRGuXQonEMmh-zQOUcYJVA9pvUuNXyZRZtP-KH8,25768
 lsst/daf/butler/registry/dimensions/__init__.py,sha256=vLzPZYAJ-9r1cnqsP64MVpFgSw2166yOpq0iPMSdAvw,1298
@@ -223,7 +223,7 @@ lsst/daf/butler/registry/interfaces/__init__.py,sha256=IBMBBb1gyAx3o9uTufhQHtMrh
 lsst/daf/butler/registry/interfaces/_attributes.py,sha256=z-njEpWLhmKU4S0KOCplrY4QeBGoKUhlPRtSdNS_4uw,7258
 lsst/daf/butler/registry/interfaces/_bridge.py,sha256=pds0AhEXZVO3YguU-tkZ8J_NZrBj55HCGkHpBTXbBFQ,15924
 lsst/daf/butler/registry/interfaces/_collections.py,sha256=q0xRy7gA4KacPvJb1Wtk-uDSHrjmv4xpD2qGmzpvdKM,28485
-lsst/daf/butler/registry/interfaces/_database.py,sha256=ibsaPp40WhutsBx9fyf3eioouvySfcX_awT0UucgL7A,84557
+lsst/daf/butler/registry/interfaces/_database.py,sha256=YmbGySh0qXf8HJe_FQd69DgBMdx0HuytbhnPVIpYCJw,84706
 lsst/daf/butler/registry/interfaces/_database_explain.py,sha256=CkALWwNeyrjRvKizWrxvcGDunIhB77kLtEuXscrXVOY,3052
 lsst/daf/butler/registry/interfaces/_datasets.py,sha256=_vvEStau34AdZ45NdnrrgBh_9uSHmth7amnH538gj7A,24975
 lsst/daf/butler/registry/interfaces/_dimensions.py,sha256=peUPhmQEdwvgZcSsEsR80OZBcKEyjBRvmAbEkvMKlKY,14504
@@ -337,13 +337,13 @@ lsst/daf/butler/transfers/__init__.py,sha256=M1YcFszSkNB5hB2pZwwGXqbJE2dKt4YXDin
 lsst/daf/butler/transfers/_context.py,sha256=Ro_nf9NDw9IAr-Pw_NtcdotQKx34RbBbNubt20zwRXU,16449
 lsst/daf/butler/transfers/_interfaces.py,sha256=Ia1NqcFR5E-Ik4zsXEe2fuMtNCJj5Yfe_gVHLTBtJDw,7490
 lsst/daf/butler/transfers/_yaml.py,sha256=w_0GmrueuHVLfOfAXGHFBbWAl18tX6eSElbTC-2jRoc,32632
-lsst_daf_butler-30.2025.5000.dist-info/licenses/COPYRIGHT,sha256=k1Vq0-Be_K-puaeW4UZnckPjksEL-MJh4XKiWcjMxJE,312
-lsst_daf_butler-30.2025.5000.dist-info/licenses/LICENSE,sha256=pRExkS03v0MQW-neNfIcaSL6aiAnoLxYgtZoFzQ6zkM,232
-lsst_daf_butler-30.2025.5000.dist-info/licenses/bsd_license.txt,sha256=7MIcv8QRX9guUtqPSBDMPz2SnZ5swI-xZMqm_VDSfxY,1606
-lsst_daf_butler-30.2025.5000.dist-info/licenses/gpl-v3.0.txt,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-lsst_daf_butler-30.2025.5000.dist-info/METADATA,sha256=LU-VqW_BssqqK4on8WCqIuI9N6QrcBI-1F3QE4O8pxU,3813
-lsst_daf_butler-30.2025.5000.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-lsst_daf_butler-30.2025.5000.dist-info/entry_points.txt,sha256=XsRxyTK3c-jGlKVuVnbpch3gtaO0lAA_fS3i2NGS5rw,59
-lsst_daf_butler-30.2025.5000.dist-info/top_level.txt,sha256=eUWiOuVVm9wwTrnAgiJT6tp6HQHXxIhj2QSZ7NYZH80,5
-lsst_daf_butler-30.2025.5000.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-lsst_daf_butler-30.2025.5000.dist-info/RECORD,,
+lsst_daf_butler-30.2025.5100.dist-info/licenses/COPYRIGHT,sha256=k1Vq0-Be_K-puaeW4UZnckPjksEL-MJh4XKiWcjMxJE,312
+lsst_daf_butler-30.2025.5100.dist-info/licenses/LICENSE,sha256=pRExkS03v0MQW-neNfIcaSL6aiAnoLxYgtZoFzQ6zkM,232
+lsst_daf_butler-30.2025.5100.dist-info/licenses/bsd_license.txt,sha256=7MIcv8QRX9guUtqPSBDMPz2SnZ5swI-xZMqm_VDSfxY,1606
+lsst_daf_butler-30.2025.5100.dist-info/licenses/gpl-v3.0.txt,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+lsst_daf_butler-30.2025.5100.dist-info/METADATA,sha256=9foKLN5p5TlcxoI2l8Pa85qZ-ayz5WpArFk_0V9VHWI,3813
+lsst_daf_butler-30.2025.5100.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+lsst_daf_butler-30.2025.5100.dist-info/entry_points.txt,sha256=XsRxyTK3c-jGlKVuVnbpch3gtaO0lAA_fS3i2NGS5rw,59
+lsst_daf_butler-30.2025.5100.dist-info/top_level.txt,sha256=eUWiOuVVm9wwTrnAgiJT6tp6HQHXxIhj2QSZ7NYZH80,5
+lsst_daf_butler-30.2025.5100.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
+lsst_daf_butler-30.2025.5100.dist-info/RECORD,,