lsst-daf-butler 29.2025.2600__py3-none-any.whl → 29.2025.2700__py3-none-any.whl

lsst/daf/butler/dimensions/_record_set.py

@@ -809,8 +809,7 @@ class DimensionDataAttacher:
             been referenced in at least one of the constructor arguments.
         data_ids : `~collections.abc.Iterable` [ `DataCoordinate` ]
             Data IDs to attach dimension records to (not in place; data
-            coordinates are immutable).  Must have full values (i.e. implied
-            as well as required dimensions).
+            coordinates are immutable).
         query : `.queries.Query`, optional
             A butler query that can be used to look up missing dimension
             records.  Records fetched via query are cached in the `records`
@@ -822,49 +821,41 @@ class DimensionDataAttacher:
             Data IDs with dimension records attached, in the same order as the
             original iterable.
         """
-        incomplete: list[_InProgressRecordDicts] = []
         lookup_helpers = [
             _DimensionRecordLookupHelper.build(dimensions, element_name, self)
-            for element_name in dimensions.elements
+            for element_name in dimensions.lookup_order
         ]
-        result: list[DataCoordinate] = []
-        for n, data_id in enumerate(data_ids):
-            records = _InProgressRecordDicts(n)
-            for lookup_helper in lookup_helpers:
-                lookup_helper.lookup(data_id.full_values, records)
-            if records.missing:
-                incomplete.append(records)
-            else:
-                data_id = data_id.expanded(records.done)
-            result.append(data_id)
-        if query is not None:
-            for lookup_helper in lookup_helpers:
-                lookup_helper.fetch_missing(query)
-            for records in incomplete:
-                for element, required_values in records.missing.items():
-                    records.done[element] = self.records[element].find_with_required_values(required_values)
-                result[records.index] = result[records.index].expanded(records.done)
-        else:
-            # Other logic branches also raise LookupError internally (e.g. in
-            # find_with_required_values); this one has a better message (i.e.
-            # about all failures, not just the first one) just because we
-            # happen to have any information at hand, while we don't in other
-            # branches.
-            for records in incomplete:
-                raise LookupError(
-                    f"No dimension record for element(s) {list(records.missing.keys())} "
-                    f"for data ID {result[records.index]}.  "
-                    f"{len(incomplete)} data ID{' was' if len(incomplete) == 1 else 's were'} "
-                    "missing at least one record."
-                )
-        return result
+        records = [_InProgressRecordDicts(data_id) for data_id in data_ids]
+        for lookup_helper in lookup_helpers:
+            for r in records:
+                lookup_helper.lookup(r)
+            incomplete = lookup_helper.incomplete_records
+            if incomplete:
+                if query is not None:
+                    lookup_helper.fetch_missing(query)
+                    # We may still be missing records at this point, if they
+                    # were not available in the database.
+                    # This is intentional, because in existing Butler
+                    # repositories dimension records are not always fully
+                    # populated. (For example, it is common for a visit to
+                    # exist without corresponding visit_detector_region
+                    # records, since these are populated at different times
+                    # by different processes.)
+                else:
+                    raise LookupError(
+                        f"No dimension record for element '{lookup_helper.element}' "
+                        f"for data ID {incomplete[0].data_id}.  "
+                        f"{len(incomplete)} data ID{' was' if len(incomplete) == 1 else 's were'} "
+                        "missing at least one record."
+                    )
+
+        return [r.data_id.expanded(r.done) for r in records]
 
 
 @dataclasses.dataclass
 class _InProgressRecordDicts:
-    index: int  # Index of the data ID these are for in the result list.
+    data_id: DataCoordinate
     done: dict[str, DimensionRecord] = dataclasses.field(default_factory=dict)
-    missing: dict[str, tuple[DataIdValue, ...]] = dataclasses.field(default_factory=dict)
 
 
 @dataclasses.dataclass
@@ -873,7 +864,7 @@ class _DimensionRecordLookupHelper:
     # tuple in the to-be-expanded data ID's full-values tuple.
     indices: list[int]
     record_set: DimensionRecordSet
-    missing: set[tuple[DataIdValue, ...]] = dataclasses.field(default_factory=set)
+    incomplete_records: list[_InProgressRecordDicts] = dataclasses.field(default_factory=list)
 
     @property
     def element(self) -> str:
@@ -902,33 +893,62 @@ class _DimensionRecordLookupHelper:
         else:
             return _DimensionRecordLookupHelper(indices, attacher.records[element])
 
-    def lookup(self, full_data_id_values: tuple[DataIdValue, ...], records: _InProgressRecordDicts) -> None:
-        required_values = tuple([full_data_id_values[i] for i in self.indices])
+    def lookup(self, records: _InProgressRecordDicts) -> None:
+        required_values = self._get_required_values(records)
         if (result := self.record_set._by_required_values.get(required_values)) is None:
             result = self.fallback(required_values)
             if result is not None:
                 self.record_set.add(result)
                 records.done[self.element] = result
             else:
-                records.missing[self.element] = required_values
-                self.missing.add(required_values)
+                self.incomplete_records.append(records)
         else:
             records.done[self.element] = result
 
+    def _get_required_values(self, records: _InProgressRecordDicts) -> tuple[DataIdValue, ...]:
+        if records.data_id.hasFull():
+            full_values = records.data_id.full_values
+            return tuple([full_values[i] for i in self.indices])
+        else:
+            values = []
+            dimensions = self.record_set.element.minimal_group.required
+            for dimension in dimensions:
+                value = records.data_id.get(dimension)
+                if value is None:
+                    value = self._find_implied_value(dimension, records)
+                values.append(value)
+            return tuple(values)
+
+    def _find_implied_value(self, implied_dimension: str, records: _InProgressRecordDicts) -> DataIdValue:
+        for rec in records.done.values():
+            if implied_dimension in rec.definition.implied:
+                return rec.get(implied_dimension)
+
+        raise LookupError(
+            f"Implied value for dimension '{implied_dimension}' not found in records for"
+            f" {list(records.done.keys())}"
+        )
+
     def fallback(self, required_values: tuple[DataIdValue, ...]) -> DimensionRecord | None:
         return None
 
     def fetch_missing(self, query: Query) -> None:
-        if self.missing:
+        if self.incomplete_records:
+            missing_values = set(self._get_required_values(r) for r in self.incomplete_records)
             self.record_set.update(
                 query.join_data_coordinates(
                     [
-                        DataCoordinate.from_required_values(self.record_set.element.minimal_group, row)
-                        for row in self.missing
+                        DataCoordinate.from_required_values(self.record_set.element.minimal_group, values)
+                        for values in missing_values
                     ]
                 ).dimension_records(self.record_set.element.name)
             )
 
+        missing = self.incomplete_records
+        self.incomplete_records = list()
+        for record in missing:
+            self.lookup(record)
+
 
 @dataclasses.dataclass
 class _DeserializingDimensionRecordLookupHelper(_DimensionRecordLookupHelper):

lsst/daf/butler/dimensions/_records.py

@@ -34,15 +34,7 @@ from collections.abc import Hashable
 from typing import TYPE_CHECKING, Any, ClassVar, TypeAlias
 
 import pydantic
-from pydantic import (
-    BaseModel,
-    Field,
-    StrictBool,
-    StrictFloat,
-    StrictInt,
-    StrictStr,
-    create_model,
-)
+from pydantic import BaseModel, Field, StrictBool, StrictFloat, StrictInt, StrictStr, create_model
 
 import lsst.sphgeom
 from lsst.utils.classes import immutable
@@ -533,6 +525,29 @@ class DimensionRecord:
                 results["datetime_end"] = timespan.end
         return results
 
+    def get(self, name: str) -> Any:
+        """Return a single metadata value associated with this record.
+
+        Parameters
+        ----------
+        name : `str`
+            Key of the metadata value to be retrieved.
+
+        Returns
+        -------
+        value
+            The metadata value.
+
+        Raises
+        ------
+        KeyError
+            If the given name is not a valid key in this dimension record.
+        """
+        if name not in self.__slots__:
+            raise KeyError(f"'{name}' is not a valid record key for dimension '{self.definition.name}'")
+
+        return getattr(self, name)
+
     def serialize_key_value(self) -> SerializedKeyValueDimensionRecord:
         """Serialize this record to a `list` that can be sliced into a key
         (data ID values) / value (everything else) pair.

lsst/daf/butler/registry/sql_registry.py

@@ -34,6 +34,7 @@ __all__ = ("SqlRegistry",)
 import contextlib
 import logging
 import warnings
+from collections import defaultdict
 from collections.abc import Iterable, Iterator, Mapping, Sequence
 from typing import TYPE_CHECKING, Any, Literal, cast
 
@@ -61,6 +62,7 @@ from ..dimensions import (
     DataCoordinate,
     DataId,
     DimensionConfig,
+    DimensionDataAttacher,
     DimensionElement,
     DimensionGroup,
     DimensionRecord,
@@ -1060,16 +1062,15 @@ class SqlRegistry:
                 f"Given collection is of type {runRecord.type.name}; RUN collection required."
             )
         assert isinstance(runRecord, RunRecord)
-        progress = Progress("daf.butler.Registry.insertDatasets", level=logging.DEBUG)
+
+        expandedDataIds = [
+            DataCoordinate.standardize(dataId, dimensions=datasetType.dimensions) for dataId in dataIds
+        ]
         if expand:
-            expandedDataIds = [
-                self.expandDataId(dataId, dimensions=datasetType.dimensions)
-                for dataId in progress.wrap(dataIds, f"Expanding {datasetType.name} data IDs")
-            ]
-        else:
-            expandedDataIds = [
-                DataCoordinate.standardize(dataId, dimensions=datasetType.dimensions) for dataId in dataIds
-            ]
+            _LOG.debug("Expanding %d data IDs", len(expandedDataIds))
+            expandedDataIds = self.expand_data_ids(expandedDataIds)
+            _LOG.debug("Finished expanding data IDs")
+
         try:
             refs = list(
                 self._managers.datasets.insert(datasetType.name, runRecord, expandedDataIds, idGenerationMode)
@@ -1161,12 +1162,10 @@ class SqlRegistry:
             )
         assert isinstance(runRecord, RunRecord)
 
-        progress = Progress("daf.butler.Registry.importDatasets", level=logging.DEBUG)
         if expand:
-            datasets = [
-                dataset.expanded(self.expandDataId(dataset.dataId, dimensions=dataset.datasetType.dimensions))
-                for dataset in progress.wrap(datasets, "Expanding data IDs")
-            ]
+            _LOG.debug("Expanding %d data IDs", len(datasets))
+            datasets = self.expand_refs(datasets)
+            _LOG.debug("Finished expanding data IDs")
 
         try:
             self._managers.datasets.import_(runRecord, datasets)
@@ -1545,6 +1544,34 @@ class SqlRegistry:
                     )
         return DataCoordinate.standardize(keys, dimensions=standardized.dimensions).expanded(records=records)
 
+    def expand_data_ids(self, data_ids: Iterable[DataCoordinate]) -> list[DataCoordinate]:
+        output = list(data_ids)
+
+        grouped_by_dimensions: defaultdict[DimensionGroup, list[int]] = defaultdict(list)
+        for i, data_id in enumerate(data_ids):
+            if not data_id.hasRecords():
+                grouped_by_dimensions[data_id.dimensions].append(i)
+
+        if not grouped_by_dimensions:
+            # All given DataCoordinate values are already expanded.
+            return output
+
+        attacher = DimensionDataAttacher(
+            cache=self.dimension_record_cache,
+            dimensions=DimensionGroup.union(*grouped_by_dimensions.keys(), universe=self.dimensions),
+        )
+        with self._query() as query:
+            for dimensions, indexes in grouped_by_dimensions.items():
+                expanded = attacher.attach(dimensions, (output[index] for index in indexes), query)
+                for index, data_id in zip(indexes, expanded):
+                    output[index] = data_id
+
+        return output
+
+    def expand_refs(self, dataset_refs: list[DatasetRef]) -> list[DatasetRef]:
+        expanded_ids = self.expand_data_ids([ref.dataId for ref in dataset_refs])
+        return [ref.expanded(data_id) for ref, data_id in zip(dataset_refs, expanded_ids)]
+
     def insertDimensionData(
         self,
         element: DimensionElement | str,

lsst/daf/butler/transfers/_context.py

@@ -174,13 +174,9 @@ class RepoExportContext:
                     element = self._butler.dimensions[element]
                 if element.has_own_table:
                     standardized_elements.add(element)
-        for dataId in dataIds:
-            # This is potentially quite slow, because it's approximately
-            # len(dataId.graph.elements) queries per data ID.  But it's a no-op
-            # if the data ID is already expanded, and DM-26692 will add (or at
-            # least start to add / unblock) query functionality that should
-            # let us speed this up internally as well.
-            dataId = self._butler.registry.expandDataId(dataId)
+
+        expanded_data_ids = self._butler._registry.expand_data_ids(dataIds)
+        for dataId in expanded_data_ids:
             for element_name in dataId.dimensions.elements:
                 record = dataId.records[element_name]
                 if record is not None and record.definition in standardized_elements:

lsst/daf/butler/version.py

@@ -1,2 +1,2 @@
 __all__ = ["__version__"]
-__version__ = "29.2025.2600"
+__version__ = "29.2025.2700"

{lsst_daf_butler-29.2025.2600.dist-info → lsst_daf_butler-29.2025.2700.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lsst-daf-butler
-Version: 29.2025.2600
+Version: 29.2025.2700
 Summary: An abstraction layer for reading and writing astronomical data to datastores.
 Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
 License: BSD 3-Clause License

{lsst_daf_butler-29.2025.2600.dist-info → lsst_daf_butler-29.2025.2700.dist-info}/RECORD

@@ -52,7 +52,7 @@ lsst/daf/butler/repo_relocation.py,sha256=Ivhx2xU4slc53Z6RExhNnquMr2Hx-S8h62emml
 lsst/daf/butler/time_utils.py,sha256=MVTfOFI2xt3IeA46pa-fWY2kJRwSzaQyq1uzeUABcTM,11805
 lsst/daf/butler/timespan_database_representation.py,sha256=MWDusjIQIL2RH1CDpWSW5sYvdHCJKzAfpg1rm1DfgEU,24302
 lsst/daf/butler/utils.py,sha256=5u50COK5z4u31grOhmQF7mFz55biNLOvSMRdQjEdsjo,5140
-lsst/daf/butler/version.py,sha256=eKEngRNTsqzq5yv4faNqXQ7KB1ya5wGTLTxJ1qbWG50,55
+lsst/daf/butler/version.py,sha256=ijvsqTn7akfUc9cY8XFVpeeoU8nGM3T0Tr7idt4aIaw,55
 lsst/daf/butler/_utilities/__init__.py,sha256=vLzPZYAJ-9r1cnqsP64MVpFgSw2166yOpq0iPMSdAvw,1298
 lsst/daf/butler/_utilities/locked_object.py,sha256=3RQf0Ish55mfQAfBy3V4Tfnfq5Q7-cxrwTlQMUhrIno,1931
 lsst/daf/butler/_utilities/named_locks.py,sha256=Zj_u1rZELaiWec3wJfkgmGD_YiZMLVxbMQmdbbVgk5E,2286
@@ -117,9 +117,9 @@ lsst/daf/butler/dimensions/_elements.py,sha256=3Hn884mHbxz6H2KkMm-SFitEG_eHVRivU
 lsst/daf/butler/dimensions/_governor.py,sha256=En0Ar_uBuUnuKG1IACbyaZ7FRwf234zI5IEjwIQmMHw,5963
 lsst/daf/butler/dimensions/_group.py,sha256=1X9PbAqyLWt-MxucVaTNFgBPzMS6migbFptqvQNSboE,21953
 lsst/daf/butler/dimensions/_packer.py,sha256=yfgw7gXMSkPC59Kr51X6Isi8j6ihxqoR1OO2xPnd8FY,6461
-lsst/daf/butler/dimensions/_record_set.py,sha256=4AGb53ECbAbEKrefJLqoJMfAWOgDCylx1jSDVHWzc7U,37417
+lsst/daf/butler/dimensions/_record_set.py,sha256=agiAZcAxueb1gqOf4723xXrQlXJ-MZI8-mh8gzvslJ8,38107
 lsst/daf/butler/dimensions/_record_table.py,sha256=b1ijXQlnXH1N8rdiAV4C4GlDmOsBbWGuzob8Gv0mtII,9733
-lsst/daf/butler/dimensions/_records.py,sha256=7MBeUESXbopEkrm4Nxq0Bin_0PlJ4z8Au9vGWA8XByg,24092
+lsst/daf/butler/dimensions/_records.py,sha256=QX8XxiHRhRKFkUO3CvoDpe_8KGGyt3nyl08ifxUWwVs,24677
 lsst/daf/butler/dimensions/_schema.py,sha256=DOuhmI98f3yuvasFFATJCFrx6cjQSx4o2kyLioMbHlU,18507
 lsst/daf/butler/dimensions/_skypix.py,sha256=Tn60mvl_n7uWcip6eltfGzMB__98zubY3m9gHbzIwKc,6853
 lsst/daf/butler/dimensions/_universe.py,sha256=HYWtfdxu09aPaZV9AJb6qLvzF0ObVq903zLAVIsx2jw,21191
@@ -186,7 +186,7 @@ lsst/daf/butler/registry/connectionString.py,sha256=MgZ_jdUAL9_8l7UFeYF0Fhvo6TBd
 lsst/daf/butler/registry/managers.py,sha256=gal-O39ViqsvrOJNWtFjdtWT2BCUaLYBvzf67blkPp8,21175
 lsst/daf/butler/registry/nameShrinker.py,sha256=7gnlYnRdeYc3XXSfrMnSyQrIZerDYVZeU8x4eppWDXI,1464
 lsst/daf/butler/registry/opaque.py,sha256=xbdOogqabRH3YaevIZ7_hbbTb54rCOGyC7Uq7SZNH-Q,9882
-lsst/daf/butler/registry/sql_registry.py,sha256=FohIriwaDu9WC5sj3FK0whOEVOIbIomRAw3s9vGDVKc,107398
+lsst/daf/butler/registry/sql_registry.py,sha256=0sgrI_w9vrmN28ybR-lMUUz1xmeMW0ofKTFidaAQkj4,108472
 lsst/daf/butler/registry/versions.py,sha256=egvrctt_1wBzZgh8iSfySaQJQ9bkx_9bUJWkvvDZONQ,9331
 lsst/daf/butler/registry/wildcards.py,sha256=akMGgqDkVM0mQ9RAFENv0IrnoUyMP3mhODYXDaWIQ8o,20277
 lsst/daf/butler/registry/bridge/__init__.py,sha256=vLzPZYAJ-9r1cnqsP64MVpFgSw2166yOpq0iPMSdAvw,1298
@@ -340,16 +340,16 @@ lsst/daf/butler/tests/registry_data/hsc-rc2-subset-v0.yaml,sha256=Y_Ihkoa2uZ2QSW
 lsst/daf/butler/tests/registry_data/spatial.py,sha256=X8kq0_MzlWxtAA8wjWGFPkmyOaWHP1XjRsSLZNyZaFU,21302
 lsst/daf/butler/tests/registry_data/spatial.yaml,sha256=B8nPSgBnVUx7USQPdIpTND-0gW1njcrXQ-0c3jVrv-8,11262
 lsst/daf/butler/transfers/__init__.py,sha256=M1YcFszSkNB5hB2pZwwGXqbJE2dKt4YXDinW4s1iHI8,1371
-lsst/daf/butler/transfers/_context.py,sha256=h1XDJpdg64R7DRHo7mb9xgaLiHDs_AIJmZbyo66qSSw,17278
+lsst/daf/butler/transfers/_context.py,sha256=1oOlKWj53bjcVioOIzRNPZSC_Q0dLDI_R4syI9gMOkg,16938
 lsst/daf/butler/transfers/_interfaces.py,sha256=Ia1NqcFR5E-Ik4zsXEe2fuMtNCJj5Yfe_gVHLTBtJDw,7490
 lsst/daf/butler/transfers/_yaml.py,sha256=w_0GmrueuHVLfOfAXGHFBbWAl18tX6eSElbTC-2jRoc,32632
-lsst_daf_butler-29.2025.2600.dist-info/licenses/COPYRIGHT,sha256=k1Vq0-Be_K-puaeW4UZnckPjksEL-MJh4XKiWcjMxJE,312
-lsst_daf_butler-29.2025.2600.dist-info/licenses/LICENSE,sha256=pRExkS03v0MQW-neNfIcaSL6aiAnoLxYgtZoFzQ6zkM,232
-lsst_daf_butler-29.2025.2600.dist-info/licenses/bsd_license.txt,sha256=7MIcv8QRX9guUtqPSBDMPz2SnZ5swI-xZMqm_VDSfxY,1606
-lsst_daf_butler-29.2025.2600.dist-info/licenses/gpl-v3.0.txt,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-lsst_daf_butler-29.2025.2600.dist-info/METADATA,sha256=cmlhIrx73oYFr2nqz5rguNpe5QIZRf949UPtZpzmcJQ,3265
-lsst_daf_butler-29.2025.2600.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-lsst_daf_butler-29.2025.2600.dist-info/entry_points.txt,sha256=XsRxyTK3c-jGlKVuVnbpch3gtaO0lAA_fS3i2NGS5rw,59
-lsst_daf_butler-29.2025.2600.dist-info/top_level.txt,sha256=eUWiOuVVm9wwTrnAgiJT6tp6HQHXxIhj2QSZ7NYZH80,5
-lsst_daf_butler-29.2025.2600.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-lsst_daf_butler-29.2025.2600.dist-info/RECORD,,
+lsst_daf_butler-29.2025.2700.dist-info/licenses/COPYRIGHT,sha256=k1Vq0-Be_K-puaeW4UZnckPjksEL-MJh4XKiWcjMxJE,312
+lsst_daf_butler-29.2025.2700.dist-info/licenses/LICENSE,sha256=pRExkS03v0MQW-neNfIcaSL6aiAnoLxYgtZoFzQ6zkM,232
+lsst_daf_butler-29.2025.2700.dist-info/licenses/bsd_license.txt,sha256=7MIcv8QRX9guUtqPSBDMPz2SnZ5swI-xZMqm_VDSfxY,1606
+lsst_daf_butler-29.2025.2700.dist-info/licenses/gpl-v3.0.txt,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+lsst_daf_butler-29.2025.2700.dist-info/METADATA,sha256=fXwwFAdYJ-5kzkL2Dnk9EPZXtgxjhe9O6pAx45PPrNc,3265
+lsst_daf_butler-29.2025.2700.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+lsst_daf_butler-29.2025.2700.dist-info/entry_points.txt,sha256=XsRxyTK3c-jGlKVuVnbpch3gtaO0lAA_fS3i2NGS5rw,59
+lsst_daf_butler-29.2025.2700.dist-info/top_level.txt,sha256=eUWiOuVVm9wwTrnAgiJT6tp6HQHXxIhj2QSZ7NYZH80,5
+lsst_daf_butler-29.2025.2700.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
+lsst_daf_butler-29.2025.2700.dist-info/RECORD,,