lsst-daf-butler 29.2025.2500__tar.gz → 29.2025.2700__tar.gz

{lsst_daf_butler-29.2025.2500/python/lsst_daf_butler.egg-info → lsst_daf_butler-29.2025.2700}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lsst-daf-butler
-Version: 29.2025.2500
+Version: 29.2025.2700
 Summary: An abstraction layer for reading and writing astronomical data to datastores.
 Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
 License: BSD 3-Clause License

{lsst_daf_butler-29.2025.2500 → lsst_daf_butler-29.2025.2700}/python/lsst/daf/butler/dimensions/_record_set.py

@@ -809,8 +809,7 @@ class DimensionDataAttacher:
             been referenced in at least one of the constructor arguments.
         data_ids : `~collections.abc.Iterable` [ `DataCoordinate` ]
             Data IDs to attach dimension records to (not in place; data
-            coordinates are immutable).  Must have full values (i.e. implied
-            as well as required dimensions).
+            coordinates are immutable).
         query : `.queries.Query`, optional
             A butler query that can be used to look up missing dimension
             records.  Records fetched via query are cached in the `records`
@@ -822,49 +821,41 @@ class DimensionDataAttacher:
             Data IDs with dimension records attached, in the same order as the
             original iterable.
         """
-        incomplete: list[_InProgressRecordDicts] = []
         lookup_helpers = [
             _DimensionRecordLookupHelper.build(dimensions, element_name, self)
-            for element_name in dimensions.elements
+            for element_name in dimensions.lookup_order
         ]
-        result: list[DataCoordinate] = []
-        for n, data_id in enumerate(data_ids):
-            records = _InProgressRecordDicts(n)
-            for lookup_helper in lookup_helpers:
-                lookup_helper.lookup(data_id.full_values, records)
-            if records.missing:
-                incomplete.append(records)
-            else:
-                data_id = data_id.expanded(records.done)
-            result.append(data_id)
-        if query is not None:
-            for lookup_helper in lookup_helpers:
-                lookup_helper.fetch_missing(query)
-            for records in incomplete:
-                for element, required_values in records.missing.items():
-                    records.done[element] = self.records[element].find_with_required_values(required_values)
-                result[records.index] = result[records.index].expanded(records.done)
-        else:
-            # Other logic branches also raise LookupError internally (e.g. in
-            # find_with_required_values); this one has a better message (i.e.
-            # about all failures, not just the first one) just because we
-            # happen to have any information at hand, while we don't in other
-            # branches.
-            for records in incomplete:
-                raise LookupError(
-                    f"No dimension record for element(s) {list(records.missing.keys())} "
-                    f"for data ID {result[records.index]}.  "
-                    f"{len(incomplete)} data ID{' was' if len(incomplete) == 1 else 's were'} "
-                    "missing at least one record."
-                )
-        return result
+        records = [_InProgressRecordDicts(data_id) for data_id in data_ids]
+        for lookup_helper in lookup_helpers:
+            for r in records:
+                lookup_helper.lookup(r)
+            incomplete = lookup_helper.incomplete_records
+            if incomplete:
+                if query is not None:
+                    lookup_helper.fetch_missing(query)
+                    # We may still be missing records at this point, if they
+                    # were not available in the database.
+                    # This is intentional, because in existing Butler
+                    # repositories dimension records are not always fully
+                    # populated. (For example, it is common for a visit to
+                    # exist without corresponding visit_detector_region
+                    # records, since these are populated at different times
+                    # by different processes.)
+                else:
+                    raise LookupError(
+                        f"No dimension record for element '{lookup_helper.element}' "
+                        f"for data ID {incomplete[0].data_id}.  "
+                        f"{len(incomplete)} data ID{' was' if len(incomplete) == 1 else 's were'} "
+                        "missing at least one record."
+                    )
+
+        return [r.data_id.expanded(r.done) for r in records]
 
 
 @dataclasses.dataclass
 class _InProgressRecordDicts:
-    index: int  # Index of the data ID these are for in the result list.
+    data_id: DataCoordinate
     done: dict[str, DimensionRecord] = dataclasses.field(default_factory=dict)
-    missing: dict[str, tuple[DataIdValue, ...]] = dataclasses.field(default_factory=dict)
 
 
 @dataclasses.dataclass
@@ -873,7 +864,7 @@ class _DimensionRecordLookupHelper:
     # tuple in the to-be-expanded data ID's full-values tuple.
     indices: list[int]
     record_set: DimensionRecordSet
-    missing: set[tuple[DataIdValue, ...]] = dataclasses.field(default_factory=set)
+    incomplete_records: list[_InProgressRecordDicts] = dataclasses.field(default_factory=list)
 
     @property
     def element(self) -> str:
@@ -902,33 +893,62 @@ class _DimensionRecordLookupHelper:
         else:
             return _DimensionRecordLookupHelper(indices, attacher.records[element])
 
-    def lookup(self, full_data_id_values: tuple[DataIdValue, ...], records: _InProgressRecordDicts) -> None:
-        required_values = tuple([full_data_id_values[i] for i in self.indices])
+    def lookup(self, records: _InProgressRecordDicts) -> None:
+        required_values = self._get_required_values(records)
         if (result := self.record_set._by_required_values.get(required_values)) is None:
             result = self.fallback(required_values)
             if result is not None:
                 self.record_set.add(result)
                 records.done[self.element] = result
             else:
-                records.missing[self.element] = required_values
-                self.missing.add(required_values)
+                self.incomplete_records.append(records)
         else:
             records.done[self.element] = result
 
+    def _get_required_values(self, records: _InProgressRecordDicts) -> tuple[DataIdValue, ...]:
+        if records.data_id.hasFull():
+            full_values = records.data_id.full_values
+            return tuple([full_values[i] for i in self.indices])
+        else:
+            values = []
+            dimensions = self.record_set.element.minimal_group.required
+            for dimension in dimensions:
+                value = records.data_id.get(dimension)
+                if value is None:
+                    value = self._find_implied_value(dimension, records)
+                values.append(value)
+            return tuple(values)
+
+    def _find_implied_value(self, implied_dimension: str, records: _InProgressRecordDicts) -> DataIdValue:
+        for rec in records.done.values():
+            if implied_dimension in rec.definition.implied:
+                return rec.get(implied_dimension)
+
+        raise LookupError(
+            f"Implied value for dimension '{implied_dimension}' not found in records for"
+            f" {list(records.done.keys())}"
+        )
+
     def fallback(self, required_values: tuple[DataIdValue, ...]) -> DimensionRecord | None:
         return None
 
     def fetch_missing(self, query: Query) -> None:
-        if self.missing:
+        if self.incomplete_records:
+            missing_values = set(self._get_required_values(r) for r in self.incomplete_records)
             self.record_set.update(
                 query.join_data_coordinates(
                     [
-                        DataCoordinate.from_required_values(self.record_set.element.minimal_group, row)
-                        for row in self.missing
+                        DataCoordinate.from_required_values(self.record_set.element.minimal_group, values)
+                        for values in missing_values
                     ]
                 ).dimension_records(self.record_set.element.name)
             )
 
+        missing = self.incomplete_records
+        self.incomplete_records = list()
+        for record in missing:
+            self.lookup(record)
+
 
 @dataclasses.dataclass
 class _DeserializingDimensionRecordLookupHelper(_DimensionRecordLookupHelper):

{lsst_daf_butler-29.2025.2500 → lsst_daf_butler-29.2025.2700}/python/lsst/daf/butler/dimensions/_records.py

@@ -34,15 +34,7 @@ from collections.abc import Hashable
 from typing import TYPE_CHECKING, Any, ClassVar, TypeAlias
 
 import pydantic
-from pydantic import (
-    BaseModel,
-    Field,
-    StrictBool,
-    StrictFloat,
-    StrictInt,
-    StrictStr,
-    create_model,
-)
+from pydantic import BaseModel, Field, StrictBool, StrictFloat, StrictInt, StrictStr, create_model
 
 import lsst.sphgeom
 from lsst.utils.classes import immutable
@@ -533,6 +525,29 @@ class DimensionRecord:
                 results["datetime_end"] = timespan.end
         return results
 
+    def get(self, name: str) -> Any:
+        """Return a single metadata value associated with this record.
+
+        Parameters
+        ----------
+        name : `str`
+            Key of the metadata value to be retrieved.
+
+        Returns
+        -------
+        value
+            The metadata value.
+
+        Raises
+        ------
+        KeyError
+            If the given name is not a valid key in this dimension record.
+        """
+        if name not in self.__slots__:
+            raise KeyError(f"'{name}' is not a valid record key for dimension '{self.definition.name}'")
+
+        return getattr(self, name)
+
     def serialize_key_value(self) -> SerializedKeyValueDimensionRecord:
         """Serialize this record to a `list` that can be sliced into a key
         (data ID values) / value (everything else) pair.

{lsst_daf_butler-29.2025.2500 → lsst_daf_butler-29.2025.2700}/python/lsst/daf/butler/registry/obscore/_config.py

@@ -101,6 +101,11 @@ class DatasetTypeConfig(pydantic.BaseModel):
     """Value for the ``obs_collection`` column, if specified it overrides
     global value in `ObsCoreConfig`."""
 
+    s_xel: tuple[int, int] | None = None
+    """The number of pixels in the first and second dimension of the dataset
+    type.
+    """
+
     extra_columns: (
         None | (dict[str, StrictFloat | StrictInt | StrictBool | StrictStr | ExtraColumnConfig])
     ) = None

{lsst_daf_butler-29.2025.2500 → lsst_daf_butler-29.2025.2700}/python/lsst/daf/butler/registry/obscore/_records.py

@@ -179,6 +179,10 @@ class RecordFactory:
             record["obs_collection"] = self.config.obs_collection
         record["access_format"] = dataset_config.access_format
 
+        if dataset_config.s_xel is not None:
+            record["s_xel1"] = dataset_config.s_xel[0]
+            record["s_xel2"] = dataset_config.s_xel[1]
+
         dataId = ref.dataId
         dataset_type_name = ref.datasetType.name
 
@@ -190,6 +194,11 @@ class RecordFactory:
         fmt_kws.update(run=ref.run)
         fmt_kws.update(dataset_type=dataset_type_name)
         fmt_kws.update(record)
+
+        # In some cases we would like some keys to be duplicated
+        # with different names. For example making an exposure be a visit.
+        self.finalize_format_keywords(fmt_kws)
+
         if dataset_config.obs_id_fmt:
             record["obs_id"] = dataset_config.obs_id_fmt.format(**fmt_kws)
             fmt_kws["obs_id"] = record["obs_id"]
@@ -330,6 +339,17 @@ class RecordFactory:
                 raise
         return record
 
+    def finalize_format_keywords(self, format_kws: dict[str, Any]) -> None:
+        """Modify the formatting dict as required after it has been populated
+        with generic content.
+
+        Parameters
+        ----------
+        format_kws : `dict`
+            Dictionary that might be modified in place.
+        """
+        pass
+
     @classmethod
     def get_record_type_from_universe(cls, universe: DimensionUniverse) -> type[RecordFactory]:
         namespace = universe.namespace
@@ -410,10 +430,10 @@ class DafButlerRecordFactory(RecordFactory):
         if timespan is not None:
             if timespan.begin is not None:
                 t_min = cast(astropy.time.Time, timespan.begin)
-                record["t_min"] = float(t_min.mjd)
+                record["t_min"] = float(t_min.utc.mjd)
             if timespan.end is not None:
                 t_max = cast(astropy.time.Time, timespan.end)
-                record["t_max"] = float(t_max.mjd)
+                record["t_max"] = float(t_max.utc.mjd)
 
         region = dataId.region
         if self.exposure.name in dataId:
@@ -469,3 +489,9 @@ class DafButlerRecordFactory(RecordFactory):
             else:
                 return None, None
         return region_dim, "region"
+
+    def finalize_format_keywords(self, format_kws: dict[str, Any]) -> None:
+        # If we have an exposure but do not have a visit, copy the exposure
+        # into the visit field.
+        if "exposure" in format_kws and "visit" not in format_kws:
+            format_kws["visit"] = format_kws["exposure"]

{lsst_daf_butler-29.2025.2500 → lsst_daf_butler-29.2025.2700}/python/lsst/daf/butler/registry/sql_registry.py

@@ -34,6 +34,7 @@ __all__ = ("SqlRegistry",)
 import contextlib
 import logging
 import warnings
+from collections import defaultdict
 from collections.abc import Iterable, Iterator, Mapping, Sequence
 from typing import TYPE_CHECKING, Any, Literal, cast
 
@@ -61,6 +62,7 @@ from ..dimensions import (
     DataCoordinate,
     DataId,
     DimensionConfig,
+    DimensionDataAttacher,
     DimensionElement,
     DimensionGroup,
     DimensionRecord,
@@ -1060,16 +1062,15 @@ class SqlRegistry:
                 f"Given collection is of type {runRecord.type.name}; RUN collection required."
             )
         assert isinstance(runRecord, RunRecord)
-        progress = Progress("daf.butler.Registry.insertDatasets", level=logging.DEBUG)
+
+        expandedDataIds = [
+            DataCoordinate.standardize(dataId, dimensions=datasetType.dimensions) for dataId in dataIds
+        ]
         if expand:
-            expandedDataIds = [
-                self.expandDataId(dataId, dimensions=datasetType.dimensions)
-                for dataId in progress.wrap(dataIds, f"Expanding {datasetType.name} data IDs")
-            ]
-        else:
-            expandedDataIds = [
-                DataCoordinate.standardize(dataId, dimensions=datasetType.dimensions) for dataId in dataIds
-            ]
+            _LOG.debug("Expanding %d data IDs", len(expandedDataIds))
+            expandedDataIds = self.expand_data_ids(expandedDataIds)
+            _LOG.debug("Finished expanding data IDs")
+
         try:
             refs = list(
                 self._managers.datasets.insert(datasetType.name, runRecord, expandedDataIds, idGenerationMode)
@@ -1161,12 +1162,10 @@ class SqlRegistry:
             )
         assert isinstance(runRecord, RunRecord)
 
-        progress = Progress("daf.butler.Registry.importDatasets", level=logging.DEBUG)
         if expand:
-            datasets = [
-                dataset.expanded(self.expandDataId(dataset.dataId, dimensions=dataset.datasetType.dimensions))
-                for dataset in progress.wrap(datasets, "Expanding data IDs")
-            ]
+            _LOG.debug("Expanding %d data IDs", len(datasets))
+            datasets = self.expand_refs(datasets)
+            _LOG.debug("Finished expanding data IDs")
 
         try:
             self._managers.datasets.import_(runRecord, datasets)
@@ -1545,6 +1544,34 @@ class SqlRegistry:
                     )
         return DataCoordinate.standardize(keys, dimensions=standardized.dimensions).expanded(records=records)
 
+    def expand_data_ids(self, data_ids: Iterable[DataCoordinate]) -> list[DataCoordinate]:
+        output = list(data_ids)
+
+        grouped_by_dimensions: defaultdict[DimensionGroup, list[int]] = defaultdict(list)
+        for i, data_id in enumerate(data_ids):
+            if not data_id.hasRecords():
+                grouped_by_dimensions[data_id.dimensions].append(i)
+
+        if not grouped_by_dimensions:
+            # All given DataCoordinate values are already expanded.
+            return output
+
+        attacher = DimensionDataAttacher(
+            cache=self.dimension_record_cache,
+            dimensions=DimensionGroup.union(*grouped_by_dimensions.keys(), universe=self.dimensions),
+        )
+        with self._query() as query:
+            for dimensions, indexes in grouped_by_dimensions.items():
+                expanded = attacher.attach(dimensions, (output[index] for index in indexes), query)
+                for index, data_id in zip(indexes, expanded):
+                    output[index] = data_id
+
+        return output
+
+    def expand_refs(self, dataset_refs: list[DatasetRef]) -> list[DatasetRef]:
+        expanded_ids = self.expand_data_ids([ref.dataId for ref in dataset_refs])
+        return [ref.expanded(data_id) for ref, data_id in zip(dataset_refs, expanded_ids)]
+
     def insertDimensionData(
         self,
         element: DimensionElement | str,

{lsst_daf_butler-29.2025.2500 → lsst_daf_butler-29.2025.2700}/python/lsst/daf/butler/transfers/_context.py

@@ -174,13 +174,9 @@ class RepoExportContext:
                     element = self._butler.dimensions[element]
                 if element.has_own_table:
                     standardized_elements.add(element)
-        for dataId in dataIds:
-            # This is potentially quite slow, because it's approximately
-            # len(dataId.graph.elements) queries per data ID.  But it's a no-op
-            # if the data ID is already expanded, and DM-26692 will add (or at
-            # least start to add / unblock) query functionality that should
-            # let us speed this up internally as well.
-            dataId = self._butler.registry.expandDataId(dataId)
+
+        expanded_data_ids = self._butler._registry.expand_data_ids(dataIds)
+        for dataId in expanded_data_ids:
             for element_name in dataId.dimensions.elements:
                 record = dataId.records[element_name]
                 if record is not None and record.definition in standardized_elements:

lsst_daf_butler-29.2025.2700/python/lsst/daf/butler/version.py

@@ -0,0 +1,2 @@
+__all__ = ["__version__"]
+__version__ = "29.2025.2700"

{lsst_daf_butler-29.2025.2500 → lsst_daf_butler-29.2025.2700/python/lsst_daf_butler.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lsst-daf-butler
-Version: 29.2025.2500
+Version: 29.2025.2700
 Summary: An abstraction layer for reading and writing astronomical data to datastores.
 Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
 License: BSD 3-Clause License

lsst_daf_butler-29.2025.2500/python/lsst/daf/butler/version.py

@@ -1,2 +0,0 @@
-__all__ = ["__version__"]
-__version__ = "29.2025.2500"