lsst-daf-butler 29.2025.4400__tar.gz → 29.2025.4700__tar.gz

{lsst_daf_butler-29.2025.4400/python/lsst_daf_butler.egg-info → lsst_daf_butler-29.2025.4700}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lsst-daf-butler
-Version: 29.2025.4400
+Version: 29.2025.4700
 Summary: An abstraction layer for reading and writing astronomical data to datastores.
 Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
 License-Expression: BSD-3-Clause OR GPL-3.0-or-later
@@ -26,7 +26,6 @@ Requires-Dist: click>=7.0
 Requires-Dist: lsst-sphgeom
 Requires-Dist: lsst-utils
 Requires-Dist: lsst-resources
-Requires-Dist: lsst-daf-relation
 Requires-Dist: deprecated>=1.2
 Requires-Dist: pydantic<3.0,>=2
 Requires-Dist: pyarrow>=0.16

{lsst_daf_butler-29.2025.4400 → lsst_daf_butler-29.2025.4700}/pyproject.toml

@@ -30,7 +30,6 @@ dependencies = [
     "lsst-sphgeom",
     "lsst-utils",
     "lsst-resources",
-    "lsst-daf-relation",
     "deprecated >= 1.2",
     "pydantic >=2,<3.0",
     "pyarrow >= 0.16",
@@ -205,7 +204,7 @@ extend-select = [
 
 [tool.ruff.lint.per-file-ignores]
 # parserYacc docstrings can not be fixed. Docstrings are used to define grammar.
-"python/lsst/daf/butler/registry/queries/expressions/parser/parserYacc.py" = ["D401", "D403"]
+"python/lsst/daf/butler/queries/expressions/parser/parserYacc.py" = ["D401", "D403"]
 
 [tool.ruff.lint.isort]
 known-first-party = ["lsst"]

{lsst_daf_butler-29.2025.4400 → lsst_daf_butler-29.2025.4700}/python/lsst/daf/butler/__init__.py

@@ -41,9 +41,6 @@ from ._butler_config import *
 from ._butler_metrics import *
 from ._butler_repo_index import *
 from ._collection_type import CollectionType
-from ._column_categorization import *
-from ._column_tags import *
-from ._column_type_info import *
 from ._config import *
 from ._config_support import LookupKey
 from ._dataset_association import *

{lsst_daf_butler-29.2025.4400 → lsst_daf_butler-29.2025.4700}/python/lsst/daf/butler/_butler.py

@@ -1012,7 +1012,7 @@ class Butler(LimitedButler):  # numpydoc ignore=PR02
     @abstractmethod
     def get_dataset(
         self,
-        id: DatasetId,
+        id: DatasetId | str,
         *,
         storage_class: str | StorageClass | None = None,
         dimension_records: bool = False,
@@ -1023,7 +1023,8 @@ class Butler(LimitedButler):  # numpydoc ignore=PR02
         Parameters
         ----------
         id : `DatasetId`
-            The unique identifier for the dataset.
+            The unique identifier for the dataset, as an instance of
+            `uuid.UUID` or a string containing a hexadecimal number.
         storage_class : `str` or `StorageClass` or `None`
             A storage class to use when creating the returned entry. If given
             it must be compatible with the default storage class.
@@ -1040,6 +1041,26 @@ class Butler(LimitedButler):  # numpydoc ignore=PR02
         """
         raise NotImplementedError()
 
+    @abstractmethod
+    def get_many_datasets(self, ids: Iterable[DatasetId | str]) -> list[DatasetRef]:
+        """Retrieve a list of dataset entries.
+
+        Parameters
+        ----------
+        ids : `~collections.abc.Iterable` [ `DatasetId` or `str` ]
+            The unique identifiers for the datasets, as instances of
+            `uuid.UUID` or strings containing a hexadecimal number.
+
+        Returns
+        -------
+        refs : `list` [ `DatasetRef` ]
+            A list containing a `DatasetRef` for each of the given dataset IDs.
+            If a dataset was not found, no error is thrown -- it is just not
+            included in the list.  The returned datasets are in no particular
+            order.
+        """
+        raise NotImplementedError()
+
     @abstractmethod
     def find_dataset(
         self,

{lsst_daf_butler-29.2025.4400 → lsst_daf_butler-29.2025.4700}/python/lsst/daf/butler/_dataset_provenance.py

@@ -112,6 +112,8 @@ class DatasetProvenance(pydantic.BaseModel):
         sep: str = ".",
         simple_types: bool = False,
         use_upper: bool | None = None,
+        max_inputs: int | None = None,
+        store_minimalist_inputs: bool = False,
     ) -> dict[str, _PROV_TYPES]:
         """Return provenance as a flattened dictionary.
 
@@ -137,6 +139,13 @@ class DatasetProvenance(pydantic.BaseModel):
             character of the prefix (defined by whether `str.isupper()` returns
             true, else they will be lower case). If `False` the case will be
             lower case, and if `True` the case will be upper case.
+        max_inputs : `int` or `None`, optional
+            Maximum number of inputs to be recorded in provenance. `None`
+            results in all inputs being recorded. If the number of inputs
+            exceeds this value no input provenance will be recorded.
+        store_minimalist_inputs : `bool`, optional
+            If `True` only the ID of the input is stored along with explicit
+            extras. If `False` the run and dataset type are also recorded.
 
         Returns
         -------
@@ -155,7 +164,13 @@ class DatasetProvenance(pydantic.BaseModel):
 
         Each input dataset will have the ``id``, ``run``, and ``datasettype``
         keys as defined above (but no ``dataid`` key) with an ``input N``
-        prefix where ``N`` starts counting at 0.
+        prefix where ``N`` starts counting at 0. It is possible to drop
+        the ``datasettype`` and ``run`` to save space by using the
+        ``store_minimalist_inputs`` flag.
+
+        If there are too many inputs (see the ``max_inputs`` parameters)
+        no inputs will be recorded. The number of inputs is always recorded
+        to indicate that the inputs were dropped.
 
         The quantum ID, if present, will use key ``quantum``.
 
@@ -171,6 +186,7 @@ class DatasetProvenance(pydantic.BaseModel):
             "lsst.butler.dataid.detector": 10,
             "lsst.butler.dataid.instrument": "LSSTCam",
             "lsst.butler.quantum": "d93a735b-08f0-477d-bc95-2cc32d6d898b",
+            "lsst.butler.n_inputs": 2,
             "lsst.butler.input.0.id": "3dfd7ba5-5e35-4565-9d87-4b33880ed06c",
             "lsst.butler.input.0.run": "other_run",
             "lsst.butler.input.0.datasettype": "astropy_parquet",
@@ -206,12 +222,28 @@ class DatasetProvenance(pydantic.BaseModel):
         if self.quantum_id is not None:
             prov[_make_key("quantum")] = self.quantum_id if not simple_types else str(self.quantum_id)
 
-        for i, input in enumerate(self.inputs):
+        # Record the number of inputs so that people can determine how many
+        # there were even if they were dropped because they exceeded the
+        # allowed maximum. Do not record the count if we have a null provenance
+        # state with no ref and no inputs.
+        if ref is not None or len(self.inputs) > 0:
+            prov[_make_key("n_inputs")] = len(self.inputs)
+
+        # Remove all inputs if the maximum is found. Truncating to the
+        # maximum (or auto switching to minimalist mode and increasing the
+        # maximum by 3) is not preferred.
+        inputs = (
+            self.inputs
+            if max_inputs is None or (max_inputs is not None and len(self.inputs) <= max_inputs)
+            else []
+        )
+        for i, input in enumerate(inputs):
             prov[_make_key("input", i, "id")] = input.id if not simple_types else str(input.id)
-            if input.run is not None:  # for mypy
-                prov[_make_key("input", i, "run")] = input.run
-            if input.datasetType is not None:  # for mypy
-                prov[_make_key("input", i, "datasettype")] = input.datasetType.name
+            if not store_minimalist_inputs:
+                if input.run is not None:  # for mypy
+                    prov[_make_key("input", i, "run")] = input.run
+                if input.datasetType is not None:  # for mypy
+                    prov[_make_key("input", i, "datasettype")] = input.datasetType.name
 
             if input.id in self.extras:
                 for xk, xv in self.extras[input.id].items():
@@ -369,7 +401,9 @@ class DatasetProvenance(pydantic.BaseModel):
             # Prefix will always include the separator if it is defined.
             prefix += sep
 
-        core_provenance = tuple(f"{prefix}{k}".lower() for k in ("run", "id", "datasettype", "quantum"))
+        core_provenance = tuple(
+            f"{prefix}{k}".lower() for k in ("run", "id", "datasettype", "quantum", "n_inputs")
+        )
 
         # Need to escape the prefix and separator for regex usage.
         esc_sep = re.escape(sep)
@@ -445,7 +479,7 @@ class DatasetProvenance(pydantic.BaseModel):
 
         quantum_id = None
         ref_id = None
-        input_ids = {}
+        input_ids: dict[int, uuid.UUID] = {}
         extras: dict[int, dict[str, Any]] = {}
 
         for k, standard in prov_keys.items():
@@ -475,8 +509,9 @@ class DatasetProvenance(pydantic.BaseModel):
 
         provenance = cls(quantum_id=quantum_id)
 
+        input_refs = {ref.id: ref for ref in butler.get_many_datasets(input_ids.values())}
         for i in sorted(input_ids):
-            input_ref = butler.get_dataset(input_ids[i])
+            input_ref = input_refs.get(input_ids[i])
             if input_ref is None:
                 raise ValueError(f"Input dataset ({input_ids[i]}) is not known to this butler.")
             provenance.add_input(input_ref)

{lsst_daf_butler-29.2025.4400 → lsst_daf_butler-29.2025.4700}/python/lsst/daf/butler/configs/datastores/formatters.yaml

@@ -97,3 +97,4 @@ QPEnsemble: lsst.meas.pz.qp_formatter.QPFormatter
 PZModel: lsst.meas.pz.model_formatter.ModelFormatter
 VisitBackgroundModel: lsst.daf.butler.formatters.json.JsonFormatter
 VignettingCorrection: lsst.ts.observatory.control.utils.extras.vignetting_storage.VignettingCorrectionFormatter
+SSPAuxiliaryFile: lsst.pipe.tasks.sspAuxiliaryFile.SSPAuxiliaryFileFormatter

{lsst_daf_butler-29.2025.4400 → lsst_daf_butler-29.2025.4700}/python/lsst/daf/butler/configs/datastores/writeRecipes.yaml

@@ -1,23 +1,12 @@
 lsst.obs.base.formatters.fitsExposure.StandardFitsImageFormatterBase: &StandardFitsImageFormatterBase
   # No compression
   noCompression: &noCompression
-    image: &noCompressionOptions
-      compression:
-        algorithm: NONE
-      scaling:
-        algorithm: NONE
-    mask:
-      <<: *noCompressionOptions
-    variance:
-      <<: *noCompressionOptions
+    null
 
   # Lossless compression
   lossless: &lossless
     image: &losslessOptions
-      compression:
-        algorithm: GZIP_SHUFFLE
-      scaling:
-        algorithm: NONE
+      algorithm: GZIP_2
     mask:
       <<: *losslessOptions
     variance:
@@ -26,14 +15,12 @@ lsst.obs.base.formatters.fitsExposure.StandardFitsImageFormatterBase: &StandardF
   # Basic lossy (quantizing) compression
   lossyBasic: &lossyBasic
     image: &lossyBasicOptions
-      compression:
-        algorithm: RICE
-      scaling:
-        algorithm: STDEV_POSITIVE
-        maskPlanes: ["NO_DATA"]
-        bitpix: 32
-        quantizeLevel: 10.0
-        quantizePad: 10.0
+      algorithm: RICE_1
+      quantization:
+        dither: SUBTRACTIVE_DITHER_2
+        scaling: STDEV_MASKED
+        mask_planes: ["NO_DATA"]
+        level: 10.0
     mask:
       <<: *losslessOptions
     variance:

{lsst_daf_butler-29.2025.4400 → lsst_daf_butler-29.2025.4700}/python/lsst/daf/butler/configs/storageClasses.yaml

@@ -437,3 +437,5 @@ storageClasses:
     pytype: lsst.drp.tasks.fit_visit_background.VisitBackgroundModel
   VignettingCorrection:
     pytype: lsst.ts.observatory.control.utils.extras.vignetting_correction.VignettingCorrection
+  SSPAuxiliaryFile:
+    pytype: lsst.pipe.tasks.sspAuxiliaryFile.SSPAuxiliaryFile

{lsst_daf_butler-29.2025.4400 → lsst_daf_butler-29.2025.4700}/python/lsst/daf/butler/delegates/arrowtable.py

@@ -229,7 +229,9 @@ def _add_arrow_provenance(
     type_string = _checkArrowCompatibleType(in_memory_dataset)
     if type_string == "astropy":
         provenance = provenance if provenance is not None else DatasetProvenance()
-        prov_dict = provenance.to_flat_dict(ref, prefix="LSST.BUTLER", sep=".", simple_types=True)
+        prov_dict = provenance.to_flat_dict(
+            ref, prefix="LSST.BUTLER", sep=".", simple_types=True, max_inputs=2000
+        )
 
         # Strip any previous provenance.
         DatasetProvenance.strip_provenance_from_flat_dict(in_memory_dataset.meta)

{lsst_daf_butler-29.2025.4400 → lsst_daf_butler-29.2025.4700}/python/lsst/daf/butler/dimensions/_schema.py

@@ -29,20 +29,17 @@ from __future__ import annotations
 __all__ = ("DimensionRecordSchema", "addDimensionForeignKey")
 
 import copy
-from collections.abc import Mapping, Set
+from collections.abc import Set
 from typing import TYPE_CHECKING
 
-from lsst.utils.classes import cached_getter, immutable
+from lsst.utils.classes import immutable
 
 from .. import arrow_utils, ddl
-from .._column_tags import DimensionKeyColumnTag, DimensionRecordColumnTag
 from .._named import NamedValueAbstractSet, NamedValueSet
 from ..column_spec import RegionColumnSpec, TimespanColumnSpec
 from ..timespan_database_representation import TimespanDatabaseRepresentation
 
 if TYPE_CHECKING:  # Imports needed only for type annotations; may be circular.
-    from lsst.daf.relation import ColumnTag
-
     from ._elements import Dimension, DimensionElement, KeyColumnSpec, MetadataColumnSpec
     from ._group import DimensionGroup
 
@@ -387,25 +384,6 @@ class DimensionElementFields:
             lines.append("  timespan: lsst.daf.butler.Timespan")
         return "\n".join(lines)
 
-    @property
-    @cached_getter
-    def columns(self) -> Mapping[ColumnTag, str]:
-        """A mapping from `ColumnTag` to field name for all fields in this
-        element's records (`~collections.abc.Mapping`).
-        """
-        result: dict[ColumnTag, str] = {}
-        for dimension_name, field_name in zip(
-            self.element.dimensions.names, self.dimensions.names, strict=True
-        ):
-            result[DimensionKeyColumnTag(dimension_name)] = field_name
-        for field_name in self.facts.names:
-            result[DimensionRecordColumnTag(self.element.name, field_name)] = field_name
-        if self.element.spatial:
-            result[DimensionRecordColumnTag(self.element.name, "region")] = "region"
-        if self.element.temporal:
-            result[DimensionRecordColumnTag(self.element.name, "timespan")] = "timespan"
-        return result
-
     element: DimensionElement
     """The dimension element these fields correspond to.
 

{lsst_daf_butler-29.2025.4400 → lsst_daf_butler-29.2025.4700}/python/lsst/daf/butler/direct_butler/_direct_butler.py

@@ -1256,12 +1256,13 @@ class DirectButler(Butler):  # numpydoc ignore=PR02
 
     def get_dataset(
         self,
-        id: DatasetId,
+        id: DatasetId | str,
         *,
         storage_class: str | StorageClass | None = None,
         dimension_records: bool = False,
         datastore_records: bool = False,
     ) -> DatasetRef | None:
+        id = _to_uuid(id)
         ref = self._registry.getDataset(id)
         if ref is not None:
             if dimension_records:
@@ -1274,6 +1275,10 @@ class DirectButler(Butler):  # numpydoc ignore=PR02
                 ref = self._registry.get_datastore_records(ref)
         return ref
 
+    def get_many_datasets(self, ids: Iterable[DatasetId | str]) -> list[DatasetRef]:
+        uuids = [_to_uuid(id) for id in ids]
+        return self._registry._managers.datasets.get_dataset_refs(uuids)
+
     def find_dataset(
         self,
         dataset_type: DatasetType | str,
@@ -1411,20 +1416,12 @@ class DirectButler(Butler):  # numpydoc ignore=PR02
         # Docstring inherited.
         existence = {ref: DatasetExistence.UNRECOGNIZED for ref in refs}
 
-        # Registry does not have a bulk API to check for a ref.
-        for ref in refs:
-            registry_ref = self._registry.getDataset(ref.id)
-            if registry_ref is not None:
-                # It is possible, albeit unlikely, that the given ref does
-                # not match the one in registry even though the UUID matches.
-                # When checking a single ref we raise, but it's impolite to
-                # do that when potentially hundreds of refs are being checked.
-                # We could change the API to only accept UUIDs and that would
-                # remove the ability to even check and remove the worry
-                # about differing storage classes. Given the ongoing discussion
-                # on refs vs UUIDs and whether to raise or have a new
-                # private flag, treat this as a private API for now.
-                existence[ref] |= DatasetExistence.RECORDED
+        # Check which refs exist in the registry.
+        id_map = {ref.id: ref for ref in existence.keys()}
+        for registry_ref in self.get_many_datasets(id_map.keys()):
+            # Consistency between the given DatasetRef and the information
+            # recorded in the registry is not verified.
+            existence[id_map[registry_ref.id]] |= DatasetExistence.RECORDED
 
         # Ask datastore if it knows about these refs.
         knows = self._datastore.knows_these(refs)
@@ -2610,3 +2607,10 @@ class _ImportDatasetsInfo(NamedTuple):
 
     grouped_refs: defaultdict[_RefGroup, list[DatasetRef]]
     dimension_records: dict[DimensionElement, dict[DataCoordinate, DimensionRecord]]
+
+
+def _to_uuid(id: DatasetId | str) -> uuid.UUID:
+    if isinstance(id, uuid.UUID):
+        return id
+    else:
+        return uuid.UUID(id)

{lsst_daf_butler-29.2025.4400 → lsst_daf_butler-29.2025.4700}/python/lsst/daf/butler/direct_query_driver/_driver.py

@@ -1437,7 +1437,7 @@ class DirectQueryDriver(QueryDriver):
         self,
         joins_builder: SqlJoinsBuilder,
         resolved_search: ResolvedDatasetSearch[list[str]],
-        fields: Set[str],
+        fields: Set[qt.AnyDatasetFieldName],
         union_dataset_type_name: str,
     ) -> None: ...
 
@@ -1446,14 +1446,14 @@ class DirectQueryDriver(QueryDriver):
         self,
         joins_builder: SqlJoinsBuilder,
         resolved_search: ResolvedDatasetSearch[str],
-        fields: Set[str],
+        fields: Set[qt.AnyDatasetFieldName],
     ) -> None: ...
 
     def join_dataset_search(
         self,
         joins_builder: SqlJoinsBuilder,
         resolved_search: ResolvedDatasetSearch[Any],
-        fields: Set[str],
+        fields: Set[qt.AnyDatasetFieldName],
         union_dataset_type_name: str | None = None,
     ) -> None:
         """Join a dataset search into an under-construction query.

{lsst_daf_butler-29.2025.4400 → lsst_daf_butler-29.2025.4700}/python/lsst/daf/butler/direct_query_driver/_query_builder.py

@@ -284,7 +284,7 @@ class QueryBuilder(ABC):
         raise NotImplementedError()
 
     def _needs_collection_key_field(
-        self, dataset_search: ResolvedDatasetSearch, fields_for_dataset: set[str]
+        self, dataset_search: ResolvedDatasetSearch, fields_for_dataset: set[qt.AnyDatasetFieldName]
     ) -> bool:
         """Return `True` if the ``collection_key`` dataset field is needed to
         provide uniqueness for rows.

{lsst_daf_butler-29.2025.4400 → lsst_daf_butler-29.2025.4700}/python/lsst/daf/butler/formatters/logs.py

@@ -66,5 +66,5 @@ class ButlerLogRecordsFormatter(FormatterV2):
         # over pre-downloading the whole file (which can be very large).
         return self._get_read_pytype().from_file(path)
 
-    def to_bytes(self, in_memory_dataset: Any) -> bytes:
-        return in_memory_dataset.model_dump_json(exclude_unset=True, exclude_defaults=True).encode()
+    def to_bytes(self, in_memory_dataset: ButlerLogRecords) -> bytes:
+        return in_memory_dataset.to_json_data().encode()