lsst-daf-butler 29.2025.3300__tar.gz → 29.2025.3500__tar.gz

{lsst_daf_butler-29.2025.3300/python/lsst_daf_butler.egg-info → lsst_daf_butler-29.2025.3500}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lsst-daf-butler
-Version: 29.2025.3300
+Version: 29.2025.3500
 Summary: An abstraction layer for reading and writing astronomical data to datastores.
 Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
 License: BSD 3-Clause License

{lsst_daf_butler-29.2025.3300 → lsst_daf_butler-29.2025.3500}/python/lsst/daf/butler/_butler.py

@@ -2040,7 +2040,7 @@ class Butler(LimitedButler):  # numpydoc ignore=PR02
             raise EmptyQueryResultError(list(result.explain_no_results()))
         return dimension_records
 
-    def _query_all_datasets(
+    def query_all_datasets(
         self,
         collections: str | Iterable[str] | None = None,
         *,

{lsst_daf_butler-29.2025.3300 → lsst_daf_butler-29.2025.3500}/python/lsst/daf/butler/_dataset_type.py

@@ -38,6 +38,7 @@ from typing import TYPE_CHECKING, Any, ClassVar
 from pydantic import BaseModel, StrictBool, StrictStr
 
 from ._config_support import LookupKey
+from ._exceptions import UnknownComponentError
 from ._storage_class import StorageClass, StorageClassFactory
 from .dimensions import DimensionGroup
 from .json import from_json_pydantic, to_json_pydantic
@@ -492,7 +493,9 @@ class DatasetType:
         """
         if component in self.storageClass.allComponents():
             return self.nameWithComponent(self.name, component)
-        raise KeyError(f"Requested component ({component}) not understood by this DatasetType ({self})")
+        raise UnknownComponentError(
+            f"Requested component ({component}) not understood by this DatasetType ({self})"
+        )
 
     def makeCompositeDatasetType(self) -> DatasetType:
         """Return a composite dataset type from the component.

{lsst_daf_butler-29.2025.3300 → lsst_daf_butler-29.2025.3500}/python/lsst/daf/butler/_exceptions.py

@@ -41,6 +41,7 @@ __all__ = (
     "InvalidQueryError",
     "MissingCollectionError",
     "MissingDatasetTypeError",
+    "UnknownComponentError",
     "ValidationError",
 )
 
@@ -170,6 +171,14 @@ class MissingDatasetTypeError(DatasetTypeError, KeyError, ButlerUserError):
     error_type = "missing_dataset_type"
 
 
+class UnknownComponentError(KeyError, ButlerUserError):
+    """Exception raised when the requested component of a DatasetType is not
+    known.
+    """
+
+    error_type = "unknown_component"
+
+
 class DatasetTypeNotSupportedError(RuntimeError):
     """A `DatasetType` is not handled by this routine.
 
@@ -227,6 +236,7 @@ _USER_ERROR_TYPES: tuple[type[ButlerUserError], ...] = (
     MissingDatasetTypeError,
     UnimplementedQueryError,
     UnknownButlerUserError,
+    UnknownComponentError,
 )
 _USER_ERROR_MAPPING = {e.error_type: e for e in _USER_ERROR_TYPES}
 assert len(_USER_ERROR_MAPPING) == len(_USER_ERROR_TYPES), (

{lsst_daf_butler-29.2025.3300 → lsst_daf_butler-29.2025.3500}/python/lsst/daf/butler/dimensions/_coordinate.py

@@ -1005,10 +1005,12 @@ class _FullTupleDataCoordinate(_BasicTupleDataCoordinate):
         dimensions = self.universe.conform(dimensions)
         if self._dimensions == dimensions:
             return self
-        return DataCoordinate.from_full_values(
-            dimensions,
-            tuple(self[k] for k in dimensions.data_coordinate_keys),
-        )
+
+        try:
+            values = tuple(self[k] for k in dimensions.data_coordinate_keys)
+        except KeyError as e:
+            raise DimensionNameError(f"Data ID is missing value for dimension {str(e)}.")
+        return DataCoordinate.from_full_values(dimensions, values)
 
     def union(self, other: DataCoordinate) -> DataCoordinate:
         # Docstring inherited from DataCoordinate.

{lsst_daf_butler-29.2025.3300 → lsst_daf_butler-29.2025.3500}/python/lsst/daf/butler/direct_butler/_direct_butler.py

@@ -1490,7 +1490,7 @@ class DirectButler(Butler):  # numpydoc ignore=PR02
             names = [info.name for info in collections_info]
 
             # Get all the datasets from these runs.
-            refs = self._query_all_datasets(names, find_first=False, limit=None)
+            refs = self.query_all_datasets(names, find_first=False, limit=None)
 
         # Call pruneDatasets since we are deliberately removing
         # datasets in chunks from the RUN collections rather than

{lsst_daf_butler-29.2025.3300 → lsst_daf_butler-29.2025.3500}/python/lsst/daf/butler/registry/dimensions/static.py

@@ -468,7 +468,7 @@ class StaticDimensionRecordStorageManager(DimensionRecordStorageManager):
                 distinct=True,
             ).into_joins_builder(postprocessing=None)
         if not element.has_own_table:
-            raise NotImplementedError(f"Cannot join dimension element {element} with no table.")
+            raise UnimplementedQueryError(f"Cannot join dimension element {element} with no table.")
         table = self._tables[element.name]
         result = SqlJoinsBuilder(db=self._db, from_clause=table)
         for dimension_name, column_name in zip(element.required.names, element.schema.required.names):
@@ -1055,7 +1055,7 @@ class _CommonSkyPixMediatedOverlapsVisitor(OverlapsVisitor):
         # Reject spatial constraints that are nested inside OR or NOT, because
         # the postprocessing needed for those would be a lot harder.
         if flags & PredicateVisitFlags.INVERTED or flags & PredicateVisitFlags.HAS_OR_SIBLINGS:
-            raise NotImplementedError(
+            raise UnimplementedQueryError(
                 "Spatial overlap constraints nested inside OR or NOT are not supported."
             )
         # Delegate to super just because that's good practice with
@@ -1114,7 +1114,7 @@ class _CommonSkyPixMediatedOverlapsVisitor(OverlapsVisitor):
                 )
                 skypix = element
             case _:
-                raise NotImplementedError(
+                raise UnimplementedQueryError(
                     f"Spatial overlap constraint for dimension {element} not supported."
                 )
         # Convert the region-overlap constraint into a skypix

{lsst_daf_butler-29.2025.3300 → lsst_daf_butler-29.2025.3500}/python/lsst/daf/butler/script/queryDatasets.py

@@ -31,7 +31,7 @@ import warnings
 from collections import defaultdict
 from collections.abc import Iterable, Iterator
 from contextlib import contextmanager
-from itertools import groupby
+from itertools import chain, groupby
 from typing import TYPE_CHECKING
 
 import numpy as np
@@ -39,7 +39,7 @@ from astropy.table import Table as AstropyTable
 
 from .._butler import Butler
 from .._exceptions import MissingDatasetTypeError
-from .._query_all_datasets import DatasetsPage, QueryAllDatasetsParameters, query_all_datasets
+from .._query_all_datasets import QueryAllDatasetsParameters
 from ..cli.utils import sortAstropyTable
 from ..utils import has_globs
 
@@ -297,9 +297,10 @@ class QueryDatasets:
             query_func = self._query_single_dataset_type
 
         try:
-            with query_func(query_collections, limit) as pages:
+            with query_func(query_collections, limit) as ref_iter:
                 datasets_found = 0
-                for dataset_type, refs in _chunk_by_dataset_type(pages):
+                for dataset_type, chunk in groupby(ref_iter, lambda ref: ref.datasetType.name):
+                    refs = list(chunk)
                     datasets_found += len(refs)
                     if warn_limit and limit is not None and datasets_found >= limit:
                         # We asked for one too many so must remove that from
@@ -324,24 +325,24 @@ class QueryDatasets:
     @contextmanager
     def _query_multiple_dataset_types(
         self, collections: list[str], limit: int | None
-    ) -> Iterator[Iterator[DatasetsPage]]:
-        with self.butler.query() as query:
-            args = QueryAllDatasetsParameters(
-                collections=collections,
-                find_first=self._find_first,
-                name=self._dataset_type_glob,
-                where=self._where,
-                limit=limit,
-                data_id={},
-                bind={},
-                with_dimension_records=self._with_dimension_records,
-            )
-            yield query_all_datasets(self.butler, query, args)
+    ) -> Iterator[Iterator[DatasetRef]]:
+        args = QueryAllDatasetsParameters(
+            collections=collections,
+            find_first=self._find_first,
+            name=self._dataset_type_glob,
+            where=self._where,
+            limit=limit,
+            data_id={},
+            bind={},
+            with_dimension_records=self._with_dimension_records,
+        )
+        with self.butler._query_all_datasets_by_page(args) as iterator:
+            yield chain.from_iterable(iterator)
 
     @contextmanager
     def _query_single_dataset_type(
         self, collections: list[str], limit: int | None
-    ) -> Iterator[Iterator[DatasetsPage]]:
+    ) -> Iterator[Iterator[DatasetRef]]:
         assert len(self._dataset_type_glob) == 1
         dataset_type = self._dataset_type_glob[0]
 
@@ -355,15 +356,4 @@ class QueryDatasets:
             with_dimension_records=self._with_dimension_records,
         )
 
-        yield iter([DatasetsPage(dataset_type=dataset_type, data=refs)])
-
-
-def _chunk_by_dataset_type(pages: Iterator[DatasetsPage]) -> Iterator[DatasetsPage]:
-    # For each dataset type in the results, collect all the refs into a
-    # single page.  This assumes that in the original iterator, the pages
-    # for each dataset type are contiguous.
-    for dataset_type, chunk in groupby(pages, lambda page: page.dataset_type):
-        refs = []
-        for page in chunk:
-            refs.extend(page.data)
-        yield DatasetsPage(dataset_type, refs)
+        yield iter(refs)

{lsst_daf_butler-29.2025.3300 → lsst_daf_butler-29.2025.3500}/python/lsst/daf/butler/tests/butler_queries.py

@@ -2377,7 +2377,7 @@ class ButlerQueryTests(ABC, TestCaseMixin):
         butler = self.make_butler("base.yaml", "datasets.yaml")
 
         # Make sure that refs are coming out well-formed.
-        datasets = butler._query_all_datasets("imported_r", where="detector = 2", instrument="Cam1")
+        datasets = butler.query_all_datasets("imported_r", where="detector = 2", instrument="Cam1")
         datasets.sort(key=lambda ref: ref.datasetType.name)
         self.assertEqual(len(datasets), 2)
         bias = datasets[0]
@@ -2396,29 +2396,29 @@ class ButlerQueryTests(ABC, TestCaseMixin):
         self.assertEqual(flat.id, UUID("c1296796-56c5-4acf-9b49-40d920c6f840"))
 
         # Querying for everything finds everything.
-        results = butler._query_all_datasets("*", find_first=False)
+        results = butler.query_all_datasets("*", find_first=False)
         self.assertEqual(len(results), 13)
 
         # constraining by data ID works
         detector_1_ids = ("d0bb04cd-d697-4a83-ba53-cdfcd58e3a0c", "e15ab039-bc8b-4135-87c5-90902a7c0b22")
-        results = butler._query_all_datasets(
+        results = butler.query_all_datasets(
             "*", data_id={"detector": 1, "instrument": "Cam1"}, find_first=False
         )
         self.assertCountEqual(detector_1_ids, _ref_uuids(results))
 
         # bind values work.
-        results = butler._query_all_datasets(
+        results = butler.query_all_datasets(
             "*", where="detector=:my_bind and instrument='Cam1'", bind={"my_bind": 1}, find_first=False
         )
         self.assertCountEqual(detector_1_ids, _ref_uuids(results))
 
         # find_first requires ordered collections.
         with self.assertRaisesRegex(InvalidQueryError, "Can not use wildcards"):
-            results = butler._query_all_datasets("*")
+            results = butler.query_all_datasets("*")
 
         butler.collections.register("chain", CollectionType.CHAINED)
         butler.collections.redefine_chain("chain", ["imported_g", "imported_r"])
-        results = butler._query_all_datasets(
+        results = butler.query_all_datasets(
             "chain", where="detector=2 and instrument = 'Cam1'", find_first=True
         )
         # find_first searches the collection chain in order.
@@ -2435,47 +2435,47 @@ class ButlerQueryTests(ABC, TestCaseMixin):
         )
 
         # collection searches work.
-        results = butler._query_all_datasets(
+        results = butler.query_all_datasets(
             "*g", where="detector=1 and instrument = 'Cam1'", find_first=False
         )
         self.assertEqual(_ref_uuids(results), ["e15ab039-bc8b-4135-87c5-90902a7c0b22"])
 
         # we raise for missing collections with explicit names.
         with self.assertRaises(MissingCollectionError):
-            results = butler._query_all_datasets("nonexistent")
+            results = butler.query_all_datasets("nonexistent")
         # we don't raise for collection wildcard searches that find nothing.
-        results = butler._query_all_datasets("nonexistent*", find_first=False)
+        results = butler.query_all_datasets("nonexistent*", find_first=False)
         self.assertEqual(results, [])
 
         # dataset type searches work.
-        results = butler._query_all_datasets(
+        results = butler.query_all_datasets(
             "*", name="b*", where="detector=1 and instrument = 'Cam1'", find_first=False
         )
         self.assertEqual(_ref_uuids(results), ["e15ab039-bc8b-4135-87c5-90902a7c0b22"])
 
         # Missing dataset types raise.
         with self.assertRaises(MissingDatasetTypeError):
-            results = butler._query_all_datasets("chain", name=["notfound", "flat"])
+            results = butler.query_all_datasets("chain", name=["notfound", "flat"])
         with self.assertRaises(MissingDatasetTypeError):
-            results = butler._query_all_datasets("chain", name="notfound*")
+            results = butler.query_all_datasets("chain", name="notfound*")
 
         # Limit of 3 lands at the boundary of a dataset type.
         # Limit of 4 is in the middle of a dataset type.
         for limit in [3, 4]:
             with self.subTest(limit=limit):
-                results = butler._query_all_datasets("imported_g", limit=limit)
+                results = butler.query_all_datasets("imported_g", limit=limit)
                 self.assertEqual(len(results), limit)
                 with self.assertLogs(level="WARNING") as log:
-                    results = butler._query_all_datasets("imported_g", limit=-limit)
+                    results = butler.query_all_datasets("imported_g", limit=-limit)
                     self.assertEqual(len(results), limit)
                 self.assertIn("requested limit", log.output[0])
 
-        results = butler._query_all_datasets("imported_g", limit=0)
+        results = butler.query_all_datasets("imported_g", limit=0)
         self.assertEqual(len(results), 0)
 
         # 'where' constraints that don't apply to all dataset types follow the
         # same rules as query_datasets.
-        results = butler._query_all_datasets(
+        results = butler.query_all_datasets(
             "*", where="detector = 2 and band = 'g' and instrument = 'Cam1'", find_first=False
         )
         self.assertCountEqual(
@@ -2491,7 +2491,7 @@ class ButlerQueryTests(ABC, TestCaseMixin):
 
         # Default collections and data ID apply.
         butler.registry.defaults = RegistryDefaults(collections="imported_g")
-        results = butler._query_all_datasets(where="detector = 2")
+        results = butler.query_all_datasets(where="detector = 2")
         self.assertCountEqual(
             _ref_uuids(results),
             ["51352db4-a47a-447c-b12d-a50b206b17cd", "60c8a65c-7290-4c38-b1de-e3b1cdcf872d"],

lsst_daf_butler-29.2025.3500/python/lsst/daf/butler/version.py

@@ -0,0 +1,2 @@
+__all__ = ["__version__"]
+__version__ = "29.2025.3500"

{lsst_daf_butler-29.2025.3300 → lsst_daf_butler-29.2025.3500/python/lsst_daf_butler.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lsst-daf-butler
-Version: 29.2025.3300
+Version: 29.2025.3500
 Summary: An abstraction layer for reading and writing astronomical data to datastores.
 Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
 License: BSD 3-Clause License

{lsst_daf_butler-29.2025.3300 → lsst_daf_butler-29.2025.3500}/tests/test_datasets.py

@@ -42,6 +42,7 @@ from lsst.daf.butler import (
     SerializedDatasetRefContainerV1,
     StorageClass,
     StorageClassFactory,
+    UnknownComponentError,
 )
 from lsst.daf.butler.datastore.stored_file_info import StoredFileInfo
 from lsst.daf.butler.datastores.file_datastore.retrieve_artifacts import ZipIndex
@@ -516,9 +517,12 @@ class DatasetTypeTestCase(unittest.TestCase):
         self.assertEqual(datasetTypeComponentB.parentStorageClass, storageClass)
         self.assertIsNone(datasetTypeComposite.parentStorageClass)
 
-        with self.assertRaises(KeyError):
+        with self.assertRaises(UnknownComponentError):
             datasetTypeComposite.makeComponentDatasetType("compF")
 
+        with self.assertRaises(UnknownComponentError):
+            datasetTypeComposite.componentTypeName("unknown")
+
 
 class DatasetRefTestCase(unittest.TestCase):
     """Test for DatasetRef."""

{lsst_daf_butler-29.2025.3300 → lsst_daf_butler-29.2025.3500}/tests/test_dimensions.py

@@ -48,6 +48,7 @@ from lsst.daf.butler import (
     DimensionConfig,
     DimensionElement,
     DimensionGroup,
+    DimensionNameError,
     DimensionPacker,
     DimensionUniverse,
     NamedKeyDict,
@@ -797,6 +798,10 @@ class DataCoordinateTestCase(unittest.TestCase):
                         self.assertEqual(dataId, newDataId)
                         self.assertTrue(newDataId.hasFull())
 
+        coord = DataCoordinate.standardize({"instrument": "HSC"}, universe=self.allDataIds.universe)
+        with self.assertRaises(DimensionNameError):
+            coord.subset(["detector"])
+
     def testUnion(self):
         """Test `DataCoordinate.union`."""
         # Make test groups to combine; mostly random, but with a few explicit

lsst_daf_butler-29.2025.3300/python/lsst/daf/butler/version.py

@@ -1,2 +0,0 @@
-__all__ = ["__version__"]
-__version__ = "29.2025.3300"