lsst-daf-butler 29.2025.4000__tar.gz → 29.2025.4100__tar.gz

{lsst_daf_butler-29.2025.4000/python/lsst_daf_butler.egg-info → lsst_daf_butler-29.2025.4100}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lsst-daf-butler
-Version: 29.2025.4000
+Version: 29.2025.4100
 Summary: An abstraction layer for reading and writing astronomical data to datastores.
 Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
 License-Expression: BSD-3-Clause OR GPL-3.0-or-later

{lsst_daf_butler-29.2025.4000 → lsst_daf_butler-29.2025.4100}/python/lsst/daf/butler/_dataset_ref.py

@@ -42,7 +42,7 @@ import enum
 import logging
 import sys
 import uuid
-from collections.abc import Iterable, Mapping
+from collections.abc import Callable, Iterable, Mapping
 from typing import (
     TYPE_CHECKING,
     Annotated,
@@ -52,6 +52,7 @@ from typing import (
     Protocol,
     Self,
     TypeAlias,
+    cast,
     runtime_checkable,
 )
 
@@ -63,6 +64,7 @@ from lsst.utils.classes import immutable
 from ._config_support import LookupKey
 from ._dataset_type import DatasetType, SerializedDatasetType
 from ._named import NamedKeyDict
+from ._uuid import generate_uuidv7
 from .datastore.stored_file_info import StoredDatastoreItemInfo
 from .dimensions import (
     DataCoordinate,
@@ -181,7 +183,12 @@ class DatasetIdFactory:
             Dataset identifier.
         """
         if idGenerationMode is DatasetIdGenEnum.UNIQUE:
-            return uuid.uuid4()
+            # Earlier versions of this code used UUIDv4. However, totally
+            # random IDs create problems for Postgres insert performance,
+            # because it scatters index updates randomly around the disk.
+            # UUIDv7 has similar uniqueness properties to v4, but IDs generated
+            # at the same time are close together in the index.
+            return generate_uuidv7()
         else:
             # WARNING: If you modify this code make sure that the order of
             # items in the `items` list below never changes.
@@ -559,7 +566,7 @@ class DatasetRef:
         return ref
 
     to_json = to_json_pydantic
-    from_json: ClassVar = classmethod(from_json_pydantic)
+    from_json: ClassVar[Callable[..., Self]] = cast(Callable[..., Self], classmethod(from_json_pydantic))
 
     @classmethod
     def _unpickle(

{lsst_daf_butler-29.2025.4000 → lsst_daf_butler-29.2025.4100}/python/lsst/daf/butler/_dataset_type.py

@@ -33,7 +33,7 @@ import re
 from collections.abc import Callable, Iterable, Mapping
 from copy import deepcopy
 from types import MappingProxyType
-from typing import TYPE_CHECKING, Any, ClassVar
+from typing import TYPE_CHECKING, Any, ClassVar, Self, cast
 
 from pydantic import BaseModel, StrictBool, StrictStr
 
@@ -756,7 +756,7 @@ class DatasetType:
         return newType
 
     to_json = to_json_pydantic
-    from_json: ClassVar = classmethod(from_json_pydantic)
+    from_json: ClassVar[Callable[..., Self]] = cast(Callable[..., Self], classmethod(from_json_pydantic))
 
     def __reduce__(
         self,

{lsst_daf_butler-29.2025.4000 → lsst_daf_butler-29.2025.4100}/python/lsst/daf/butler/_registry_shim.py

@@ -27,16 +27,16 @@
 
 from __future__ import annotations
 
-__all__ = ("Registry",)
+__all__ = ("RegistryShim",)
 
 import contextlib
 from collections.abc import Iterable, Iterator, Mapping, Sequence
 from typing import TYPE_CHECKING, Any
 
 from ._collection_type import CollectionType
-from ._dataset_association import DatasetAssociation
 from ._dataset_ref import DatasetId, DatasetIdGenEnum, DatasetRef
 from ._dataset_type import DatasetType
+from ._storage_class import StorageClassFactory
 from ._timespan import Timespan
 from .dimensions import (
     DataCoordinate,
@@ -46,10 +46,9 @@ from .dimensions import (
     DimensionRecord,
     DimensionUniverse,
 )
-from .registry import Registry
 from .registry._collection_summary import CollectionSummary
 from .registry._defaults import RegistryDefaults
-from .registry.queries import DataCoordinateQueryResults, DatasetQueryResults, DimensionRecordQueryResults
+from .registry._registry_base import RegistryBase
 
 if TYPE_CHECKING:
     from .direct_butler import DirectButler
@@ -57,7 +56,7 @@ if TYPE_CHECKING:
     from .registry.interfaces import ObsCoreTableManager
 
 
-class RegistryShim(Registry):
+class RegistryShim(RegistryBase):
     """Implementation of `Registry` interface exposed to clients by `Butler`.
 
     Parameters
@@ -74,7 +73,7 @@ class RegistryShim(Registry):
     """
 
     def __init__(self, butler: DirectButler):
-        self._butler = butler
+        super().__init__(butler)
         self._registry = butler._registry
 
     def isWriteable(self) -> bool:
@@ -299,97 +298,11 @@ class RegistryShim(Registry):
             expression, datasetType, collectionTypes, flattenChains, includeChains
         )
 
-    def queryDatasets(
-        self,
-        datasetType: Any,
-        *,
-        collections: CollectionArgType | None = None,
-        dimensions: Iterable[str] | None = None,
-        dataId: DataId | None = None,
-        where: str = "",
-        findFirst: bool = False,
-        bind: Mapping[str, Any] | None = None,
-        check: bool = True,
-        **kwargs: Any,
-    ) -> DatasetQueryResults:
-        # Docstring inherited from a base class.
-        return self._registry.queryDatasets(
-            datasetType,
-            collections=collections,
-            dimensions=dimensions,
-            dataId=dataId,
-            where=where,
-            findFirst=findFirst,
-            bind=bind,
-            check=check,
-            **kwargs,
-        )
-
-    def queryDataIds(
-        self,
-        dimensions: DimensionGroup | Iterable[str] | str,
-        *,
-        dataId: DataId | None = None,
-        datasets: Any = None,
-        collections: CollectionArgType | None = None,
-        where: str = "",
-        bind: Mapping[str, Any] | None = None,
-        check: bool = True,
-        **kwargs: Any,
-    ) -> DataCoordinateQueryResults:
-        # Docstring inherited from a base class.
-        return self._registry.queryDataIds(
-            dimensions,
-            dataId=dataId,
-            datasets=datasets,
-            collections=collections,
-            where=where,
-            bind=bind,
-            check=check,
-            **kwargs,
-        )
-
-    def queryDimensionRecords(
-        self,
-        element: DimensionElement | str,
-        *,
-        dataId: DataId | None = None,
-        datasets: Any = None,
-        collections: CollectionArgType | None = None,
-        where: str = "",
-        bind: Mapping[str, Any] | None = None,
-        check: bool = True,
-        **kwargs: Any,
-    ) -> DimensionRecordQueryResults:
-        # Docstring inherited from a base class.
-        return self._registry.queryDimensionRecords(
-            element,
-            dataId=dataId,
-            datasets=datasets,
-            collections=collections,
-            where=where,
-            bind=bind,
-            check=check,
-            **kwargs,
-        )
-
-    def queryDatasetAssociations(
-        self,
-        datasetType: str | DatasetType,
-        collections: CollectionArgType | None = ...,
-        *,
-        collectionTypes: Iterable[CollectionType] = CollectionType.all(),
-        flattenChains: bool = False,
-    ) -> Iterator[DatasetAssociation]:
-        # Docstring inherited from a base class.
-        return self._registry.queryDatasetAssociations(
-            datasetType,
-            collections,
-            collectionTypes=collectionTypes,
-            flattenChains=flattenChains,
-        )
-
     @property
     def obsCoreTableManager(self) -> ObsCoreTableManager | None:
         # Docstring inherited from a base class.
         return self._registry.obsCoreTableManager
+
+    @property
+    def storageClasses(self) -> StorageClassFactory:
+        return self._registry.storageClasses

{lsst_daf_butler-29.2025.4000 → lsst_daf_butler-29.2025.4100}/python/lsst/daf/butler/_rubin/__init__.py

@@ -29,3 +29,6 @@
 by other LSST packages.  The interfaces and behavior of these functions are
 subject to change at any time.
 """
+
+# UUIDv7 generation is used by pipe_base.
+from .._uuid import generate_uuidv7

lsst_daf_butler-29.2025.4100/python/lsst/daf/butler/_uuid.py

@@ -0,0 +1,79 @@
+# This file is part of daf_butler.
+#
+# Developed for the LSST Data Management System.
+# This product includes software developed by the LSST Project
+# (http://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# This software is dual licensed under the GNU General Public License and also
+# under a 3-clause BSD license. Recipients may choose which of these licenses
+# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
+# respectively.  If you choose the GPL option then the following text applies
+# (but note that there is still no warranty even if you opt for BSD instead):
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from __future__ import annotations
+
+__all__ = ("generate_uuidv7",)
+
+import time
+from uuid import UUID, uuid4
+
+
+def generate_uuidv7() -> UUID:
+    """Generate a v7 UUID compliant with IETF RFC-9562.
+
+    Returns
+    -------
+    uuid : `uuid.UUID`
+        Version 7 UUID.
+    """
+    # Python 3.14 will include an implementation of UUIDv7 that can replace
+    # this, but at the time of writing 3.14 hasn't been released and we're
+    # still supporting 3.12. There are a few libraries on PyPI with an
+    # implementation of v7 UUIDs, but none of them look well-maintained.
+    #
+    # This is the format of a v7 UUID:
+    #  0                   1                   2                   3
+    #  0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+    # +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+    # |                           unix_ts_ms                          |
+    # +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+    # |          unix_ts_ms           |  ver  |       rand_a          |
+    # +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+    # |var|                        rand_b                             |
+    # +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+    # |                            rand_b                             |
+    # +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+    #
+    # It's basically identical to a UUID v4, but with the top 6 bytes
+    # replaced with a millisecond UNIX timestamp.
+
+    # Generate a UUIDv4 for the random portion of the data.
+    # A little wasteful, but means we inherit the best practices from
+    # the standard library for generating this random data.
+    byte_data = bytearray(uuid4().bytes)
+
+    # Replace high 6 bytes with millisecond UNIX timestamp.
+    timestamp = time.time_ns() // 1_000_000
+    timestamp_bytes = timestamp.to_bytes(length=6, byteorder="big")
+    byte_data[0:6] = timestamp_bytes
+
+    # Set 4-bit version field to 7.
+    byte_data[6] &= 0x0F
+    byte_data[6] |= 0x70
+
+    return UUID(bytes=bytes(byte_data))

{lsst_daf_butler-29.2025.4000 → lsst_daf_butler-29.2025.4100}/python/lsst/daf/butler/dimensions/_coordinate.py

@@ -43,8 +43,8 @@ __all__ = (
 
 import numbers
 from abc import abstractmethod
-from collections.abc import Iterable, Iterator, Mapping
-from typing import TYPE_CHECKING, Any, ClassVar, TypeAlias, overload
+from collections.abc import Callable, Iterable, Iterator, Mapping
+from typing import TYPE_CHECKING, Any, ClassVar, Self, TypeAlias, cast, overload
 
 import pydantic
 
@@ -753,7 +753,7 @@ class DataCoordinate:
         return dataId
 
     to_json = to_json_pydantic
-    from_json: ClassVar = classmethod(from_json_pydantic)
+    from_json: ClassVar[Callable[..., Self]] = cast(Callable[..., Self], classmethod(from_json_pydantic))
 
 
 DataId = DataCoordinate | Mapping[str, Any]

{lsst_daf_butler-29.2025.4000 → lsst_daf_butler-29.2025.4100}/python/lsst/daf/butler/dimensions/_elements.py

@@ -34,7 +34,8 @@ __all__ = (
 )
 
 from abc import abstractmethod
-from typing import TYPE_CHECKING, Annotated, Any, ClassVar, TypeAlias, Union, cast
+from collections.abc import Callable
+from typing import TYPE_CHECKING, Annotated, Any, ClassVar, Self, TypeAlias, Union, cast
 
 import pydantic
 from pydantic_core import core_schema
@@ -212,7 +213,7 @@ class DimensionElement(TopologicalRelationshipEndpoint):
         return universe[simple]
 
     to_json = to_json_generic
-    from_json: ClassVar = classmethod(from_json_generic)
+    from_json: ClassVar[Callable[..., Self]] = cast(Callable[..., Self], classmethod(from_json_generic))
 
     def hasTable(self) -> bool:
         """Indicate if this element is associated with a table.

{lsst_daf_butler-29.2025.4000 → lsst_daf_butler-29.2025.4100}/python/lsst/daf/butler/dimensions/_records.py

@@ -30,8 +30,8 @@ from __future__ import annotations
 __all__ = ("DimensionRecord", "SerializedDimensionRecord", "SerializedKeyValueDimensionRecord")
 
 import itertools
-from collections.abc import Hashable
-from typing import TYPE_CHECKING, Any, ClassVar, TypeAlias
+from collections.abc import Callable, Hashable
+from typing import TYPE_CHECKING, Any, ClassVar, Self, TypeAlias, cast
 
 import pydantic
 from pydantic import BaseModel, Field, StrictBool, StrictFloat, StrictInt, StrictStr, create_model
@@ -505,7 +505,7 @@ class DimensionRecord:
         return dimRec
 
     to_json = to_json_pydantic
-    from_json: ClassVar = classmethod(from_json_pydantic)
+    from_json: ClassVar[Callable[..., Self]] = cast(Callable[..., Self], classmethod(from_json_pydantic))
 
     def toDict(self, splitTimespan: bool = False) -> dict[str, Any]:
         """Return a vanilla `dict` representation of this record.

{lsst_daf_butler-29.2025.4000 → lsst_daf_butler-29.2025.4100}/python/lsst/daf/butler/direct_query_driver/_driver.py

@@ -641,7 +641,11 @@ class DirectQueryDriver(QueryDriver):
         # datasets later.
         predicate_constraints = PredicateConstraintsSummary(tree.predicate)
         # Use the default data ID to apply additional constraints where needed.
-        predicate_constraints.apply_default_data_id(self._default_data_id, tree.dimensions)
+        predicate_constraints.apply_default_data_id(
+            self._default_data_id,
+            tree.dimensions,
+            validate_governor_constraints=tree.validateGovernorConstraints,
+        )
         predicate = predicate_constraints.predicate
         # Delegate to the dimensions manager to rewrite the predicate and start
         # a SqlSelectBuilder to cover any spatial overlap joins or constraints.

{lsst_daf_butler-29.2025.4000 → lsst_daf_butler-29.2025.4100}/python/lsst/daf/butler/queries/_query.py

@@ -746,6 +746,10 @@ class Query(QueryBase):
             driver=self._driver,
         )
 
+    def _skip_governor_validation(self) -> Query:
+        tree = self._tree.model_copy(update={"validateGovernorConstraints": False})
+        return Query(tree=tree, driver=self._driver)
+
     def _join_dataset_search_impl(
         self,
         dataset_type: str | DatasetType,

{lsst_daf_butler-29.2025.4000 → lsst_daf_butler-29.2025.4100}/python/lsst/daf/butler/queries/predicate_constraints_summary.py

@@ -71,7 +71,10 @@ class PredicateConstraintsSummary:
         )
 
     def apply_default_data_id(
-        self, default_data_id: DataCoordinate, query_dimensions: DimensionGroup
+        self,
+        default_data_id: DataCoordinate,
+        query_dimensions: DimensionGroup,
+        validate_governor_constraints: bool,
     ) -> None:
         """Augment the predicate and summary by adding missing constraints for
         governor dimensions using a default data ID.
@@ -81,9 +84,11 @@ class PredicateConstraintsSummary:
         default_data_id : `DataCoordinate`
             Data ID values that will be used to constrain the query if governor
             dimensions have not already been constrained by the predicate.
-
         query_dimensions : `DimensionGroup`
             The set of dimensions returned in result rows from the query.
+        validate_governor_constraints : `bool`
+            If `True`, enforce the requirement that governor dimensions must be
+            constrained if any dimensions that depend on them have constraints.
         """
         # Find governor dimensions required by the predicate.
         # If these are not constrained by the predicate or the default data ID,
@@ -108,7 +113,7 @@ class PredicateConstraintsSummary:
                     self.predicate = self.predicate.logical_and(
                         _create_data_id_predicate(governor, data_id_value, query_dimensions.universe)
                     )
-                elif governor in where_governors:
+                elif governor in where_governors and validate_governor_constraints:
                     # Check that the predicate doesn't reference any dimensions
                     # without constraining their governor dimensions, since
                     # that's a particularly easy mistake to make and it's

{lsst_daf_butler-29.2025.4000 → lsst_daf_butler-29.2025.4100}/python/lsst/daf/butler/queries/tree/_query_tree.py

@@ -146,6 +146,11 @@ class QueryTree(QueryTreeBase):
     predicate: Predicate = Predicate.from_bool(True)
     """Boolean expression trees whose logical AND defines a row filter."""
 
+    validateGovernorConstraints: bool = True
+    """If True, enforce the requirement that governor dimensions must be
+    constrained if any dimensions that depend on them have constraints.
+    """
+
     def iter_all_dataset_searches(self) -> Iterator[tuple[str | AnyDatasetType, DatasetSearch]]:
         yield from self.datasets.items()
         if self.any_dataset is not None:

{lsst_daf_butler-29.2025.4000 → lsst_daf_butler-29.2025.4100}/python/lsst/daf/butler/registry/_registry.py

@@ -1449,6 +1449,9 @@ class Registry(ABC):
         """
         return None
 
-    storageClasses: StorageClassFactory
-    """All storage classes known to the registry (`StorageClassFactory`).
-    """
+    @property
+    def storageClasses(self) -> StorageClassFactory:
+        """All storage classes known to the registry
+        (`StorageClassFactory`).
+        """
+        raise NotImplementedError()