lsst-daf-butler 30.2026.200__py3-none-any.whl → 30.2026.500__py3-none-any.whl

lsst/daf/butler/_butler.py

@@ -138,7 +138,10 @@ class Butler(LimitedButler):  # numpydoc ignore=PR02
     without_datastore : `bool`, optional
         If `True` do not attach a datastore to this butler. Any attempts
         to use a datastore will fail.
-    **kwargs : `Any`
+    metrics : `ButlerMetrics` or `None`
+        External metrics object to be used for tracking butler usage. If `None`
+        a new metrics object is created.
+    **kwargs : `typing.Any`
         Additional keyword arguments passed to a constructor of actual butler
         class.
 
@@ -240,7 +243,7 @@ class Butler(LimitedButler):  # numpydoc ignore=PR02
             to use a datastore will fail.
         metrics : `ButlerMetrics` or `None`, optional
             Metrics object to record butler usage statistics.
-        **kwargs : `Any`
+        **kwargs : `typing.Any`
             Default data ID key-value pairs.  These may only identify
             "governor" dimensions like ``instrument`` and ``skymap``.
 
@@ -1390,6 +1393,10 @@ class Butler(LimitedButler):  # numpydoc ignore=PR02
             raised if any datasets with the same dataset ID already exist
             in the datastore.
 
+        Returns
+        -------
+        None
+
         Raises
         ------
         TypeError
@@ -1566,7 +1573,7 @@ class Butler(LimitedButler):  # numpydoc ignore=PR02
 
     @abstractmethod
     def transfer_dimension_records_from(
-        self, source_butler: LimitedButler | Butler, source_refs: Iterable[DatasetRef]
+        self, source_butler: LimitedButler | Butler, source_refs: Iterable[DatasetRef | DataCoordinate]
     ) -> None:
         """Transfer dimension records to this Butler from another Butler.
 
@@ -1578,10 +1585,9 @@ class Butler(LimitedButler):  # numpydoc ignore=PR02
             `Butler` whose registry will be used to expand data IDs. If the
             source refs contain coordinates that are used to populate other
             records then this will also need to be a full `Butler`.
-        source_refs : iterable of `DatasetRef`
-            Datasets defined in the source butler whose dimension records
-            should be transferred to this butler. In most circumstances.
-            transfer is faster if the dataset refs are expanded.
+        source_refs : iterable of `DatasetRef` or `DataCoordinate`
+            Datasets or data IDs defined in the source butler whose dimension
+            records should be transferred to this butler.
         """
         raise NotImplementedError()
 
@@ -2025,7 +2031,7 @@ class Butler(LimitedButler):  # numpydoc ignore=PR02
 
         Returns
         -------
-        records : `list`[`DimensionRecord`]
+        records : `list` [`DimensionRecord`]
             Dimension records matching the given query parameters.
 
         Raises
@@ -2227,3 +2233,7 @@ class Butler(LimitedButler):  # numpydoc ignore=PR02
     @abstractmethod
     def close(self) -> None:
         raise NotImplementedError()
+
+    @abstractmethod
+    def _expand_data_ids(self, data_ids: Iterable[DataCoordinate]) -> list[DataCoordinate]:
+        raise NotImplementedError()

lsst/daf/butler/_butler_collections.py

@@ -360,10 +360,10 @@ class ButlerCollections(ABC, Sequence):
         name : `str`
             The name of the collection of interest.
         include_parents : `bool`, optional
-           If `True` any parents of this collection will be included.
+            If `True` any parents of this collection will be included.
         include_summary : `bool`, optional
-           If `True` dataset type names and governor dimensions of datasets
-           stored in this collection will be included in the result.
+            If `True` dataset type names and governor dimensions of datasets
+            stored in this collection will be included in the result.
 
         Returns
         -------
@@ -464,7 +464,7 @@ class ButlerCollections(ABC, Sequence):
 
         Returns
         -------
-        filtered : `~collections.abc.Mapping` [`str`, `list`[`str`]]
+        filtered : `~collections.abc.Mapping` [`str`, `list` [`str`]]
             Mapping of the dataset type name to its corresponding list of
             collection names.
         """

lsst/daf/butler/_butler_metrics.py

@@ -27,14 +27,19 @@
 
 from __future__ import annotations
 
+__all__ = ["ButlerMetrics"]
+
 from collections.abc import Callable, Iterator
 from contextlib import contextmanager
+from typing import Concatenate, ParamSpec
 
 from pydantic import BaseModel
 
 from lsst.utils.logging import LsstLoggers
 from lsst.utils.timer import time_this
 
+P = ParamSpec("P")
+
 
 class ButlerMetrics(BaseModel):
     """Metrics collected during Butler operations."""
@@ -45,18 +50,26 @@ class ButlerMetrics(BaseModel):
     time_in_get: float = 0.0
     """Wall-clock time, in seconds, spent in get()."""
 
+    time_in_ingest: float = 0.0
+    """Wall-clock time, in seconds, spent in ingest()."""
+
     n_get: int = 0
     """Number of datasets retrieved with get()."""
 
     n_put: int = 0
     """Number of datasets stored with put()."""
 
+    n_ingest: int = 0
+    """Number of datasets ingested."""
+
     def reset(self) -> None:
         """Reset all metrics."""
         self.time_in_put = 0.0
         self.time_in_get = 0.0
+        self.time_in_ingest = 0.0
         self.n_get = 0
         self.n_put = 0
+        self.n_ingest = 0
 
     def increment_get(self, duration: float) -> None:
         """Increment time for get().
@@ -80,13 +93,31 @@ class ButlerMetrics(BaseModel):
         self.time_in_put += duration
         self.n_put += 1
 
+    def increment_ingest(self, duration: float, n_datasets: int) -> None:
+        """Increment time and datasets for ingest().
+
+        Parameters
+        ----------
+        duration : `float`
+            Duration to add to the ingest() statistics.
+        n_datasets : `int`
+            Number of datasets to be ingested for this call.
+        """
+        self.time_in_ingest += duration
+        self.n_ingest += n_datasets
+
     @contextmanager
     def _timer(
-        self, handler: Callable[[float], None], log: LsstLoggers | None = None, msg: str | None = None
+        self,
+        handler: Callable[Concatenate[float, P], None],
+        log: LsstLoggers | None = None,
+        msg: str | None = None,
+        *args: P.args,
+        **kwargs: P.kwargs,
     ) -> Iterator[None]:
         with time_this(log=log, msg=msg) as timer:
             yield
-        handler(timer.duration)
+        handler(timer.duration, *args, **kwargs)
 
     @contextmanager
     def instrument_get(self, log: LsstLoggers | None = None, msg: str | None = None) -> Iterator[None]:
@@ -115,3 +146,21 @@ class ButlerMetrics(BaseModel):
         """
         with self._timer(self.increment_put, log=log, msg=msg):
             yield
+
+    @contextmanager
+    def instrument_ingest(
+        self, n_datasets: int, log: LsstLoggers | None = None, msg: str | None = None
+    ) -> Iterator[None]:
+        """Run code and increment ingest statistics.
+
+        Parameters
+        ----------
+        n_datasets : `int`
+            Number of datasets being ingested.
+        log : `logging.Logger` or `None`
+            Logger to use for any timing information.
+        msg : `str` or `None`
+            Any message to be included in log output.
+        """
+        with self._timer(self.increment_ingest, n_datasets=n_datasets, log=log, msg=msg):
+            yield

lsst/daf/butler/_dataset_provenance.py

@@ -267,7 +267,7 @@ class DatasetProvenance(pydantic.BaseModel):
         use_upper : `bool` or `None`
             If `True` use upper case for provenance keys, if `False` use lower
             case, if `None` match the case of the prefix.
-        keys : `tuple` of `str` | `int`
+        *keys : `tuple` of `str` | `int`
             Components of key to combine with prefix and separator.
 
         Returns

lsst/daf/butler/_dataset_ref.py

@@ -479,7 +479,7 @@ class DatasetRef:
 
         Parameters
         ----------
-        simple : `dict` of [`str`, `Any`]
+        simple : `dict` of [`str`, `typing.Any`]
             The value returned by `to_simple()`.
         universe : `DimensionUniverse`
             The special graph of all known dimensions.

lsst/daf/butler/_exceptions.py

@@ -196,8 +196,8 @@ class ValidationError(RuntimeError):
 
 
 class EmptyQueryResultError(Exception):
-    """Exception raised when query methods return an empty result and `explain`
-    flag is set.
+    """Exception raised when query methods return an empty result and
+    ``explain`` flag is set.
 
     Parameters
     ----------

lsst/daf/butler/_file_dataset.py

@@ -129,7 +129,8 @@ class FileDataset:
         ----------
         dataset : `SerializedFileDataset`
             Object to deserialize.
-        dataset_type_loader : `Callable` [[ `str` ], `DatasetType` ]
+        dataset_type_loader : `~collections.abc.Callable` \
+              [[ `str` ], `DatasetType` ]
             Function that takes a string dataset type name as its
             only parameter, and returns an instance of `DatasetType`.
             Used to deserialize the `DatasetRef` instances contained

lsst/daf/butler/_formatter.py

@@ -910,6 +910,10 @@ class FormatterV2:
         provenance : `DatasetProvenance` | `None`, optional
             Provenance to attach to the file being written.
 
+        Returns
+        -------
+        None
+
         Raises
         ------
         FormatterNotImplementedError
@@ -1137,6 +1141,10 @@ class FormatterV2:
         location : `Location`
             Location from which to extract a file extension.
 
+        Returns
+        -------
+        None
+
         Raises
         ------
         ValueError
@@ -1583,6 +1591,10 @@ class Formatter(metaclass=ABCMeta):
         location : `Location`
             Location from which to extract a file extension.
 
+        Returns
+        -------
+        None
+
         Raises
         ------
         NotImplementedError

lsst/daf/butler/_labeled_butler_factory.py

@@ -30,7 +30,9 @@ from __future__ import annotations
 __all__ = ("LabeledButlerFactory", "LabeledButlerFactoryProtocol")
 
 from collections.abc import Mapping
-from typing import Protocol
+from contextlib import AbstractContextManager
+from logging import getLogger
+from typing import Any, Literal, Protocol, Self
 
 from lsst.resources import ResourcePathExpression
 
@@ -40,6 +42,8 @@ from ._butler_repo_index import ButlerRepoIndex
 from ._utilities.named_locks import NamedLocks
 from ._utilities.thread_safe_cache import ThreadSafeCache
 
+_LOG = getLogger(__name__)
+
 
 class LabeledButlerFactoryProtocol(Protocol):
     """Callable to retrieve a butler from a label."""
@@ -47,7 +51,7 @@ class LabeledButlerFactoryProtocol(Protocol):
     def __call__(self, label: str) -> Butler: ...
 
 
-class LabeledButlerFactory:
+class LabeledButlerFactory(AbstractContextManager):
     """Factory for efficiently instantiating Butler instances from the
     repository index file.  This is intended for use from long-lived services
     that want to instantiate a separate Butler instance for each end user
@@ -60,6 +64,9 @@ class LabeledButlerFactory:
         files.  If not provided, defaults to the global repository index
         configured by the ``DAF_BUTLER_REPOSITORY_INDEX`` environment variable
         --  see `ButlerRepoIndex`.
+    writeable : `bool`, optional
+        If `True`, Butler instances created by this factory will be writeable.
+        If `False` (the default), instances will be read-only.
 
     Notes
     -----
@@ -76,11 +83,12 @@ class LabeledButlerFactory:
     safely be used by separate threads.
     """
 
-    def __init__(self, repositories: Mapping[str, str] | None = None) -> None:
+    def __init__(self, repositories: Mapping[str, str] | None = None, writeable: bool = False) -> None:
         if repositories is None:
             self._repositories = None
         else:
             self._repositories = dict(repositories)
+        self._writeable = writeable
 
         self._factories = ThreadSafeCache[str, _ButlerFactory]()
         self._initialization_locks = NamedLocks()
@@ -88,6 +96,16 @@ class LabeledButlerFactory:
         # This may be overridden by unit tests.
         self._preload_unsafe_direct_butler_caches = True
 
+    def __enter__(self) -> Self:
+        return self
+
+    def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> Literal[False]:
+        try:
+            self.close()
+        except Exception:
+            _LOG.exception("An exception occurred during LabeledButlerFactory.close()")
+        return False
+
     def bind(self, access_token: str | None) -> LabeledButlerFactoryProtocol:
         """Create a callable factory function for generating Butler instances
         with out needing to specify access tokans again.
@@ -109,7 +127,7 @@ class LabeledButlerFactory:
 
         return create
 
-    def create_butler(self, *, label: str, access_token: str | None) -> Butler:
+    def create_butler(self, label: str, *, access_token: str | None = None) -> Butler:
         """Create a Butler instance.
 
         Parameters
@@ -118,7 +136,7 @@ class LabeledButlerFactory:
             Label of the repository to instantiate, from the ``repositories``
             parameter to the `LabeledButlerFactory` constructor or the global
             repository index file.
-        access_token : `str` | `None`
+        access_token : `str` | `None`, optional
             Gafaelfawr access token used to authenticate to a Butler server.
             This is required for any repositories configured to use
             `RemoteButler`.  If you only use `DirectButler`, this may be
@@ -167,7 +185,9 @@ class LabeledButlerFactory:
 
         match butler_type:
             case ButlerType.DIRECT:
-                return _DirectButlerFactory(config, self._preload_unsafe_direct_butler_caches)
+                return _DirectButlerFactory(
+                    config, self._preload_unsafe_direct_butler_caches, self._writeable
+                )
             case ButlerType.REMOTE:
                 return _RemoteButlerFactory(config)
             case _:
@@ -189,12 +209,12 @@ class _ButlerFactory(Protocol):
 
 
 class _DirectButlerFactory(_ButlerFactory):
-    def __init__(self, config: ButlerConfig, preload_unsafe_caches: bool) -> None:
+    def __init__(self, config: ButlerConfig, preload_unsafe_caches: bool, writeable: bool) -> None:
         import lsst.daf.butler.direct_butler
 
         # Create a 'template' Butler that will be cloned when callers request
         # an instance.
-        self._butler = Butler.from_config(config)
+        self._butler = Butler.from_config(config, writeable=writeable)
         assert isinstance(self._butler, lsst.daf.butler.direct_butler.DirectButler)
 
         # Load caches so that data is available in cloned instances without

lsst/daf/butler/_query_all_datasets.py

@@ -151,6 +151,8 @@ def _filter_collections_and_dataset_types(
 
     Parameters
     ----------
+    butler
+        Butler repository to use.
     collections
         List of collection names or collection search globs.
     dataset_type_query

lsst/daf/butler/cli/cmd/_remove_runs.py

@@ -114,18 +114,7 @@ def remove_runs(context: click.Context, confirm: bool, force: bool, **kwargs: An
 
     This command can be used to remove RUN collections and the datasets within
     them.
-
-    Parameters
-    ----------
-    context : `click.Context`
-        Context provided by Click.
-    confirm : `bool`
-        Confirmation for removal of the run.
-    force : `bool`
-        Force removal.
-    **kwargs : `dict` [`str`, `str`]
-        The parameters to pass to `~lsst.daf.butler.script.removeRuns`.
-    """
+    """  # numpydoc ignore=PR01
     result = script.removeRuns(**kwargs)
     canRemoveRuns = len(result.runs)
     if not canRemoveRuns:

lsst/daf/butler/column_spec.py

@@ -109,12 +109,12 @@ class ColumnValueSerializer(ABC):
 
         Parameters
         ----------
-        value : `Any`
+        value : `typing.Any`
             Column value to be serialized.
 
         Returns
         -------
-        value : `Any`
+        value : `typing.Any`
             Column value in serializable format.
         """
         raise NotImplementedError
@@ -125,12 +125,12 @@ class ColumnValueSerializer(ABC):
 
         Parameters
         ----------
-        value : `Any`
+        value : `typing.Any`
             Serialized column value.
 
         Returns
         -------
-        value : `Any`
+        value : `typing.Any`
             Deserialized column value.
         """
         raise NotImplementedError

lsst/daf/butler/configs/datastores/formatters.yaml

@@ -100,3 +100,4 @@ VisitBackgroundModel: lsst.daf.butler.formatters.json.JsonFormatter
 VignettingCorrection: lsst.ts.observatory.control.utils.extras.vignetting_storage.VignettingCorrectionFormatter
 SSPAuxiliaryFile: lsst.pipe.tasks.sspAuxiliaryFile.SSPAuxiliaryFileFormatter
 VisitGeometry: lsst.daf.butler.formatters.json.JsonFormatter
+ProvenanceQuantumGraph: lsst.pipe.base.quantum_graph.formatter.ProvenanceFormatter

lsst/daf/butler/configs/storageClasses.yaml

@@ -443,3 +443,18 @@ storageClasses:
     pytype: lsst.pipe.tasks.sspAuxiliaryFile.SSPAuxiliaryFile
   VisitGeometry:
     pytype: lsst.obs.base.visit_geometry.VisitGeometry
+  ProvenanceQuantumGraph:
+    pytype: lsst.pipe.base.quantum_graph.ProvenanceQuantumGraph
+    parameters:
+      - import_mode   # lsst.pipe.base.pipeline_graph.TaskImportMode
+      - quanta  # iterable of uuid.UUID; quanta to read
+      - datasets  # iterable of uuid.UUID; datasets to read
+      - read_init_quanta  # bool, defaults to True; whether to read pre-exec-init info
+    derivedComponents:
+      packages: Packages  # ignores node parameters
+
+      # UUID keys can be quantum or data IDs (whichever is passed in via
+      # parameters).  Nested lists are attempts to run the quantum (last is
+      # most recent).
+      logs: StructuredDataDict  # dict[uuid.UUID, list[ButlerLogRecords]]
+      metadata: StructuredDataDict  # dict[uuid.UUID, list[TaskMetadata]]

lsst/daf/butler/datastore/_datastore.py

@@ -284,6 +284,14 @@ class DatasetRefURIs(abc.Sequence):
     def __repr__(self) -> str:
         return f"DatasetRefURIs({repr(self.primaryURI)}, {repr(self.componentURIs)})"
 
+    def iter_all(self) -> Iterator[ResourcePath]:
+        """Iterate over all URIs without regard to whether they are primary
+        or component.
+        """
+        if self.primaryURI is not None:
+            yield self.primaryURI
+        yield from self.componentURIs.values()
+
 
 class Datastore(FileTransferSource, metaclass=ABCMeta):
     """Datastore interface.
@@ -536,7 +544,7 @@ class Datastore(FileTransferSource, metaclass=ABCMeta):
 
         Returns
         -------
-        exists : `dict`[`DatasetRef`, `bool`]
+        exists : `dict` [`DatasetRef`, `bool`]
             Mapping of dataset to boolean indicating whether the dataset
             is known to the datastore.
         """
@@ -825,6 +833,10 @@ class Datastore(FileTransferSource, metaclass=ABCMeta):
             in an external system or if the file is to be compressed in place.
             It is up to the datastore whether this parameter is relevant.
 
+        Returns
+        -------
+        None
+
         Raises
         ------
         NotImplementedError
@@ -1143,6 +1155,10 @@ class Datastore(FileTransferSource, metaclass=ABCMeta):
             Determine whether errors should be ignored. When multiple
             refs are being trashed there will be no per-ref check.
 
+        Returns
+        -------
+        None
+
         Raises
         ------
         FileNotFoundError
@@ -1278,6 +1294,10 @@ class Datastore(FileTransferSource, metaclass=ABCMeta):
             Entity to compare with configuration retrieved using the
             specified lookup key.
 
+        Returns
+        -------
+        None
+
         Raises
         ------
         DatastoreValidationError

lsst/daf/butler/datastore/stored_file_info.py

@@ -423,8 +423,8 @@ def make_datastore_path_relative(path: str) -> str:
     path : `str`
         The file path from a `StoredFileInfo`.
 
-    Return
-    ------
+    Returns
+    -------
     normalized_path : `str`
         The original path, if it was relative. Otherwise, a version of it that
         was converted to a relative path, stripping URI scheme and netloc from

lsst/daf/butler/datastores/chainedDatastore.py

@@ -1077,6 +1077,10 @@ class ChainedDatastore(Datastore):
             If `True`, output a log message for every validation error
             detected.
 
+        Returns
+        -------
+        None
+
         Raises
         ------
         DatastoreValidationError

lsst/daf/butler/datastores/fileDatastore.py

@@ -2976,6 +2976,10 @@ class FileDatastore(GenericBaseDatastore[StoredFileInfo]):
             If `True`, output a log message for every validation error
             detected.
 
+        Returns
+        -------
+        None
+
         Raises
         ------
         DatastoreValidationError

lsst/daf/butler/datastores/file_datastore/get.py

@@ -97,12 +97,12 @@ def generate_datastore_get_information(
 
     Parameters
     ----------
-    fileLocations : `list`[`DatasetLocationInformation`]
+    fileLocations : `list` [`DatasetLocationInformation`]
         List of file locations for this artifact and their associated datastore
         records.
     ref : `DatasetRef`
         The registry information associated with this artifact.
-    parameters : `Mapping`[`str`, `Any`]
+    parameters : `~collections.abc.Mapping` [`str`, `typing.Any`]
         `StorageClass` and `Formatter` parameters.
     readStorageClass : `StorageClass` | `None`, optional
         The StorageClass to use when ultimately returning the resulting object
@@ -255,12 +255,12 @@ def get_dataset_as_python_object_from_get_info(
 
     Parameters
     ----------
-    allGetInfo : `list`[`DatastoreFileGetInformation`]
+    allGetInfo : `list` [`DatastoreFileGetInformation`]
         Pre-processed information about each file associated with this
         artifact.
     ref : `DatasetRef`
         The registry information associated with this artifact.
-    parameters : `Mapping`[`str`, `Any`]
+    parameters : `~collections.abc.Mapping` [`str`, `typing.Any`]
         `StorageClass` and `Formatter` parameters.
     cache_manager : `AbstractDatastoreCacheManager`
         The cache manager to use for caching retrieved files.

lsst/daf/butler/datastores/file_datastore/transfer.py

@@ -55,8 +55,8 @@ def retrieve_file_transfer_records(
         Cache mapping datastore artifact to existence. Updated by
         this method with details of all artifacts tested.
 
-    Return
-    ------
+    Returns
+    -------
     files : `FileTransferMap`
         A dictionary from `DatasetId` to a list of `FileTransferRecord`,
         containing information about the files that were found for these

lsst/daf/butler/datastores/inMemoryDatastore.py

@@ -590,6 +590,10 @@ class InMemoryDatastore(GenericBaseDatastore[StoredMemoryItemInfo]):
         ignore_errors : `bool`, optional
             Indicate that errors should be ignored.
 
+        Returns
+        -------
+        None
+
         Raises
         ------
         FileNotFoundError
@@ -721,6 +725,10 @@ class InMemoryDatastore(GenericBaseDatastore[StoredMemoryItemInfo]):
             If `True`, output a log message for every validation error
             detected.
 
+        Returns
+        -------
+        None
+
         Raises
         ------
         DatastoreValidationError

lsst/daf/butler/ddl.py

@@ -537,7 +537,7 @@ class IndexSpec:
     ----------
     *columns : `str`
         Names of the columns to index.
-    **kwargs : `Any`
+    **kwargs : `typing.Any`
         Additional keyword arguments to pass directly to
         `sqlalchemy.schema.Index` constructor. This could be used to provide
         backend-specific options, e.g. to create a ``GIST`` index in PostgreSQL
@@ -556,7 +556,7 @@ class IndexSpec:
 
     kwargs: dict[str, Any]
     """Additional keyword arguments passed directly to
-    `sqlalchemy.schema.Index` constructor (`dict` [ `str`, `Any` ]).
+    `sqlalchemy.schema.Index` constructor (`dict` [ `str`, `typing.Any` ]).
     """