lsst-daf-butler 29.2025.1900__tar.gz → 29.2025.2100__tar.gz

{lsst_daf_butler-29.2025.1900/python/lsst_daf_butler.egg-info → lsst_daf_butler-29.2025.2100}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lsst-daf-butler
-Version: 29.2025.1900
+Version: 29.2025.2100
 Summary: An abstraction layer for reading and writing astronomical data to datastores.
 Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
 License: BSD 3-Clause License

{lsst_daf_butler-29.2025.1900 → lsst_daf_butler-29.2025.2100}/python/lsst/daf/butler/_butler.py

@@ -90,6 +90,10 @@ class SpecificButlerDataset:
     dataset: DatasetRef | None
 
 
+class _DeprecatedDefault:
+    """Default value for a deprecated parameter."""
+
+
 class Butler(LimitedButler):  # numpydoc ignore=PR02
     """Interface for data butler and factory for Butler instances.
 
@@ -1288,7 +1292,13 @@ class Butler(LimitedButler):  # numpydoc ignore=PR02
         raise NotImplementedError()
 
     @abstractmethod
-    def removeRuns(self, names: Iterable[str], unstore: bool = True) -> None:
+    def removeRuns(
+        self,
+        names: Iterable[str],
+        unstore: bool | type[_DeprecatedDefault] = _DeprecatedDefault,
+        *,
+        unlink_from_chains: bool = False,
+    ) -> None:
         """Remove one or more `~CollectionType.RUN` collections and the
         datasets within them.
 
@@ -1301,7 +1311,13 @@ class Butler(LimitedButler):  # numpydoc ignore=PR02
             they are present, and attempt to rollback the registry deletions if
             datastore deletions fail (which may not always be possible).  If
             `False`, datastore records for these datasets are still removed,
-            but any artifacts (e.g. files) will not be.
+            but any artifacts (e.g. files) will not be. This parameter is now
+            deprecated and no longer has any effect. Files are always deleted
+            from datastores unless they were ingested using full URIs.
+        unlink_from_chains : `bool`, optional
+            If `True` remove the RUN collection from any chains prior to
+            removing the RUN. If `False` the removal will fail if any chains
+            still refer to the RUN.
 
         Raises
         ------
@@ -1374,7 +1390,14 @@ class Butler(LimitedButler):  # numpydoc ignore=PR02
         raise NotImplementedError()
 
     @abstractmethod
-    def ingest_zip(self, zip_file: ResourcePathExpression, transfer: str = "auto") -> None:
+    def ingest_zip(
+        self,
+        zip_file: ResourcePathExpression,
+        transfer: str = "auto",
+        *,
+        transfer_dimensions: bool = False,
+        dry_run: bool = False,
+    ) -> None:
         """Ingest a Zip file into this butler.
 
         The Zip file must have been created by `retrieve_artifacts_zip`.
@@ -1385,10 +1408,17 @@ class Butler(LimitedButler):  # numpydoc ignore=PR02
             Path to the Zip file.
         transfer : `str`, optional
             Method to use to transfer the Zip into the datastore.
+        transfer_dimensions : `bool`, optional
+            If `True`, dimension record data associated with the new datasets
+            will be transferred from the Zip file, if present.
+        dry_run : `bool`, optional
+            If `True` the ingest will be processed without any modifications
+            made to the target butler and as if the target butler did not
+            have any of the datasets.
 
         Notes
         -----
-        Run collections are created as needed.
+        Run collections and dataset types are created as needed.
         """
         raise NotImplementedError()
 

{lsst_daf_butler-29.2025.1900 → lsst_daf_butler-29.2025.2100}/python/lsst/daf/butler/_butler_collections.py

@@ -108,6 +108,10 @@ class ButlerCollections(ABC, Sequence):
         """Collection defaults associated with this butler."""
         raise NotImplementedError("Defaults must be implemented by a subclass")
 
+    def __str__(self) -> str:
+        """Return string representation."""
+        return f"{self.__class__.__name__}(defaults={self.defaults})"
+
     @abstractmethod
     def extend_chain(self, parent_collection_name: str, child_collection_names: str | Iterable[str]) -> None:
         """Add children to the end of a CHAINED collection.

{lsst_daf_butler-29.2025.1900 → lsst_daf_butler-29.2025.2100}/python/lsst/daf/butler/_quantum_backed.py

@@ -516,8 +516,9 @@ class QuantumBackedButler(LimitedButler):
                 self._actual_output_refs.discard(ref)
 
         if unstore:
-            # Point of no return for removing artifacts
-            self._datastore.emptyTrash()
+            # Point of no return for removing artifacts. Only try to remove
+            # refs associated with this pruning.
+            self._datastore.emptyTrash(refs=refs)
 
     def retrieve_artifacts_zip(
         self,

{lsst_daf_butler-29.2025.1900 → lsst_daf_butler-29.2025.2100}/python/lsst/daf/butler/cli/butler.py

@@ -102,7 +102,7 @@ class PluginCommand:
     """Where the command came from (`str`)."""
 
 
-class LoaderCLI(click.MultiCommand, abc.ABC):
+class LoaderCLI(click.Group, abc.ABC):
     """Extends `click.MultiCommand`, which dispatches to subcommands, to load
     subcommands at runtime.
 

{lsst_daf_butler-29.2025.1900 → lsst_daf_butler-29.2025.2100}/python/lsst/daf/butler/cli/cmd/_remove_runs.py

@@ -81,7 +81,9 @@ def _print_remove(will: bool, runs: Sequence[script.RemoveRun], datasets: Mappin
         else:
             print(run.name)
     print("\n" + willRemoveDatasetsMsg if will else didRemoveDatasetsMsg)
+    total = sum(datasets.values())
     print(", ".join([f"{i[0]}({i[1]})" for i in datasets.items()]))
+    print("Total number of datasets to remove: ", total)
 
 
 def _print_requires_confirmation(runs: Sequence[script.RemoveRun], datasets: Mapping[str, int]) -> None:

{lsst_daf_butler-29.2025.1900 → lsst_daf_butler-29.2025.2100}/python/lsst/daf/butler/cli/cmd/commands.py

@@ -829,6 +829,12 @@ def export_calibs(*args: Any, **kwargs: Any) -> None:
 @repo_argument(required=True)
 @click.argument("zip", required=True)
 @transfer_option()
+@transfer_dimensions_option(
+    default=False, help="Attempt to register missing dimension records during ingest."
+)
+@click.option(
+    "--dry-run/--no-dry-run", default=False, help="Enable dry run mode and do not ingest any datasets."
+)
 def ingest_zip(**kwargs: Any) -> None:
     """Ingest a Zip file created by retrieve-artifacts.
 

{lsst_daf_butler-29.2025.1900 → lsst_daf_butler-29.2025.2100}/python/lsst/daf/butler/cli/utils.py

@@ -55,6 +55,7 @@ __all__ = (
 )
 
 
+import importlib.metadata
 import itertools
 import logging
 import os
@@ -76,6 +77,7 @@ import click
 import click.exceptions
 import click.testing
 import yaml
+from packaging.version import Version
 
 from lsst.utils.iteration import ensure_iterable
 
@@ -87,6 +89,12 @@ if TYPE_CHECKING:
 
     from lsst.daf.butler import Dimension
 
+_click_version = Version(importlib.metadata.version("click"))
+if _click_version >= Version("8.2.0"):
+    _click_make_metavar_has_context = True
+else:
+    _click_make_metavar_has_context = False
+
 log = logging.getLogger(__name__)
 
 # This is used as the metavar argument to Options that accept multiple string
@@ -741,9 +749,16 @@ class MWPath(click.Path):
 class MWOption(click.Option):
     """Overrides click.Option with desired behaviors."""
 
-    def make_metavar(self) -> str:
+    def make_metavar(self, ctx: click.Context | None = None) -> str:
         """Make the metavar for the help menu.
 
+        Parameters
+        ----------
+        ctx : `click.Context` or `None`
+            Context from the command.
+
+        Notes
+        -----
         Overrides `click.Option.make_metavar`.
         Adds a space and an ellipsis after the metavar name if
         the option accepts multiple inputs, otherwise defers to the base
@@ -758,7 +773,10 @@ class MWOption(click.Option):
         transformation that must apply to all types should be applied in
         get_help_record.
         """
-        metavar = super().make_metavar()
+        if _click_make_metavar_has_context:
+            metavar = super().make_metavar(ctx=ctx)  # type: ignore
+        else:
+            metavar = super().make_metavar()  # type: ignore
         if self.multiple and self.nargs == 1:
             metavar += " ..."
         elif self.nargs != 1:
@@ -769,9 +787,16 @@ class MWOption(click.Option):
 class MWArgument(click.Argument):
     """Overrides click.Argument with desired behaviors."""
 
-    def make_metavar(self) -> str:
+    def make_metavar(self, ctx: click.Context | None = None) -> str:
         """Make the metavar for the help menu.
 
+        Parameters
+        ----------
+        ctx : `click.Context` or `None`
+            Context from the command.
+
+        Notes
+        -----
         Overrides `click.Option.make_metavar`.
         Always adds a space and an ellipsis (' ...') after the
         metavar name if the option accepts multiple inputs.
@@ -784,7 +809,10 @@ class MWArgument(click.Argument):
         metavar : `str`
             The metavar value.
         """
-        metavar = super().make_metavar()
+        if _click_make_metavar_has_context:
+            metavar = super().make_metavar(ctx=ctx)  # type: ignore
+        else:
+            metavar = super().make_metavar()  # type: ignore
         if self.nargs != 1:
             metavar = f"{metavar[:-3]} ..."
         return metavar

{lsst_daf_butler-29.2025.1900 → lsst_daf_butler-29.2025.2100}/python/lsst/daf/butler/configs/datastores/formatters.yaml

@@ -94,3 +94,4 @@ Timespan: lsst.daf.butler.formatters.json.JsonFormatter
 RegionTimeInfo: lsst.daf.butler.formatters.json.JsonFormatter
 QPEnsemble: lsst.meas.pz.qp_formatter.QPFormatter
 PZModel: lsst.meas.pz.model_formatter.ModelFormatter
+VisitBackgroundModel: lsst.daf.butler.formatters.json.JsonFormatter

{lsst_daf_butler-29.2025.1900 → lsst_daf_butler-29.2025.2100}/python/lsst/daf/butler/configs/storageClasses.yaml

@@ -426,3 +426,5 @@ storageClasses:
     pytype: qp.Ensemble
   PZModel:
     pytype: rail.core.model.Model
+  VisitBackgroundModel:
+    pytype: lsst.drp.tasks.fit_visit_background.VisitBackgroundModel

{lsst_daf_butler-29.2025.1900 → lsst_daf_butler-29.2025.2100}/python/lsst/daf/butler/datastore/_datastore.py

@@ -1082,7 +1082,7 @@ class Datastore(metaclass=ABCMeta):
         raise NotImplementedError()
 
     @abstractmethod
-    def ingest_zip(self, zip_path: ResourcePath, transfer: str | None) -> None:
+    def ingest_zip(self, zip_path: ResourcePath, transfer: str | None, *, dry_run: bool = False) -> None:
         """Ingest an indexed Zip file and contents.
 
         The Zip file must have an index file as created by `retrieveArtifacts`.
@@ -1093,6 +1093,10 @@ class Datastore(metaclass=ABCMeta):
             Path to the Zip file.
         transfer : `str`
             Method to use for transferring the Zip file into the datastore.
+        dry_run : `bool`, optional
+            If `True` the ingest will be processed without any modifications
+            made to the target datastore and as if the target datastore did not
+            have any of the datasets.
         """
         raise NotImplementedError()
 
@@ -1160,13 +1164,29 @@ class Datastore(metaclass=ABCMeta):
         raise NotImplementedError("Must be implemented by subclass")
 
     @abstractmethod
-    def emptyTrash(self, ignore_errors: bool = True) -> None:
+    def emptyTrash(
+        self, ignore_errors: bool = True, refs: Collection[DatasetRef] | None = None, dry_run: bool = False
+    ) -> set[ResourcePath]:
         """Remove all datasets from the trash.
 
         Parameters
         ----------
         ignore_errors : `bool`, optional
             Determine whether errors should be ignored.
+        refs : `collections.abc.Collection` [ `DatasetRef` ] or `None`
+            Explicit list of datasets that can be removed from trash. If listed
+            datasets are not already stored in the trash table they will be
+            ignored. If `None` every entry in the trash table will be
+            processed.
+        dry_run : `bool`, optional
+            If `True`, the trash table will be queried and results reported
+            but no artifacts will be removed.
+
+        Returns
+        -------
+        removed : `set` [ `lsst.resources.ResourcePath` ]
+            List of artifacts that were removed. Can return nothing if
+            artifacts cannot be represented by URIs.
 
         Notes
         -----
@@ -1488,7 +1508,7 @@ class NullDatastore(Datastore):
     def getURI(self, datasetRef: DatasetRef, predict: bool = False) -> ResourcePath:
         raise FileNotFoundError("This is a no-op datastore that can not access a real datastore")
 
-    def ingest_zip(self, zip_path: ResourcePath, transfer: str | None) -> None:
+    def ingest_zip(self, zip_path: ResourcePath, transfer: str | None, *, dry_run: bool = False) -> None:
         raise NotImplementedError("Can only ingest a Zip into a real datastore.")
 
     def retrieveArtifacts(
@@ -1512,7 +1532,9 @@ class NullDatastore(Datastore):
     def trash(self, ref: DatasetRef | Iterable[DatasetRef], ignore_errors: bool = True) -> None:
         raise NotImplementedError("This is a no-op datastore that can not access a real datastore")
 
-    def emptyTrash(self, ignore_errors: bool = True) -> None:
+    def emptyTrash(
+        self, ignore_errors: bool = True, refs: Collection[DatasetRef] | None = None, dry_run: bool = False
+    ) -> set[ResourcePath]:
         raise NotImplementedError("This is a no-op datastore that can not access a real datastore")
 
     def transfer(self, inputDatastore: Datastore, datasetRef: DatasetRef) -> None:

{lsst_daf_butler-29.2025.1900 → lsst_daf_butler-29.2025.2100}/python/lsst/daf/butler/datastore/generic_base.py

@@ -80,7 +80,7 @@ class GenericBaseDatastore(Datastore, Generic[_InfoType]):
         encountered during removal are not ignored.
         """
         self.trash(ref, ignore_errors=False)
-        self.emptyTrash(ignore_errors=False)
+        self.emptyTrash(ignore_errors=False, refs=[ref])
 
     def transfer(self, inputDatastore: Datastore, ref: DatasetRef) -> None:
         """Retrieve a dataset from an input `Datastore`,
@@ -89,7 +89,7 @@ class GenericBaseDatastore(Datastore, Generic[_InfoType]):
         Parameters
         ----------
         inputDatastore : `Datastore`
-            The external `Datastore` from which to retreive the Dataset.
+            The external `Datastore` from which to retrieve the Dataset.
         ref : `DatasetRef`
             Reference to the required dataset in the input data store.
         """

{lsst_daf_butler-29.2025.1900 → lsst_daf_butler-29.2025.2100}/python/lsst/daf/butler/datastores/chainedDatastore.py

@@ -32,7 +32,6 @@ from __future__ import annotations
 __all__ = ("ChainedDatastore",)
 
 import itertools
-import logging
 import time
 import warnings
 from collections.abc import Callable, Collection, Iterable, Mapping, Sequence
@@ -51,13 +50,14 @@ from lsst.daf.butler.datastore.record_data import DatastoreRecordData
 from lsst.daf.butler.datastores.file_datastore.retrieve_artifacts import ArtifactIndexInfo, ZipIndex
 from lsst.resources import ResourcePath
 from lsst.utils import doImportType
+from lsst.utils.logging import getLogger
 
 if TYPE_CHECKING:
     from lsst.daf.butler import Config, DatasetProvenance, DatasetType, LookupKey, StorageClass
     from lsst.daf.butler.registry.interfaces import DatasetIdRef, DatastoreRegistryBridgeManager
     from lsst.resources import ResourcePathExpression
 
-log = logging.getLogger(__name__)
+log = getLogger(__name__)
 
 
 class _IngestPrepData(Datastore.IngestPrepData):
@@ -912,7 +912,7 @@ class ChainedDatastore(Datastore):
 
         return merged_artifact_map
 
-    def ingest_zip(self, zip_path: ResourcePath, transfer: str | None) -> None:
+    def ingest_zip(self, zip_path: ResourcePath, transfer: str | None, *, dry_run: bool = False) -> None:
         """Ingest an indexed Zip file and contents.
 
         The Zip file must have an index file as created by `retrieveArtifacts`.
@@ -923,6 +923,10 @@ class ChainedDatastore(Datastore):
             Path to the Zip file.
         transfer : `str`
             Method to use for transferring the Zip file into the datastore.
+        dry_run : `bool`, optional
+            If `True` the ingest will be processed without any modifications
+            made to the target datastore and as if the target datastore did not
+            have any of the datasets.
 
         Notes
         -----
@@ -966,7 +970,7 @@ class ChainedDatastore(Datastore):
                 log.debug("Datastore %s skipping zip ingest due to constraints", datastore.name)
                 continue
             try:
-                datastore.ingest_zip(zip_path, transfer=transfer)
+                datastore.ingest_zip(zip_path, transfer=transfer, dry_run=dry_run)
             except NotImplementedError:
                 continue
             except Exception as e:
@@ -999,7 +1003,7 @@ class ChainedDatastore(Datastore):
         """
         log.debug("Removing %s", ref)
         self.trash(ref, ignore_errors=False)
-        self.emptyTrash(ignore_errors=False)
+        self.emptyTrash(ignore_errors=False, refs=[ref])
 
     def forget(self, refs: Iterable[DatasetRef]) -> None:
         for datastore in tuple(self.datastores):
@@ -1028,9 +1032,13 @@ class ChainedDatastore(Datastore):
             else:
                 raise FileNotFoundError(err_msg)
 
-    def emptyTrash(self, ignore_errors: bool = True) -> None:
+    def emptyTrash(
+        self, ignore_errors: bool = True, refs: Collection[DatasetRef] | None = None, dry_run: bool = False
+    ) -> set[ResourcePath]:
+        removed = set()
         for datastore in self.datastores:
-            datastore.emptyTrash(ignore_errors=ignore_errors)
+            removed.update(datastore.emptyTrash(ignore_errors=ignore_errors, refs=refs, dry_run=dry_run))
+        return removed
 
     def transfer(self, inputDatastore: Datastore, ref: DatasetRef) -> None:
         """Retrieve a dataset from an input `Datastore`,
@@ -1039,7 +1047,7 @@ class ChainedDatastore(Datastore):
         Parameters
         ----------
         inputDatastore : `Datastore`
-            The external `Datastore` from which to retreive the Dataset.
+            The external `Datastore` from which to retrieve the Dataset.
         ref : `DatasetRef`
             Reference to the required dataset in the input data store.
 
@@ -1235,102 +1243,65 @@ class ChainedDatastore(Datastore):
         dry_run: bool = False,
     ) -> tuple[set[DatasetRef], set[DatasetRef]]:
         # Docstring inherited
-        # mypy does not understand "type(self) is not type(source)"
-        if isinstance(source_datastore, ChainedDatastore):
-            # Both the source and destination are chained datastores.
-            source_datastores = tuple(source_datastore.datastores)
-        else:
-            # The source datastore is different, forward everything to the
-            # child datastores.
-            source_datastores = (source_datastore,)
-
         if not refs:
-            # Nothing to transfer.
             return set(), set()
 
-        # Need to know the set of all possible refs that could be transferred.
-        remaining_refs = set(refs)
-
-        missing_from_source: set[DatasetRef] | None = None
-        all_accepted = set()
+        # Assume that each child datastore knows how to look inside a chained
+        # datastore for compatible datastores (and so there is no need to
+        # unpack the source datastores here).
+        # Need to decide if a ref accepted by one datastore should be sent to
+        # later datastores (as is done in put()). More efficient to filter out
+        # accepted datasets.
+        if artifact_existence is None:
+            artifact_existence = {}
+        available_refs = set(refs)
+        accepted: set[DatasetRef] = set()
+        rejected: set[DatasetRef] = set()
         nsuccess = 0
-        for source_child in source_datastores:
-            # If we are reading from a chained datastore, it's possible that
-            # only a subset of the datastores know about the dataset. We can't
-            # ask the receiving datastore to copy it when it doesn't exist
-            # so we have to filter again based on what the source datastore
-            # understands.
-            known_to_source = source_child.knows_these(list(refs))
-
-            # Need to know that there is a possibility that some of these
-            # datasets exist but are unknown to the source datastore if
-            # trust is enabled.
-            if getattr(source_child, "trustGetRequest", False):
-                unknown = [ref for ref, known in known_to_source.items() if not known]
-                existence = source_child.mexists(unknown, artifact_existence)
-                for ref, exists in existence.items():
-                    known_to_source[ref] = exists
-
-            missing = {ref for ref, known in known_to_source.items() if not known}
-            if missing:
-                if missing_from_source is None:
-                    missing_from_source = missing
-                else:
-                    missing_from_source &= missing
-
-            # Try to transfer from each source datastore to each child
-            # datastore. Have to make sure we don't transfer something
-            # we've already transferred to this destination on later passes.
-
-            # Filter the initial list based on the datasets we have
-            # not yet transferred.
-            these_refs = []
-            for ref in refs:
-                if ref in remaining_refs and known_to_source[ref]:
-                    these_refs.append(ref)
-
-            if not these_refs:
-                # Already transferred all datasets known to this datastore.
-                continue
 
-            for datastore, constraints in zip(self.datastores, self.datastoreConstraints, strict=True):
-                if constraints is not None:
-                    filtered_refs = []
-                    for ref in these_refs:
-                        if constraints.isAcceptable(ref):
-                            filtered_refs.append(ref)
-                        else:
-                            log.debug("Rejecting ref by constraints: %s", ref)
-                else:
-                    filtered_refs = list(these_refs)
-                try:
-                    accepted, _ = datastore.transfer_from(
-                        source_child,
-                        filtered_refs,
-                        transfer,
-                        artifact_existence,
-                        dry_run=dry_run,
-                    )
-                except (TypeError, NotImplementedError):
-                    # The datastores were incompatible.
-                    continue
-                else:
-                    nsuccess += 1
+        log.debug("Initiating transfer to chained datastore %s from %s", self.name, source_datastore.name)
+        for datastore in self.datastores:
+            # Rejections from this datastore might be acceptances in the next.
+            # We add them all up but then recalculate at the end.
+            if not available_refs:
+                break
+            log.verbose("Transferring %d datasets to %s from chain", len(available_refs), datastore.name)
 
-                # Remove the accepted datasets from those remaining.
-                remaining_refs = remaining_refs - accepted
+            try:
+                current_accepted, current_rejected = datastore.transfer_from(
+                    source_datastore,
+                    available_refs,
+                    transfer=transfer,
+                    artifact_existence=artifact_existence,
+                    dry_run=dry_run,
+                )
+            except (TypeError, NotImplementedError):
+                # The datastores were incompatible.
+                continue
+            else:
+                nsuccess += 1
 
-                # Keep track of everything we have accepted.
-                all_accepted.update(accepted)
+                # Do not send accepted refs to later datastores.
+                available_refs -= current_accepted
 
-        if missing_from_source:
-            for ref in missing_from_source:
-                log.warning("Asked to transfer dataset %s but no file artifacts exist for it", ref)
+                accepted.update(current_accepted)
+                rejected.update(current_rejected)
 
         if nsuccess == 0:
             raise TypeError(f"None of the child datastores could accept transfers from {source_datastore!r}")
 
-        return all_accepted, remaining_refs
+        # It's not rejected if some other datastore accepted it.
+        rejected -= accepted
+        log.verbose(
+            "Finished transfer_from %s to %s with %d accepted, %d rejected from %d requested.",
+            source_datastore.name,
+            self.name,
+            len(accepted),
+            len(rejected),
+            len(refs),
+        )
+
+        return accepted, rejected
 
     def get_opaque_table_definitions(self) -> Mapping[str, DatastoreOpaqueTable]:
         # Docstring inherited from the base class.