lsst-daf-butler 29.2025.1900__tar.gz → 29.2025.2000__tar.gz

{lsst_daf_butler-29.2025.1900/python/lsst_daf_butler.egg-info → lsst_daf_butler-29.2025.2000}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lsst-daf-butler
-Version: 29.2025.1900
+Version: 29.2025.2000
 Summary: An abstraction layer for reading and writing astronomical data to datastores.
 Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
 License: BSD 3-Clause License

{lsst_daf_butler-29.2025.1900 → lsst_daf_butler-29.2025.2000}/python/lsst/daf/butler/_quantum_backed.py

@@ -516,8 +516,9 @@ class QuantumBackedButler(LimitedButler):
                 self._actual_output_refs.discard(ref)
 
         if unstore:
-            # Point of no return for removing artifacts
-            self._datastore.emptyTrash()
+            # Point of no return for removing artifacts. Only try to remove
+            # refs associated with this pruning.
+            self._datastore.emptyTrash(refs=refs)
 
     def retrieve_artifacts_zip(
         self,

{lsst_daf_butler-29.2025.1900 → lsst_daf_butler-29.2025.2000}/python/lsst/daf/butler/cli/butler.py

@@ -102,7 +102,7 @@ class PluginCommand:
     """Where the command came from (`str`)."""
 
 
-class LoaderCLI(click.MultiCommand, abc.ABC):
+class LoaderCLI(click.Group, abc.ABC):
     """Extends `click.MultiCommand`, which dispatches to subcommands, to load
     subcommands at runtime.
 

{lsst_daf_butler-29.2025.1900 → lsst_daf_butler-29.2025.2000}/python/lsst/daf/butler/cli/cmd/_remove_runs.py

@@ -81,7 +81,9 @@ def _print_remove(will: bool, runs: Sequence[script.RemoveRun], datasets: Mappin
         else:
             print(run.name)
     print("\n" + willRemoveDatasetsMsg if will else didRemoveDatasetsMsg)
+    total = sum(datasets.values())
     print(", ".join([f"{i[0]}({i[1]})" for i in datasets.items()]))
+    print("Total number of datasets to remove: ", total)
 
 
 def _print_requires_confirmation(runs: Sequence[script.RemoveRun], datasets: Mapping[str, int]) -> None:

{lsst_daf_butler-29.2025.1900 → lsst_daf_butler-29.2025.2000}/python/lsst/daf/butler/cli/utils.py

@@ -55,6 +55,7 @@ __all__ = (
 )
 
 
+import importlib.metadata
 import itertools
 import logging
 import os
@@ -76,6 +77,7 @@ import click
 import click.exceptions
 import click.testing
 import yaml
+from packaging.version import Version
 
 from lsst.utils.iteration import ensure_iterable
 
@@ -87,6 +89,12 @@ if TYPE_CHECKING:
 
     from lsst.daf.butler import Dimension
 
+_click_version = Version(importlib.metadata.version("click"))
+if _click_version >= Version("8.2.0"):
+    _click_make_metavar_has_context = True
+else:
+    _click_make_metavar_has_context = False
+
 log = logging.getLogger(__name__)
 
 # This is used as the metavar argument to Options that accept multiple string
@@ -741,9 +749,16 @@ class MWPath(click.Path):
 class MWOption(click.Option):
     """Overrides click.Option with desired behaviors."""
 
-    def make_metavar(self) -> str:
+    def make_metavar(self, ctx: click.Context | None = None) -> str:
         """Make the metavar for the help menu.
 
+        Parameters
+        ----------
+        ctx : `click.Context` or `None`
+            Context from the command.
+
+        Notes
+        -----
         Overrides `click.Option.make_metavar`.
         Adds a space and an ellipsis after the metavar name if
         the option accepts multiple inputs, otherwise defers to the base
@@ -758,7 +773,10 @@ class MWOption(click.Option):
         transformation that must apply to all types should be applied in
         get_help_record.
         """
-        metavar = super().make_metavar()
+        if _click_make_metavar_has_context:
+            metavar = super().make_metavar(ctx=ctx)  # type: ignore
+        else:
+            metavar = super().make_metavar()  # type: ignore
         if self.multiple and self.nargs == 1:
             metavar += " ..."
         elif self.nargs != 1:
@@ -769,9 +787,16 @@ class MWOption(click.Option):
 class MWArgument(click.Argument):
     """Overrides click.Argument with desired behaviors."""
 
-    def make_metavar(self) -> str:
+    def make_metavar(self, ctx: click.Context | None = None) -> str:
         """Make the metavar for the help menu.
 
+        Parameters
+        ----------
+        ctx : `click.Context` or `None`
+            Context from the command.
+
+        Notes
+        -----
         Overrides `click.Option.make_metavar`.
         Always adds a space and an ellipsis (' ...') after the
         metavar name if the option accepts multiple inputs.
@@ -784,7 +809,10 @@ class MWArgument(click.Argument):
         metavar : `str`
             The metavar value.
         """
-        metavar = super().make_metavar()
+        if _click_make_metavar_has_context:
+            metavar = super().make_metavar(ctx=ctx)  # type: ignore
+        else:
+            metavar = super().make_metavar()  # type: ignore
         if self.nargs != 1:
             metavar = f"{metavar[:-3]} ..."
         return metavar

{lsst_daf_butler-29.2025.1900 → lsst_daf_butler-29.2025.2000}/python/lsst/daf/butler/configs/datastores/formatters.yaml

@@ -94,3 +94,4 @@ Timespan: lsst.daf.butler.formatters.json.JsonFormatter
 RegionTimeInfo: lsst.daf.butler.formatters.json.JsonFormatter
 QPEnsemble: lsst.meas.pz.qp_formatter.QPFormatter
 PZModel: lsst.meas.pz.model_formatter.ModelFormatter
+VisitBackgroundModel: lsst.daf.butler.formatters.json.JsonFormatter

{lsst_daf_butler-29.2025.1900 → lsst_daf_butler-29.2025.2000}/python/lsst/daf/butler/configs/storageClasses.yaml

@@ -426,3 +426,5 @@ storageClasses:
     pytype: qp.Ensemble
   PZModel:
     pytype: rail.core.model.Model
+  VisitBackgroundModel:
+    pytype: lsst.drp.tasks.fit_visit_background.VisitBackgroundModel

{lsst_daf_butler-29.2025.1900 → lsst_daf_butler-29.2025.2000}/python/lsst/daf/butler/datastore/_datastore.py

@@ -1160,13 +1160,29 @@ class Datastore(metaclass=ABCMeta):
         raise NotImplementedError("Must be implemented by subclass")
 
     @abstractmethod
-    def emptyTrash(self, ignore_errors: bool = True) -> None:
+    def emptyTrash(
+        self, ignore_errors: bool = True, refs: Collection[DatasetRef] | None = None, dry_run: bool = False
+    ) -> set[ResourcePath]:
         """Remove all datasets from the trash.
 
         Parameters
         ----------
         ignore_errors : `bool`, optional
             Determine whether errors should be ignored.
+        refs : `collections.abc.Collection` [ `DatasetRef` ] or `None`
+            Explicit list of datasets that can be removed from trash. If listed
+            datasets are not already stored in the trash table they will be
+            ignored. If `None` every entry in the trash table will be
+            processed.
+        dry_run : `bool`, optional
+            If `True`, the trash table will be queried and results reported
+            but no artifacts will be removed.
+
+        Returns
+        -------
+        removed : `set` [ `lsst.resources.ResourcePath` ]
+            List of artifacts that were removed. Can return nothing if
+            artifacts cannot be represented by URIs.
 
         Notes
         -----
@@ -1512,7 +1528,9 @@ class NullDatastore(Datastore):
     def trash(self, ref: DatasetRef | Iterable[DatasetRef], ignore_errors: bool = True) -> None:
         raise NotImplementedError("This is a no-op datastore that can not access a real datastore")
 
-    def emptyTrash(self, ignore_errors: bool = True) -> None:
+    def emptyTrash(
+        self, ignore_errors: bool = True, refs: Collection[DatasetRef] | None = None, dry_run: bool = False
+    ) -> set[ResourcePath]:
         raise NotImplementedError("This is a no-op datastore that can not access a real datastore")
 
     def transfer(self, inputDatastore: Datastore, datasetRef: DatasetRef) -> None:

{lsst_daf_butler-29.2025.1900 → lsst_daf_butler-29.2025.2000}/python/lsst/daf/butler/datastore/generic_base.py

@@ -80,7 +80,7 @@ class GenericBaseDatastore(Datastore, Generic[_InfoType]):
         encountered during removal are not ignored.
         """
         self.trash(ref, ignore_errors=False)
-        self.emptyTrash(ignore_errors=False)
+        self.emptyTrash(ignore_errors=False, refs=[ref])
 
     def transfer(self, inputDatastore: Datastore, ref: DatasetRef) -> None:
         """Retrieve a dataset from an input `Datastore`,
@@ -89,7 +89,7 @@ class GenericBaseDatastore(Datastore, Generic[_InfoType]):
         Parameters
         ----------
         inputDatastore : `Datastore`
-            The external `Datastore` from which to retreive the Dataset.
+            The external `Datastore` from which to retrieve the Dataset.
         ref : `DatasetRef`
             Reference to the required dataset in the input data store.
         """

{lsst_daf_butler-29.2025.1900 → lsst_daf_butler-29.2025.2000}/python/lsst/daf/butler/datastores/chainedDatastore.py

@@ -999,7 +999,7 @@ class ChainedDatastore(Datastore):
         """
         log.debug("Removing %s", ref)
         self.trash(ref, ignore_errors=False)
-        self.emptyTrash(ignore_errors=False)
+        self.emptyTrash(ignore_errors=False, refs=[ref])
 
     def forget(self, refs: Iterable[DatasetRef]) -> None:
         for datastore in tuple(self.datastores):
@@ -1028,9 +1028,13 @@ class ChainedDatastore(Datastore):
             else:
                 raise FileNotFoundError(err_msg)
 
-    def emptyTrash(self, ignore_errors: bool = True) -> None:
+    def emptyTrash(
+        self, ignore_errors: bool = True, refs: Collection[DatasetRef] | None = None, dry_run: bool = False
+    ) -> set[ResourcePath]:
+        removed = set()
         for datastore in self.datastores:
-            datastore.emptyTrash(ignore_errors=ignore_errors)
+            removed.update(datastore.emptyTrash(ignore_errors=ignore_errors, refs=refs, dry_run=dry_run))
+        return removed
 
     def transfer(self, inputDatastore: Datastore, ref: DatasetRef) -> None:
         """Retrieve a dataset from an input `Datastore`,
@@ -1039,7 +1043,7 @@ class ChainedDatastore(Datastore):
         Parameters
         ----------
         inputDatastore : `Datastore`
-            The external `Datastore` from which to retreive the Dataset.
+            The external `Datastore` from which to retrieve the Dataset.
         ref : `DatasetRef`
             Reference to the required dataset in the input data store.
 

{lsst_daf_butler-29.2025.1900 → lsst_daf_butler-29.2025.2000}/python/lsst/daf/butler/datastores/fileDatastore.py

@@ -401,53 +401,6 @@ class FileDatastore(GenericBaseDatastore[StoredFileInfo]):
         log.debug("Checking if resource exists: %s", location.uri)
         return location.uri.exists()
 
-    def _delete_artifact(self, location: Location) -> None:
-        """Delete the artifact from the datastore.
-
-        Parameters
-        ----------
-        location : `Location`
-            Location of the artifact associated with this datastore.
-        """
-        if location.pathInStore.isabs():
-            raise RuntimeError(f"Cannot delete artifact with absolute uri {location.uri}.")
-
-        try:
-            location.uri.remove()
-        except FileNotFoundError:
-            log.debug("File %s did not exist and so could not be deleted.", location.uri)
-            raise
-        except Exception as e:
-            log.critical("Failed to delete file: %s (%s)", location.uri, e)
-            raise
-        log.debug("Successfully deleted file: %s", location.uri)
-
-    def _delete_artifacts(self, locations: list[Location]) -> None:
-        """Delete multiple artifacts from the datastore."""
-        # Filter out all artifacts that are direct.
-        n_locations = len(locations)
-        filtered = [loc.uri for loc in locations if not loc.pathInStore.isabs()]
-        n_direct = n_locations - len(filtered)
-        if n_direct > 0:
-            log.info(
-                "Not deleting %d artifact%s out of %d using absolute URIs",
-                n_direct,
-                "s" if n_direct != 1 else "",
-                n_locations,
-            )
-        if not filtered:
-            return
-
-        try:
-            ResourcePath.mremove(filtered, do_raise=True)
-        except* FileNotFoundError as e:
-            # File not there is counted as "success" because we don't want
-            # the file to be there.
-            n_except = len(e.exceptions)
-            s = "s" if n_except != 1 else ""
-            log.debug("%d artifact%s did not exist and so could not be deleted.", n_except, s)
-        log.debug("Successfully deleted %d file%s", len(filtered), "s" if len(filtered) != 1 else "")
-
     def addStoredItemInfo(
         self,
         refs: Iterable[DatasetRef],
@@ -2557,8 +2510,9 @@ class FileDatastore(GenericBaseDatastore[StoredFileInfo]):
             else:
                 raise
 
-    @transactional
-    def emptyTrash(self, ignore_errors: bool = True) -> None:
+    def emptyTrash(
+        self, ignore_errors: bool = True, refs: Collection[DatasetRef] | None = None, dry_run: bool = False
+    ) -> set[ResourcePath]:
         """Remove all datasets from the trash.
 
         Parameters
@@ -2567,14 +2521,91 @@ class FileDatastore(GenericBaseDatastore[StoredFileInfo]):
             If `True` return without error even if something went wrong.
             Problems could occur if another process is simultaneously trying
             to delete.
+        refs : `collections.abc.Collection` [ `DatasetRef` ] or `None`
+            Explicit list of datasets that can be removed from trash. If listed
+            datasets are not already stored in the trash table they will be
+            ignored. If `None` every entry in the trash table will be
+            processed.
+        dry_run : `bool`, optional
+            If `True`, the trash table will be queried and results reported
+            but no artifacts will be removed.
+
+        Returns
+        -------
+        removed : `set` [ `lsst.resources.ResourcePath` ]
+            List of artifacts that were removed.
+
+        Notes
+        -----
+        Will empty the records from the trash tables only if this call finishes
+        without raising.
         """
-        log.debug("Emptying trash in datastore %s", self.name)
+        removed = set()
+        if refs:
+            selected_ids = {ref.id for ref in refs}
+            n_chunks = 0
+            for chunk in chunk_iterable(selected_ids, chunk_size=50_000):
+                n_chunks += 1
+                log.verbose("Emptying trash for chunk %d of size %d", n_chunks, len(chunk))
+                removed.update(
+                    self._empty_trash_subset(ignore_errors=ignore_errors, selected_ids=chunk, dry_run=dry_run)
+                )
+        else:
+            log.verbose("Emptying all trash in datastore %s", self.name)
+            removed = self._empty_trash_subset(ignore_errors=ignore_errors, dry_run=dry_run)
+        log.info(
+            "%sRemoved %d file artifact%s from datastore %s",
+            "Would have " if dry_run else "",
+            len(removed),
+            "s" if len(removed) != 1 else "",
+            self.name,
+        )
+        return removed
+
+    @transactional
+    def _empty_trash_subset(
+        self,
+        *,
+        ignore_errors: bool = True,
+        selected_ids: Collection[DatasetId] | None = None,
+        dry_run: bool = False,
+    ) -> set[ResourcePath]:
+        """Empty trash table in transaction.
 
+        Parameters
+        ----------
+        ignore_errors : `bool`
+            If `True` return without error even if something went wrong.
+            Problems could occur if another process is simultaneously trying
+            to delete.
+        selected_ids : `collections.abc.collection` [`DatasetId`] or `None`
+            Explicit list of dataset IDs that can be removed from the trash.
+            If listed datasets are not already included in the trash table
+            they will be ignored. If `None` every entry in the  trash table
+            will be processed.
+        dry_run : `bool`, optional
+            If `True`, the trash table will be queried and results reported
+            but no artifacts will be removed.
+
+        Returns
+        -------
+        removed : `set` [ `lsst.resources.ResourcePath` ]
+            Artifacts successfully removed.
+
+        Notes
+        -----
+        Will empty the records from the trash tables only if this call finishes
+        without raising.
+        """
         # Context manager will empty trash iff we finish it without raising.
         # It will also automatically delete the relevant rows from the
         # trash table and the records table.
         with self.bridge.emptyTrash(
-            self._table, record_class=StoredFileInfo, record_column="path"
+            self._table,
+            record_class=StoredFileInfo,
+            record_column="path",
+            selected_ids=selected_ids,
+            dry_run=dry_run,
         ) as trash_data:
             # Removing the artifacts themselves requires that the files are
             # not also associated with refs that are not to be trashed.
@@ -2622,7 +2653,8 @@ class FileDatastore(GenericBaseDatastore[StoredFileInfo]):
                 else:
                     artifacts_to_keep = slow_artifacts_to_keep
 
-            artifacts_to_delete: list[Location] = []
+            n_direct = 0
+            artifacts_to_delete: set[ResourcePath] = set()
             for ref, info in trashed_list:
                 # Should not happen for this implementation but need
                 # to keep mypy happy.
@@ -2636,36 +2668,71 @@ class FileDatastore(GenericBaseDatastore[StoredFileInfo]):
                 # Only trashed refs still known to datastore will be returned.
                 location = info.file_location(self.locationFactory)
 
-                artifacts_to_delete.append(location)
+                if location.pathInStore.isabs():
+                    n_direct += 1
+                    continue
 
-            # Now do the deleting.
-            s_del = "s" if len(artifacts_to_delete) != 1 else ""
-            log.debug(
-                "Now removing %d file artifact%s from datastore %s",
-                len(artifacts_to_delete),
-                s_del,
-                self.name,
-            )
+                # Strip fragment before storing since it is the artifact
+                # we are deleting and we do not want repeats for every member
+                # in a zip.
+                artifacts_to_delete.add(location.uri.replace(fragment=""))
 
-            try:
-                self._delete_artifacts(artifacts_to_delete)
-            except* FileNotFoundError:
-                # If the file itself has been deleted there is nothing
-                # we can do about it. It is possible that trash has
-                # been run in parallel in another process or someone
-                # decided to delete the file. It is unlikely to come
-                # back and so we should still continue with the removal
-                # of the entry from the trash table. It is also possible
-                # we removed it in a previous iteration if it was
-                # a multi-dataset artifact. The delete artifact method
-                # will log a debug message in this scenario.
-                # Distinguishing file missing before trash started and
-                # file already removed previously as part of this trash
-                # is not worth the distinction with regards to potential
-                # memory cost. These should already be caught but just in case
-                # filter them out here.
-                pass
-            except* Exception as e:
+            if n_direct > 0:
+                s = "s" if n_direct != 1 else ""
+                log.verbose("Not deleting %d artifact%s using absolute URI%s", n_direct, s, s)
+
+            if artifacts_to_keep:
+                log.verbose(
+                    "%d artifact%s were not deleted because they are associated with other datasets",
+                    len(artifacts_to_keep),
+                    "s" if len(artifacts_to_keep) != 1 else "",
+                )
+
+            if not artifacts_to_delete:
+                return set()
+
+            # Now do the deleting. Special case the log message for a single
+            # artifact.
+            if len(artifacts_to_delete) == 1:
+                log.verbose(
+                    "%s removing file artifact %s from datastore %s",
+                    "Would be" if dry_run else "Now",
+                    list(artifacts_to_delete)[0],
+                    self.name,
+                )
+            else:
+                log.verbose(
+                    "%s removing %d file artifacts from datastore %s",
+                    "Would be" if dry_run else "Now",
+                    len(artifacts_to_delete),
+                    self.name,
+                )
+
+            # For dry-run mode do not attempt to search the file store for
+            # the artifacts to determine whether they exist or not. Simply
+            # report that an attempt would be made to delete them. Never
+            # report direct imports.
+            if dry_run:
+                return artifacts_to_delete
+
+            # Now remove the actual file artifacts.
+            remove_result = ResourcePath.mremove(artifacts_to_delete, do_raise=False)
+
+            removed: set[ResourcePath] = set()
+            exceptions: list[Exception] = []
+            for uri, result in remove_result.items():
+                if result.exception is None or isinstance(result.exception, FileNotFoundError):
+                    # File not existing is not an error since some other
+                    # process might have been trying to clean it and we do not
+                    # want to raise an error for a situation where the file
+                    # is not there and we do not want it to be there.
+                    removed.add(uri)
+                else:
+                    exceptions.append(result.exception)
+
+            if exceptions:
+                s_err = "s" if len(exceptions) != 1 else ""
+                e = ExceptionGroup(f"Error{s_err} removing {len(exceptions)} artifact{s_err}", exceptions)
                 if ignore_errors:
                     # Use a debug message here even though it's not
                     # a good situation. In some cases this can be
@@ -2674,18 +2741,18 @@ class FileDatastore(GenericBaseDatastore[StoredFileInfo]):
                     # other's files. Butler does not know about users
                     # and trash has no idea what collections these
                     # files were in (without guessing from a path).
-                    s_err = "s" if len(e.exceptions) != 1 else ""
                     log.debug(
                         "Encountered %d error%s removing %d artifact%s from datastore %s: %s",
-                        len(e.exceptions),
+                        len(exceptions),
                         s_err,
                         len(artifacts_to_delete),
-                        s_del,
+                        "s" if len(artifacts_to_delete) != 1 else "",
                         self.name,
                         e,
                     )
                 else:
-                    raise
+                    raise e
+        return removed
 
     @transactional
     def transfer_from(

{lsst_daf_butler-29.2025.1900 → lsst_daf_butler-29.2025.2000}/python/lsst/daf/butler/datastores/inMemoryDatastore.py

@@ -33,7 +33,7 @@ __all__ = ("InMemoryDatastore", "StoredMemoryItemInfo")
 
 import logging
 import time
-from collections.abc import Iterable, Mapping
+from collections.abc import Collection, Iterable, Mapping
 from dataclasses import dataclass
 from typing import TYPE_CHECKING, Any
 from urllib.parse import urlencode
@@ -627,13 +627,28 @@ class InMemoryDatastore(GenericBaseDatastore[StoredMemoryItemInfo]):
         with self._transaction.undoWith(f"Trash {len(ref_list)} datasets", _rollbackMoveToTrash, ref_list):
             self._trashedIds.update(ref.id for ref in ref_list)
 
-    def emptyTrash(self, ignore_errors: bool = False) -> None:
+    def emptyTrash(
+        self, ignore_errors: bool = False, refs: Collection[DatasetRef] | None = None, dry_run: bool = False
+    ) -> set[ResourcePath]:
         """Remove all datasets from the trash.
 
         Parameters
         ----------
         ignore_errors : `bool`, optional
             Ignore errors.
+        refs : `collections.abc.Collection` [ `DatasetRef` ] or `None`
+            Explicit list of datasets that can be removed from trash. If listed
+            datasets are not already stored in the trash table they will be
+            ignored. If `None` every entry in the trash table will be
+            processed.
+        dry_run : `bool`, optional
+            If `True`, the trash table will be queried and results reported
+            but no artifacts will be removed.
+
+        Returns
+        -------
+        removed : `set` [ `lsst.resources.ResourcePath` ]
+            List of artifacts that were removed. Empty for this datastore.
 
         Notes
         -----
@@ -647,7 +662,19 @@ class InMemoryDatastore(GenericBaseDatastore[StoredMemoryItemInfo]):
         """
         log.debug("Emptying trash in datastore %s", self.name)
 
-        for dataset_id in self._trashedIds:
+        trashed_ids = self._trashedIds
+        if refs:
+            selected_ids = {ref.id for ref in refs}
+            if selected_ids is not None:
+                trashed_ids = {tid for tid in trashed_ids if tid in selected_ids}
+
+        if dry_run:
+            log.info(
+                "Would attempt remove %s dataset%s.", len(trashed_ids), "s" if len(trashed_ids) != 1 else ""
+            )
+            return set()
+
+        for dataset_id in trashed_ids:
             try:
                 realID, _ = self._get_dataset_info(dataset_id)
             except FileNotFoundError:
@@ -677,7 +704,8 @@ class InMemoryDatastore(GenericBaseDatastore[StoredMemoryItemInfo]):
             self._remove_stored_item_info(dataset_id)
 
         # Empty the trash table
-        self._trashedIds = set()
+        self._trashedIds = self._trashedIds - trashed_ids
+        return set()
 
     def validateConfiguration(
         self, entities: Iterable[DatasetRef | DatasetType | StorageClass], logFailures: bool = False