lsst-daf-butler 29.2025.2100__tar.gz → 29.2025.2400__tar.gz

{lsst_daf_butler-29.2025.2100/python/lsst_daf_butler.egg-info → lsst_daf_butler-29.2025.2400}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lsst-daf-butler
-Version: 29.2025.2100
+Version: 29.2025.2400
 Summary: An abstraction layer for reading and writing astronomical data to datastores.
 Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
 License: BSD 3-Clause License

{lsst_daf_butler-29.2025.2100 → lsst_daf_butler-29.2025.2400}/python/lsst/daf/butler/_limited_butler.py

@@ -42,7 +42,7 @@ from ._dataset_provenance import DatasetProvenance
 from ._dataset_ref import DatasetRef
 from ._deferredDatasetHandle import DeferredDatasetHandle
 from ._storage_class import StorageClass, StorageClassFactory
-from .datastore import DatasetRefURIs, Datastore
+from .datastore import DatasetRefURIs, Datastore, FileTransferSource
 from .dimensions import DimensionUniverse
 
 log = logging.getLogger(__name__)
@@ -450,6 +450,13 @@ class LimitedButler(ABC):
         """
         raise NotImplementedError()
 
+    @property
+    def _file_transfer_source(self) -> FileTransferSource:
+        """Object that manages the transfer of files between Butler
+        repositories.
+        """
+        return self._datastore
+
     _datastore: Datastore
     """The object that manages actual dataset storage (`Datastore`)."""
 

{lsst_daf_butler-29.2025.2100 → lsst_daf_butler-29.2025.2400}/python/lsst/daf/butler/cli/cmd/commands.py

@@ -424,6 +424,21 @@ def prune_datasets(**kwargs: Any) -> None:
         case_sensitive=False,
     ),
 )
+@click.option(
+    "-t",
+    "--show-dataset-types",
+    is_flag=True,
+    help="Also show the dataset types registered within each collection.",
+)
+@click.option(
+    "--exclude-dataset-types",
+    type=click.STRING,
+    multiple=True,
+    default=["*_config,*_log,*_metadata,packages"],
+    callback=split_commas,
+    show_default=True,
+    help="Dataset types (comma-separated) to exclude. Only valid with --show-dataset-types.",
+)
 @options_file_option()
 def query_collections(*args: Any, **kwargs: Any) -> None:
     """Get the collections whose names match an expression."""
@@ -454,7 +469,7 @@ def query_dataset_types(*args: Any, **kwargs: Any) -> None:
     """Get the dataset types in a repository."""
     table = script.queryDatasetTypes(*args, **kwargs)
     if table:
-        table.pprint_all()
+        table.pprint_all(align="<")
     else:
         print("No results. Try --help for more information.")
 

{lsst_daf_butler-29.2025.2100 → lsst_daf_butler-29.2025.2400}/python/lsst/daf/butler/cli/utils.py

@@ -1057,7 +1057,7 @@ class MWCommand(click.Command):
         return ret
 
     @epilog.setter
-    def epilog(self, val: str) -> None:
+    def epilog(self, val: str | None) -> None:
         self._epilog = val
 
 

{lsst_daf_butler-29.2025.2100 → lsst_daf_butler-29.2025.2400}/python/lsst/daf/butler/datastore/__init__.py

@@ -26,3 +26,4 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 from ._datastore import *
+from ._transfer import *

{lsst_daf_butler-29.2025.2100 → lsst_daf_butler-29.2025.2400}/python/lsst/daf/butler/datastore/_datastore.py

@@ -54,6 +54,7 @@ from .._config import Config, ConfigSubset
 from .._exceptions import DatasetTypeNotSupportedError, ValidationError
 from .._file_dataset import FileDataset
 from .._storage_class import StorageClassFactory
+from ._transfer import FileTransferMap, FileTransferSource
 from .constraints import Constraints
 
 if TYPE_CHECKING:
@@ -62,7 +63,7 @@ if TYPE_CHECKING:
     from .. import ddl
     from .._config_support import LookupKey
     from .._dataset_provenance import DatasetProvenance
-    from .._dataset_ref import DatasetRef
+    from .._dataset_ref import DatasetId, DatasetRef
     from .._dataset_type import DatasetType
     from .._storage_class import StorageClass
     from ..datastores.file_datastore.retrieve_artifacts import ArtifactIndexInfo
@@ -283,7 +284,7 @@ class DatasetRefURIs(abc.Sequence):
         return f"DatasetRefURIs({repr(self.primaryURI)}, {repr(self.componentURIs)})"
 
 
-class Datastore(metaclass=ABCMeta):
+class Datastore(FileTransferSource, metaclass=ABCMeta):
     """Datastore interface.
 
     Parameters
@@ -868,7 +869,7 @@ class Datastore(metaclass=ABCMeta):
 
     def transfer_from(
         self,
-        source_datastore: Datastore,
+        source_records: FileTransferMap,
         refs: Collection[DatasetRef],
         transfer: str = "auto",
         artifact_existence: dict[ResourcePath, bool] | None = None,
@@ -878,9 +879,8 @@ class Datastore(metaclass=ABCMeta):
 
         Parameters
         ----------
-        source_datastore : `Datastore`
-            The datastore from which to transfer artifacts. That datastore
-            must be compatible with this datastore receiving the artifacts.
+        source_records : `FileTransferMap`
+            The artifacts to be transferred into this datastore.
         refs : `~collections.abc.Collection` of `DatasetRef`
             The datasets to transfer from the source datastore.
         transfer : `str`, optional
@@ -915,13 +915,7 @@ class Datastore(metaclass=ABCMeta):
         TypeError
             Raised if the two datastores are not compatible.
         """
-        if type(self) is not type(source_datastore):
-            raise TypeError(
-                f"Datastore mismatch between this datastore ({type(self)}) and the "
-                f"source datastore ({type(source_datastore)})."
-            )
-
-        raise NotImplementedError(f"Datastore {type(self)} must implement a transfer_from method.")
+        raise NotImplementedError(f"Datastore {type(self)} does not implement a transfer_from method.")
 
     def getManyURIs(
         self,
@@ -1420,6 +1414,14 @@ class Datastore(metaclass=ABCMeta):
         """
         raise NotImplementedError()
 
+    def get_file_info_for_transfer(self, dataset_ids: Iterable[DatasetId]) -> FileTransferMap:
+        raise NotImplementedError(f"Transferring files is not supported by datastore {self}")
+
+    def locate_missing_files_for_transfer(
+        self, refs: Iterable[DatasetRef], artifact_existence: dict[ResourcePath, bool]
+    ) -> FileTransferMap:
+        return {}
+
 
 class NullDatastore(Datastore):
     """A datastore that implements the `Datastore` API but always fails when
@@ -1494,7 +1496,7 @@ class NullDatastore(Datastore):
 
     def transfer_from(
         self,
-        source_datastore: Datastore,
+        source_records: FileTransferMap,
         refs: Iterable[DatasetRef],
         transfer: str = "auto",
         artifact_existence: dict[ResourcePath, bool] | None = None,

lsst_daf_butler-29.2025.2400/python/lsst/daf/butler/datastore/_transfer.py

@@ -0,0 +1,102 @@
+# This file is part of daf_butler.
+#
+# Developed for the LSST Data Management System.
+# This product includes software developed by the LSST Project
+# (http://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# This software is dual licensed under the GNU General Public License and also
+# under a 3-clause BSD license. Recipients may choose which of these licenses
+# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
+# respectively.  If you choose the GPL option then the following text applies
+# (but note that there is still no warranty even if you opt for BSD instead):
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from __future__ import annotations
+
+from collections.abc import Iterable
+from typing import NamedTuple, Protocol, TypeAlias
+
+from lsst.resources import ResourcePath
+
+from .._dataset_ref import DatasetId, DatasetRef
+from .stored_file_info import Location, StoredFileInfo
+
+__all__ = ("FileTransferMap", "FileTransferRecord", "FileTransferSource")
+
+
+class FileTransferSource(Protocol):
+    """Protocol for an object that can return information about files that need
+    to be transferred to copy datasets from one Butler repository to another.
+    """
+
+    name: str
+    """A human-readable, descriptive name for this file transfer source."""
+
+    def get_file_info_for_transfer(self, dataset_ids: Iterable[DatasetId]) -> FileTransferMap:
+        """Given a list of dataset IDs, return all file information associated
+        with the datasets that can be determined without searching the
+        filesystem.
+
+        Parameters
+        ----------
+        dataset_ids : `~collections.abc.Iterable` [ `DatasetId` ]
+            List of dataset IDs for which we will retrieve file information.
+
+        Return
+        ------
+        transfer_map : `FileTransferMap`
+            Dictionary from `DatasetId` to a list of files found for that
+            dataset.  If information about any  given dataset IDs could not
+            be found, the missing IDs are omitted from the dictionary.
+        """
+
+    def locate_missing_files_for_transfer(
+        self, refs: Iterable[DatasetRef], artifact_existence: dict[ResourcePath, bool]
+    ) -> FileTransferMap:
+        """Given a list of `DatasetRef`, search the filesystem to locate
+        artifacts associated with the dataset.
+
+        Parameters
+        ----------
+        refs : iterable of `DatasetRef`
+            The datasets to be checked.
+        artifact_existence : `dict` [`lsst.resources.ResourcePath`, `bool`]
+            Optional mapping of datastore artifact to existence. Updated by
+            this method with details of all artifacts tested.
+
+        Return
+        ------
+        transfer_map : `FileTransferMap`
+            Dictionary from `DatasetId` to a list of files found for that
+            dataset.  If information about any  given dataset IDs could not
+            be found, the missing IDs are omitted from the dictionary.
+        """
+
+
+class FileTransferRecord(NamedTuple):
+    """Information needed to transfer a file from one Butler repository to
+    another.
+    """
+
+    location: Location
+    file_info: StoredFileInfo
+
+
+FileTransferMap: TypeAlias = dict[DatasetId, list[FileTransferRecord]]
+"""A dictionary from `DatasetId` to a list of `FileTransferRecord`, containing
+the datastore information about a set of files to be transferred.
+"""

{lsst_daf_butler-29.2025.2100 → lsst_daf_butler-29.2025.2400}/python/lsst/daf/butler/datastore/stored_file_info.py

@@ -395,3 +395,37 @@ class SerializedStoredFileInfo(pydantic.BaseModel):
 
     file_size: int
     """Size of the serialized dataset in bytes."""
+
+
+def make_datastore_path_relative(path: str) -> str:
+    """Normalize a path from a `StoredFileInfo` object so
+    that it is always relative.
+
+    Parameters
+    ----------
+    path : `str`
+        The file path from a `StoredFileInfo`.
+
+    Return
+    ------
+    normalized_path : `str`
+        The original path, if it was relative. Otherwise, a version of it that
+        was converted to a relative path, stripping URI scheme and netloc from
+        it.
+    """
+    # Force the datastore file path sent to the client to be relative, since
+    # absolute URLs in the server will generally not be reachable by the
+    # client.  If an absolute URL is sent, it (or a portion of it) can end up
+    # baked into the FileDatastore that is the target of the transfer in some
+    # cases.
+    rpath = ResourcePath(path, forceAbsolute=False, forceDirectory=False)
+    if rpath.isabs():
+        relative = rpath.relativeToPathRoot
+        if rpath.fragment:
+            # Preserve the fragment, since this used to indicate special
+            # processing like zip extraction.
+            return f"{relative}#{rpath.fragment}"
+        else:
+            return relative
+    else:
+        return path

{lsst_daf_butler-29.2025.2100 → lsst_daf_butler-29.2025.2400}/python/lsst/daf/butler/datastores/chainedDatastore.py

@@ -50,8 +50,12 @@ from lsst.daf.butler.datastore.record_data import DatastoreRecordData
 from lsst.daf.butler.datastores.file_datastore.retrieve_artifacts import ArtifactIndexInfo, ZipIndex
 from lsst.resources import ResourcePath
 from lsst.utils import doImportType
+from lsst.utils.introspection import get_full_type_name
 from lsst.utils.logging import getLogger
 
+from .._dataset_ref import DatasetId
+from ..datastore import FileTransferMap
+
 if TYPE_CHECKING:
     from lsst.daf.butler import Config, DatasetProvenance, DatasetType, LookupKey, StorageClass
     from lsst.daf.butler.registry.interfaces import DatasetIdRef, DatastoreRegistryBridgeManager
@@ -1234,9 +1238,50 @@ class ChainedDatastore(Datastore):
                 raise FileNotFoundError(f"Failed to export dataset {refs[i]}.")
             yield dataset
 
+    def get_file_info_for_transfer(self, dataset_ids: Iterable[DatasetId]) -> FileTransferMap:
+        unassigned_ids = set(dataset_ids)
+        output: FileTransferMap = {}
+        found_acceptable_datastore = False
+        for datastore in self.datastores:
+            try:
+                found = datastore.get_file_info_for_transfer(unassigned_ids)
+                found_acceptable_datastore = True
+                output.update(found)
+                unassigned_ids -= found.keys()
+            except NotImplementedError:
+                pass
+
+        if not found_acceptable_datastore:
+            types = {get_full_type_name(d) for d in self.datastores}
+            raise TypeError(
+                "ChainedDatastore had no datastores able to provide file transfer information."
+                f" Had {','.join(types)}"
+            )
+
+        return output
+
+    def locate_missing_files_for_transfer(
+        self, refs: Iterable[DatasetRef], artifact_existence: dict[ResourcePath, bool]
+    ) -> FileTransferMap:
+        missing_refs = {ref.id: ref for ref in refs}
+        output: FileTransferMap = {}
+        for datastore in self.datastores:
+            # Have to check each datastore in turn. If we do not do
+            # this warnings will be issued further down for datasets
+            # that are in one and not the other. The existence cache
+            # will prevent repeat checks.
+
+            found = datastore.locate_missing_files_for_transfer(missing_refs.values(), artifact_existence)
+            output.update(found)
+            for id in found.keys():
+                missing_refs.pop(id)
+            log.debug("Adding %d missing refs to list for transfer from %s", len(found), datastore.name)
+
+        return output
+
     def transfer_from(
         self,
-        source_datastore: Datastore,
+        source_records: FileTransferMap,
         refs: Collection[DatasetRef],
         transfer: str = "auto",
         artifact_existence: dict[ResourcePath, bool] | None = None,
@@ -1259,7 +1304,7 @@ class ChainedDatastore(Datastore):
         rejected: set[DatasetRef] = set()
         nsuccess = 0
 
-        log.debug("Initiating transfer to chained datastore %s from %s", self.name, source_datastore.name)
+        log.debug("Initiating transfer to chained datastore %s", self.name)
         for datastore in self.datastores:
             # Rejections from this datastore might be acceptances in the next.
             # We add them all up but then recalculate at the end.
@@ -1269,7 +1314,7 @@ class ChainedDatastore(Datastore):
 
             try:
                 current_accepted, current_rejected = datastore.transfer_from(
-                    source_datastore,
+                    source_records,
                     available_refs,
                     transfer=transfer,
                     artifact_existence=artifact_existence,
@@ -1288,13 +1333,12 @@ class ChainedDatastore(Datastore):
                 rejected.update(current_rejected)
 
         if nsuccess == 0:
-            raise TypeError(f"None of the child datastores could accept transfers from {source_datastore!r}")
+            raise TypeError("None of the child datastores could accept file transfers")
 
         # It's not rejected if some other datastore accepted it.
         rejected -= accepted
         log.verbose(
-            "Finished transfer_from %s to %s with %d accepted, %d rejected from %d requested.",
-            source_datastore.name,
+            "Finished transfer_from to %s with %d accepted, %d rejected from %d requested.",
             self.name,
             len(accepted),
             len(rejected),