lsst-resources 29.2025.1800__tar.gz → 29.2025.1900__tar.gz

{lsst_resources-29.2025.1800/python/lsst_resources.egg-info → lsst_resources-29.2025.1900}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lsst-resources
-Version: 29.2025.1800
+Version: 29.2025.1900
 Summary: An abstraction layer for reading and writing from URI file resources.
 Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
 License: BSD 3-Clause License

{lsst_resources-29.2025.1800 → lsst_resources-29.2025.1900}/python/lsst/resources/_resourcePath.py

@@ -23,6 +23,7 @@ import os
 import posixpath
 import re
 import urllib.parse
+from collections import defaultdict
 from pathlib import Path, PurePath, PurePosixPath
 from random import Random
 from typing import TypeAlias
@@ -53,8 +54,8 @@ ESCAPES_RE = re.compile(r"%[A-F0-9]{2}")
 ESCAPED_HASH = urllib.parse.quote("#")
 
 
-class MTransferResult(NamedTuple):
-    """Report on a bulk transfer."""
+class MBulkResult(NamedTuple):
+    """Report on a bulk operation."""
 
     success: bool
     exception: Exception | None
@@ -912,6 +913,14 @@ class ResourcePath:  # numpydoc ignore=PR02
         """
         raise NotImplementedError()
 
+    @classmethod
+    def _group_uris(cls, uris: Iterable[ResourcePath]) -> dict[type[ResourcePath], list[ResourcePath]]:
+        """Group URIs by class/scheme."""
+        grouped: dict[type, list[ResourcePath]] = defaultdict(list)
+        for uri in uris:
+            grouped[uri.__class__].append(uri)
+        return grouped
+
     @classmethod
     def mexists(
         cls, uris: Iterable[ResourcePath], *, num_workers: int | None = None
@@ -933,18 +942,9 @@ class ResourcePath:  # numpydoc ignore=PR02
         existence : `dict` of [`ResourcePath`, `bool`]
             Mapping of original URI to boolean indicating existence.
         """
-        # Group by scheme to allow a subclass to be able to use
-        # specialized implementations.
-        grouped: dict[type, list[ResourcePath]] = {}
-        for uri in uris:
-            uri_class = uri.__class__
-            if uri_class not in grouped:
-                grouped[uri_class] = []
-            grouped[uri_class].append(uri)
-
         existence: dict[ResourcePath, bool] = {}
-        for uri_class in grouped:
-            existence.update(uri_class._mexists(grouped[uri_class], num_workers=num_workers))
+        for uri_class, group in cls._group_uris(uris).items():
+            existence.update(uri_class._mexists(group, num_workers=num_workers))
 
         return existence
 
@@ -1029,7 +1029,7 @@ class ResourcePath:  # numpydoc ignore=PR02
         overwrite: bool = False,
         transaction: TransactionProtocol | None = None,
         do_raise: bool = True,
-    ) -> dict[ResourcePath, MTransferResult]:
+    ) -> dict[ResourcePath, MBulkResult]:
         """Transfer many files in bulk.
 
         Parameters
@@ -1048,14 +1048,16 @@ class ResourcePath:  # numpydoc ignore=PR02
             The transaction object must be thread safe.
         do_raise : `bool`, optional
             If `True` an `ExceptionGroup` will be raised containing any
-            exceptions raised by the individual transfers. Else a dict
-            reporting the status of each `ResourcePath` will be returned.
+            exceptions raised by the individual transfers. If `False`, or if
+            there were no exceptions, a dict reporting the status of each
+            `ResourcePath` will be returned.
 
         Returns
         -------
-        copy_status : `dict` [ `ResourcePath`, `MTransferResult` ]
+        copy_status : `dict` [ `ResourcePath`, `MBulkResult` ]
             A dict of all the transfer attempts with a value indicating
-            whether the transfer succeeded for the target URI.
+            whether the transfer succeeded for the target URI. If ``do_raise``
+            is `True`, this will only be returned if there are no errors.
         """
         pool_executor_class = _get_executor_class()
         if issubclass(pool_executor_class, concurrent.futures.ProcessPoolExecutor):
@@ -1088,7 +1090,7 @@ class ResourcePath:  # numpydoc ignore=PR02
         overwrite: bool = False,
         transaction: TransactionProtocol | None = None,
         do_raise: bool = True,
-    ) -> dict[ResourcePath, MTransferResult]:
+    ) -> dict[ResourcePath, MBulkResult]:
         """Transfer many files in bulk.
 
         Parameters
@@ -1112,7 +1114,7 @@ class ResourcePath:  # numpydoc ignore=PR02
 
         Returns
         -------
-        copy_status : `dict` [ `ResourcePath`, `MTransferResult` ]
+        copy_status : `dict` [ `ResourcePath`, `MBulkResult` ]
             A dict of all the transfer attempts with a value indicating
             whether the transfer succeeded for the target URI.
         """
@@ -1128,17 +1130,17 @@ class ResourcePath:  # numpydoc ignore=PR02
                 ): to_uri
                 for from_uri, to_uri in from_to
             }
-            results: dict[ResourcePath, MTransferResult] = {}
+            results: dict[ResourcePath, MBulkResult] = {}
             failed = False
             for future in concurrent.futures.as_completed(future_transfers):
                 to_uri = future_transfers[future]
                 try:
                     future.result()
                 except Exception as e:
-                    transferred = MTransferResult(False, e)
+                    transferred = MBulkResult(False, e)
                     failed = True
                 else:
-                    transferred = MTransferResult(True, None)
+                    transferred = MBulkResult(True, None)
                 results[to_uri] = transferred
 
         if do_raise and failed:
@@ -1153,6 +1155,81 @@ class ResourcePath:  # numpydoc ignore=PR02
         """Remove the resource."""
         raise NotImplementedError()
 
+    @classmethod
+    def mremove(
+        cls, uris: Iterable[ResourcePath], *, do_raise: bool = True
+    ) -> dict[ResourcePath, MBulkResult]:
+        """Remove multiple URIs at once.
+
+        Parameters
+        ----------
+        uris : iterable of `ResourcePath`
+            URIs to remove.
+        do_raise : `bool`, optional
+            If `True` an `ExceptionGroup` will be raised containing any
+            exceptions raised by the individual transfers. If `False`, or if
+            there were no exceptions, a dict reporting the status of each
+            `ResourcePath` will be returned.
+
+        Returns
+        -------
+        results : `dict` [ `ResourcePath`, `MBulkResult` ]
+            Dictionary mapping each URI to a result object indicating whether
+            the removal succeeded or resulted in an exception. If ``do_raise``
+            is `True` this will only be returned if everything succeeded.
+        """
+        # Group URIs by scheme since some URI schemes support native bulk
+        # APIs.
+        results: dict[ResourcePath, MBulkResult] = {}
+        for uri_class, group in cls._group_uris(uris).items():
+            results.update(uri_class._mremove(group))
+        if do_raise:
+            failed = any(not r.success for r in results.values())
+            if failed:
+                s = "s" if len(results) != 1 else ""
+                raise ExceptionGroup(
+                    f"Error{s} removing {len(results)} artifact{s}",
+                    tuple(res.exception for res in results.values() if res.exception is not None),
+                )
+
+        return results
+
+    @classmethod
+    def _mremove(cls, uris: Iterable[ResourcePath]) -> dict[ResourcePath, MBulkResult]:
+        """Remove multiple URIs using futures."""
+        pool_executor_class = _get_executor_class()
+        if issubclass(pool_executor_class, concurrent.futures.ProcessPoolExecutor):
+            # Patch the environment to make it think there is only one worker
+            # for each subprocess.
+            with _patch_environ({"LSST_RESOURCES_NUM_WORKERS": "1"}):
+                return cls._mremove_pool(pool_executor_class, uris)
+        else:
+            return cls._mremove_pool(pool_executor_class, uris)
+
+    @classmethod
+    def _mremove_pool(
+        cls,
+        pool_executor_class: _EXECUTOR_TYPE,
+        uris: Iterable[ResourcePath],
+        *,
+        num_workers: int | None = None,
+    ) -> dict[ResourcePath, MBulkResult]:
+        """Remove URIs using a futures pool."""
+        max_workers = num_workers if num_workers is not None else _get_num_workers()
+        results: dict[ResourcePath, MBulkResult] = {}
+        with pool_executor_class(max_workers=max_workers) as remove_executor:
+            future_remove = {remove_executor.submit(uri.remove): uri for uri in uris}
+            for future in concurrent.futures.as_completed(future_remove):
+                try:
+                    future.result()
+                except Exception as e:
+                    removed = MBulkResult(False, e)
+                else:
+                    removed = MBulkResult(True, None)
+                uri = future_remove[future]
+                results[uri] = removed
+        return results
+
     def isabs(self) -> bool:
         """Indicate that the resource is fully specified.
 

{lsst_resources-29.2025.1800 → lsst_resources-29.2025.1900}/python/lsst/resources/s3.py

@@ -20,17 +20,19 @@ import os
 import re
 import sys
 import threading
+from collections import defaultdict
 from collections.abc import Iterable, Iterator
 from functools import cache, cached_property
 from typing import IO, TYPE_CHECKING, cast
 
 from botocore.exceptions import ClientError
 
+from lsst.utils.iteration import chunk_iterable
 from lsst.utils.timer import time_this
 
 from ._resourceHandles._baseResourceHandle import ResourceHandleProtocol
 from ._resourceHandles._s3ResourceHandle import S3ResourceHandle
-from ._resourcePath import ResourcePath
+from ._resourcePath import MBulkResult, ResourcePath
 from .s3utils import (
     _get_s3_connection_parameters,
     _s3_disable_bucket_validation,
@@ -234,6 +236,55 @@ class S3ResourcePath(ResourcePath):
 
         return super()._mexists(uris, num_workers=num_workers)
 
+    @classmethod
+    def _mremove(cls, uris: Iterable[ResourcePath]) -> dict[ResourcePath, MBulkResult]:
+        # Delete multiple objects in one API call.
+        # Must group by profile and bucket.
+        grouped_uris: dict[tuple[str | None, str], list[S3ResourcePath]] = defaultdict(list)
+        for uri in uris:
+            uri = cast(S3ResourcePath, uri)
+            grouped_uris[uri._profile, uri._bucket].append(uri)
+
+        results: dict[ResourcePath, MBulkResult] = {}
+        for related_uris in grouped_uris.values():
+            # The client and bucket are the same for each of the remaining
+            # URIs.
+            first_uri = related_uris[0]
+            # API requires no more than 1000 per call.
+            for chunk in chunk_iterable(related_uris, chunk_size=1_000):
+                key_to_uri: dict[str, ResourcePath] = {}
+                keys: list[dict[str, str]] = []
+                for uri in chunk:
+                    key = uri.relativeToPathRoot
+                    key_to_uri[key] = uri
+                    keys.append({"Key": key})
+                    # Default to assuming everything worked.
+                    results[uri] = MBulkResult(True, None)
+                errored = cls._delete_related_objects(first_uri.client, first_uri._bucket, keys)
+
+                # Update with error information.
+                for key, bulk_result in errored.items():
+                    results[key_to_uri[key]] = bulk_result
+
+        return results
+
+    @classmethod
+    @backoff.on_exception(backoff.expo, retryable_io_errors, max_time=max_retry_time)
+    def _delete_related_objects(
+        cls, client: boto3.client, bucket: str, keys: list[dict[str, str]]
+    ) -> dict[str, MBulkResult]:
+        # Delete multiple objects from the same bucket, allowing for backoff
+        # retry.
+        response = client.delete_objects(Bucket=bucket, Delete={"Objects": keys, "Quiet": True})
+        # Use Quiet mode so we assume everything worked unless told otherwise.
+        # Only returning errors -- indexed by Key name.
+        errors: dict[str, MBulkResult] = {}
+        for errored_key in response.get("Errors", []):
+            errors[errored_key["Key"]] = MBulkResult(
+                False, ClientError({"Error": errored_key}, f"delete_objects: {errored_key['Key']}")
+            )
+        return errors
+
     @backoff.on_exception(backoff.expo, retryable_io_errors, max_time=max_retry_time)
     def exists(self) -> bool:
         """Check that the S3 resource exists."""

{lsst_resources-29.2025.1800 → lsst_resources-29.2025.1900}/python/lsst/resources/s3utils.py

@@ -247,9 +247,11 @@ def _s3_disable_bucket_validation(client: boto3.client) -> None:
 @functools.lru_cache
 def _get_s3_client(endpoint_config: _EndpointConfig, skip_validation: bool) -> boto3.client:
     # Helper function to cache the client for this endpoint
+    # boto seems to assume it will always have at least 10 available.
+    max_pool_size = max(_get_num_workers(), 10)
     config = botocore.config.Config(
         read_timeout=180,
-        max_pool_connections=_get_num_workers(),
+        max_pool_connections=max_pool_size,
         retries={"mode": "adaptive", "max_attempts": 10},
     )
 

{lsst_resources-29.2025.1800 → lsst_resources-29.2025.1900}/python/lsst/resources/tests.py

@@ -1045,3 +1045,12 @@ class GenericReadWriteTestCase(_GenericTestCase):
                 self.assertFalse(is_there)
             else:
                 self.assertTrue(is_there)
+
+        # Clean up. Unfortunately POSIX raises a FileNotFoundError but
+        # S3 boto does not complain if there is no key.
+        ResourcePath.mremove(expected_uris, do_raise=False)
+
+        # Check they were really removed.
+        multi = ResourcePath.mexists(expected_uris, num_workers=3)
+        for uri, is_there in multi.items():
+            self.assertFalse(is_there)

lsst_resources-29.2025.1900/python/lsst/resources/version.py

@@ -0,0 +1,2 @@
+__all__ = ["__version__"]
+__version__ = "29.2025.1900"

{lsst_resources-29.2025.1800 → lsst_resources-29.2025.1900/python/lsst_resources.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lsst-resources
-Version: 29.2025.1800
+Version: 29.2025.1900
 Summary: An abstraction layer for reading and writing from URI file resources.
 Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
 License: BSD 3-Clause License

lsst_resources-29.2025.1800/python/lsst/resources/version.py

@@ -1,2 +0,0 @@
-__all__ = ["__version__"]
-__version__ = "29.2025.1800"