lsst-resources 29.2025.2000__py3-none-any.whl → 29.2025.2100__py3-none-any.whl

lsst/resources/s3.py

@@ -13,6 +13,7 @@ from __future__ import annotations
 
 __all__ = ("S3ResourcePath",)
 
+import concurrent.futures
 import contextlib
 import io
 import logging
@@ -32,7 +33,7 @@ from lsst.utils.timer import time_this
 
 from ._resourceHandles._baseResourceHandle import ResourceHandleProtocol
 from ._resourceHandles._s3ResourceHandle import S3ResourceHandle
-from ._resourcePath import MBulkResult, ResourcePath
+from ._resourcePath import _EXECUTOR_TYPE, MBulkResult, ResourcePath, _get_executor_class, _patch_environ
 from .s3utils import (
     _get_s3_connection_parameters,
     _s3_disable_bucket_validation,
@@ -46,6 +47,7 @@ from .s3utils import (
     s3CheckFileExists,
     translate_client_error,
 )
+from .utils import _get_num_workers
 
 try:
     from boto3.s3.transfer import TransferConfig  # type: ignore
@@ -247,34 +249,96 @@ class S3ResourcePath(ResourcePath):
 
         results: dict[ResourcePath, MBulkResult] = {}
         for related_uris in grouped_uris.values():
-            # The client and bucket are the same for each of the remaining
-            # URIs.
-            first_uri = related_uris[0]
             # API requires no more than 1000 per call.
             chunk_num = 0
+            chunks: list[tuple[ResourcePath, ...]] = []
+            key_to_uri: dict[str, ResourcePath] = {}
             for chunk in chunk_iterable(related_uris, chunk_size=1_000):
-                key_to_uri: dict[str, ResourcePath] = {}
-                keys: list[dict[str, str]] = []
                 for uri in chunk:
                     key = uri.relativeToPathRoot
                     key_to_uri[key] = uri
-                    keys.append({"Key": key})
                     # Default to assuming everything worked.
                     results[uri] = MBulkResult(True, None)
                 chunk_num += 1
-                with time_this(
-                    log,
-                    msg="Bulk delete; chunk number %d; %d dataset%s",
-                    args=(chunk_num, len(chunk), "s" if len(chunk) != 1 else ""),
-                ):
-                    errored = cls._delete_related_objects(first_uri.client, first_uri._bucket, keys)
+                chunks.append(chunk)
+
+            # Bulk remove.
+            with time_this(
+                log,
+                msg="Bulk delete; %d chunk%s; totalling %d dataset%s",
+                args=(
+                    len(chunks),
+                    "s" if len(chunks) != 1 else "",
+                    len(related_uris),
+                    "s" if len(related_uris) != 1 else "",
+                ),
+            ):
+                errored = cls._mremove_select(chunks)
+
+            # Update with error information.
+            results.update(errored)
 
-                # Update with error information.
-                for key, bulk_result in errored.items():
-                    results[key_to_uri[key]] = bulk_result
+        return results
 
+    @classmethod
+    def _mremove_select(cls, chunks: list[tuple[ResourcePath, ...]]) -> dict[ResourcePath, MBulkResult]:
+        if len(chunks) == 1:
+            # Do the removal directly without futures.
+            return cls._delete_objects_wrapper(chunks[0])
+        pool_executor_class = _get_executor_class()
+        if issubclass(pool_executor_class, concurrent.futures.ProcessPoolExecutor):
+            # Patch the environment to make it think there is only one worker
+            # for each subprocess.
+            with _patch_environ({"LSST_RESOURCES_NUM_WORKERS": "1"}):
+                return cls._mremove_with_pool(pool_executor_class, chunks)
+        else:
+            return cls._mremove_with_pool(pool_executor_class, chunks)
+
+    @classmethod
+    def _mremove_with_pool(
+        cls,
+        pool_executor_class: _EXECUTOR_TYPE,
+        chunks: list[tuple[ResourcePath, ...]],
+        *,
+        num_workers: int | None = None,
+    ) -> dict[ResourcePath, MBulkResult]:
+        # Different name because different API to base class.
+        # No need to make more workers than we have chunks.
+        max_workers = num_workers if num_workers is not None else min(len(chunks), _get_num_workers())
+        results: dict[ResourcePath, MBulkResult] = {}
+        with pool_executor_class(max_workers=max_workers) as remove_executor:
+            future_remove = {
+                remove_executor.submit(cls._delete_objects_wrapper, chunk): i
+                for i, chunk in enumerate(chunks)
+            }
+            for future in concurrent.futures.as_completed(future_remove):
+                try:
+                    results.update(future.result())
+                except Exception as e:
+                    # The chunk utterly failed.
+                    chunk = chunks[future_remove[future]]
+                    for uri in chunk:
+                        results[uri] = MBulkResult(False, e)
         return results
 
+    @classmethod
+    def _delete_objects_wrapper(cls, uris: tuple[ResourcePath, ...]) -> dict[ResourcePath, MBulkResult]:
+        """Convert URIs to keys and call low-level API."""
+        if not uris:
+            return {}
+        keys: list[dict[str, str]] = []
+        key_to_uri: dict[str, ResourcePath] = {}
+        for uri in uris:
+            key = uri.relativeToPathRoot
+            key_to_uri[key] = uri
+            keys.append({"Key": key})
+
+        first_uri = cast(S3ResourcePath, uris[0])
+        results = cls._delete_related_objects(first_uri.client, first_uri._bucket, keys)
+
+        # Remap error object keys to uris.
+        return {key_to_uri[key]: result for key, result in results.items()}
+
     @classmethod
     @backoff.on_exception(backoff.expo, retryable_io_errors, max_time=max_retry_time)
     def _delete_related_objects(

lsst/resources/tests.py

@@ -557,6 +557,10 @@ class GenericReadWriteTestCase(_GenericTestCase):
 
     transfer_modes: tuple[str, ...] = ("copy", "move")
     testdir: str | None = None
+    # Number of files to use for mremove() testing to ensure difference code
+    # paths are hit. Do not want to generically use many files for schemes
+    # where it makes no difference.
+    n_mremove_files: int = 15
 
     def setUp(self) -> None:
         if self.scheme is None:
@@ -1023,19 +1027,14 @@ class GenericReadWriteTestCase(_GenericTestCase):
         # A file that is not there.
         file = root.join("config/basic/butler.yaml")
 
-        # Create some files.
-        expected_files = {
-            "dir1/a.yaml",
-            "dir1/b.yaml",
-            "dir1/c.yaml",
-            "dir1/d.yaml",
-            "dir2/e.yaml",
-        }
-        expected_uris = {root.join(f) for f in expected_files}
+        # Create some files. Most schemes the code paths do not change for 10
+        # vs 1000 files but in some schemes it does.
+        expected_files = [f"dir1/f{n}.yaml" for n in range(self.n_mremove_files)]
+        expected_uris = [root.join(f) for f in expected_files]
         for uri in expected_uris:
             uri.write(b"")
             self.assertTrue(uri.exists())
-        expected_uris.add(file)
+        expected_uris.append(file)
 
         # Force to run with fewer workers than there are files.
         multi = ResourcePath.mexists(expected_uris, num_workers=3)
@@ -1054,3 +1053,7 @@ class GenericReadWriteTestCase(_GenericTestCase):
         multi = ResourcePath.mexists(expected_uris, num_workers=3)
         for uri, is_there in multi.items():
             self.assertFalse(is_there)
+
+        # Clean up a subset of files that are already gone, but this can
+        # trigger a different code path.
+        ResourcePath.mremove(expected_uris[:5], do_raise=False)

lsst/resources/version.py

@@ -1,2 +1,2 @@
 __all__ = ["__version__"]
-__version__ = "29.2025.2000"
+__version__ = "29.2025.2100"

{lsst_resources-29.2025.2000.dist-info → lsst_resources-29.2025.2100.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lsst-resources
-Version: 29.2025.2000
+Version: 29.2025.2100
 Summary: An abstraction layer for reading and writing from URI file resources.
 Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
 License: BSD 3-Clause License

{lsst_resources-29.2025.2000.dist-info → lsst_resources-29.2025.2100.dist-info}/RECORD

@@ -10,22 +10,22 @@ lsst/resources/location.py,sha256=x3Tq0x5o1OXYmZDxYBenUG1N71wtDhnjVAr3s2ZEiu8,79
 lsst/resources/mem.py,sha256=VOWh7XxJPfqKcFdLZSjKEAfORQ2AHZHpxmjT8LniV60,1008
 lsst/resources/packageresource.py,sha256=vnfeRlpVwpC5cDQZE6Lnh8EH6oZy1sH2vLz9ONYjJ4k,6817
 lsst/resources/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-lsst/resources/s3.py,sha256=n9m4BHEVJgj2_4yAMydwX-w5hmnOPZTihWEItHi0kQI,26923
+lsst/resources/s3.py,sha256=IPhNqKzseHsMS9RF9TAl8vhs-sFvXfdFsJCb7VMvuNg,29572
 lsst/resources/s3utils.py,sha256=ojWf9BPrK9mhGQ8jvs4_8Nsqf9360e79U5FnPTxe24A,14576
 lsst/resources/schemeless.py,sha256=GfJcKzZ0XIeepfQdW4HPZWiZlSp_ej0SEtSiJTrDUQs,10666
-lsst/resources/tests.py,sha256=SqYLbDG6QkZTB-0UvrsiPtfmdL1TcglGeqBTPQxu9GE,44027
+lsst/resources/tests.py,sha256=nTRvfAGrPnh2tsR8jzA-XPuWN1XECOW1f4dDveADmb8,44439
 lsst/resources/utils.py,sha256=6O3Mq7JbPEtqyD2lM77pRpwcPMfV5SxiNMknw-F2vNs,8097
-lsst/resources/version.py,sha256=bT9RvvieKvOl5VImh838xbe0jWwzezTNwq19FWhF5NU,55
+lsst/resources/version.py,sha256=czd5myijXlfzN1l1OFP7x6hn5ASHoE02wXhI32IqLec,55
 lsst/resources/_resourceHandles/__init__.py,sha256=zOcZ8gVEBdAWcHJaZabA8Vdq-wAVcxjbmA_1b1IWM6M,76
 lsst/resources/_resourceHandles/_baseResourceHandle.py,sha256=lQwxDOmFUNJndTxsjpz-HxrQBL0L-z4aXQocHdOEI7c,4676
 lsst/resources/_resourceHandles/_davResourceHandle.py,sha256=12X5-K5KqzG4EV78ZkIIrjcZcFroXy3Y2JQ_N-SDqF0,6616
 lsst/resources/_resourceHandles/_fileResourceHandle.py,sha256=A7_WQPzD0ZlOzNmaI_TPdZybrNxrXPkNHWVla3UFxfs,3676
 lsst/resources/_resourceHandles/_httpResourceHandle.py,sha256=Yami8IVGeru4bLQCag-OvGG0ltz1qyEg57FY4IEB87Y,10995
 lsst/resources/_resourceHandles/_s3ResourceHandle.py,sha256=NkDmPb9bm_zMvr6mMnb-tBmqJDt0yUJrt2gZXR8l7ok,12923
-lsst_resources-29.2025.2000.dist-info/licenses/COPYRIGHT,sha256=yazVsoMmFwhiw5itGrdT4YPmXbpsQyUFjlpOyZIa77M,148
-lsst_resources-29.2025.2000.dist-info/licenses/LICENSE,sha256=7wrtgl8meQ0_RIuv2TjIKpAnNrl-ODH-QLwyHe9citI,1516
-lsst_resources-29.2025.2000.dist-info/METADATA,sha256=47lnuHxjeCYuBH9Kj_2545yFIgE5DkwUy6AWnUox_ws,2237
-lsst_resources-29.2025.2000.dist-info/WHEEL,sha256=Nw36Djuh_5VDukK0H78QzOX-_FQEo6V37m3nkm96gtU,91
-lsst_resources-29.2025.2000.dist-info/top_level.txt,sha256=eUWiOuVVm9wwTrnAgiJT6tp6HQHXxIhj2QSZ7NYZH80,5
-lsst_resources-29.2025.2000.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-lsst_resources-29.2025.2000.dist-info/RECORD,,
+lsst_resources-29.2025.2100.dist-info/licenses/COPYRIGHT,sha256=yazVsoMmFwhiw5itGrdT4YPmXbpsQyUFjlpOyZIa77M,148
+lsst_resources-29.2025.2100.dist-info/licenses/LICENSE,sha256=7wrtgl8meQ0_RIuv2TjIKpAnNrl-ODH-QLwyHe9citI,1516
+lsst_resources-29.2025.2100.dist-info/METADATA,sha256=h1FqE1jzjXciOQ4whXXfG_UTucO0hZLDLboM-9IV-v8,2237
+lsst_resources-29.2025.2100.dist-info/WHEEL,sha256=zaaOINJESkSfm_4HQVc5ssNzHCPXhJm0kEUakpsEHaU,91
+lsst_resources-29.2025.2100.dist-info/top_level.txt,sha256=eUWiOuVVm9wwTrnAgiJT6tp6HQHXxIhj2QSZ7NYZH80,5
+lsst_resources-29.2025.2100.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
+lsst_resources-29.2025.2100.dist-info/RECORD,,

{lsst_resources-29.2025.2000.dist-info → lsst_resources-29.2025.2100.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (80.7.1)
+Generator: setuptools (80.8.0)
 Root-Is-Purelib: true
 Tag: py3-none-any