lsst-resources 29.2025.2000__tar.gz → 29.2025.2100__tar.gz

{lsst_resources-29.2025.2000/python/lsst_resources.egg-info → lsst_resources-29.2025.2100}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lsst-resources
-Version: 29.2025.2000
+Version: 29.2025.2100
 Summary: An abstraction layer for reading and writing from URI file resources.
 Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
 License: BSD 3-Clause License

{lsst_resources-29.2025.2000 → lsst_resources-29.2025.2100}/python/lsst/resources/s3.py

@@ -13,6 +13,7 @@ from __future__ import annotations
 
 __all__ = ("S3ResourcePath",)
 
+import concurrent.futures
 import contextlib
 import io
 import logging
@@ -32,7 +33,7 @@ from lsst.utils.timer import time_this
 
 from ._resourceHandles._baseResourceHandle import ResourceHandleProtocol
 from ._resourceHandles._s3ResourceHandle import S3ResourceHandle
-from ._resourcePath import MBulkResult, ResourcePath
+from ._resourcePath import _EXECUTOR_TYPE, MBulkResult, ResourcePath, _get_executor_class, _patch_environ
 from .s3utils import (
     _get_s3_connection_parameters,
     _s3_disable_bucket_validation,
@@ -46,6 +47,7 @@ from .s3utils import (
     s3CheckFileExists,
     translate_client_error,
 )
+from .utils import _get_num_workers
 
 try:
     from boto3.s3.transfer import TransferConfig  # type: ignore
@@ -247,34 +249,96 @@ class S3ResourcePath(ResourcePath):
 
         results: dict[ResourcePath, MBulkResult] = {}
         for related_uris in grouped_uris.values():
-            # The client and bucket are the same for each of the remaining
-            # URIs.
-            first_uri = related_uris[0]
             # API requires no more than 1000 per call.
             chunk_num = 0
+            chunks: list[tuple[ResourcePath, ...]] = []
+            key_to_uri: dict[str, ResourcePath] = {}
             for chunk in chunk_iterable(related_uris, chunk_size=1_000):
-                key_to_uri: dict[str, ResourcePath] = {}
-                keys: list[dict[str, str]] = []
                 for uri in chunk:
                     key = uri.relativeToPathRoot
                     key_to_uri[key] = uri
-                    keys.append({"Key": key})
                     # Default to assuming everything worked.
                     results[uri] = MBulkResult(True, None)
                 chunk_num += 1
-                with time_this(
-                    log,
-                    msg="Bulk delete; chunk number %d; %d dataset%s",
-                    args=(chunk_num, len(chunk), "s" if len(chunk) != 1 else ""),
-                ):
-                    errored = cls._delete_related_objects(first_uri.client, first_uri._bucket, keys)
+                chunks.append(chunk)
+
+            # Bulk remove.
+            with time_this(
+                log,
+                msg="Bulk delete; %d chunk%s; totalling %d dataset%s",
+                args=(
+                    len(chunks),
+                    "s" if len(chunks) != 1 else "",
+                    len(related_uris),
+                    "s" if len(related_uris) != 1 else "",
+                ),
+            ):
+                errored = cls._mremove_select(chunks)
+
+            # Update with error information.
+            results.update(errored)
 
-                # Update with error information.
-                for key, bulk_result in errored.items():
-                    results[key_to_uri[key]] = bulk_result
+        return results
 
+    @classmethod
+    def _mremove_select(cls, chunks: list[tuple[ResourcePath, ...]]) -> dict[ResourcePath, MBulkResult]:
+        if len(chunks) == 1:
+            # Do the removal directly without futures.
+            return cls._delete_objects_wrapper(chunks[0])
+        pool_executor_class = _get_executor_class()
+        if issubclass(pool_executor_class, concurrent.futures.ProcessPoolExecutor):
+            # Patch the environment to make it think there is only one worker
+            # for each subprocess.
+            with _patch_environ({"LSST_RESOURCES_NUM_WORKERS": "1"}):
+                return cls._mremove_with_pool(pool_executor_class, chunks)
+        else:
+            return cls._mremove_with_pool(pool_executor_class, chunks)
+
+    @classmethod
+    def _mremove_with_pool(
+        cls,
+        pool_executor_class: _EXECUTOR_TYPE,
+        chunks: list[tuple[ResourcePath, ...]],
+        *,
+        num_workers: int | None = None,
+    ) -> dict[ResourcePath, MBulkResult]:
+        # Different name because different API to base class.
+        # No need to make more workers than we have chunks.
+        max_workers = num_workers if num_workers is not None else min(len(chunks), _get_num_workers())
+        results: dict[ResourcePath, MBulkResult] = {}
+        with pool_executor_class(max_workers=max_workers) as remove_executor:
+            future_remove = {
+                remove_executor.submit(cls._delete_objects_wrapper, chunk): i
+                for i, chunk in enumerate(chunks)
+            }
+            for future in concurrent.futures.as_completed(future_remove):
+                try:
+                    results.update(future.result())
+                except Exception as e:
+                    # The chunk utterly failed.
+                    chunk = chunks[future_remove[future]]
+                    for uri in chunk:
+                        results[uri] = MBulkResult(False, e)
         return results
 
+    @classmethod
+    def _delete_objects_wrapper(cls, uris: tuple[ResourcePath, ...]) -> dict[ResourcePath, MBulkResult]:
+        """Convert URIs to keys and call low-level API."""
+        if not uris:
+            return {}
+        keys: list[dict[str, str]] = []
+        key_to_uri: dict[str, ResourcePath] = {}
+        for uri in uris:
+            key = uri.relativeToPathRoot
+            key_to_uri[key] = uri
+            keys.append({"Key": key})
+
+        first_uri = cast(S3ResourcePath, uris[0])
+        results = cls._delete_related_objects(first_uri.client, first_uri._bucket, keys)
+
+        # Remap error object keys to uris.
+        return {key_to_uri[key]: result for key, result in results.items()}
+
     @classmethod
     @backoff.on_exception(backoff.expo, retryable_io_errors, max_time=max_retry_time)
     def _delete_related_objects(

{lsst_resources-29.2025.2000 → lsst_resources-29.2025.2100}/python/lsst/resources/tests.py

@@ -557,6 +557,10 @@ class GenericReadWriteTestCase(_GenericTestCase):
 
     transfer_modes: tuple[str, ...] = ("copy", "move")
     testdir: str | None = None
+    # Number of files to use for mremove() testing to ensure difference code
+    # paths are hit. Do not want to generically use many files for schemes
+    # where it makes no difference.
+    n_mremove_files: int = 15
 
     def setUp(self) -> None:
         if self.scheme is None:
@@ -1023,19 +1027,14 @@ class GenericReadWriteTestCase(_GenericTestCase):
         # A file that is not there.
         file = root.join("config/basic/butler.yaml")
 
-        # Create some files.
-        expected_files = {
-            "dir1/a.yaml",
-            "dir1/b.yaml",
-            "dir1/c.yaml",
-            "dir1/d.yaml",
-            "dir2/e.yaml",
-        }
-        expected_uris = {root.join(f) for f in expected_files}
+        # Create some files. Most schemes the code paths do not change for 10
+        # vs 1000 files but in some schemes it does.
+        expected_files = [f"dir1/f{n}.yaml" for n in range(self.n_mremove_files)]
+        expected_uris = [root.join(f) for f in expected_files]
         for uri in expected_uris:
             uri.write(b"")
             self.assertTrue(uri.exists())
-        expected_uris.add(file)
+        expected_uris.append(file)
 
         # Force to run with fewer workers than there are files.
         multi = ResourcePath.mexists(expected_uris, num_workers=3)
@@ -1054,3 +1053,7 @@ class GenericReadWriteTestCase(_GenericTestCase):
         multi = ResourcePath.mexists(expected_uris, num_workers=3)
         for uri, is_there in multi.items():
             self.assertFalse(is_there)
+
+        # Clean up a subset of files that are already gone, but this can
+        # trigger a different code path.
+        ResourcePath.mremove(expected_uris[:5], do_raise=False)

lsst_resources-29.2025.2100/python/lsst/resources/version.py

@@ -0,0 +1,2 @@
+__all__ = ["__version__"]
+__version__ = "29.2025.2100"

{lsst_resources-29.2025.2000 → lsst_resources-29.2025.2100/python/lsst_resources.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lsst-resources
-Version: 29.2025.2000
+Version: 29.2025.2100
 Summary: An abstraction layer for reading and writing from URI file resources.
 Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
 License: BSD 3-Clause License

{lsst_resources-29.2025.2000 → lsst_resources-29.2025.2100}/tests/test_s3.py

@@ -55,6 +55,8 @@ class S3ReadWriteTestCaseBase(GenericReadWriteTestCase):
 
     scheme = "s3"
     s3_endpoint_url: str | None = None
+    # S3 batches in 1000 files so need more than that.
+    n_mremove_files: int = 1015
 
     def setUp(self):
         self.enterContext(clean_test_environment_for_s3())

lsst_resources-29.2025.2000/python/lsst/resources/version.py

@@ -1,2 +0,0 @@
-__all__ = ["__version__"]
-__version__ = "29.2025.2000"