lsst-resources 29.2025.1700__py3-none-any.whl → 29.2025.1900__py3-none-any.whl

lsst/resources/_resourceHandles/_httpResourceHandle.py

@@ -168,7 +168,9 @@ class HttpReadResourceHandle(BaseResourceHandle[bytes]):
             # return the result
             self._completeBuffer = io.BytesIO()
             with time_this(self._log, msg="Read from remote resource %s", args=(self._url,)):
-                resp = self._session.get(_dav_to_http(self._url), stream=False, timeout=self._timeout)
+                with self._session as session:
+                    resp = session.get(_dav_to_http(self._url), stream=False, timeout=self._timeout)
+
             if (code := resp.status_code) not in (requests.codes.ok, requests.codes.partial):
                 raise FileNotFoundError(f"Unable to read resource {self._url}; status code: {code}")
             self._completeBuffer.write(resp.content)
@@ -190,9 +192,10 @@ class HttpReadResourceHandle(BaseResourceHandle[bytes]):
         with time_this(
             self._log, msg="Read from remote resource %s using headers %s", args=(self._url, headers)
         ):
-            resp = self._session.get(
-                _dav_to_http(self._url), stream=False, timeout=self._timeout, headers=headers
-            )
+            with self._session as session:
+                resp = session.get(
+                    _dav_to_http(self._url), stream=False, timeout=self._timeout, headers=headers
+                )
 
         if resp.status_code == requests.codes.range_not_satisfiable:
             # Must have run off the end of the file. A standard file handle

lsst/resources/_resourcePath.py

@@ -19,14 +19,14 @@ import copy
 import io
 import locale
 import logging
-import multiprocessing
 import os
 import posixpath
 import re
 import urllib.parse
-from functools import cache
+from collections import defaultdict
 from pathlib import Path, PurePath, PurePosixPath
 from random import Random
+from typing import TypeAlias
 
 try:
     import fsspec
@@ -39,7 +39,7 @@ from collections.abc import Iterable, Iterator
 from typing import TYPE_CHECKING, Any, Literal, NamedTuple, overload
 
 from ._resourceHandles._baseResourceHandle import ResourceHandleProtocol
-from .utils import get_tempdir
+from .utils import _get_num_workers, get_tempdir
 
 if TYPE_CHECKING:
     from .utils import TransactionProtocol
@@ -53,49 +53,81 @@ ESCAPES_RE = re.compile(r"%[A-F0-9]{2}")
 # Precomputed escaped hash
 ESCAPED_HASH = urllib.parse.quote("#")
 
-# Maximum number of worker threads for parallelized operations.
-# If greater than 10, be aware that this number has to be consistent
-# with connection pool sizing (for example in urllib3).
-MAX_WORKERS = 10
 
-
-class MTransferResult(NamedTuple):
-    """Report on a bulk transfer."""
+class MBulkResult(NamedTuple):
+    """Report on a bulk operation."""
 
     success: bool
     exception: Exception | None
 
 
-def _get_int_env_var(env_var: str) -> int | None:
-    int_value = None
-    env_value = os.getenv(env_var)
-    if env_value is not None:
-        with contextlib.suppress(TypeError):
-            int_value = int(env_value)
-    return int_value
+_EXECUTOR_TYPE: TypeAlias = type[
+    concurrent.futures.ThreadPoolExecutor | concurrent.futures.ProcessPoolExecutor
+]
+
+# Cache value for executor class so as not to issue warning multiple
+# times but still allow tests to override the value.
+_POOL_EXECUTOR_CLASS: _EXECUTOR_TYPE | None = None
 
 
-@cache
-def _get_num_workers() -> int:
-    f"""Calculate the number of workers to use.
+def _get_executor_class() -> _EXECUTOR_TYPE:
+    """Return the executor class used for parallelized execution.
 
     Returns
     -------
-    num : `int`
-        The number of workers to use. Will use the value of the
-        ``LSST_RESOURCES_NUM_WORKERS`` environment variable if set. Will fall
-        back to using the CPU count (plus 2) but capped at {MAX_WORKERS}.
+    cls : `concurrent.futures.Executor`
+        The ``Executor`` class. Default is
+        `concurrent.futures.ThreadPoolExecutor`. Can be set explicitly by
+        setting the ``$LSST_RESOURCES_EXECUTOR`` environment variable to
+        "thread" or "process". Returns "thread" pool if the value of the
+        variable is not recognized.
+    """
+    global _POOL_EXECUTOR_CLASS
+
+    if _POOL_EXECUTOR_CLASS is not None:
+        return _POOL_EXECUTOR_CLASS
+
+    pool_executor_classes = {
+        "threads": concurrent.futures.ThreadPoolExecutor,
+        "process": concurrent.futures.ProcessPoolExecutor,
+    }
+    default_executor = "threads"
+    external = os.getenv("LSST_RESOURCES_EXECUTOR", default_executor)
+    if not external:
+        external = default_executor
+    if external not in pool_executor_classes:
+        log.warning(
+            "Unrecognized value of '%s' for LSST_RESOURCES_EXECUTOR env var. Using '%s'",
+            external,
+            default_executor,
+        )
+        external = default_executor
+    _POOL_EXECUTOR_CLASS = pool_executor_classes[external]
+    return _POOL_EXECUTOR_CLASS
+
+
+@contextlib.contextmanager
+def _patch_environ(new_values: dict[str, str]) -> Iterator[None]:
+    """Patch os.environ temporarily using the supplied values.
+
+    Parameters
+    ----------
+    new_values : `dict` [ `str`, `str` ]
+        New values to be stored in the environment.
     """
-    num_workers: int | None = None
-    num_workers = _get_int_env_var("LSST_RESOURCES_NUM_WORKERS")
-    if num_workers is None:
-        # CPU_LIMIT is used on nublado.
-        cpu_limit = _get_int_env_var("CPU_LIMIT") or multiprocessing.cpu_count()
-        if cpu_limit is not None:
-            num_workers = cpu_limit + 2
+    old_values: dict[str, str] = {}
+    for k, v in new_values.items():
+        if k in os.environ:
+            old_values[k] = os.environ[k]
+        os.environ[k] = v
 
-    # But don't ever return more than the maximum allowed.
-    return min([num_workers, MAX_WORKERS])
+    try:
+        yield
+    finally:
+        for k in new_values:
+            del os.environ[k]
+            if k in old_values:
+                os.environ[k] = old_values[k]
 
 
 class ResourcePath:  # numpydoc ignore=PR02
@@ -882,51 +914,101 @@ class ResourcePath:  # numpydoc ignore=PR02
         raise NotImplementedError()
 
     @classmethod
-    def mexists(cls, uris: Iterable[ResourcePath]) -> dict[ResourcePath, bool]:
+    def _group_uris(cls, uris: Iterable[ResourcePath]) -> dict[type[ResourcePath], list[ResourcePath]]:
+        """Group URIs by class/scheme."""
+        grouped: dict[type, list[ResourcePath]] = defaultdict(list)
+        for uri in uris:
+            grouped[uri.__class__].append(uri)
+        return grouped
+
+    @classmethod
+    def mexists(
+        cls, uris: Iterable[ResourcePath], *, num_workers: int | None = None
+    ) -> dict[ResourcePath, bool]:
         """Check for existence of multiple URIs at once.
 
         Parameters
         ----------
         uris : iterable of `ResourcePath`
             The URIs to test.
+        num_workers : `int` or `None`, optional
+            The number of parallel workers to use when checking for existence
+            If `None`, the default value will be taken from the environment.
+            If this number is higher than the default and a thread pool is
+            used, there may not be enough cached connections available.
 
         Returns
         -------
         existence : `dict` of [`ResourcePath`, `bool`]
             Mapping of original URI to boolean indicating existence.
         """
-        # Group by scheme to allow a subclass to be able to use
-        # specialized implementations.
-        grouped: dict[type, list[ResourcePath]] = {}
-        for uri in uris:
-            uri_class = uri.__class__
-            if uri_class not in grouped:
-                grouped[uri_class] = []
-            grouped[uri_class].append(uri)
-
         existence: dict[ResourcePath, bool] = {}
-        for uri_class in grouped:
-            existence.update(uri_class._mexists(grouped[uri_class]))
+        for uri_class, group in cls._group_uris(uris).items():
+            existence.update(uri_class._mexists(group, num_workers=num_workers))
 
         return existence
 
     @classmethod
-    def _mexists(cls, uris: Iterable[ResourcePath]) -> dict[ResourcePath, bool]:
+    def _mexists(
+        cls, uris: Iterable[ResourcePath], *, num_workers: int | None = None
+    ) -> dict[ResourcePath, bool]:
         """Check for existence of multiple URIs at once.
 
         Implementation helper method for `mexists`.
 
+
         Parameters
         ----------
         uris : iterable of `ResourcePath`
             The URIs to test.
+        num_workers : `int` or `None`, optional
+            The number of parallel workers to use when checking for existence
+            If `None`, the default value will be taken from the environment.
 
         Returns
         -------
         existence : `dict` of [`ResourcePath`, `bool`]
             Mapping of original URI to boolean indicating existence.
         """
-        with concurrent.futures.ThreadPoolExecutor(max_workers=_get_num_workers()) as exists_executor:
+        pool_executor_class = _get_executor_class()
+        if issubclass(pool_executor_class, concurrent.futures.ProcessPoolExecutor):
+            # Patch the environment to make it think there is only one worker
+            # for each subprocess.
+            with _patch_environ({"LSST_RESOURCES_NUM_WORKERS": "1"}):
+                return cls._mexists_pool(pool_executor_class, uris)
+        else:
+            return cls._mexists_pool(pool_executor_class, uris, num_workers=num_workers)
+
+    @classmethod
+    def _mexists_pool(
+        cls,
+        pool_executor_class: _EXECUTOR_TYPE,
+        uris: Iterable[ResourcePath],
+        *,
+        num_workers: int | None = None,
+    ) -> dict[ResourcePath, bool]:
+        """Check for existence of multiple URIs at once using specified pool
+        executor.
+
+        Implementation helper method for `_mexists`.
+
+        Parameters
+        ----------
+        pool_executor_class : `type` [ `concurrent.futures.Executor` ]
+            Type of executor pool to use.
+        uris : iterable of `ResourcePath`
+            The URIs to test.
+        num_workers : `int` or `None`, optional
+            The number of parallel workers to use when checking for existence
+            If `None`, the default value will be taken from the environment.
+
+        Returns
+        -------
+        existence : `dict` of [`ResourcePath`, `bool`]
+            Mapping of original URI to boolean indicating existence.
+        """
+        max_workers = num_workers if num_workers is not None else _get_num_workers()
+        with pool_executor_class(max_workers=max_workers) as exists_executor:
             future_exists = {exists_executor.submit(uri.exists): uri for uri in uris}
 
             results: dict[ResourcePath, bool] = {}
@@ -947,7 +1029,68 @@ class ResourcePath:  # numpydoc ignore=PR02
         overwrite: bool = False,
         transaction: TransactionProtocol | None = None,
         do_raise: bool = True,
-    ) -> dict[ResourcePath, MTransferResult]:
+    ) -> dict[ResourcePath, MBulkResult]:
+        """Transfer many files in bulk.
+
+        Parameters
+        ----------
+        transfer : `str`
+            Mode to use for transferring the resource. Generically there are
+            many standard options: copy, link, symlink, hardlink, relsymlink.
+            Not all URIs support all modes.
+        from_to : `list` [ `tuple` [ `ResourcePath`, `ResourcePath` ] ]
+            A sequence of the source URIs and the target URIs.
+        overwrite : `bool`, optional
+            Allow an existing file to be overwritten. Defaults to `False`.
+        transaction : `~lsst.resources.utils.TransactionProtocol`, optional
+            A transaction object that can (depending on implementation)
+            rollback transfers on error.  Not guaranteed to be implemented.
+            The transaction object must be thread safe.
+        do_raise : `bool`, optional
+            If `True` an `ExceptionGroup` will be raised containing any
+            exceptions raised by the individual transfers. If `False`, or if
+            there were no exceptions, a dict reporting the status of each
+            `ResourcePath` will be returned.
+
+        Returns
+        -------
+        copy_status : `dict` [ `ResourcePath`, `MBulkResult` ]
+            A dict of all the transfer attempts with a value indicating
+            whether the transfer succeeded for the target URI. If ``do_raise``
+            is `True`, this will only be returned if there are no errors.
+        """
+        pool_executor_class = _get_executor_class()
+        if issubclass(pool_executor_class, concurrent.futures.ProcessPoolExecutor):
+            # Patch the environment to make it think there is only one worker
+            # for each subprocess.
+            with _patch_environ({"LSST_RESOURCES_NUM_WORKERS": "1"}):
+                return cls._mtransfer(
+                    pool_executor_class,
+                    transfer,
+                    from_to,
+                    overwrite=overwrite,
+                    transaction=transaction,
+                    do_raise=do_raise,
+                )
+        return cls._mtransfer(
+            pool_executor_class,
+            transfer,
+            from_to,
+            overwrite=overwrite,
+            transaction=transaction,
+            do_raise=do_raise,
+        )
+
+    @classmethod
+    def _mtransfer(
+        cls,
+        pool_executor_class: _EXECUTOR_TYPE,
+        transfer: str,
+        from_to: Iterable[tuple[ResourcePath, ResourcePath]],
+        overwrite: bool = False,
+        transaction: TransactionProtocol | None = None,
+        do_raise: bool = True,
+    ) -> dict[ResourcePath, MBulkResult]:
         """Transfer many files in bulk.
 
         Parameters
@@ -971,11 +1114,11 @@ class ResourcePath:  # numpydoc ignore=PR02
 
         Returns
         -------
-        copy_status : `dict` [ `ResourcePath`, `MTransferResult` ]
+        copy_status : `dict` [ `ResourcePath`, `MBulkResult` ]
             A dict of all the transfer attempts with a value indicating
             whether the transfer succeeded for the target URI.
         """
-        with concurrent.futures.ThreadPoolExecutor(max_workers=_get_num_workers()) as transfer_executor:
+        with pool_executor_class(max_workers=_get_num_workers()) as transfer_executor:
             future_transfers = {
                 transfer_executor.submit(
                     to_uri.transfer_from,
@@ -987,17 +1130,17 @@ class ResourcePath:  # numpydoc ignore=PR02
                 ): to_uri
                 for from_uri, to_uri in from_to
             }
-            results: dict[ResourcePath, MTransferResult] = {}
+            results: dict[ResourcePath, MBulkResult] = {}
             failed = False
             for future in concurrent.futures.as_completed(future_transfers):
                 to_uri = future_transfers[future]
                 try:
                     future.result()
                 except Exception as e:
-                    transferred = MTransferResult(False, e)
+                    transferred = MBulkResult(False, e)
                     failed = True
                 else:
-                    transferred = MTransferResult(True, None)
+                    transferred = MBulkResult(True, None)
                 results[to_uri] = transferred
 
         if do_raise and failed:
@@ -1012,6 +1155,81 @@ class ResourcePath:  # numpydoc ignore=PR02
         """Remove the resource."""
         raise NotImplementedError()
 
+    @classmethod
+    def mremove(
+        cls, uris: Iterable[ResourcePath], *, do_raise: bool = True
+    ) -> dict[ResourcePath, MBulkResult]:
+        """Remove multiple URIs at once.
+
+        Parameters
+        ----------
+        uris : iterable of `ResourcePath`
+            URIs to remove.
+        do_raise : `bool`, optional
+            If `True` an `ExceptionGroup` will be raised containing any
+            exceptions raised by the individual transfers. If `False`, or if
+            there were no exceptions, a dict reporting the status of each
+            `ResourcePath` will be returned.
+
+        Returns
+        -------
+        results : `dict` [ `ResourcePath`, `MBulkResult` ]
+            Dictionary mapping each URI to a result object indicating whether
+            the removal succeeded or resulted in an exception. If ``do_raise``
+            is `True` this will only be returned if everything succeeded.
+        """
+        # Group URIs by scheme since some URI schemes support native bulk
+        # APIs.
+        results: dict[ResourcePath, MBulkResult] = {}
+        for uri_class, group in cls._group_uris(uris).items():
+            results.update(uri_class._mremove(group))
+        if do_raise:
+            failed = any(not r.success for r in results.values())
+            if failed:
+                s = "s" if len(results) != 1 else ""
+                raise ExceptionGroup(
+                    f"Error{s} removing {len(results)} artifact{s}",
+                    tuple(res.exception for res in results.values() if res.exception is not None),
+                )
+
+        return results
+
+    @classmethod
+    def _mremove(cls, uris: Iterable[ResourcePath]) -> dict[ResourcePath, MBulkResult]:
+        """Remove multiple URIs using futures."""
+        pool_executor_class = _get_executor_class()
+        if issubclass(pool_executor_class, concurrent.futures.ProcessPoolExecutor):
+            # Patch the environment to make it think there is only one worker
+            # for each subprocess.
+            with _patch_environ({"LSST_RESOURCES_NUM_WORKERS": "1"}):
+                return cls._mremove_pool(pool_executor_class, uris)
+        else:
+            return cls._mremove_pool(pool_executor_class, uris)
+
+    @classmethod
+    def _mremove_pool(
+        cls,
+        pool_executor_class: _EXECUTOR_TYPE,
+        uris: Iterable[ResourcePath],
+        *,
+        num_workers: int | None = None,
+    ) -> dict[ResourcePath, MBulkResult]:
+        """Remove URIs using a futures pool."""
+        max_workers = num_workers if num_workers is not None else _get_num_workers()
+        results: dict[ResourcePath, MBulkResult] = {}
+        with pool_executor_class(max_workers=max_workers) as remove_executor:
+            future_remove = {remove_executor.submit(uri.remove): uri for uri in uris}
+            for future in concurrent.futures.as_completed(future_remove):
+                try:
+                    future.result()
+                except Exception as e:
+                    removed = MBulkResult(False, e)
+                else:
+                    removed = MBulkResult(True, None)
+                uri = future_remove[future]
+                results[uri] = removed
+        return results
+
     def isabs(self) -> bool:
         """Indicate that the resource is fully specified.
 

lsst/resources/http.py

@@ -59,7 +59,7 @@ from lsst.utils.timer import time_this
 from ._resourceHandles import ResourceHandleProtocol
 from ._resourceHandles._httpResourceHandle import HttpReadResourceHandle, parse_content_range_header
 from ._resourcePath import ResourcePath
-from .utils import get_tempdir
+from .utils import _get_num_workers, get_tempdir
 
 if TYPE_CHECKING:
     from .utils import TransactionProtocol
@@ -165,14 +165,14 @@ class HttpResourcePathConfig:
         if self._front_end_connections is not None:
             return self._front_end_connections
 
+        default_pool_size = max(_get_num_workers(), self.DEFAULT_FRONTEND_PERSISTENT_CONNECTIONS)
+
         try:
             self._front_end_connections = int(
-                os.environ.get(
-                    "LSST_HTTP_FRONTEND_PERSISTENT_CONNECTIONS", self.DEFAULT_FRONTEND_PERSISTENT_CONNECTIONS
-                )
+                os.environ.get("LSST_HTTP_FRONTEND_PERSISTENT_CONNECTIONS", default_pool_size)
             )
         except ValueError:
-            self._front_end_connections = self.DEFAULT_FRONTEND_PERSISTENT_CONNECTIONS
+            self._front_end_connections = default_pool_size
 
         return self._front_end_connections
 
@@ -182,14 +182,14 @@ class HttpResourcePathConfig:
         if self._back_end_connections is not None:
             return self._back_end_connections
 
+        default_pool_size = max(_get_num_workers(), self.DEFAULT_FRONTEND_PERSISTENT_CONNECTIONS)
+
         try:
             self._back_end_connections = int(
-                os.environ.get(
-                    "LSST_HTTP_BACKEND_PERSISTENT_CONNECTIONS", self.DEFAULT_BACKEND_PERSISTENT_CONNECTIONS
-                )
+                os.environ.get("LSST_HTTP_BACKEND_PERSISTENT_CONNECTIONS", default_pool_size)
             )
         except ValueError:
-            self._back_end_connections = self.DEFAULT_BACKEND_PERSISTENT_CONNECTIONS
+            self._back_end_connections = default_pool_size
 
         return self._back_end_connections
 
@@ -587,7 +587,7 @@ class SessionStore:
         Note that "https://www.example.org" and "https://www.example.org:12345"
         will have different sessions since the port number is not identical.
         """
-        root_uri = str(rpath.root_uri())
+        root_uri = _dav_to_http(str(rpath.root_uri()))
         if root_uri not in self._sessions:
             # We don't have yet a session for this endpoint: create a new one.
             self._sessions[root_uri] = self._make_session(rpath)

lsst/resources/s3.py

@@ -20,17 +20,19 @@ import os
 import re
 import sys
 import threading
+from collections import defaultdict
 from collections.abc import Iterable, Iterator
 from functools import cache, cached_property
 from typing import IO, TYPE_CHECKING, cast
 
 from botocore.exceptions import ClientError
 
+from lsst.utils.iteration import chunk_iterable
 from lsst.utils.timer import time_this
 
 from ._resourceHandles._baseResourceHandle import ResourceHandleProtocol
 from ._resourceHandles._s3ResourceHandle import S3ResourceHandle
-from ._resourcePath import ResourcePath
+from ._resourcePath import MBulkResult, ResourcePath
 from .s3utils import (
     _get_s3_connection_parameters,
     _s3_disable_bucket_validation,
@@ -220,7 +222,9 @@ class S3ResourcePath(ResourcePath):
         return bucket
 
     @classmethod
-    def _mexists(cls, uris: Iterable[ResourcePath]) -> dict[ResourcePath, bool]:
+    def _mexists(
+        cls, uris: Iterable[ResourcePath], *, num_workers: int | None = None
+    ) -> dict[ResourcePath, bool]:
         # Force client to be created for each profile before creating threads.
         profiles = set[str | None]()
         for path in uris:
@@ -230,7 +234,56 @@ class S3ResourcePath(ResourcePath):
         for profile in profiles:
             getS3Client(profile)
 
-        return super()._mexists(uris)
+        return super()._mexists(uris, num_workers=num_workers)
+
+    @classmethod
+    def _mremove(cls, uris: Iterable[ResourcePath]) -> dict[ResourcePath, MBulkResult]:
+        # Delete multiple objects in one API call.
+        # Must group by profile and bucket.
+        grouped_uris: dict[tuple[str | None, str], list[S3ResourcePath]] = defaultdict(list)
+        for uri in uris:
+            uri = cast(S3ResourcePath, uri)
+            grouped_uris[uri._profile, uri._bucket].append(uri)
+
+        results: dict[ResourcePath, MBulkResult] = {}
+        for related_uris in grouped_uris.values():
+            # The client and bucket are the same for each of the remaining
+            # URIs.
+            first_uri = related_uris[0]
+            # API requires no more than 1000 per call.
+            for chunk in chunk_iterable(related_uris, chunk_size=1_000):
+                key_to_uri: dict[str, ResourcePath] = {}
+                keys: list[dict[str, str]] = []
+                for uri in chunk:
+                    key = uri.relativeToPathRoot
+                    key_to_uri[key] = uri
+                    keys.append({"Key": key})
+                    # Default to assuming everything worked.
+                    results[uri] = MBulkResult(True, None)
+                errored = cls._delete_related_objects(first_uri.client, first_uri._bucket, keys)
+
+                # Update with error information.
+                for key, bulk_result in errored.items():
+                    results[key_to_uri[key]] = bulk_result
+
+        return results
+
+    @classmethod
+    @backoff.on_exception(backoff.expo, retryable_io_errors, max_time=max_retry_time)
+    def _delete_related_objects(
+        cls, client: boto3.client, bucket: str, keys: list[dict[str, str]]
+    ) -> dict[str, MBulkResult]:
+        # Delete multiple objects from the same bucket, allowing for backoff
+        # retry.
+        response = client.delete_objects(Bucket=bucket, Delete={"Objects": keys, "Quiet": True})
+        # Use Quiet mode so we assume everything worked unless told otherwise.
+        # Only returning errors -- indexed by Key name.
+        errors: dict[str, MBulkResult] = {}
+        for errored_key in response.get("Errors", []):
+            errors[errored_key["Key"]] = MBulkResult(
+                False, ClientError({"Error": errored_key}, f"delete_objects: {errored_key['Key']}")
+            )
+        return errors
 
     @backoff.on_exception(backoff.expo, retryable_io_errors, max_time=max_retry_time)
     def exists(self) -> bool:

lsst/resources/s3utils.py

@@ -53,6 +53,7 @@ except ImportError:
 
 from ._resourcePath import ResourcePath
 from .location import Location
+from .utils import _get_num_workers
 
 # https://pypi.org/project/backoff/
 try:
@@ -246,7 +247,13 @@ def _s3_disable_bucket_validation(client: boto3.client) -> None:
 @functools.lru_cache
 def _get_s3_client(endpoint_config: _EndpointConfig, skip_validation: bool) -> boto3.client:
     # Helper function to cache the client for this endpoint
-    config = botocore.config.Config(read_timeout=180, retries={"mode": "adaptive", "max_attempts": 10})
+    # boto seems to assume it will always have at least 10 available.
+    max_pool_size = max(_get_num_workers(), 10)
+    config = botocore.config.Config(
+        read_timeout=180,
+        max_pool_connections=max_pool_size,
+        retries={"mode": "adaptive", "max_attempts": 10},
+    )
 
     session = boto3.Session(profile_name=endpoint_config.profile)
 

lsst/resources/tests.py

@@ -1027,6 +1027,8 @@ class GenericReadWriteTestCase(_GenericTestCase):
         expected_files = {
             "dir1/a.yaml",
             "dir1/b.yaml",
+            "dir1/c.yaml",
+            "dir1/d.yaml",
             "dir2/e.yaml",
         }
         expected_uris = {root.join(f) for f in expected_files}
@@ -1035,10 +1037,20 @@ class GenericReadWriteTestCase(_GenericTestCase):
             self.assertTrue(uri.exists())
         expected_uris.add(file)
 
-        multi = ResourcePath.mexists(expected_uris)
+        # Force to run with fewer workers than there are files.
+        multi = ResourcePath.mexists(expected_uris, num_workers=3)
 
         for uri, is_there in multi.items():
             if uri == file:
                 self.assertFalse(is_there)
             else:
                 self.assertTrue(is_there)
+
+        # Clean up. Unfortunately POSIX raises a FileNotFoundError but
+        # S3 boto does not complain if there is no key.
+        ResourcePath.mremove(expected_uris, do_raise=False)
+
+        # Check they were really removed.
+        multi = ResourcePath.mexists(expected_uris, num_workers=3)
+        for uri, is_there in multi.items():
+            self.assertFalse(is_there)

lsst/resources/utils.py

@@ -15,6 +15,7 @@ __all__ = ("NoTransaction", "TransactionProtocol", "get_tempdir", "os2posix", "p
 
 import contextlib
 import logging
+import multiprocessing
 import os
 import posixpath
 import shutil
@@ -33,6 +34,11 @@ IS_POSIX = os.sep == posixpath.sep
 # posix means posix and only determine explicitly in the non-posix case.
 OS_ROOT_PATH = posixpath.sep if IS_POSIX else Path().resolve().root
 
+# Maximum number of worker threads for parallelized operations.
+# If greater than 10, be aware that this number has to be consistent
+# with connection pool sizing (for example in urllib3).
+MAX_WORKERS = 10
+
 log = logging.getLogger(__name__)
 
 
@@ -226,3 +232,40 @@ def ensure_directory_is_writeable(directory_path: str | bytes) -> None:
     desired_mode = current_mode | stat.S_IWUSR | stat.S_IXUSR
     if current_mode != desired_mode:
         os.chmod(directory_path, desired_mode)
+
+
+def _get_int_env_var(env_var: str) -> int | None:
+    int_value = None
+    env_value = os.getenv(env_var)
+    if env_value is not None:
+        with contextlib.suppress(TypeError):
+            int_value = int(env_value)
+    return int_value
+
+
+@cache
+def _get_num_workers() -> int:
+    f"""Calculate the number of workers to use.
+
+    Returns
+    -------
+    num : `int`
+        The number of workers to use. Will use the value of the
+        ``LSST_RESOURCES_NUM_WORKERS`` environment variable if set. Will fall
+        back to using the CPU count (plus 2) but capped at {MAX_WORKERS}.
+    """
+    num_workers: int | None = None
+    num_workers = _get_int_env_var("LSST_RESOURCES_NUM_WORKERS")
+
+    # If someone is explicitly specifying a number, let them use that number.
+    if num_workers is not None:
+        return num_workers
+
+    if num_workers is None:
+        # CPU_LIMIT is used on nublado.
+        cpu_limit = _get_int_env_var("CPU_LIMIT") or multiprocessing.cpu_count()
+        if cpu_limit is not None:
+            num_workers = cpu_limit + 2
+
+    # But don't ever return more than the maximum allowed.
+    return min([num_workers, MAX_WORKERS])

lsst/resources/version.py

@@ -1,2 +1,2 @@
 __all__ = ["__version__"]
-__version__ = "29.2025.1700"
+__version__ = "29.2025.1900"

{lsst_resources-29.2025.1700.dist-info → lsst_resources-29.2025.1900.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lsst-resources
-Version: 29.2025.1700
+Version: 29.2025.1900
 Summary: An abstraction layer for reading and writing from URI file resources.
 Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
 License: BSD 3-Clause License

{lsst_resources-29.2025.1700.dist-info → lsst_resources-29.2025.1900.dist-info}/RECORD

@@ -1,28 +1,28 @@
 lsst/__init__.py,sha256=9I6UQ9gj-ZcPlvsa0OPBo76UujxXVehVzw9yMAOQvyM,466
 lsst/resources/__init__.py,sha256=BDj6uokvd0ZQNGl-Xgz5gZd83Z0L2gFqGSk0KJpylP8,778
-lsst/resources/_resourcePath.py,sha256=xTVyDHD-UHlF5FeDvSXXnsmOuoSFnORZD_wMksxiFfA,64926
+lsst/resources/_resourcePath.py,sha256=AnyqWz6TVgDjuZO4DjFj6q4J0-zbIduCFuxwopSS6Nk,73905
 lsst/resources/file.py,sha256=-jPuoHvTEtx5tnDyNkfwhWAyX0cTwkuMd-JvJn9EGdE,23226
 lsst/resources/gs.py,sha256=Lpo5GAzH7R7HG8E5RMGOdP4j4hjWJn-k6M3OXj0nHQM,12783
-lsst/resources/http.py,sha256=JW3cBe4MERyjopFKkELui1BRr4b4Mkgp0Iqt9YFIxuc,88227
+lsst/resources/http.py,sha256=9a_VadSabznPC0FTQtDtfV041zH25wZgXFtJ7HkvHp0,88275
 lsst/resources/location.py,sha256=x3Tq0x5o1OXYmZDxYBenUG1N71wtDhnjVAr3s2ZEiu8,7937
 lsst/resources/mem.py,sha256=VOWh7XxJPfqKcFdLZSjKEAfORQ2AHZHpxmjT8LniV60,1008
 lsst/resources/packageresource.py,sha256=vnfeRlpVwpC5cDQZE6Lnh8EH6oZy1sH2vLz9ONYjJ4k,6817
 lsst/resources/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-lsst/resources/s3.py,sha256=wrOMdFWltxpGEWeL--kPCbk5Le_viCIsEn4lOPZbXhM,24124
-lsst/resources/s3utils.py,sha256=cKJ9GWHETHhn1djezyikWwAaw4k1B3hFvfif96THHDQ,14355
+lsst/resources/s3.py,sha256=KH9oPThUMG6bvkE_gEgJU3KXY8RgIVbKsLu0orgv1Mo,26634
+lsst/resources/s3utils.py,sha256=ojWf9BPrK9mhGQ8jvs4_8Nsqf9360e79U5FnPTxe24A,14576
 lsst/resources/schemeless.py,sha256=GfJcKzZ0XIeepfQdW4HPZWiZlSp_ej0SEtSiJTrDUQs,10666
-lsst/resources/tests.py,sha256=MLB8hERKuNredzzg3Qq9M_U7IesV3xrbcjFwKuMp3Ok,43513
-lsst/resources/utils.py,sha256=IHVrOdj0szNWxiXk-jbZu1RhTR8WXks1vI9JCpBxeBA,6706
-lsst/resources/version.py,sha256=k6PcnewHb2vETZCsCS6q8mJ_uyMg-otfwuQ8B0oDDwQ,55
+lsst/resources/tests.py,sha256=SqYLbDG6QkZTB-0UvrsiPtfmdL1TcglGeqBTPQxu9GE,44027
+lsst/resources/utils.py,sha256=6O3Mq7JbPEtqyD2lM77pRpwcPMfV5SxiNMknw-F2vNs,8097
+lsst/resources/version.py,sha256=duakYcQBT5t4KBaOX14WCtq_sSvpoXvNcxi3sUb8GvI,55
 lsst/resources/_resourceHandles/__init__.py,sha256=zOcZ8gVEBdAWcHJaZabA8Vdq-wAVcxjbmA_1b1IWM6M,76
 lsst/resources/_resourceHandles/_baseResourceHandle.py,sha256=lQwxDOmFUNJndTxsjpz-HxrQBL0L-z4aXQocHdOEI7c,4676
 lsst/resources/_resourceHandles/_fileResourceHandle.py,sha256=A7_WQPzD0ZlOzNmaI_TPdZybrNxrXPkNHWVla3UFxfs,3676
-lsst/resources/_resourceHandles/_httpResourceHandle.py,sha256=JRjpE-ZQfgKX5OyVLulIbzW38FdhovcoOd1D4rhb5vk,10900
+lsst/resources/_resourceHandles/_httpResourceHandle.py,sha256=Yami8IVGeru4bLQCag-OvGG0ltz1qyEg57FY4IEB87Y,10995
 lsst/resources/_resourceHandles/_s3ResourceHandle.py,sha256=NkDmPb9bm_zMvr6mMnb-tBmqJDt0yUJrt2gZXR8l7ok,12923
-lsst_resources-29.2025.1700.dist-info/licenses/COPYRIGHT,sha256=yazVsoMmFwhiw5itGrdT4YPmXbpsQyUFjlpOyZIa77M,148
-lsst_resources-29.2025.1700.dist-info/licenses/LICENSE,sha256=7wrtgl8meQ0_RIuv2TjIKpAnNrl-ODH-QLwyHe9citI,1516
-lsst_resources-29.2025.1700.dist-info/METADATA,sha256=gybCKJmJyKdf0oqRgCbF6IKA0SzNalaw1nhptr1Y51I,2237
-lsst_resources-29.2025.1700.dist-info/WHEEL,sha256=SmOxYU7pzNKBqASvQJ7DjX3XGUF92lrGhMb3R6_iiqI,91
-lsst_resources-29.2025.1700.dist-info/top_level.txt,sha256=eUWiOuVVm9wwTrnAgiJT6tp6HQHXxIhj2QSZ7NYZH80,5
-lsst_resources-29.2025.1700.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-lsst_resources-29.2025.1700.dist-info/RECORD,,
+lsst_resources-29.2025.1900.dist-info/licenses/COPYRIGHT,sha256=yazVsoMmFwhiw5itGrdT4YPmXbpsQyUFjlpOyZIa77M,148
+lsst_resources-29.2025.1900.dist-info/licenses/LICENSE,sha256=7wrtgl8meQ0_RIuv2TjIKpAnNrl-ODH-QLwyHe9citI,1516
+lsst_resources-29.2025.1900.dist-info/METADATA,sha256=AsHgImBvDFy_E373VzU-E3sARIx2y1E7-gfLP-991is,2237
+lsst_resources-29.2025.1900.dist-info/WHEEL,sha256=0CuiUZ_p9E4cD6NyLD6UG80LBXYyiSYZOKDm5lp32xk,91
+lsst_resources-29.2025.1900.dist-info/top_level.txt,sha256=eUWiOuVVm9wwTrnAgiJT6tp6HQHXxIhj2QSZ7NYZH80,5
+lsst_resources-29.2025.1900.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
+lsst_resources-29.2025.1900.dist-info/RECORD,,

{lsst_resources-29.2025.1700.dist-info → lsst_resources-29.2025.1900.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (79.0.1)
+Generator: setuptools (80.3.1)
 Root-Is-Purelib: true
 Tag: py3-none-any