lsst-resources 29.2025.1700__py3-none-any.whl → 29.2025.4600__py3-none-any.whl

lsst/resources/_resourcePath.py

@@ -19,14 +19,15 @@ import copy
 import io
 import locale
 import logging
-import multiprocessing
 import os
 import posixpath
 import re
+import sys
 import urllib.parse
-from functools import cache
+from collections import defaultdict
 from pathlib import Path, PurePath, PurePosixPath
 from random import Random
+from typing import TypeAlias
 
 try:
     import fsspec
@@ -39,7 +40,7 @@ from collections.abc import Iterable, Iterator
 from typing import TYPE_CHECKING, Any, Literal, NamedTuple, overload
 
 from ._resourceHandles._baseResourceHandle import ResourceHandleProtocol
-from .utils import get_tempdir
+from .utils import _get_num_workers, get_tempdir
 
 if TYPE_CHECKING:
     from .utils import TransactionProtocol
@@ -53,49 +54,81 @@ ESCAPES_RE = re.compile(r"%[A-F0-9]{2}")
 # Precomputed escaped hash
 ESCAPED_HASH = urllib.parse.quote("#")
 
-# Maximum number of worker threads for parallelized operations.
-# If greater than 10, be aware that this number has to be consistent
-# with connection pool sizing (for example in urllib3).
-MAX_WORKERS = 10
 
-
-class MTransferResult(NamedTuple):
-    """Report on a bulk transfer."""
+class MBulkResult(NamedTuple):
+    """Report on a bulk operation."""
 
     success: bool
     exception: Exception | None
 
 
-def _get_int_env_var(env_var: str) -> int | None:
-    int_value = None
-    env_value = os.getenv(env_var)
-    if env_value is not None:
-        with contextlib.suppress(TypeError):
-            int_value = int(env_value)
-    return int_value
+_EXECUTOR_TYPE: TypeAlias = type[
+    concurrent.futures.ThreadPoolExecutor | concurrent.futures.ProcessPoolExecutor
+]
+
+# Cache value for executor class so as not to issue warning multiple
+# times but still allow tests to override the value.
+_POOL_EXECUTOR_CLASS: _EXECUTOR_TYPE | None = None
 
 
-@cache
-def _get_num_workers() -> int:
-    f"""Calculate the number of workers to use.
+def _get_executor_class() -> _EXECUTOR_TYPE:
+    """Return the executor class used for parallelized execution.
 
     Returns
     -------
-    num : `int`
-        The number of workers to use. Will use the value of the
-        ``LSST_RESOURCES_NUM_WORKERS`` environment variable if set. Will fall
-        back to using the CPU count (plus 2) but capped at {MAX_WORKERS}.
+    cls : `concurrent.futures.Executor`
+        The ``Executor`` class. Default is
+        `concurrent.futures.ThreadPoolExecutor`. Can be set explicitly by
+        setting the ``$LSST_RESOURCES_EXECUTOR`` environment variable to
+        "thread" or "process". Returns "thread" pool if the value of the
+        variable is not recognized.
+    """
+    global _POOL_EXECUTOR_CLASS
+
+    if _POOL_EXECUTOR_CLASS is not None:
+        return _POOL_EXECUTOR_CLASS
+
+    pool_executor_classes = {
+        "threads": concurrent.futures.ThreadPoolExecutor,
+        "process": concurrent.futures.ProcessPoolExecutor,
+    }
+    default_executor = "threads"
+    external = os.getenv("LSST_RESOURCES_EXECUTOR", default_executor)
+    if not external:
+        external = default_executor
+    if external not in pool_executor_classes:
+        log.warning(
+            "Unrecognized value of '%s' for LSST_RESOURCES_EXECUTOR env var. Using '%s'",
+            external,
+            default_executor,
+        )
+        external = default_executor
+    _POOL_EXECUTOR_CLASS = pool_executor_classes[external]
+    return _POOL_EXECUTOR_CLASS
+
+
+@contextlib.contextmanager
+def _patch_environ(new_values: dict[str, str]) -> Iterator[None]:
+    """Patch os.environ temporarily using the supplied values.
+
+    Parameters
+    ----------
+    new_values : `dict` [ `str`, `str` ]
+        New values to be stored in the environment.
     """
-    num_workers: int | None = None
-    num_workers = _get_int_env_var("LSST_RESOURCES_NUM_WORKERS")
-    if num_workers is None:
-        # CPU_LIMIT is used on nublado.
-        cpu_limit = _get_int_env_var("CPU_LIMIT") or multiprocessing.cpu_count()
-        if cpu_limit is not None:
-            num_workers = cpu_limit + 2
+    old_values: dict[str, str] = {}
+    for k, v in new_values.items():
+        if k in os.environ:
+            old_values[k] = os.environ[k]
+        os.environ[k] = v
 
-    # But don't ever return more than the maximum allowed.
-    return min([num_workers, MAX_WORKERS])
+    try:
+        yield
+    finally:
+        for k in new_values:
+            del os.environ[k]
+            if k in old_values:
+                os.environ[k] = old_values[k]
 
 
 class ResourcePath:  # numpydoc ignore=PR02
@@ -336,9 +369,9 @@ class ResourcePath:  # numpydoc ignore=PR02
 
                 subclass = HttpResourcePath
             elif parsed.scheme in {"dav", "davs"}:
-                from .http import HttpResourcePath
+                from .dav import DavResourcePath
 
-                subclass = HttpResourcePath
+                subclass = DavResourcePath
             elif parsed.scheme == "gs":
                 from .gs import GSResourcePath
 
@@ -517,7 +550,7 @@ class ResourcePath:  # numpydoc ignore=PR02
             return self, ""
 
         head, tail = self._pathModule.split(self.path)
-        headuri = self._uri._replace(path=head)
+        headuri = self._uri._replace(path=head, fragment="", query="", params="")
 
         # The file part should never include quoted metacharacters
         tail = urllib.parse.unquote(tail)
@@ -587,7 +620,7 @@ class ResourcePath:  # numpydoc ignore=PR02
         # regardless of the presence of a trailing separator
         originalPath = self._pathLib(self.path)
         parentPath = originalPath.parent
-        return self.replace(path=str(parentPath), forceDirectory=True)
+        return self.replace(path=str(parentPath), forceDirectory=True, fragment="", query="", params="")
 
     def replace(
         self, forceDirectory: bool | None = None, isTemporary: bool = False, **kwargs: Any
@@ -617,9 +650,11 @@ class ResourcePath:  # numpydoc ignore=PR02
         # Disallow a change in scheme
         if "scheme" in kwargs:
             raise ValueError(f"Can not use replace() method to change URI scheme for {self}")
-        return self.__class__(
+        result = self.__class__(
             self._uri._replace(**kwargs), forceDirectory=forceDirectory, isTemporary=isTemporary
         )
+        result._copy_extra_attributes(self)
+        return result
 
     def updatedFile(self, newfile: str) -> ResourcePath:
         """Return new URI with an updated final component of the path.
@@ -832,9 +867,11 @@ class ResourcePath:  # numpydoc ignore=PR02
             forceDirectory=forceDirectory,
             isTemporary=isTemporary,
             fragment=path_uri.fragment,
+            query=path_uri.query,
+            params=path_uri.params,
         )
 
-    def relative_to(self, other: ResourcePath) -> str | None:
+    def relative_to(self, other: ResourcePath, walk_up: bool = False) -> str | None:
         """Return the relative path from this URI to the other URI.
 
         Parameters
@@ -842,6 +879,9 @@ class ResourcePath:  # numpydoc ignore=PR02
         other : `ResourcePath`
             URI to use to calculate the relative path. Must be a parent
             of this URI.
+        walk_up : `bool`, optional
+            Control whether "``..``" can be used to resolve a relative path.
+            Default is `False`. Can not be `True` on Python version 3.11.
 
         Returns
         -------
@@ -860,11 +900,22 @@ class ResourcePath:  # numpydoc ignore=PR02
             if not {self.netloc, other.netloc}.issubset(local_netlocs):
                 return None
 
+        # Rather than trying to guess a failure reason from the TypeError
+        # explicitly check for python 3.11. Doing this will simplify the
+        # rediscovery of a useless python version check when we set a new
+        # minimum version.
+        kwargs = {}
+        if walk_up:
+            if sys.version_info < (3, 12, 0):
+                raise TypeError("walk_up parameter can not be true in python 3.11 and older")
+
+            kwargs["walk_up"] = True
+
         enclosed_path = self._pathLib(self.relativeToPathRoot)
         parent_path = other.relativeToPathRoot
         subpath: str | None
         try:
-            subpath = str(enclosed_path.relative_to(parent_path))
+            subpath = str(enclosed_path.relative_to(parent_path, **kwargs))
         except ValueError:
             subpath = None
         else:
@@ -882,51 +933,101 @@ class ResourcePath:  # numpydoc ignore=PR02
         raise NotImplementedError()
 
     @classmethod
-    def mexists(cls, uris: Iterable[ResourcePath]) -> dict[ResourcePath, bool]:
+    def _group_uris(cls, uris: Iterable[ResourcePath]) -> dict[type[ResourcePath], list[ResourcePath]]:
+        """Group URIs by class/scheme."""
+        grouped: dict[type, list[ResourcePath]] = defaultdict(list)
+        for uri in uris:
+            grouped[uri.__class__].append(uri)
+        return grouped
+
+    @classmethod
+    def mexists(
+        cls, uris: Iterable[ResourcePath], *, num_workers: int | None = None
+    ) -> dict[ResourcePath, bool]:
         """Check for existence of multiple URIs at once.
 
         Parameters
         ----------
         uris : iterable of `ResourcePath`
             The URIs to test.
+        num_workers : `int` or `None`, optional
+            The number of parallel workers to use when checking for existence
+            If `None`, the default value will be taken from the environment.
+            If this number is higher than the default and a thread pool is
+            used, there may not be enough cached connections available.
 
         Returns
         -------
         existence : `dict` of [`ResourcePath`, `bool`]
             Mapping of original URI to boolean indicating existence.
         """
-        # Group by scheme to allow a subclass to be able to use
-        # specialized implementations.
-        grouped: dict[type, list[ResourcePath]] = {}
-        for uri in uris:
-            uri_class = uri.__class__
-            if uri_class not in grouped:
-                grouped[uri_class] = []
-            grouped[uri_class].append(uri)
-
         existence: dict[ResourcePath, bool] = {}
-        for uri_class in grouped:
-            existence.update(uri_class._mexists(grouped[uri_class]))
+        for uri_class, group in cls._group_uris(uris).items():
+            existence.update(uri_class._mexists(group, num_workers=num_workers))
 
         return existence
 
     @classmethod
-    def _mexists(cls, uris: Iterable[ResourcePath]) -> dict[ResourcePath, bool]:
+    def _mexists(
+        cls, uris: Iterable[ResourcePath], *, num_workers: int | None = None
+    ) -> dict[ResourcePath, bool]:
         """Check for existence of multiple URIs at once.
 
         Implementation helper method for `mexists`.
 
+
         Parameters
         ----------
         uris : iterable of `ResourcePath`
             The URIs to test.
+        num_workers : `int` or `None`, optional
+            The number of parallel workers to use when checking for existence
+            If `None`, the default value will be taken from the environment.
 
         Returns
         -------
         existence : `dict` of [`ResourcePath`, `bool`]
             Mapping of original URI to boolean indicating existence.
         """
-        with concurrent.futures.ThreadPoolExecutor(max_workers=_get_num_workers()) as exists_executor:
+        pool_executor_class = _get_executor_class()
+        if issubclass(pool_executor_class, concurrent.futures.ProcessPoolExecutor):
+            # Patch the environment to make it think there is only one worker
+            # for each subprocess.
+            with _patch_environ({"LSST_RESOURCES_NUM_WORKERS": "1"}):
+                return cls._mexists_pool(pool_executor_class, uris)
+        else:
+            return cls._mexists_pool(pool_executor_class, uris, num_workers=num_workers)
+
+    @classmethod
+    def _mexists_pool(
+        cls,
+        pool_executor_class: _EXECUTOR_TYPE,
+        uris: Iterable[ResourcePath],
+        *,
+        num_workers: int | None = None,
+    ) -> dict[ResourcePath, bool]:
+        """Check for existence of multiple URIs at once using specified pool
+        executor.
+
+        Implementation helper method for `_mexists`.
+
+        Parameters
+        ----------
+        pool_executor_class : `type` [ `concurrent.futures.Executor` ]
+            Type of executor pool to use.
+        uris : iterable of `ResourcePath`
+            The URIs to test.
+        num_workers : `int` or `None`, optional
+            The number of parallel workers to use when checking for existence
+            If `None`, the default value will be taken from the environment.
+
+        Returns
+        -------
+        existence : `dict` of [`ResourcePath`, `bool`]
+            Mapping of original URI to boolean indicating existence.
+        """
+        max_workers = num_workers if num_workers is not None else _get_num_workers()
+        with pool_executor_class(max_workers=max_workers) as exists_executor:
             future_exists = {exists_executor.submit(uri.exists): uri for uri in uris}
 
             results: dict[ResourcePath, bool] = {}
@@ -947,7 +1048,68 @@ class ResourcePath:  # numpydoc ignore=PR02
         overwrite: bool = False,
         transaction: TransactionProtocol | None = None,
         do_raise: bool = True,
-    ) -> dict[ResourcePath, MTransferResult]:
+    ) -> dict[ResourcePath, MBulkResult]:
+        """Transfer many files in bulk.
+
+        Parameters
+        ----------
+        transfer : `str`
+            Mode to use for transferring the resource. Generically there are
+            many standard options: copy, link, symlink, hardlink, relsymlink.
+            Not all URIs support all modes.
+        from_to : `list` [ `tuple` [ `ResourcePath`, `ResourcePath` ] ]
+            A sequence of the source URIs and the target URIs.
+        overwrite : `bool`, optional
+            Allow an existing file to be overwritten. Defaults to `False`.
+        transaction : `~lsst.resources.utils.TransactionProtocol`, optional
+            A transaction object that can (depending on implementation)
+            rollback transfers on error.  Not guaranteed to be implemented.
+            The transaction object must be thread safe.
+        do_raise : `bool`, optional
+            If `True` an `ExceptionGroup` will be raised containing any
+            exceptions raised by the individual transfers. If `False`, or if
+            there were no exceptions, a dict reporting the status of each
+            `ResourcePath` will be returned.
+
+        Returns
+        -------
+        copy_status : `dict` [ `ResourcePath`, `MBulkResult` ]
+            A dict of all the transfer attempts with a value indicating
+            whether the transfer succeeded for the target URI. If ``do_raise``
+            is `True`, this will only be returned if there are no errors.
+        """
+        pool_executor_class = _get_executor_class()
+        if issubclass(pool_executor_class, concurrent.futures.ProcessPoolExecutor):
+            # Patch the environment to make it think there is only one worker
+            # for each subprocess.
+            with _patch_environ({"LSST_RESOURCES_NUM_WORKERS": "1"}):
+                return cls._mtransfer(
+                    pool_executor_class,
+                    transfer,
+                    from_to,
+                    overwrite=overwrite,
+                    transaction=transaction,
+                    do_raise=do_raise,
+                )
+        return cls._mtransfer(
+            pool_executor_class,
+            transfer,
+            from_to,
+            overwrite=overwrite,
+            transaction=transaction,
+            do_raise=do_raise,
+        )
+
+    @classmethod
+    def _mtransfer(
+        cls,
+        pool_executor_class: _EXECUTOR_TYPE,
+        transfer: str,
+        from_to: Iterable[tuple[ResourcePath, ResourcePath]],
+        overwrite: bool = False,
+        transaction: TransactionProtocol | None = None,
+        do_raise: bool = True,
+    ) -> dict[ResourcePath, MBulkResult]:
         """Transfer many files in bulk.
 
         Parameters
@@ -971,11 +1133,11 @@ class ResourcePath:  # numpydoc ignore=PR02
 
         Returns
         -------
-        copy_status : `dict` [ `ResourcePath`, `MTransferResult` ]
+        copy_status : `dict` [ `ResourcePath`, `MBulkResult` ]
             A dict of all the transfer attempts with a value indicating
             whether the transfer succeeded for the target URI.
         """
-        with concurrent.futures.ThreadPoolExecutor(max_workers=_get_num_workers()) as transfer_executor:
+        with pool_executor_class(max_workers=_get_num_workers()) as transfer_executor:
             future_transfers = {
                 transfer_executor.submit(
                     to_uri.transfer_from,
@@ -987,17 +1149,17 @@ class ResourcePath:  # numpydoc ignore=PR02
                 ): to_uri
                 for from_uri, to_uri in from_to
             }
-            results: dict[ResourcePath, MTransferResult] = {}
+            results: dict[ResourcePath, MBulkResult] = {}
             failed = False
             for future in concurrent.futures.as_completed(future_transfers):
                 to_uri = future_transfers[future]
                 try:
                     future.result()
                 except Exception as e:
-                    transferred = MTransferResult(False, e)
+                    transferred = MBulkResult(False, e)
                     failed = True
                 else:
-                    transferred = MTransferResult(True, None)
+                    transferred = MBulkResult(True, None)
                 results[to_uri] = transferred
 
         if do_raise and failed:
@@ -1012,6 +1174,81 @@ class ResourcePath:  # numpydoc ignore=PR02
         """Remove the resource."""
         raise NotImplementedError()
 
+    @classmethod
+    def mremove(
+        cls, uris: Iterable[ResourcePath], *, do_raise: bool = True
+    ) -> dict[ResourcePath, MBulkResult]:
+        """Remove multiple URIs at once.
+
+        Parameters
+        ----------
+        uris : iterable of `ResourcePath`
+            URIs to remove.
+        do_raise : `bool`, optional
+            If `True` an `ExceptionGroup` will be raised containing any
+            exceptions raised by the individual transfers. If `False`, or if
+            there were no exceptions, a dict reporting the status of each
+            `ResourcePath` will be returned.
+
+        Returns
+        -------
+        results : `dict` [ `ResourcePath`, `MBulkResult` ]
+            Dictionary mapping each URI to a result object indicating whether
+            the removal succeeded or resulted in an exception. If ``do_raise``
+            is `True` this will only be returned if everything succeeded.
+        """
+        # Group URIs by scheme since some URI schemes support native bulk
+        # APIs.
+        results: dict[ResourcePath, MBulkResult] = {}
+        for uri_class, group in cls._group_uris(uris).items():
+            results.update(uri_class._mremove(group))
+        if do_raise:
+            failed = any(not r.success for r in results.values())
+            if failed:
+                s = "s" if len(results) != 1 else ""
+                raise ExceptionGroup(
+                    f"Error{s} removing {len(results)} artifact{s}",
+                    tuple(res.exception for res in results.values() if res.exception is not None),
+                )
+
+        return results
+
+    @classmethod
+    def _mremove(cls, uris: Iterable[ResourcePath]) -> dict[ResourcePath, MBulkResult]:
+        """Remove multiple URIs using futures."""
+        pool_executor_class = _get_executor_class()
+        if issubclass(pool_executor_class, concurrent.futures.ProcessPoolExecutor):
+            # Patch the environment to make it think there is only one worker
+            # for each subprocess.
+            with _patch_environ({"LSST_RESOURCES_NUM_WORKERS": "1"}):
+                return cls._mremove_pool(pool_executor_class, uris)
+        else:
+            return cls._mremove_pool(pool_executor_class, uris)
+
+    @classmethod
+    def _mremove_pool(
+        cls,
+        pool_executor_class: _EXECUTOR_TYPE,
+        uris: Iterable[ResourcePath],
+        *,
+        num_workers: int | None = None,
+    ) -> dict[ResourcePath, MBulkResult]:
+        """Remove URIs using a futures pool."""
+        max_workers = num_workers if num_workers is not None else _get_num_workers()
+        results: dict[ResourcePath, MBulkResult] = {}
+        with pool_executor_class(max_workers=max_workers) as remove_executor:
+            future_remove = {remove_executor.submit(uri.remove): uri for uri in uris}
+            for future in concurrent.futures.as_completed(future_remove):
+                try:
+                    future.result()
+                except Exception as e:
+                    removed = MBulkResult(False, e)
+                else:
+                    removed = MBulkResult(True, None)
+                uri = future_remove[future]
+                results[uri] = removed
+        return results
+
     def isabs(self) -> bool:
         """Indicate that the resource is fully specified.
 
@@ -1035,7 +1272,10 @@ class ResourcePath:  # numpydoc ignore=PR02
         """
         return self
 
-    def _as_local(self, multithreaded: bool = True, tmpdir: ResourcePath | None = None) -> tuple[str, bool]:
+    @contextlib.contextmanager
+    def _as_local(
+        self, multithreaded: bool = True, tmpdir: ResourcePath | None = None
+    ) -> Iterator[ResourcePath]:
         """Return the location of the (possibly remote) resource as local file.
 
         This is a helper function for `as_local` context manager.
@@ -1054,13 +1294,9 @@ class ResourcePath:  # numpydoc ignore=PR02
 
         Returns
         -------
-        path : `str`
-            If this is a remote resource, it will be a copy of the resource
-            on the local file system, probably in a temporary directory.
-            For a local resource this should be the actual path to the
-            resource.
-        is_temporary : `bool`
-            Indicates if the local path is a temporary file or not.
+        local_uri : `ResourcePath`
+            A URI to a local POSIX file. This can either be the same resource
+            or a local downloaded copy of the resource.
         """
         raise NotImplementedError()
 
@@ -1110,18 +1346,8 @@ class ResourcePath:  # numpydoc ignore=PR02
         temp_dir = ResourcePath(tmpdir, forceDirectory=True) if tmpdir is not None else None
         if temp_dir is not None and not temp_dir.isLocal:
             raise ValueError(f"Temporary directory for as_local must be local resource not {temp_dir}")
-        local_src, is_temporary = self._as_local(multithreaded=multithreaded, tmpdir=temp_dir)
-        local_uri = ResourcePath(local_src, isTemporary=is_temporary)
-
-        try:
+        with self._as_local(multithreaded=multithreaded, tmpdir=temp_dir) as local_uri:
             yield local_uri
-        finally:
-            # The caller might have relocated the temporary file.
-            # Do not ever delete if the temporary matches self
-            # (since it may have been that a temporary file was made local
-            # but already was local).
-            if self != local_uri and is_temporary and local_uri.exists():
-                local_uri.remove()
 
     @classmethod
     @contextlib.contextmanager
@@ -1685,6 +1911,12 @@ class ResourcePath:  # numpydoc ignore=PR02
         """
         raise NotImplementedError(f"URL signing is not supported for '{self.scheme}'")
 
+    def _copy_extra_attributes(self, original_uri: ResourcePath) -> None:
+        # May be overridden by subclasses to transfer attributes when a
+        # ResourcePath is constructed using the "clone" version of the
+        # ResourcePath constructor by passing in a ResourcePath object.
+        pass
+
 
 ResourcePathExpression = str | urllib.parse.ParseResult | ResourcePath | Path
 """Type-annotation alias for objects that can be coerced to ResourcePath.