lsst-resources 29.2025.2100__tar.gz → 29.2025.2500__tar.gz

{lsst_resources-29.2025.2100/python/lsst_resources.egg-info → lsst_resources-29.2025.2500}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lsst-resources
-Version: 29.2025.2100
+Version: 29.2025.2500
 Summary: An abstraction layer for reading and writing from URI file resources.
 Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
 License: BSD 3-Clause License

{lsst_resources-29.2025.2100 → lsst_resources-29.2025.2500}/doc/lsst.resources/CHANGES.rst

@@ -1,3 +1,34 @@
+Resources v29.1.0 (2025-06-13)
+==============================
+
+Miscellaneous Changes of Minor Interest
+---------------------------------------
+
+New Features
+------------
+
+- * Added ``ResourcePath.mtransfer()`` for doing multiple transfers in parallel.
+    The number of workers can be controlled using the ``$LSST_RESOURCES_NUM_WORKERS`` environment variable.
+  * ``transfer_from`` and ``as_local`` now have an additional parameter that can control whether implicit multithreading should be used for a single download.
+  * ``as_local`` has a new parameter that can be used to explicitly specify the local download location.
+    This can be used for ``transfer_from`` to allow the file to be downloaded to the local destination directory immediately. (`DM-31824 <https://rubinobs.atlassian.net/browse/DM-31824>`_)
+- Added specialized support for schemes ``davs://`` and ``dav://`` hosted by storage endpoints implementing WebDAV protocol as described in `RFC-4918 HTTP Extensions for Web Distributed Authoring and Versioning (WebDAV) <http://www.webdav.org/specs/rfc4918.html>`_. (`DM-49784 <https://rubinobs.atlassian.net/browse/DM-49784>`_)
+- Added new bulk removal API: ``ResourcePath.mremove()``.
+  This can be 10 times faster than calling ``remove()`` in a loop. (`DM-50724 <https://rubinobs.atlassian.net/browse/DM-50724>`_)
+
+
+Miscellaneous Changes of Minor Interest
+---------------------------------------
+
+- It is now possible to control how bulk APIs such as ``mexists()`` and ``mtransfer()`` work.
+  Added ``$LSST_RESOURCES_NUM_WORKERS`` environment variable to specify how many workers should be used.
+  The default is derived from the number of CPUs but capped at 10.
+  Also the ``mexists()`` method has an explicit parameter to allow the number of workers to be specified.
+  Added ``$LSST_RESOURCES_EXECUTOR`` to specify how the jobs should be executed.
+  The default is ``threads`` (which is the same as used previously) but on Linux more performance may be achievable by setting this environment variable to ``process``. (`DM-50074 <https://rubinobs.atlassian.net/browse/DM-50074>`_)
+- * Fixed problem with multiple ``flush()`` calls with S3 resource handle for small chunks.
+  * Fixed bug in File resource handle where ``flush()`` was mistakenly calling ``close()``. (`DM-51087 <https://rubinobs.atlassian.net/browse/DM-51087>`_)
+
 Resources v29.0.0 (2025-03-25)
 ==============================
 

{lsst_resources-29.2025.2100 → lsst_resources-29.2025.2500}/python/lsst/resources/_resourceHandles/_fileResourceHandle.py

@@ -79,7 +79,7 @@ class FileResourceHandle(BaseResourceHandle[U]):
         return self._fileHandle.fileno()
 
     def flush(self) -> None:
-        self._fileHandle.close()
+        self._fileHandle.flush()
 
     @property
     def isatty(self) -> bool:

{lsst_resources-29.2025.2100 → lsst_resources-29.2025.2500}/python/lsst/resources/_resourceHandles/_s3ResourceHandle.py

@@ -14,14 +14,12 @@ from __future__ import annotations
 __all__ = ("S3ResourceHandle",)
 
 import logging
-import warnings
 from collections.abc import Iterable, Mapping
 from io import SEEK_CUR, SEEK_END, SEEK_SET, BytesIO, UnsupportedOperation
 from typing import TYPE_CHECKING
 
 from botocore.exceptions import ClientError
 
-from lsst.utils.introspection import find_outside_stacklevel
 from lsst.utils.timer import time_this
 
 from ..s3utils import all_retryable_errors, backoff, max_retry_time, translate_client_error
@@ -168,21 +166,9 @@ class S3ResourceHandle(BaseResourceHandle[bytes]):
         # written to.
         s3_min_bits = 5 * 1024 * 1024  # S3 flush threshold is 5 Mib.
         if (
-            (self.tell() - (self._last_flush_position or 0)) < s3_min_bits
-            and self._closed != CloseStatus.CLOSING
-            and not self._warned
-        ):
-            amount = s3_min_bits / (1024 * 1024)
-            warnings.warn(
-                f"S3 does not support flushing objects less than {amount} Mib, skipping",
-                stacklevel=find_outside_stacklevel(
-                    "lsst.resources",
-                    "backoff",
-                    "contextlib",
-                    allow_modules={"lsst.resources.tests"},
-                ),
-            )
-            self._warned = True
+            self.tell() - (self._last_flush_position or 0)
+        ) < s3_min_bits and self._closed != CloseStatus.CLOSING:
+            # Return until the buffer is big enough.
             return
         # nothing to write, don't create an empty upload
         if self.tell() == 0:

{lsst_resources-29.2025.2100 → lsst_resources-29.2025.2500}/python/lsst/resources/_resourcePath.py

@@ -649,9 +649,11 @@ class ResourcePath:  # numpydoc ignore=PR02
         # Disallow a change in scheme
         if "scheme" in kwargs:
             raise ValueError(f"Can not use replace() method to change URI scheme for {self}")
-        return self.__class__(
+        result = self.__class__(
             self._uri._replace(**kwargs), forceDirectory=forceDirectory, isTemporary=isTemporary
         )
+        result._copy_extra_attributes(self)
+        return result
 
     def updatedFile(self, newfile: str) -> ResourcePath:
         """Return new URI with an updated final component of the path.
@@ -1253,7 +1255,10 @@ class ResourcePath:  # numpydoc ignore=PR02
         """
         return self
 
-    def _as_local(self, multithreaded: bool = True, tmpdir: ResourcePath | None = None) -> tuple[str, bool]:
+    @contextlib.contextmanager
+    def _as_local(
+        self, multithreaded: bool = True, tmpdir: ResourcePath | None = None
+    ) -> Iterator[ResourcePath]:
         """Return the location of the (possibly remote) resource as local file.
 
         This is a helper function for `as_local` context manager.
@@ -1272,13 +1277,9 @@ class ResourcePath:  # numpydoc ignore=PR02
 
         Returns
         -------
-        path : `str`
-            If this is a remote resource, it will be a copy of the resource
-            on the local file system, probably in a temporary directory.
-            For a local resource this should be the actual path to the
-            resource.
-        is_temporary : `bool`
-            Indicates if the local path is a temporary file or not.
+        local_uri : `ResourcePath`
+            A URI to a local POSIX file. This can either be the same resource
+            or a local downloaded copy of the resource.
         """
         raise NotImplementedError()
 
@@ -1328,18 +1329,8 @@ class ResourcePath:  # numpydoc ignore=PR02
         temp_dir = ResourcePath(tmpdir, forceDirectory=True) if tmpdir is not None else None
         if temp_dir is not None and not temp_dir.isLocal:
             raise ValueError(f"Temporary directory for as_local must be local resource not {temp_dir}")
-        local_src, is_temporary = self._as_local(multithreaded=multithreaded, tmpdir=temp_dir)
-        local_uri = ResourcePath(local_src, isTemporary=is_temporary)
-
-        try:
+        with self._as_local(multithreaded=multithreaded, tmpdir=temp_dir) as local_uri:
             yield local_uri
-        finally:
-            # The caller might have relocated the temporary file.
-            # Do not ever delete if the temporary matches self
-            # (since it may have been that a temporary file was made local
-            # but already was local).
-            if self != local_uri and is_temporary and local_uri.exists():
-                local_uri.remove()
 
     @classmethod
     @contextlib.contextmanager
@@ -1903,6 +1894,12 @@ class ResourcePath:  # numpydoc ignore=PR02
         """
         raise NotImplementedError(f"URL signing is not supported for '{self.scheme}'")
 
+    def _copy_extra_attributes(self, original_uri: ResourcePath) -> None:
+        # May be overridden by subclasses to transfer attributes when a
+        # ResourcePath is constructed using the "clone" version of the
+        # ResourcePath constructor by passing in a ResourcePath object.
+        pass
+
 
 ResourcePathExpression = str | urllib.parse.ParseResult | ResourcePath | Path
 """Type-annotation alias for objects that can be coerced to ResourcePath.

{lsst_resources-29.2025.2100 → lsst_resources-29.2025.2500}/python/lsst/resources/dav.py

@@ -172,7 +172,21 @@ dav_globals: DavGlobals = DavGlobals()
 
 
 class DavResourcePath(ResourcePath):
-    """WebDAV resource."""
+    """WebDAV resource.
+
+    Parameters
+    ----------
+    uri : `ResourcePathExpression`
+        URI to store in object.
+    root : `str` or `ResourcePath` or `None`, optional
+        Root for relative URIs. Not used in this constructor.
+    forceAbsolute : `bool`
+        Whether to force absolute URI. A WebDAV URI is always absolute.
+    forceDirectory : `bool` or `None`, optional
+        Whether this URI represents a directory.
+    isTemporary : `bool` or `None`, optional
+        Whether this URI represents a temporary resource.
+    """
 
     def __init__(
         self,
@@ -382,7 +396,10 @@ class DavResourcePath(ResourcePath):
         headers.update({"Accept-Encoding": "identity"})
         return self._client.read_range(self._internal_url, start=start, end=end, headers=headers)
 
-    def _as_local(self, multithreaded: bool = True, tmpdir: ResourcePath | None = None) -> tuple[str, bool]:
+    @contextlib.contextmanager
+    def _as_local(
+        self, multithreaded: bool = True, tmpdir: ResourcePath | None = None
+    ) -> Iterator[ResourcePath]:
         """Download object and place in temporary directory.
 
         Parameters
@@ -399,10 +416,9 @@ class DavResourcePath(ResourcePath):
 
         Returns
         -------
-        path : `str`
-            Path to local temporary file.
-        temporary : `bool`
-            Always returns `True`. This is always a temporary file.
+        local_uri : `ResourcePath`
+            A URI to a local POSIX file corresponding to a local temporary
+            downloaded copy of the resource.
         """
         # We need to ensure that this resource is actually a file. dCache
         # responds with a HTML-formatted content to a HTTP GET request to a
@@ -417,9 +433,9 @@ class DavResourcePath(ResourcePath):
         else:
             buffer_size = _calc_tmpdir_buffer_size(tmpdir.ospath)
 
-        with ResourcePath.temporary_uri(suffix=self.getExtension(), prefix=tmpdir, delete=False) as tmp_uri:
+        with ResourcePath.temporary_uri(suffix=self.getExtension(), prefix=tmpdir, delete=True) as tmp_uri:
             self._client.download(self._internal_url, tmp_uri.ospath, buffer_size)
-            return tmp_uri.ospath, True
+            yield tmp_uri
 
     def write(self, data: BinaryIO | bytes, overwrite: bool = True) -> None:
         """Write the supplied bytes to the new resource.
@@ -470,7 +486,7 @@ class DavResourcePath(ResourcePath):
 
         Parameters
         ----------
-        recursive: `bool`
+        recursive : `bool`
             If `True` recursively remove all files and directories under this
             directory.
 

{lsst_resources-29.2025.2100 → lsst_resources-29.2025.2500}/python/lsst/resources/file.py

@@ -79,7 +79,10 @@ class FileResourcePath(ResourcePath):
         """Remove the resource."""
         os.remove(self.ospath)
 
-    def _as_local(self, multithreaded: bool = True, tmpdir: ResourcePath | None = None) -> tuple[str, bool]:
+    @contextlib.contextmanager
+    def _as_local(
+        self, multithreaded: bool = True, tmpdir: ResourcePath | None = None
+    ) -> Iterator[ResourcePath]:
         """Return the local path of the file.
 
         This is an internal helper for ``as_local()``.
@@ -93,12 +96,10 @@ class FileResourcePath(ResourcePath):
 
         Returns
         -------
-        path : `str`
-            The local path to this file.
-        temporary : `bool`
-            Always returns the temporary nature of the input file resource.
+        local_uri : `ResourcePath`
+            A local URI. In this case it will be itself.
         """
-        return self.ospath, self.isTemporary
+        yield self
 
     def read(self, size: int = -1) -> bytes:
         with open(self.ospath, "rb") as fh:

{lsst_resources-29.2025.2100 → lsst_resources-29.2025.2500}/python/lsst/resources/gs.py

@@ -202,17 +202,20 @@ class GSResourcePath(ResourcePath):
         # Should this method do anything at all?
         self.blob.upload_from_string(b"", retry=_RETRY_POLICY)
 
-    def _as_local(self, multithreaded: bool = True, tmpdir: ResourcePath | None = None) -> tuple[str, bool]:
+    @contextlib.contextmanager
+    def _as_local(
+        self, multithreaded: bool = True, tmpdir: ResourcePath | None = None
+    ) -> Iterator[ResourcePath]:
         with (
-            ResourcePath.temporary_uri(prefix=tmpdir, suffix=self.getExtension(), delete=False) as tmp_uri,
+            ResourcePath.temporary_uri(prefix=tmpdir, suffix=self.getExtension(), delete=True) as tmp_uri,
             time_this(log, msg="Downloading %s to local file", args=(self,)),
         ):
             try:
                 with tmp_uri.open("wb") as tmpFile:
                     self.blob.download_to_file(tmpFile, retry=_RETRY_POLICY)
+                yield tmp_uri
             except NotFound as e:
                 raise FileNotFoundError(f"No such resource: {self}") from e
-        return tmp_uri.ospath, True
 
     def transfer_from(
         self,

{lsst_resources-29.2025.2100 → lsst_resources-29.2025.2500}/python/lsst/resources/http.py

@@ -759,6 +759,42 @@ class HttpResourcePath(ResourcePath):
         a HTTP URL. The value of the variable is not inspected.
     """
 
+    @staticmethod
+    def create_http_resource_path(
+        path: str, *, extra_headers: dict[str, str] | None = None
+    ) -> HttpResourcePath:
+        """Create an instance of `HttpResourcePath` with additional
+        HTTP-specific configuration.
+
+        Parameters
+        ----------
+        path : `str`
+            HTTP URL to be wrapped in a `ResourcePath` instance.
+        extra_headers : `dict` [ `str`, `str` ], optional
+            Additional headers that will be sent with every HTTP request made
+            by this `ResourcePath`.  These override any headers that may be
+            generated internally by `HttpResourcePath` (e.g. authentication
+            headers).
+
+        Return
+        ------
+        instance : `ResourcePath`
+            Newly-created `HttpResourcePath` instance.
+
+        Notes
+        -----
+        Most users should use the `ResourcePath` constructor, instead.
+        """
+        # Make sure we instantiate ResourcePath using a string to guarantee we
+        # get a new ResourcePath.  If we accidentally provided a ResourcePath
+        # instance instead, the ResourcePath constructor sometimes returns
+        # the original object and we would be modifying an object that is
+        # supposed to be immutable.
+        instance = ResourcePath(str(path))
+        assert isinstance(instance, HttpResourcePath)
+        instance._extra_headers = extra_headers
+        return instance
+
     # WebDAV servers known to be able to sign URLs. The values are lowercased
     # server identifiers retrieved from the 'Server' header included in
     # the response to a HTTP OPTIONS request.
@@ -805,39 +841,48 @@ class HttpResourcePath(ResourcePath):
     # and is shared by all instances of this class.
     _tcp_connector: TCPConnector | None = None
 
+    # Additional headers added to every request.
+    _extra_headers: dict[str, str] | None = None
+
     @property
-    def metadata_session(self) -> requests.Session:
+    def metadata_session(self) -> _SessionWrapper:
         """Client session to send requests which do not require upload or
         download of data, i.e. mostly metadata requests.
         """
+        session = None
         if hasattr(self, "_metadata_session"):
             if HttpResourcePath._pid == os.getpid():
-                return self._metadata_session
+                session = self._metadata_session
             else:
                 # The metadata session we have in cache was likely created by
                 # a parent process. Discard all the sessions in that store.
                 self._metadata_session_store.clear()
 
         # Retrieve a new metadata session.
-        HttpResourcePath._pid = os.getpid()
-        self._metadata_session: requests.Session = self._metadata_session_store.get(self)
-        return self._metadata_session
+        if session is None:
+            HttpResourcePath._pid = os.getpid()
+            session = self._metadata_session_store.get(self)
+            self._metadata_session: requests.Session = session
+        return _SessionWrapper(session, extra_headers=self._extra_headers)
 
     @property
-    def data_session(self) -> requests.Session:
+    def data_session(self) -> _SessionWrapper:
         """Client session for uploading and downloading data."""
+        session = None
         if hasattr(self, "_data_session"):
             if HttpResourcePath._pid == os.getpid():
-                return self._data_session
+                session = self._data_session
             else:
                 # The data session we have in cache was likely created by
                 # a parent process. Discard all the sessions in that store.
                 self._data_session_store.clear()
 
         # Retrieve a new data session.
-        HttpResourcePath._pid = os.getpid()
-        self._data_session: requests.Session = self._data_session_store.get(self)
-        return self._data_session
+        if session is None:
+            HttpResourcePath._pid = os.getpid()
+            session = self._data_session_store.get(self)
+            self._data_session: requests.Session = session
+        return _SessionWrapper(session, extra_headers=self._extra_headers)
 
     def _clear_sessions(self) -> None:
         """Close the socket connections that are still open.
@@ -1486,7 +1531,10 @@ class HttpResourcePath(ResourcePath):
         except json.JSONDecodeError:
             raise ValueError(f"could not deserialize response to POST request for URL {self}")
 
-    def _as_local(self, multithreaded: bool = True, tmpdir: ResourcePath | None = None) -> tuple[str, bool]:
+    @contextlib.contextmanager
+    def _as_local(
+        self, multithreaded: bool = True, tmpdir: ResourcePath | None = None
+    ) -> Iterator[ResourcePath]:
         """Download object over HTTP and place in temporary directory.
 
         Parameters
@@ -1503,10 +1551,9 @@ class HttpResourcePath(ResourcePath):
 
         Returns
         -------
-        path : `str`
-            Path to local temporary file.
-        temporary : `bool`
-            Always returns `True`. This is always a temporary file.
+        local_uri : `ResourcePath`
+            A URI to a local POSIX file corresponding to a local temporary
+            downloaded copy of the resource.
         """
         # Use the session as a context manager to ensure that connections
         # to both the front end and back end servers are closed after the
@@ -1525,7 +1572,7 @@ class HttpResourcePath(ResourcePath):
                 buffer_size = _calc_tmpdir_buffer_size(tmpdir.ospath)
 
             with ResourcePath.temporary_uri(
-                suffix=self.getExtension(), prefix=tmpdir, delete=False
+                suffix=self.getExtension(), prefix=tmpdir, delete=True
             ) as tmp_uri:
                 expected_length = int(resp.headers.get("Content-Length", "-1"))
                 with time_this(
@@ -1541,20 +1588,20 @@ class HttpResourcePath(ResourcePath):
                             tmpFile.write(chunk)
                             content_length += len(chunk)
 
-            # Check that the expected and actual content lengths match. Perform
-            # this check only when the contents of the file was not encoded by
-            # the server.
-            if (
-                "Content-Encoding" not in resp.headers
-                and expected_length >= 0
-                and expected_length != content_length
-            ):
-                raise ValueError(
-                    f"Size of downloaded file does not match value in Content-Length header for {self}: "
-                    f"expecting {expected_length} and got {content_length} bytes"
-                )
+                # Check that the expected and actual content lengths match.
+                # Perform this check only when the contents of the file was not
+                # encoded by the server.
+                if (
+                    "Content-Encoding" not in resp.headers
+                    and expected_length >= 0
+                    and expected_length != content_length
+                ):
+                    raise ValueError(
+                        f"Size of downloaded file does not match value in Content-Length header for {self}: "
+                        f"expecting {expected_length} and got {content_length} bytes"
+                    )
 
-            return tmpFile.name, True
+                yield tmp_uri
 
     def _send_webdav_request(
         self,
@@ -1562,7 +1609,7 @@ class HttpResourcePath(ResourcePath):
         url: str | None = None,
         headers: dict[str, str] | None = None,
         body: str | None = None,
-        session: requests.Session | None = None,
+        session: _SessionWrapper | None = None,
         timeout: tuple[float, float] | None = None,
     ) -> requests.Response:
         """Send a webDAV request and correctly handle redirects.
@@ -1983,6 +2030,10 @@ class HttpResourcePath(ResourcePath):
             with super()._openImpl(mode, encoding=encoding) as http_handle:
                 yield http_handle
 
+    def _copy_extra_attributes(self, original_uri: ResourcePath) -> None:
+        assert isinstance(original_uri, HttpResourcePath)
+        self._extra_headers = original_uri._extra_headers
+
 
 def _dump_response(resp: requests.Response) -> None:
     """Log the contents of a HTTP or webDAV request and its response.
@@ -2193,3 +2244,95 @@ class DavProperty:
     @property
     def href(self) -> str:
         return self._href
+
+
+class _SessionWrapper(contextlib.AbstractContextManager):
+    """Wraps a `requests.Session` to allow header values to be injected with
+    all requests.
+
+    Notes
+    -----
+    `requests.Session` already has a feature for setting headers globally, but
+    our session objects are global and authorization headers can vary for each
+    HttpResourcePath instance.
+    """
+
+    def __init__(self, session: requests.Session, *, extra_headers: dict[str, str] | None) -> None:
+        self._session = session
+        self._extra_headers = extra_headers
+
+    def __enter__(self) -> _SessionWrapper:
+        self._session.__enter__()
+        return self
+
+    def __exit__(
+        self,
+        exc_type: Any,
+        exc_value: Any,
+        traceback: Any,
+    ) -> None:
+        return self._session.__exit__(exc_type, exc_value, traceback)
+
+    def get(
+        self,
+        url: str,
+        *,
+        timeout: tuple[float, float],
+        allow_redirects: bool = True,
+        stream: bool,
+        headers: dict[str, str] | None = None,
+    ) -> requests.Response:
+        return self._session.get(
+            url,
+            timeout=timeout,
+            allow_redirects=allow_redirects,
+            stream=stream,
+            headers=self._augment_headers(headers),
+        )
+
+    def head(
+        self,
+        url: str,
+        *,
+        timeout: tuple[float, float],
+        allow_redirects: bool,
+        stream: bool,
+        headers: dict[str, str] | None = None,
+    ) -> requests.Response:
+        return self._session.head(
+            url,
+            timeout=timeout,
+            allow_redirects=allow_redirects,
+            stream=stream,
+            headers=self._augment_headers(headers),
+        )
+
+    def request(
+        self,
+        method: str,
+        url: str,
+        *,
+        data: str | bytes | BinaryIO | None,
+        timeout: tuple[float, float],
+        allow_redirects: bool,
+        stream: bool,
+        headers: dict[str, str] | None = None,
+    ) -> requests.Response:
+        return self._session.request(
+            method,
+            url,
+            data=data,
+            timeout=timeout,
+            allow_redirects=allow_redirects,
+            stream=stream,
+            headers=self._augment_headers(headers),
+        )
+
+    def _augment_headers(self, headers: dict[str, str] | None) -> dict[str, str]:
+        if headers is None:
+            headers = {}
+
+        if self._extra_headers is not None:
+            headers = headers | self._extra_headers
+
+        return headers

{lsst_resources-29.2025.2100 → lsst_resources-29.2025.2500}/python/lsst/resources/mem.py

@@ -13,6 +13,9 @@ from __future__ import annotations
 
 __all__ = ("InMemoryResourcePath",)
 
+import contextlib
+from collections.abc import Iterator
+
 from ._resourcePath import ResourcePath
 
 
@@ -27,5 +30,8 @@ class InMemoryResourcePath(ResourcePath):
         """Test for existence and always return False."""
         return True
 
-    def _as_local(self, multithreaded: bool = True, tmpdir: ResourcePath | None = None) -> tuple[str, bool]:
+    @contextlib.contextmanager
+    def _as_local(
+        self, multithreaded: bool = True, tmpdir: ResourcePath | None = None
+    ) -> Iterator[ResourcePath]:
         raise RuntimeError(f"Do not know how to retrieve data for URI '{self}'")

{lsst_resources-29.2025.2100 → lsst_resources-29.2025.2500}/python/lsst/resources/s3.py

@@ -477,7 +477,10 @@ class S3ResourcePath(ResourcePath):
 
         return s3, f"{self._bucket}/{self.relativeToPathRoot}"
 
-    def _as_local(self, multithreaded: bool = True, tmpdir: ResourcePath | None = None) -> tuple[str, bool]:
+    @contextlib.contextmanager
+    def _as_local(
+        self, multithreaded: bool = True, tmpdir: ResourcePath | None = None
+    ) -> Iterator[ResourcePath]:
         """Download object from S3 and place in temporary directory.
 
         Parameters
@@ -494,13 +497,12 @@ class S3ResourcePath(ResourcePath):
 
         Returns
         -------
-        path : `str`
-            Path to local temporary file.
-        temporary : `bool`
-            Always returns `True`. This is always a temporary file.
+        local_uri : `ResourcePath`
+            A URI to a local POSIX file corresponding to a local temporary
+            downloaded copy of the resource.
         """
         with (
-            ResourcePath.temporary_uri(prefix=tmpdir, suffix=self.getExtension(), delete=False) as tmp_uri,
+            ResourcePath.temporary_uri(prefix=tmpdir, suffix=self.getExtension(), delete=True) as tmp_uri,
             self._use_threads_temp_override(multithreaded),
             time_this(log, msg="Downloading %s to local file", args=(self,)),
         ):
@@ -511,7 +513,7 @@ class S3ResourcePath(ResourcePath):
             )
             with tmp_uri.open("wb") as tmpFile:
                 self._download_file(tmpFile, progress)
-        return tmp_uri.ospath, True
+            yield tmp_uri
 
     @backoff.on_exception(backoff.expo, all_retryable_errors, max_time=max_retry_time)
     def _upload_file(self, local_file: ResourcePath, progress: ProgressPercentage | None) -> None:
@@ -542,7 +544,7 @@ class S3ResourcePath(ResourcePath):
         try:
             self.client.copy_object(CopySource=copy_source, Bucket=self._bucket, Key=self.relativeToPathRoot)
         except (self.client.exceptions.NoSuchKey, self.client.exceptions.NoSuchBucket) as err:
-            raise FileNotFoundError("No such resource to transfer: {self}") from err
+            raise FileNotFoundError(f"No such resource to transfer: {src} -> {self}") from err
         except ClientError as err:
             translate_client_error(err, self)
             raise
@@ -609,10 +611,10 @@ class S3ResourcePath(ResourcePath):
         timer_msg = "Transfer from %s to %s"
         timer_args = (src, self)
 
-        if isinstance(src, type(self)):
-            # Looks like an S3 remote uri so we can use direct copy
-            # note that boto3.resource.meta.copy is cleverer than the low
-            # level copy_object
+        if isinstance(src, type(self)) and self.client == src.client:
+            # Looks like an S3 remote uri so we can use direct copy.
+            # This only works if the source and destination are using the same
+            # S3 endpoint and profile.
             with time_this(log, msg=timer_msg, args=timer_args):
                 self._copy_from(src)
 

{lsst_resources-29.2025.2100 → lsst_resources-29.2025.2500}/python/lsst/resources/tests.py

@@ -61,18 +61,18 @@ def _check_open(
     """
     text_content = "abcdefghijklmnopqrstuvwxyz🙂"
     bytes_content = uuid.uuid4().bytes
-    content_by_mode_suffix = {
+    content_by_mode_suffix: dict[str, str | bytes] = {
         "": text_content,
         "t": text_content,
         "b": bytes_content,
     }
-    empty_content_by_mode_suffix = {
+    empty_content_by_mode_suffix: dict[str, str | bytes] = {
         "": "",
         "t": "",
         "b": b"",
     }
     # To appease mypy
-    double_content_by_mode_suffix = {
+    double_content_by_mode_suffix: dict[str, str | bytes] = {
         "": text_content + text_content,
         "t": text_content + text_content,
         "b": bytes_content + bytes_content,
@@ -143,6 +143,16 @@ def _check_open(
             content_read = read_buffer.read()
             test_case.assertEqual(len(content_read), 0, f"Read: {content_read!r}, expected empty.")
 
+        # Write multiple chunks with flushing to ensure that any handles that
+        # cache without flushing work properly.
+        n = 3
+        with uri.open("w" + mode_suffix, **kwargs) as write_buffer:
+            for _ in range(n):
+                write_buffer.write(content)
+                write_buffer.flush()
+        with uri.open("r" + mode_suffix, **kwargs) as read_buffer:
+            test_case.assertEqual(read_buffer.read(), content * n)
+
         # Write two copies of the content, overwriting the single copy there.
         with uri.open("w" + mode_suffix, **kwargs) as write_buffer:
             write_buffer.write(double_content)

lsst_resources-29.2025.2500/python/lsst/resources/version.py

@@ -0,0 +1,2 @@
+__all__ = ["__version__"]
+__version__ = "29.2025.2500"

{lsst_resources-29.2025.2100 → lsst_resources-29.2025.2500/python/lsst_resources.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lsst-resources
-Version: 29.2025.2100
+Version: 29.2025.2500
 Summary: An abstraction layer for reading and writing from URI file resources.
 Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
 License: BSD 3-Clause License

{lsst_resources-29.2025.2100 → lsst_resources-29.2025.2500}/tests/test_dav.py

@@ -284,12 +284,12 @@ class DavReadWriteTestCase(GenericReadWriteTestCase, unittest.TestCase):
         self.assertTrue(remote_file.exists())
         self.assertEqual(remote_file.size(), len(contents))
 
-        local_path, is_temp = remote_file._as_local()
-        self.assertTrue(is_temp)
-        self.assertTrue(os.path.exists(local_path))
-        self.assertTrue(os.stat(local_path).st_size, len(contents))
-        self.assertEqual(ResourcePath(local_path).read(), contents)
-        os.remove(local_path)
+        with remote_file._as_local() as local_uri:
+            self.assertTrue(local_uri.isTemporary)
+            self.assertTrue(os.path.exists(local_uri.ospath))
+            self.assertTrue(os.stat(local_uri.ospath).st_size, len(contents))
+            self.assertEqual(local_uri.read(), contents)
+        self.assertFalse(local_uri.exists())
 
     def test_dav_size(self):
         # Retrieving the size of a non-existent file must raise.

{lsst_resources-29.2025.2100 → lsst_resources-29.2025.2500}/tests/test_http.py

@@ -12,6 +12,7 @@
 import hashlib
 import io
 import os.path
+import pickle
 import random
 import shutil
 import socket
@@ -20,6 +21,7 @@ import string
 import tempfile
 import time
 import unittest
+import unittest.mock
 import warnings
 from collections.abc import Callable
 from threading import Thread
@@ -33,6 +35,7 @@ except ImportError:
 
 import requests
 import responses
+import responses.matchers
 
 import lsst.resources
 from lsst.resources import ResourcePath
@@ -82,6 +85,38 @@ class GenericHttpTestCase(GenericTestCase, unittest.TestCase):
             ResourcePath("http://user:password@server.com:3000/"),
         )
 
+    @responses.activate
+    def test_extra_headers(self):
+        url = "http://test.example/something.txt"
+        path = HttpResourcePath.create_http_resource_path(
+            url, extra_headers={"Authorization": "Bearer my-token"}
+        )
+
+        self.assertEqual(str(path), "http://test.example/something.txt")
+        self.assertEqual(path._extra_headers, {"Authorization": "Bearer my-token"})
+
+        # Make sure that headers are added to requests.
+        responses.add(
+            responses.GET,
+            url,
+            b"test",
+            match=[responses.matchers.header_matcher({"Authorization": "Bearer my-token"})],
+        )
+        self.assertEqual(path.read(), b"test")
+
+        # Extra headers should be preserved through pickle, to ensure that
+        # `mtransfer` and similar methods work in multi-process mode.
+        dump = pickle.dumps(path)
+        restored = pickle.loads(dump)
+        self.assertEqual(restored._extra_headers, {"Authorization": "Bearer my-token"})
+
+        # Extra headers should be preserved when making a modified copy of the
+        # ResourcePath using replace() or the ResourcePath constructor.
+        replacement = path.replace(forceDirectory=True)
+        self.assertEqual(replacement._extra_headers, {"Authorization": "Bearer my-token"})
+        copy = ResourcePath(path, forceDirectory=True)
+        self.assertEqual(copy._extra_headers, {"Authorization": "Bearer my-token"})
+
 
 class HttpReadWriteWebdavTestCase(GenericReadWriteTestCase, unittest.TestCase):
     """Test with a real webDAV server, as opposed to mocking responses."""
@@ -280,12 +315,12 @@ class HttpReadWriteWebdavTestCase(GenericReadWriteTestCase, unittest.TestCase):
         remote_file = self.tmpdir.join(self._get_file_name())
         self.assertIsNone(remote_file.write(data=contents, overwrite=True))
 
-        local_path, is_temp = remote_file._as_local()
-        self.assertTrue(is_temp)
-        self.assertTrue(os.path.exists(local_path))
-        self.assertTrue(os.stat(local_path).st_size, len(contents))
-        self.assertEqual(ResourcePath(local_path).read(), contents)
-        os.remove(local_path)
+        with remote_file._as_local() as local_uri:
+            self.assertTrue(local_uri.isTemporary)
+            self.assertTrue(os.path.exists(local_uri.ospath))
+            self.assertTrue(os.stat(local_uri.ospath).st_size, len(contents))
+            self.assertEqual(local_uri.read(), contents)
+        self.assertFalse(local_uri.exists())
 
     def test_dav_size(self):
         # Size of a non-existent file must raise.

{lsst_resources-29.2025.2100 → lsst_resources-29.2025.2500}/tests/test_location.py

@@ -98,7 +98,7 @@ class LocationTestCase(unittest.TestCase):
 
         for uriInfo in uriStrings:
             uri = ResourcePath(uriInfo[0], root=testRoot, forceAbsolute=uriInfo[1], forceDirectory=uriInfo[2])
-            with self.subTest(in_uri=uriInfo[0], out_uri=uri):
+            with self.subTest(in_uri=repr(uriInfo[0]), out_uri=repr(uri)):
                 self.assertEqual(uri.scheme, uriInfo[3], "test scheme")
                 self.assertEqual(uri.netloc, uriInfo[4], "test netloc")
                 self.assertEqual(uri.path, uriInfo[5], "test path")
@@ -115,7 +115,7 @@ class LocationTestCase(unittest.TestCase):
 
         for uriInfo in uriStrings:
             uri = ResourcePath(uriInfo[0], forceAbsolute=uriInfo[1], forceDirectory=uriInfo[2])
-            with self.subTest(in_uri=uriInfo[0], out_uri=uri):
+            with self.subTest(in_uri=repr(uriInfo[0]), out_uri=repr(uri)):
                 self.assertEqual(uri.scheme, uriInfo[3], "test scheme")
                 self.assertEqual(uri.netloc, uriInfo[4], "test netloc")
                 # Use ospath here to ensure that we have unquoted any
@@ -133,7 +133,7 @@ class LocationTestCase(unittest.TestCase):
 
         for uriInfo in uriStrings:
             uri = ResourcePath(uriInfo[0], forceAbsolute=False).updatedFile(uriInfo[1])
-            with self.subTest(in_uri=uriInfo[0], out_uri=uri):
+            with self.subTest(in_uri=repr(uriInfo[0]), out_uri=repr(uri)):
                 self.assertEqual(uri.path, uriInfo[2])
 
         # Check that schemeless can become file scheme.
@@ -333,7 +333,7 @@ class LocationTestCase(unittest.TestCase):
         """Test round tripping of the posix to os.path conversion helpers."""
         testPaths = ("/a/b/c.e", "a/b", "a/b/", "/a/b", "/a/b/", "a/b/c.e")
         for p in testPaths:
-            with self.subTest(path=p):
+            with self.subTest(path=repr(p)):
                 self.assertEqual(os2posix(posix2os(p)), p)
 
     def testSplit(self):
@@ -368,7 +368,7 @@ class LocationTestCase(unittest.TestCase):
         )
 
         for p, e in zip(testPaths, expected, strict=True):
-            with self.subTest(path=p):
+            with self.subTest(path=repr(p)):
                 uri = ResourcePath(p, testRoot)
                 head, tail = uri.split()
                 self.assertEqual((head.geturl(), tail), e)

{lsst_resources-29.2025.2100 → lsst_resources-29.2025.2500}/tests/test_s3.py

@@ -351,6 +351,17 @@ class S3WithProfileReadWriteTestCase(S3ReadWriteTestCaseBase, unittest.TestCase)
         self.assertEqual(path2._bucket, "ceph:bucket2")
         self.assertIsNone(path2._profile)
 
+    def test_transfer_from_different_endpoints(self):
+        # Create a bucket using a different endpoint (the default endpoint.)
+        boto3.resource("s3").create_bucket(Bucket="source-bucket")
+        source_path = ResourcePath("s3://source-bucket/file.txt")
+        source_path.write(b"123")
+        target_path = ResourcePath(f"s3://{self.netloc}/target.txt")
+        # Transfer from default endpoint to custom endpoint with custom
+        # profile.
+        target_path.transfer_from(source_path)
+        self.assertEqual(target_path.read(), b"123")
+
 
 if __name__ == "__main__":
     unittest.main()

lsst_resources-29.2025.2100/python/lsst/resources/version.py

@@ -1,2 +0,0 @@
-__all__ = ["__version__"]
-__version__ = "29.2025.2100"