lsst-resources 29.2025.1700__py3-none-any.whl → 29.2025.4600__py3-none-any.whl

lsst/resources/file.py

@@ -21,6 +21,7 @@ import os.path
 import posixpath
 import re
 import shutil
+import stat
 import urllib.parse
 from collections.abc import Iterator
 from typing import IO, TYPE_CHECKING
@@ -79,7 +80,10 @@ class FileResourcePath(ResourcePath):
         """Remove the resource."""
         os.remove(self.ospath)
 
-    def _as_local(self, multithreaded: bool = True, tmpdir: ResourcePath | None = None) -> tuple[str, bool]:
+    @contextlib.contextmanager
+    def _as_local(
+        self, multithreaded: bool = True, tmpdir: ResourcePath | None = None
+    ) -> Iterator[ResourcePath]:
         """Return the local path of the file.
 
         This is an internal helper for ``as_local()``.
@@ -93,12 +97,10 @@ class FileResourcePath(ResourcePath):
 
         Returns
         -------
-        path : `str`
-            The local path to this file.
-        temporary : `bool`
-            Always returns the temporary nature of the input file resource.
+        local_uri : `ResourcePath`
+            A local URI. In this case it will be itself.
         """
-        return self.ospath, self.isTemporary
+        yield self
 
     def read(self, size: int = -1) -> bytes:
         with open(self.ospath, "rb") as fh:
@@ -106,7 +108,7 @@ class FileResourcePath(ResourcePath):
 
     def write(self, data: bytes, overwrite: bool = True) -> None:
         dir = os.path.dirname(self.ospath)
-        if not os.path.exists(dir):
+        if dir and not os.path.exists(dir):
             _create_directories(dir)
         mode = "wb" if overwrite else "xb"
         with open(self.ospath, mode) as f:
@@ -327,15 +329,38 @@ class FileResourcePath(ResourcePath):
                 # the same output directory. This at least guarantees that
                 # if multiple processes are writing to the same file
                 # simultaneously the file we end up with will not be corrupt.
-                with self.temporary_uri(prefix=self.parent(), suffix=self.getExtension()) as temp_copy:
-                    shutil.copy(local_src, temp_copy.ospath)
-                    with transaction.undoWith(f"copy from {local_src}", os.remove, newFullPath):
-                        # os.rename works even if the file exists.
-                        # It's possible that another process has copied a file
-                        # in whilst this one was copying. If overwrite
-                        # protection is needed then another stat() call should
-                        # happen here.
-                        os.rename(temp_copy.ospath, newFullPath)
+                if overwrite:
+                    with self.temporary_uri(prefix=self.parent(), suffix=self.getExtension()) as temp_copy:
+                        shutil.copy(local_src, temp_copy.ospath)
+                        with transaction.undoWith(f"copy from {local_src}", os.remove, newFullPath):
+                            os.rename(temp_copy.ospath, newFullPath)
+                else:
+                    # Create the file exclusively to ensure that no others are
+                    # trying to write.
+                    temp_path = newFullPath + ".transfer-tmp"
+                    try:
+                        with open(temp_path, "x"):
+                            pass
+                    except FileExistsError:
+                        raise FileExistsError(
+                            f"Another process is writing to '{self}'."
+                            f" Transfer from {src} cannot be completed."
+                        )
+                    with transaction.undoWith(f"copy from {local_src}", os.remove, temp_path):
+                        # Make sure file is writable, no matter the umask.
+                        st = os.stat(temp_path)
+                        os.chmod(temp_path, st.st_mode | stat.S_IWUSR)
+                        shutil.copy(local_src, temp_path)
+                    # Use link/remove to atomically and exclusively move the
+                    # file into place (only one concurrent linker can win).
+                    try:
+                        os.link(temp_path, newFullPath)
+                    except FileExistsError:
+                        raise FileExistsError(
+                            f"Another process wrote to '{self}'. Transfer from {src} cannot be completed."
+                        )
+                    finally:
+                        os.remove(temp_path)
             elif transfer == "link":
                 # Try hard link and if that fails use a symlink
                 with transaction.undoWith(f"link to {local_src}", os.remove, newFullPath):

lsst/resources/gs.py

@@ -202,17 +202,20 @@ class GSResourcePath(ResourcePath):
         # Should this method do anything at all?
         self.blob.upload_from_string(b"", retry=_RETRY_POLICY)
 
-    def _as_local(self, multithreaded: bool = True, tmpdir: ResourcePath | None = None) -> tuple[str, bool]:
+    @contextlib.contextmanager
+    def _as_local(
+        self, multithreaded: bool = True, tmpdir: ResourcePath | None = None
+    ) -> Iterator[ResourcePath]:
         with (
-            ResourcePath.temporary_uri(prefix=tmpdir, suffix=self.getExtension(), delete=False) as tmp_uri,
+            ResourcePath.temporary_uri(prefix=tmpdir, suffix=self.getExtension(), delete=True) as tmp_uri,
             time_this(log, msg="Downloading %s to local file", args=(self,)),
         ):
             try:
                 with tmp_uri.open("wb") as tmpFile:
                     self.blob.download_to_file(tmpFile, retry=_RETRY_POLICY)
+                yield tmp_uri
             except NotFound as e:
                 raise FileNotFoundError(f"No such resource: {self}") from e
-        return tmp_uri.ospath, True
 
     def transfer_from(
         self,

lsst/resources/http.py

@@ -59,7 +59,7 @@ from lsst.utils.timer import time_this
 from ._resourceHandles import ResourceHandleProtocol
 from ._resourceHandles._httpResourceHandle import HttpReadResourceHandle, parse_content_range_header
 from ._resourcePath import ResourcePath
-from .utils import get_tempdir
+from .utils import _get_num_workers, get_tempdir
 
 if TYPE_CHECKING:
     from .utils import TransactionProtocol
@@ -112,13 +112,6 @@ def _calc_tmpdir_buffer_size(tmpdir: str) -> int:
     return max(10 * fsstats.f_bsize, 256 * 4096)
 
 
-def _dav_to_http(url: str) -> str:
-    """Convert dav scheme in URL to http scheme."""
-    if url.startswith("dav"):
-        url = "http" + url.removeprefix("dav")
-    return url
-
-
 class HttpResourcePathConfig:
     """Configuration class to encapsulate the configurable items used by class
     HttpResourcePath.
@@ -165,14 +158,14 @@ class HttpResourcePathConfig:
         if self._front_end_connections is not None:
             return self._front_end_connections
 
+        default_pool_size = max(_get_num_workers(), self.DEFAULT_FRONTEND_PERSISTENT_CONNECTIONS)
+
         try:
             self._front_end_connections = int(
-                os.environ.get(
-                    "LSST_HTTP_FRONTEND_PERSISTENT_CONNECTIONS", self.DEFAULT_FRONTEND_PERSISTENT_CONNECTIONS
-                )
+                os.environ.get("LSST_HTTP_FRONTEND_PERSISTENT_CONNECTIONS", default_pool_size)
             )
         except ValueError:
-            self._front_end_connections = self.DEFAULT_FRONTEND_PERSISTENT_CONNECTIONS
+            self._front_end_connections = default_pool_size
 
         return self._front_end_connections
 
@@ -182,14 +175,14 @@ class HttpResourcePathConfig:
         if self._back_end_connections is not None:
             return self._back_end_connections
 
+        default_pool_size = max(_get_num_workers(), self.DEFAULT_FRONTEND_PERSISTENT_CONNECTIONS)
+
         try:
             self._back_end_connections = int(
-                os.environ.get(
-                    "LSST_HTTP_BACKEND_PERSISTENT_CONNECTIONS", self.DEFAULT_BACKEND_PERSISTENT_CONNECTIONS
-                )
+                os.environ.get("LSST_HTTP_BACKEND_PERSISTENT_CONNECTIONS", default_pool_size)
             )
         except ValueError:
-            self._back_end_connections = self.DEFAULT_BACKEND_PERSISTENT_CONNECTIONS
+            self._back_end_connections = default_pool_size
 
         return self._back_end_connections
 
@@ -437,9 +430,7 @@ def _get_dav_and_server_headers(path: ResourcePath | str) -> tuple[str | None, s
         config = HttpResourcePathConfig()
         with SessionStore(config=config).get(path) as session:
             resp = session.options(
-                _dav_to_http(str(path)),
-                stream=False,
-                timeout=config.timeout,
+                str(path), stream=False, timeout=config.timeout, headers=path._extra_headers
             )
 
             dav_header = server_header = None
@@ -597,7 +588,7 @@ class SessionStore:
     def _make_session(self, rpath: ResourcePath) -> requests.Session:
         """Make a new session configured from values from the environment."""
         session = requests.Session()
-        root_uri = _dav_to_http(str(rpath.root_uri()))
+        root_uri = str(rpath.root_uri())
         log.debug("Creating new HTTP session for endpoint %s ...", root_uri)
         retries = Retry(
             # Total number of retries to allow. Takes precedence over other
@@ -657,9 +648,8 @@ class SessionStore:
         # from request to request. Systematically persisting connections to
         # those servers may exhaust their capabilities when there are thousands
         # of simultaneous clients.
-        scheme = _dav_to_http(rpath.scheme)
         session.mount(
-            f"{scheme}://",
+            f"{rpath.scheme}://",
             HTTPAdapter(
                 pool_connections=self._num_pools,
                 pool_maxsize=0,
@@ -767,6 +757,42 @@ class HttpResourcePath(ResourcePath):
         a HTTP URL. The value of the variable is not inspected.
     """
 
+    @staticmethod
+    def create_http_resource_path(
+        path: str, *, extra_headers: dict[str, str] | None = None
+    ) -> HttpResourcePath:
+        """Create an instance of `HttpResourcePath` with additional
+        HTTP-specific configuration.
+
+        Parameters
+        ----------
+        path : `str`
+            HTTP URL to be wrapped in a `ResourcePath` instance.
+        extra_headers : `dict` [ `str`, `str` ], optional
+            Additional headers that will be sent with every HTTP request made
+            by this `ResourcePath`.  These override any headers that may be
+            generated internally by `HttpResourcePath` (e.g. authentication
+            headers).
+
+        Returns
+        -------
+        instance : `ResourcePath`
+            Newly-created `HttpResourcePath` instance.
+
+        Notes
+        -----
+        Most users should use the `ResourcePath` constructor, instead.
+        """
+        # Make sure we instantiate ResourcePath using a string to guarantee we
+        # get a new ResourcePath.  If we accidentally provided a ResourcePath
+        # instance instead, the ResourcePath constructor sometimes returns
+        # the original object and we would be modifying an object that is
+        # supposed to be immutable.
+        instance = ResourcePath(str(path))
+        assert isinstance(instance, HttpResourcePath)
+        instance._extra_headers = extra_headers
+        return instance
+
     # WebDAV servers known to be able to sign URLs. The values are lowercased
     # server identifiers retrieved from the 'Server' header included in
     # the response to a HTTP OPTIONS request.
@@ -813,39 +839,48 @@ class HttpResourcePath(ResourcePath):
     # and is shared by all instances of this class.
     _tcp_connector: TCPConnector | None = None
 
+    # Additional headers added to every request.
+    _extra_headers: dict[str, str] | None = None
+
     @property
-    def metadata_session(self) -> requests.Session:
+    def metadata_session(self) -> _SessionWrapper:
         """Client session to send requests which do not require upload or
         download of data, i.e. mostly metadata requests.
         """
+        session = None
         if hasattr(self, "_metadata_session"):
             if HttpResourcePath._pid == os.getpid():
-                return self._metadata_session
+                session = self._metadata_session
             else:
                 # The metadata session we have in cache was likely created by
                 # a parent process. Discard all the sessions in that store.
                 self._metadata_session_store.clear()
 
         # Retrieve a new metadata session.
-        HttpResourcePath._pid = os.getpid()
-        self._metadata_session: requests.Session = self._metadata_session_store.get(self)
-        return self._metadata_session
+        if session is None:
+            HttpResourcePath._pid = os.getpid()
+            session = self._metadata_session_store.get(self)
+            self._metadata_session: requests.Session = session
+        return _SessionWrapper(session, extra_headers=self._extra_headers)
 
     @property
-    def data_session(self) -> requests.Session:
+    def data_session(self) -> _SessionWrapper:
         """Client session for uploading and downloading data."""
+        session = None
         if hasattr(self, "_data_session"):
             if HttpResourcePath._pid == os.getpid():
-                return self._data_session
+                session = self._data_session
             else:
                 # The data session we have in cache was likely created by
                 # a parent process. Discard all the sessions in that store.
                 self._data_session_store.clear()
 
         # Retrieve a new data session.
-        HttpResourcePath._pid = os.getpid()
-        self._data_session: requests.Session = self._data_session_store.get(self)
-        return self._data_session
+        if session is None:
+            HttpResourcePath._pid = os.getpid()
+            session = self._data_session_store.get(self)
+            self._data_session: requests.Session = session
+        return _SessionWrapper(session, extra_headers=self._extra_headers)
 
     def _clear_sessions(self) -> None:
         """Close the socket connections that are still open.
@@ -1127,7 +1162,7 @@ class HttpResourcePath(ResourcePath):
         stream = size > 0
         with self.data_session as session:
             with time_this(log, msg="GET %s", args=(self,)):
-                resp = session.get(_dav_to_http(self.geturl()), stream=stream, timeout=self._config.timeout)
+                resp = session.get(self.geturl(), stream=stream, timeout=self._config.timeout)
 
             if resp.status_code != requests.codes.ok:  # 200
                 raise FileNotFoundError(
@@ -1343,16 +1378,12 @@ class HttpResourcePath(ResourcePath):
         path : `str`
             A path that can be opened by the file system object.
         """
-        if (
-            fsspec is None
-            or not self.is_webdav_endpoint
-            or self.server not in HttpResourcePath.SUPPORTED_URL_SIGNERS
-        ):
-            if self.scheme.startswith("dav") and fsspec:
-                # Not webdav so convert to http.
-                return fsspec.url_to_fs(_dav_to_http(self.geturl()))
+        if fsspec is None:
             return super().to_fsspec()
 
+        if not self.is_webdav_endpoint or self.server not in HttpResourcePath.SUPPORTED_URL_SIGNERS:
+            return fsspec.url_to_fs(self.geturl(), client_kwargs={"headers": self._extra_headers})
+
         if self.isdir():
             raise NotImplementedError(
                 f"method HttpResourcePath.to_fsspec() not implemented for directory {self}"
@@ -1494,7 +1525,10 @@ class HttpResourcePath(ResourcePath):
         except json.JSONDecodeError:
             raise ValueError(f"could not deserialize response to POST request for URL {self}")
 
-    def _as_local(self, multithreaded: bool = True, tmpdir: ResourcePath | None = None) -> tuple[str, bool]:
+    @contextlib.contextmanager
+    def _as_local(
+        self, multithreaded: bool = True, tmpdir: ResourcePath | None = None
+    ) -> Iterator[ResourcePath]:
         """Download object over HTTP and place in temporary directory.
 
         Parameters
@@ -1511,16 +1545,15 @@ class HttpResourcePath(ResourcePath):
 
         Returns
         -------
-        path : `str`
-            Path to local temporary file.
-        temporary : `bool`
-            Always returns `True`. This is always a temporary file.
+        local_uri : `ResourcePath`
+            A URI to a local POSIX file corresponding to a local temporary
+            downloaded copy of the resource.
         """
         # Use the session as a context manager to ensure that connections
         # to both the front end and back end servers are closed after the
         # download operation is finished.
         with self.data_session as session:
-            resp = session.get(_dav_to_http(self.geturl()), stream=True, timeout=self._config.timeout)
+            resp = session.get(self.geturl(), stream=True, timeout=self._config.timeout)
             if resp.status_code != requests.codes.ok:
                 raise FileNotFoundError(
                     f"Unable to download resource {self}; status: {resp.status_code} {resp.reason}"
@@ -1533,7 +1566,7 @@ class HttpResourcePath(ResourcePath):
                 buffer_size = _calc_tmpdir_buffer_size(tmpdir.ospath)
 
             with ResourcePath.temporary_uri(
-                suffix=self.getExtension(), prefix=tmpdir, delete=False
+                suffix=self.getExtension(), prefix=tmpdir, delete=True
             ) as tmp_uri:
                 expected_length = int(resp.headers.get("Content-Length", "-1"))
                 with time_this(
@@ -1549,20 +1582,20 @@ class HttpResourcePath(ResourcePath):
                             tmpFile.write(chunk)
                             content_length += len(chunk)
 
-            # Check that the expected and actual content lengths match. Perform
-            # this check only when the contents of the file was not encoded by
-            # the server.
-            if (
-                "Content-Encoding" not in resp.headers
-                and expected_length >= 0
-                and expected_length != content_length
-            ):
-                raise ValueError(
-                    f"Size of downloaded file does not match value in Content-Length header for {self}: "
-                    f"expecting {expected_length} and got {content_length} bytes"
-                )
+                # Check that the expected and actual content lengths match.
+                # Perform this check only when the contents of the file was not
+                # encoded by the server.
+                if (
+                    "Content-Encoding" not in resp.headers
+                    and expected_length >= 0
+                    and expected_length != content_length
+                ):
+                    raise ValueError(
+                        f"Size of downloaded file does not match value in Content-Length header for {self}: "
+                        f"expecting {expected_length} and got {content_length} bytes"
+                    )
 
-            return tmpFile.name, True
+                yield tmp_uri
 
     def _send_webdav_request(
         self,
@@ -1570,7 +1603,7 @@ class HttpResourcePath(ResourcePath):
         url: str | None = None,
         headers: dict[str, str] | None = None,
         body: str | None = None,
-        session: requests.Session | None = None,
+        session: _SessionWrapper | None = None,
         timeout: tuple[float, float] | None = None,
     ) -> requests.Response:
         """Send a webDAV request and correctly handle redirects.
@@ -1633,7 +1666,7 @@ class HttpResourcePath(ResourcePath):
             for _ in range(max_redirects := 5):
                 resp = session.request(
                     method,
-                    _dav_to_http(url),
+                    url,
                     data=body,
                     headers=headers,
                     stream=False,
@@ -1791,7 +1824,7 @@ class HttpResourcePath(ResourcePath):
         src : `HttpResourcePath`
             The source of the contents to move to `self`.
         """
-        headers = {"Destination": _dav_to_http(self.geturl())}
+        headers = {"Destination": self.geturl()}
         resp = self._send_webdav_request(method, url=src.geturl(), headers=headers, session=self.data_session)
         if resp.status_code in (requests.codes.created, requests.codes.no_content):
             return
@@ -1898,7 +1931,7 @@ class HttpResourcePath(ResourcePath):
         if self._config.send_expect_on_put or self.server == "dcache":
             headers["Expect"] = "100-continue"
 
-        url = _dav_to_http(self.geturl())
+        url = self.geturl()
 
         # Use the session as a context manager to ensure the underlying
         # connections are closed after finishing uploading the data.
@@ -1991,6 +2024,10 @@ class HttpResourcePath(ResourcePath):
             with super()._openImpl(mode, encoding=encoding) as http_handle:
                 yield http_handle
 
+    def _copy_extra_attributes(self, original_uri: ResourcePath) -> None:
+        assert isinstance(original_uri, HttpResourcePath)
+        self._extra_headers = original_uri._extra_headers
+
 
 def _dump_response(resp: requests.Response) -> None:
     """Log the contents of a HTTP or webDAV request and its response.
@@ -2201,3 +2238,95 @@ class DavProperty:
     @property
     def href(self) -> str:
         return self._href
+
+
+class _SessionWrapper(contextlib.AbstractContextManager):
+    """Wraps a `requests.Session` to allow header values to be injected with
+    all requests.
+
+    Notes
+    -----
+    `requests.Session` already has a feature for setting headers globally, but
+    our session objects are global and authorization headers can vary for each
+    HttpResourcePath instance.
+    """
+
+    def __init__(self, session: requests.Session, *, extra_headers: dict[str, str] | None) -> None:
+        self._session = session
+        self._extra_headers = extra_headers
+
+    def __enter__(self) -> _SessionWrapper:
+        self._session.__enter__()
+        return self
+
+    def __exit__(
+        self,
+        exc_type: Any,
+        exc_value: Any,
+        traceback: Any,
+    ) -> None:
+        return self._session.__exit__(exc_type, exc_value, traceback)
+
+    def get(
+        self,
+        url: str,
+        *,
+        timeout: tuple[float, float],
+        allow_redirects: bool = True,
+        stream: bool,
+        headers: dict[str, str] | None = None,
+    ) -> requests.Response:
+        return self._session.get(
+            url,
+            timeout=timeout,
+            allow_redirects=allow_redirects,
+            stream=stream,
+            headers=self._augment_headers(headers),
+        )
+
+    def head(
+        self,
+        url: str,
+        *,
+        timeout: tuple[float, float],
+        allow_redirects: bool,
+        stream: bool,
+        headers: dict[str, str] | None = None,
+    ) -> requests.Response:
+        return self._session.head(
+            url,
+            timeout=timeout,
+            allow_redirects=allow_redirects,
+            stream=stream,
+            headers=self._augment_headers(headers),
+        )
+
+    def request(
+        self,
+        method: str,
+        url: str,
+        *,
+        data: str | bytes | BinaryIO | None,
+        timeout: tuple[float, float],
+        allow_redirects: bool,
+        stream: bool,
+        headers: dict[str, str] | None = None,
+    ) -> requests.Response:
+        return self._session.request(
+            method,
+            url,
+            data=data,
+            timeout=timeout,
+            allow_redirects=allow_redirects,
+            stream=stream,
+            headers=self._augment_headers(headers),
+        )
+
+    def _augment_headers(self, headers: dict[str, str] | None) -> dict[str, str]:
+        if headers is None:
+            headers = {}
+
+        if self._extra_headers is not None:
+            headers = headers | self._extra_headers
+
+        return headers

lsst/resources/mem.py

@@ -13,6 +13,9 @@ from __future__ import annotations
 
 __all__ = ("InMemoryResourcePath",)
 
+import contextlib
+from collections.abc import Iterator
+
 from ._resourcePath import ResourcePath
 
 
@@ -27,5 +30,8 @@ class InMemoryResourcePath(ResourcePath):
         """Test for existence and always return False."""
         return True
 
-    def _as_local(self, multithreaded: bool = True, tmpdir: ResourcePath | None = None) -> tuple[str, bool]:
+    @contextlib.contextmanager
+    def _as_local(
+        self, multithreaded: bool = True, tmpdir: ResourcePath | None = None
+    ) -> Iterator[ResourcePath]:
         raise RuntimeError(f"Do not know how to retrieve data for URI '{self}'")