lsst-resources 29.0.0rc7__py3-none-any.whl → 29.2025.4600__py3-none-any.whl

lsst/resources/http.py

@@ -26,7 +26,6 @@ import random
 import re
 import ssl
 import stat
-import tempfile
 from collections.abc import Iterator
 from typing import TYPE_CHECKING, Any, BinaryIO, cast
 
@@ -60,6 +59,7 @@ from lsst.utils.timer import time_this
 from ._resourceHandles import ResourceHandleProtocol
 from ._resourceHandles._httpResourceHandle import HttpReadResourceHandle, parse_content_range_header
 from ._resourcePath import ResourcePath
+from .utils import _get_num_workers, get_tempdir
 
 if TYPE_CHECKING:
     from .utils import TransactionProtocol
@@ -98,6 +98,20 @@ def _timeout_from_environment(env_var: str, default_value: float) -> float:
     return timeout
 
 
+@functools.lru_cache
+def _calc_tmpdir_buffer_size(tmpdir: str) -> int:
+    """Compute the block size as 256 blocks of typical size
+    (i.e. 4096 bytes) or 10 times the file system block size,
+    whichever is higher.
+
+    This is a reasonable compromise between
+    using memory for buffering and the number of system calls
+    issued to read from or write to temporary files.
+    """
+    fsstats = os.statvfs(tmpdir)
+    return max(10 * fsstats.f_bsize, 256 * 4096)
+
+
 class HttpResourcePathConfig:
     """Configuration class to encapsulate the configurable items used by class
     HttpResourcePath.
@@ -144,14 +158,14 @@ class HttpResourcePathConfig:
         if self._front_end_connections is not None:
             return self._front_end_connections
 
+        default_pool_size = max(_get_num_workers(), self.DEFAULT_FRONTEND_PERSISTENT_CONNECTIONS)
+
         try:
             self._front_end_connections = int(
-                os.environ.get(
-                    "LSST_HTTP_FRONTEND_PERSISTENT_CONNECTIONS", self.DEFAULT_FRONTEND_PERSISTENT_CONNECTIONS
-                )
+                os.environ.get("LSST_HTTP_FRONTEND_PERSISTENT_CONNECTIONS", default_pool_size)
             )
         except ValueError:
-            self._front_end_connections = self.DEFAULT_FRONTEND_PERSISTENT_CONNECTIONS
+            self._front_end_connections = default_pool_size
 
         return self._front_end_connections
 
@@ -161,14 +175,14 @@ class HttpResourcePathConfig:
         if self._back_end_connections is not None:
             return self._back_end_connections
 
+        default_pool_size = max(_get_num_workers(), self.DEFAULT_FRONTEND_PERSISTENT_CONNECTIONS)
+
         try:
             self._back_end_connections = int(
-                os.environ.get(
-                    "LSST_HTTP_BACKEND_PERSISTENT_CONNECTIONS", self.DEFAULT_BACKEND_PERSISTENT_CONNECTIONS
-                )
+                os.environ.get("LSST_HTTP_BACKEND_PERSISTENT_CONNECTIONS", default_pool_size)
             )
         except ValueError:
-            self._back_end_connections = self.DEFAULT_BACKEND_PERSISTENT_CONNECTIONS
+            self._back_end_connections = default_pool_size
 
         return self._back_end_connections
 
@@ -363,26 +377,15 @@ class HttpResourcePathConfig:
         if self._tmpdir_buffersize is not None:
             return self._tmpdir_buffersize
 
-        # Use the value of environment variables 'LSST_RESOURCES_TMPDIR' or
-        # 'TMPDIR', if defined. Otherwise use the system temporary directory,
-        # with a last-resort fallback to the current working directory if
-        # nothing else is available.
-        tmpdir = None
-        for dir in (os.getenv(v) for v in ("LSST_RESOURCES_TMPDIR", "TMPDIR")):
-            if dir and os.path.isdir(dir):
-                tmpdir = dir
-                break
-
-        if tmpdir is None:
-            tmpdir = tempfile.gettempdir()
+        tmpdir = get_tempdir()
 
         # Compute the block size as 256 blocks of typical size
         # (i.e. 4096 bytes) or 10 times the file system block size,
         # whichever is higher. This is a reasonable compromise between
         # using memory for buffering and the number of system calls
         # issued to read from or write to temporary files.
-        fsstats = os.statvfs(tmpdir)
-        self._tmpdir_buffersize = (tmpdir, max(10 * fsstats.f_bsize, 256 * 4096))
+        bufsize = _calc_tmpdir_buffer_size(tmpdir)
+        self._tmpdir_buffersize = (tmpdir, bufsize)
 
         return self._tmpdir_buffersize
 
@@ -427,9 +430,7 @@ def _get_dav_and_server_headers(path: ResourcePath | str) -> tuple[str | None, s
         config = HttpResourcePathConfig()
         with SessionStore(config=config).get(path) as session:
             resp = session.options(
-                str(path),
-                stream=False,
-                timeout=config.timeout,
+                str(path), stream=False, timeout=config.timeout, headers=path._extra_headers
             )
 
             dav_header = server_header = None
@@ -756,6 +757,42 @@ class HttpResourcePath(ResourcePath):
         a HTTP URL. The value of the variable is not inspected.
     """
 
+    @staticmethod
+    def create_http_resource_path(
+        path: str, *, extra_headers: dict[str, str] | None = None
+    ) -> HttpResourcePath:
+        """Create an instance of `HttpResourcePath` with additional
+        HTTP-specific configuration.
+
+        Parameters
+        ----------
+        path : `str`
+            HTTP URL to be wrapped in a `ResourcePath` instance.
+        extra_headers : `dict` [ `str`, `str` ], optional
+            Additional headers that will be sent with every HTTP request made
+            by this `ResourcePath`.  These override any headers that may be
+            generated internally by `HttpResourcePath` (e.g. authentication
+            headers).
+
+        Returns
+        -------
+        instance : `ResourcePath`
+            Newly-created `HttpResourcePath` instance.
+
+        Notes
+        -----
+        Most users should use the `ResourcePath` constructor, instead.
+        """
+        # Make sure we instantiate ResourcePath using a string to guarantee we
+        # get a new ResourcePath.  If we accidentally provided a ResourcePath
+        # instance instead, the ResourcePath constructor sometimes returns
+        # the original object and we would be modifying an object that is
+        # supposed to be immutable.
+        instance = ResourcePath(str(path))
+        assert isinstance(instance, HttpResourcePath)
+        instance._extra_headers = extra_headers
+        return instance
+
     # WebDAV servers known to be able to sign URLs. The values are lowercased
     # server identifiers retrieved from the 'Server' header included in
     # the response to a HTTP OPTIONS request.
@@ -802,39 +839,48 @@ class HttpResourcePath(ResourcePath):
     # and is shared by all instances of this class.
     _tcp_connector: TCPConnector | None = None
 
+    # Additional headers added to every request.
+    _extra_headers: dict[str, str] | None = None
+
     @property
-    def metadata_session(self) -> requests.Session:
+    def metadata_session(self) -> _SessionWrapper:
         """Client session to send requests which do not require upload or
         download of data, i.e. mostly metadata requests.
         """
+        session = None
         if hasattr(self, "_metadata_session"):
             if HttpResourcePath._pid == os.getpid():
-                return self._metadata_session
+                session = self._metadata_session
             else:
                 # The metadata session we have in cache was likely created by
                 # a parent process. Discard all the sessions in that store.
                 self._metadata_session_store.clear()
 
         # Retrieve a new metadata session.
-        HttpResourcePath._pid = os.getpid()
-        self._metadata_session: requests.Session = self._metadata_session_store.get(self)
-        return self._metadata_session
+        if session is None:
+            HttpResourcePath._pid = os.getpid()
+            session = self._metadata_session_store.get(self)
+            self._metadata_session: requests.Session = session
+        return _SessionWrapper(session, extra_headers=self._extra_headers)
 
     @property
-    def data_session(self) -> requests.Session:
+    def data_session(self) -> _SessionWrapper:
         """Client session for uploading and downloading data."""
+        session = None
         if hasattr(self, "_data_session"):
             if HttpResourcePath._pid == os.getpid():
-                return self._data_session
+                session = self._data_session
             else:
                 # The data session we have in cache was likely created by
                 # a parent process. Discard all the sessions in that store.
                 self._data_session_store.clear()
 
         # Retrieve a new data session.
-        HttpResourcePath._pid = os.getpid()
-        self._data_session: requests.Session = self._data_session_store.get(self)
-        return self._data_session
+        if session is None:
+            HttpResourcePath._pid = os.getpid()
+            session = self._data_session_store.get(self)
+            self._data_session: requests.Session = session
+        return _SessionWrapper(session, extra_headers=self._extra_headers)
 
     def _clear_sessions(self) -> None:
         """Close the socket connections that are still open.
@@ -1160,6 +1206,7 @@ class HttpResourcePath(ResourcePath):
         transfer: str = "copy",
         overwrite: bool = False,
         transaction: TransactionProtocol | None = None,
+        multithreaded: bool = True,
     ) -> None:
         """Transfer the current resource to a Webdav repository.
 
@@ -1174,6 +1221,12 @@ class HttpResourcePath(ResourcePath):
             Whether overwriting the remote resource is allowed or not.
         transaction : `~lsst.resources.utils.TransactionProtocol`, optional
             Currently unused.
+        multithreaded : `bool`, optional
+            If `True` the transfer will be allowed to attempt to improve
+            throughput by using parallel download streams. This may of no
+            effect if the URI scheme does not support parallel streams or
+            if a global override has been applied. If `False` parallel
+            streams will be disabled.
         """
         # Fail early to prevent delays if remote resources are requested.
         if transfer not in self.transferModes:
@@ -1325,13 +1378,12 @@ class HttpResourcePath(ResourcePath):
         path : `str`
             A path that can be opened by the file system object.
         """
-        if (
-            fsspec is None
-            or not self.is_webdav_endpoint
-            or self.server not in HttpResourcePath.SUPPORTED_URL_SIGNERS
-        ):
+        if fsspec is None:
             return super().to_fsspec()
 
+        if not self.is_webdav_endpoint or self.server not in HttpResourcePath.SUPPORTED_URL_SIGNERS:
+            return fsspec.url_to_fs(self.geturl(), client_kwargs={"headers": self._extra_headers})
+
         if self.isdir():
             raise NotImplementedError(
                 f"method HttpResourcePath.to_fsspec() not implemented for directory {self}"
@@ -1473,15 +1525,29 @@ class HttpResourcePath(ResourcePath):
         except json.JSONDecodeError:
             raise ValueError(f"could not deserialize response to POST request for URL {self}")
 
-    def _as_local(self) -> tuple[str, bool]:
+    @contextlib.contextmanager
+    def _as_local(
+        self, multithreaded: bool = True, tmpdir: ResourcePath | None = None
+    ) -> Iterator[ResourcePath]:
         """Download object over HTTP and place in temporary directory.
 
+        Parameters
+        ----------
+        multithreaded : `bool`, optional
+            If `True` the transfer will be allowed to attempt to improve
+            throughput by using parallel download streams. This may of no
+            effect if the URI scheme does not support parallel streams or
+            if a global override has been applied. If `False` parallel
+            streams will be disabled.
+        tmpdir : `ResourcePath` or `None`, optional
+            Explicit override of the temporary directory to use for remote
+            downloads.
+
         Returns
         -------
-        path : `str`
-            Path to local temporary file.
-        temporary : `bool`
-            Always returns `True`. This is always a temporary file.
+        local_uri : `ResourcePath`
+            A URI to a local POSIX file corresponding to a local temporary
+            downloaded copy of the resource.
         """
         # Use the session as a context manager to ensure that connections
         # to both the front end and back end servers are closed after the
@@ -1493,9 +1559,14 @@ class HttpResourcePath(ResourcePath):
                     f"Unable to download resource {self}; status: {resp.status_code} {resp.reason}"
                 )
 
-            tmpdir, buffer_size = self._config.tmpdir_buffersize
+            if tmpdir is None:
+                temp_dir, buffer_size = self._config.tmpdir_buffersize
+                tmpdir = ResourcePath(temp_dir, forceDirectory=True)
+            else:
+                buffer_size = _calc_tmpdir_buffer_size(tmpdir.ospath)
+
             with ResourcePath.temporary_uri(
-                suffix=self.getExtension(), prefix=ResourcePath(tmpdir, forceDirectory=True), delete=False
+                suffix=self.getExtension(), prefix=tmpdir, delete=True
             ) as tmp_uri:
                 expected_length = int(resp.headers.get("Content-Length", "-1"))
                 with time_this(
@@ -1511,20 +1582,20 @@ class HttpResourcePath(ResourcePath):
                             tmpFile.write(chunk)
                             content_length += len(chunk)
 
-            # Check that the expected and actual content lengths match. Perform
-            # this check only when the contents of the file was not encoded by
-            # the server.
-            if (
-                "Content-Encoding" not in resp.headers
-                and expected_length >= 0
-                and expected_length != content_length
-            ):
-                raise ValueError(
-                    f"Size of downloaded file does not match value in Content-Length header for {self}: "
-                    f"expecting {expected_length} and got {content_length} bytes"
-                )
+                # Check that the expected and actual content lengths match.
+                # Perform this check only when the contents of the file was not
+                # encoded by the server.
+                if (
+                    "Content-Encoding" not in resp.headers
+                    and expected_length >= 0
+                    and expected_length != content_length
+                ):
+                    raise ValueError(
+                        f"Size of downloaded file does not match value in Content-Length header for {self}: "
+                        f"expecting {expected_length} and got {content_length} bytes"
+                    )
 
-            return tmpFile.name, True
+                yield tmp_uri
 
     def _send_webdav_request(
         self,
@@ -1532,7 +1603,7 @@ class HttpResourcePath(ResourcePath):
         url: str | None = None,
         headers: dict[str, str] | None = None,
         body: str | None = None,
-        session: requests.Session | None = None,
+        session: _SessionWrapper | None = None,
         timeout: tuple[float, float] | None = None,
     ) -> requests.Response:
         """Send a webDAV request and correctly handle redirects.
@@ -1953,6 +2024,10 @@ class HttpResourcePath(ResourcePath):
             with super()._openImpl(mode, encoding=encoding) as http_handle:
                 yield http_handle
 
+    def _copy_extra_attributes(self, original_uri: ResourcePath) -> None:
+        assert isinstance(original_uri, HttpResourcePath)
+        self._extra_headers = original_uri._extra_headers
+
 
 def _dump_response(resp: requests.Response) -> None:
     """Log the contents of a HTTP or webDAV request and its response.
@@ -2163,3 +2238,95 @@ class DavProperty:
     @property
     def href(self) -> str:
         return self._href
+
+
+class _SessionWrapper(contextlib.AbstractContextManager):
+    """Wraps a `requests.Session` to allow header values to be injected with
+    all requests.
+
+    Notes
+    -----
+    `requests.Session` already has a feature for setting headers globally, but
+    our session objects are global and authorization headers can vary for each
+    HttpResourcePath instance.
+    """
+
+    def __init__(self, session: requests.Session, *, extra_headers: dict[str, str] | None) -> None:
+        self._session = session
+        self._extra_headers = extra_headers
+
+    def __enter__(self) -> _SessionWrapper:
+        self._session.__enter__()
+        return self
+
+    def __exit__(
+        self,
+        exc_type: Any,
+        exc_value: Any,
+        traceback: Any,
+    ) -> None:
+        return self._session.__exit__(exc_type, exc_value, traceback)
+
+    def get(
+        self,
+        url: str,
+        *,
+        timeout: tuple[float, float],
+        allow_redirects: bool = True,
+        stream: bool,
+        headers: dict[str, str] | None = None,
+    ) -> requests.Response:
+        return self._session.get(
+            url,
+            timeout=timeout,
+            allow_redirects=allow_redirects,
+            stream=stream,
+            headers=self._augment_headers(headers),
+        )
+
+    def head(
+        self,
+        url: str,
+        *,
+        timeout: tuple[float, float],
+        allow_redirects: bool,
+        stream: bool,
+        headers: dict[str, str] | None = None,
+    ) -> requests.Response:
+        return self._session.head(
+            url,
+            timeout=timeout,
+            allow_redirects=allow_redirects,
+            stream=stream,
+            headers=self._augment_headers(headers),
+        )
+
+    def request(
+        self,
+        method: str,
+        url: str,
+        *,
+        data: str | bytes | BinaryIO | None,
+        timeout: tuple[float, float],
+        allow_redirects: bool,
+        stream: bool,
+        headers: dict[str, str] | None = None,
+    ) -> requests.Response:
+        return self._session.request(
+            method,
+            url,
+            data=data,
+            timeout=timeout,
+            allow_redirects=allow_redirects,
+            stream=stream,
+            headers=self._augment_headers(headers),
+        )
+
+    def _augment_headers(self, headers: dict[str, str] | None) -> dict[str, str]:
+        if headers is None:
+            headers = {}
+
+        if self._extra_headers is not None:
+            headers = headers | self._extra_headers
+
+        return headers

lsst/resources/mem.py

@@ -13,6 +13,9 @@ from __future__ import annotations
 
 __all__ = ("InMemoryResourcePath",)
 
+import contextlib
+from collections.abc import Iterator
+
 from ._resourcePath import ResourcePath
 
 
@@ -27,5 +30,8 @@ class InMemoryResourcePath(ResourcePath):
         """Test for existence and always return False."""
         return True
 
-    def _as_local(self) -> tuple[str, bool]:
+    @contextlib.contextmanager
+    def _as_local(
+        self, multithreaded: bool = True, tmpdir: ResourcePath | None = None
+    ) -> Iterator[ResourcePath]:
         raise RuntimeError(f"Do not know how to retrieve data for URI '{self}'")

lsst/resources/packageresource.py

@@ -29,7 +29,7 @@ if TYPE_CHECKING:
         AbstractFileSystem = type
 
 from ._resourceHandles._baseResourceHandle import ResourceHandleProtocol
-from ._resourcePath import ResourcePath
+from ._resourcePath import ResourcePath, ResourcePathExpression
 
 log = logging.getLogger(__name__)
 
@@ -85,14 +85,25 @@ class PackageResourcePath(ResourcePath):
             return fh.read(size)
 
     @contextlib.contextmanager
-    def as_local(self) -> Iterator[ResourcePath]:
+    def as_local(
+        self, multithreaded: bool = True, tmpdir: ResourcePathExpression | None = None
+    ) -> Iterator[ResourcePath]:
         """Return the location of the Python resource as local file.
 
+        Parameters
+        ----------
+        multithreaded : `bool`, optional
+            Unused.
+        tmpdir : `ResourcePathExpression` or `None`, optional
+            Unused.
+
         Yields
         ------
         local : `ResourcePath`
             This might be the original resource or a copy on the local file
             system.
+        multithreaded : `bool`, optional
+            Unused.
 
         Notes
         -----