lsst-resources 29.2025.2100__tar.gz → 29.2025.2400__tar.gz

{lsst_resources-29.2025.2100/python/lsst_resources.egg-info → lsst_resources-29.2025.2400}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lsst-resources
-Version: 29.2025.2100
+Version: 29.2025.2400
 Summary: An abstraction layer for reading and writing from URI file resources.
 Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
 License: BSD 3-Clause License

{lsst_resources-29.2025.2100 → lsst_resources-29.2025.2400}/python/lsst/resources/_resourceHandles/_fileResourceHandle.py

@@ -79,7 +79,7 @@ class FileResourceHandle(BaseResourceHandle[U]):
         return self._fileHandle.fileno()
 
     def flush(self) -> None:
-        self._fileHandle.close()
+        self._fileHandle.flush()
 
     @property
     def isatty(self) -> bool:

{lsst_resources-29.2025.2100 → lsst_resources-29.2025.2400}/python/lsst/resources/_resourceHandles/_s3ResourceHandle.py

@@ -14,14 +14,12 @@ from __future__ import annotations
 __all__ = ("S3ResourceHandle",)
 
 import logging
-import warnings
 from collections.abc import Iterable, Mapping
 from io import SEEK_CUR, SEEK_END, SEEK_SET, BytesIO, UnsupportedOperation
 from typing import TYPE_CHECKING
 
 from botocore.exceptions import ClientError
 
-from lsst.utils.introspection import find_outside_stacklevel
 from lsst.utils.timer import time_this
 
 from ..s3utils import all_retryable_errors, backoff, max_retry_time, translate_client_error
@@ -168,21 +166,9 @@ class S3ResourceHandle(BaseResourceHandle[bytes]):
         # written to.
         s3_min_bits = 5 * 1024 * 1024  # S3 flush threshold is 5 Mib.
         if (
-            (self.tell() - (self._last_flush_position or 0)) < s3_min_bits
-            and self._closed != CloseStatus.CLOSING
-            and not self._warned
-        ):
-            amount = s3_min_bits / (1024 * 1024)
-            warnings.warn(
-                f"S3 does not support flushing objects less than {amount} Mib, skipping",
-                stacklevel=find_outside_stacklevel(
-                    "lsst.resources",
-                    "backoff",
-                    "contextlib",
-                    allow_modules={"lsst.resources.tests"},
-                ),
-            )
-            self._warned = True
+            self.tell() - (self._last_flush_position or 0)
+        ) < s3_min_bits and self._closed != CloseStatus.CLOSING:
+            # Return until the buffer is big enough.
             return
         # nothing to write, don't create an empty upload
         if self.tell() == 0:

{lsst_resources-29.2025.2100 → lsst_resources-29.2025.2400}/python/lsst/resources/_resourcePath.py

@@ -649,9 +649,11 @@ class ResourcePath:  # numpydoc ignore=PR02
         # Disallow a change in scheme
         if "scheme" in kwargs:
             raise ValueError(f"Can not use replace() method to change URI scheme for {self}")
-        return self.__class__(
+        result = self.__class__(
             self._uri._replace(**kwargs), forceDirectory=forceDirectory, isTemporary=isTemporary
         )
+        result._copy_extra_attributes(self)
+        return result
 
     def updatedFile(self, newfile: str) -> ResourcePath:
         """Return new URI with an updated final component of the path.
@@ -1903,6 +1905,12 @@ class ResourcePath:  # numpydoc ignore=PR02
         """
         raise NotImplementedError(f"URL signing is not supported for '{self.scheme}'")
 
+    def _copy_extra_attributes(self, original_uri: ResourcePath) -> None:
+        # May be overridden by subclasses to transfer attributes when a
+        # ResourcePath is constructed using the "clone" version of the
+        # ResourcePath constructor by passing in a ResourcePath object.
+        pass
+
 
 ResourcePathExpression = str | urllib.parse.ParseResult | ResourcePath | Path
 """Type-annotation alias for objects that can be coerced to ResourcePath.

{lsst_resources-29.2025.2100 → lsst_resources-29.2025.2400}/python/lsst/resources/http.py

@@ -759,6 +759,42 @@ class HttpResourcePath(ResourcePath):
         a HTTP URL. The value of the variable is not inspected.
     """
 
+    @staticmethod
+    def create_http_resource_path(
+        path: str, *, extra_headers: dict[str, str] | None = None
+    ) -> HttpResourcePath:
+        """Create an instance of `HttpResourcePath` with additional
+        HTTP-specific configuration.
+
+        Parameters
+        ----------
+        path : `str`
+            HTTP URL to be wrapped in a `ResourcePath` instance.
+        extra_headers : `dict` [ `str`, `str` ], optional
+            Additional headers that will be sent with every HTTP request made
+            by this `ResourcePath`.  These override any headers that may be
+            generated internally by `HttpResourcePath` (e.g. authentication
+            headers).
+
+        Return
+        ------
+        instance : `ResourcePath`
+            Newly-created `HttpResourcePath` instance.
+
+        Notes
+        -----
+        Most users should use the `ResourcePath` constructor, instead.
+        """
+        # Make sure we instantiate ResourcePath using a string to guarantee we
+        # get a new ResourcePath.  If we accidentally provided a ResourcePath
+        # instance instead, the ResourcePath constructor sometimes returns
+        # the original object and we would be modifying an object that is
+        # supposed to be immutable.
+        instance = ResourcePath(str(path))
+        assert isinstance(instance, HttpResourcePath)
+        instance._extra_headers = extra_headers
+        return instance
+
     # WebDAV servers known to be able to sign URLs. The values are lowercased
     # server identifiers retrieved from the 'Server' header included in
     # the response to a HTTP OPTIONS request.
@@ -805,39 +841,48 @@ class HttpResourcePath(ResourcePath):
     # and is shared by all instances of this class.
     _tcp_connector: TCPConnector | None = None
 
+    # Additional headers added to every request.
+    _extra_headers: dict[str, str] | None = None
+
     @property
-    def metadata_session(self) -> requests.Session:
+    def metadata_session(self) -> _SessionWrapper:
         """Client session to send requests which do not require upload or
         download of data, i.e. mostly metadata requests.
         """
+        session = None
         if hasattr(self, "_metadata_session"):
             if HttpResourcePath._pid == os.getpid():
-                return self._metadata_session
+                session = self._metadata_session
             else:
                 # The metadata session we have in cache was likely created by
                 # a parent process. Discard all the sessions in that store.
                 self._metadata_session_store.clear()
 
         # Retrieve a new metadata session.
-        HttpResourcePath._pid = os.getpid()
-        self._metadata_session: requests.Session = self._metadata_session_store.get(self)
-        return self._metadata_session
+        if session is None:
+            HttpResourcePath._pid = os.getpid()
+            session = self._metadata_session_store.get(self)
+            self._metadata_session: requests.Session = session
+        return _SessionWrapper(session, extra_headers=self._extra_headers)
 
     @property
-    def data_session(self) -> requests.Session:
+    def data_session(self) -> _SessionWrapper:
         """Client session for uploading and downloading data."""
+        session = None
         if hasattr(self, "_data_session"):
             if HttpResourcePath._pid == os.getpid():
-                return self._data_session
+                session = self._data_session
             else:
                 # The data session we have in cache was likely created by
                 # a parent process. Discard all the sessions in that store.
                 self._data_session_store.clear()
 
         # Retrieve a new data session.
-        HttpResourcePath._pid = os.getpid()
-        self._data_session: requests.Session = self._data_session_store.get(self)
-        return self._data_session
+        if session is None:
+            HttpResourcePath._pid = os.getpid()
+            session = self._data_session_store.get(self)
+            self._data_session: requests.Session = session
+        return _SessionWrapper(session, extra_headers=self._extra_headers)
 
     def _clear_sessions(self) -> None:
         """Close the socket connections that are still open.
@@ -1562,7 +1607,7 @@ class HttpResourcePath(ResourcePath):
         url: str | None = None,
         headers: dict[str, str] | None = None,
         body: str | None = None,
-        session: requests.Session | None = None,
+        session: _SessionWrapper | None = None,
         timeout: tuple[float, float] | None = None,
     ) -> requests.Response:
         """Send a webDAV request and correctly handle redirects.
@@ -1983,6 +2028,10 @@ class HttpResourcePath(ResourcePath):
             with super()._openImpl(mode, encoding=encoding) as http_handle:
                 yield http_handle
 
+    def _copy_extra_attributes(self, original_uri: ResourcePath) -> None:
+        assert isinstance(original_uri, HttpResourcePath)
+        self._extra_headers = original_uri._extra_headers
+
 
 def _dump_response(resp: requests.Response) -> None:
     """Log the contents of a HTTP or webDAV request and its response.
@@ -2193,3 +2242,95 @@ class DavProperty:
     @property
     def href(self) -> str:
         return self._href
+
+
+class _SessionWrapper(contextlib.AbstractContextManager):
+    """Wraps a `requests.Session` to allow header values to be injected with
+    all requests.
+
+    Notes
+    -----
+    `requests.Session` already has a feature for setting headers globally, but
+    our session objects are global and authorization headers can vary for each
+    HttpResourcePath instance.
+    """
+
+    def __init__(self, session: requests.Session, *, extra_headers: dict[str, str] | None) -> None:
+        self._session = session
+        self._extra_headers = extra_headers
+
+    def __enter__(self) -> _SessionWrapper:
+        self._session.__enter__()
+        return self
+
+    def __exit__(
+        self,
+        exc_type: Any,
+        exc_value: Any,
+        traceback: Any,
+    ) -> None:
+        return self._session.__exit__(exc_type, exc_value, traceback)
+
+    def get(
+        self,
+        url: str,
+        *,
+        timeout: tuple[float, float],
+        allow_redirects: bool = True,
+        stream: bool,
+        headers: dict[str, str] | None = None,
+    ) -> requests.Response:
+        return self._session.get(
+            url,
+            timeout=timeout,
+            allow_redirects=allow_redirects,
+            stream=stream,
+            headers=self._augment_headers(headers),
+        )
+
+    def head(
+        self,
+        url: str,
+        *,
+        timeout: tuple[float, float],
+        allow_redirects: bool,
+        stream: bool,
+        headers: dict[str, str] | None = None,
+    ) -> requests.Response:
+        return self._session.head(
+            url,
+            timeout=timeout,
+            allow_redirects=allow_redirects,
+            stream=stream,
+            headers=self._augment_headers(headers),
+        )
+
+    def request(
+        self,
+        method: str,
+        url: str,
+        *,
+        data: str | bytes | BinaryIO | None,
+        timeout: tuple[float, float],
+        allow_redirects: bool,
+        stream: bool,
+        headers: dict[str, str] | None = None,
+    ) -> requests.Response:
+        return self._session.request(
+            method,
+            url,
+            data=data,
+            timeout=timeout,
+            allow_redirects=allow_redirects,
+            stream=stream,
+            headers=self._augment_headers(headers),
+        )
+
+    def _augment_headers(self, headers: dict[str, str] | None) -> dict[str, str]:
+        if headers is None:
+            headers = {}
+
+        if self._extra_headers is not None:
+            headers = headers | self._extra_headers
+
+        return headers

{lsst_resources-29.2025.2100 → lsst_resources-29.2025.2400}/python/lsst/resources/s3.py

@@ -542,7 +542,7 @@ class S3ResourcePath(ResourcePath):
         try:
             self.client.copy_object(CopySource=copy_source, Bucket=self._bucket, Key=self.relativeToPathRoot)
         except (self.client.exceptions.NoSuchKey, self.client.exceptions.NoSuchBucket) as err:
-            raise FileNotFoundError("No such resource to transfer: {self}") from err
+            raise FileNotFoundError(f"No such resource to transfer: {self}") from err
         except ClientError as err:
             translate_client_error(err, self)
             raise

{lsst_resources-29.2025.2100 → lsst_resources-29.2025.2400}/python/lsst/resources/tests.py

@@ -61,18 +61,18 @@ def _check_open(
     """
     text_content = "abcdefghijklmnopqrstuvwxyz🙂"
     bytes_content = uuid.uuid4().bytes
-    content_by_mode_suffix = {
+    content_by_mode_suffix: dict[str, str | bytes] = {
         "": text_content,
         "t": text_content,
         "b": bytes_content,
     }
-    empty_content_by_mode_suffix = {
+    empty_content_by_mode_suffix: dict[str, str | bytes] = {
         "": "",
         "t": "",
         "b": b"",
     }
     # To appease mypy
-    double_content_by_mode_suffix = {
+    double_content_by_mode_suffix: dict[str, str | bytes] = {
         "": text_content + text_content,
         "t": text_content + text_content,
         "b": bytes_content + bytes_content,
@@ -143,6 +143,16 @@ def _check_open(
             content_read = read_buffer.read()
             test_case.assertEqual(len(content_read), 0, f"Read: {content_read!r}, expected empty.")
 
+        # Write multiple chunks with flushing to ensure that any handles that
+        # cache without flushing work properly.
+        n = 3
+        with uri.open("w" + mode_suffix, **kwargs) as write_buffer:
+            for _ in range(n):
+                write_buffer.write(content)
+                write_buffer.flush()
+        with uri.open("r" + mode_suffix, **kwargs) as read_buffer:
+            test_case.assertEqual(read_buffer.read(), content * n)
+
         # Write two copies of the content, overwriting the single copy there.
         with uri.open("w" + mode_suffix, **kwargs) as write_buffer:
             write_buffer.write(double_content)

lsst_resources-29.2025.2400/python/lsst/resources/version.py

@@ -0,0 +1,2 @@
+__all__ = ["__version__"]
+__version__ = "29.2025.2400"

{lsst_resources-29.2025.2100 → lsst_resources-29.2025.2400/python/lsst_resources.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lsst-resources
-Version: 29.2025.2100
+Version: 29.2025.2400
 Summary: An abstraction layer for reading and writing from URI file resources.
 Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
 License: BSD 3-Clause License

{lsst_resources-29.2025.2100 → lsst_resources-29.2025.2400}/tests/test_http.py

@@ -12,6 +12,7 @@
 import hashlib
 import io
 import os.path
+import pickle
 import random
 import shutil
 import socket
@@ -20,6 +21,7 @@ import string
 import tempfile
 import time
 import unittest
+import unittest.mock
 import warnings
 from collections.abc import Callable
 from threading import Thread
@@ -33,6 +35,7 @@ except ImportError:
 
 import requests
 import responses
+import responses.matchers
 
 import lsst.resources
 from lsst.resources import ResourcePath
@@ -82,6 +85,38 @@ class GenericHttpTestCase(GenericTestCase, unittest.TestCase):
             ResourcePath("http://user:password@server.com:3000/"),
         )
 
+    @responses.activate
+    def test_extra_headers(self):
+        url = "http://test.example/something.txt"
+        path = HttpResourcePath.create_http_resource_path(
+            url, extra_headers={"Authorization": "Bearer my-token"}
+        )
+
+        self.assertEqual(str(path), "http://test.example/something.txt")
+        self.assertEqual(path._extra_headers, {"Authorization": "Bearer my-token"})
+
+        # Make sure that headers are added to requests.
+        responses.add(
+            responses.GET,
+            url,
+            b"test",
+            match=[responses.matchers.header_matcher({"Authorization": "Bearer my-token"})],
+        )
+        self.assertEqual(path.read(), b"test")
+
+        # Extra headers should be preserved through pickle, to ensure that
+        # `mtransfer` and similar methods work in multi-process mode.
+        dump = pickle.dumps(path)
+        restored = pickle.loads(dump)
+        self.assertEqual(restored._extra_headers, {"Authorization": "Bearer my-token"})
+
+        # Extra headers should be preserved when making a modified copy of the
+        # ResourcePath using replace() or the ResourcePath constructor.
+        replacement = path.replace(forceDirectory=True)
+        self.assertEqual(replacement._extra_headers, {"Authorization": "Bearer my-token"})
+        copy = ResourcePath(path, forceDirectory=True)
+        self.assertEqual(copy._extra_headers, {"Authorization": "Bearer my-token"})
+
 
 class HttpReadWriteWebdavTestCase(GenericReadWriteTestCase, unittest.TestCase):
     """Test with a real webDAV server, as opposed to mocking responses."""

lsst_resources-29.2025.2100/python/lsst/resources/version.py

@@ -1,2 +0,0 @@
-__all__ = ["__version__"]
-__version__ = "29.2025.2100"