lsst-resources 29.2025.2100__py3-none-any.whl → 29.2025.2400__py3-none-any.whl

lsst/resources/_resourceHandles/_fileResourceHandle.py

@@ -79,7 +79,7 @@ class FileResourceHandle(BaseResourceHandle[U]):
         return self._fileHandle.fileno()
 
     def flush(self) -> None:
-        self._fileHandle.close()
+        self._fileHandle.flush()
 
     @property
     def isatty(self) -> bool:

lsst/resources/_resourceHandles/_s3ResourceHandle.py

@@ -14,14 +14,12 @@ from __future__ import annotations
 __all__ = ("S3ResourceHandle",)
 
 import logging
-import warnings
 from collections.abc import Iterable, Mapping
 from io import SEEK_CUR, SEEK_END, SEEK_SET, BytesIO, UnsupportedOperation
 from typing import TYPE_CHECKING
 
 from botocore.exceptions import ClientError
 
-from lsst.utils.introspection import find_outside_stacklevel
 from lsst.utils.timer import time_this
 
 from ..s3utils import all_retryable_errors, backoff, max_retry_time, translate_client_error
@@ -168,21 +166,9 @@ class S3ResourceHandle(BaseResourceHandle[bytes]):
         # written to.
         s3_min_bits = 5 * 1024 * 1024  # S3 flush threshold is 5 Mib.
         if (
-            (self.tell() - (self._last_flush_position or 0)) < s3_min_bits
-            and self._closed != CloseStatus.CLOSING
-            and not self._warned
-        ):
-            amount = s3_min_bits / (1024 * 1024)
-            warnings.warn(
-                f"S3 does not support flushing objects less than {amount} Mib, skipping",
-                stacklevel=find_outside_stacklevel(
-                    "lsst.resources",
-                    "backoff",
-                    "contextlib",
-                    allow_modules={"lsst.resources.tests"},
-                ),
-            )
-            self._warned = True
+            self.tell() - (self._last_flush_position or 0)
+        ) < s3_min_bits and self._closed != CloseStatus.CLOSING:
+            # Return until the buffer is big enough.
             return
         # nothing to write, don't create an empty upload
         if self.tell() == 0:

lsst/resources/_resourcePath.py

@@ -649,9 +649,11 @@ class ResourcePath:  # numpydoc ignore=PR02
         # Disallow a change in scheme
         if "scheme" in kwargs:
             raise ValueError(f"Can not use replace() method to change URI scheme for {self}")
-        return self.__class__(
+        result = self.__class__(
             self._uri._replace(**kwargs), forceDirectory=forceDirectory, isTemporary=isTemporary
         )
+        result._copy_extra_attributes(self)
+        return result
 
     def updatedFile(self, newfile: str) -> ResourcePath:
         """Return new URI with an updated final component of the path.
@@ -1903,6 +1905,12 @@ class ResourcePath:  # numpydoc ignore=PR02
         """
         raise NotImplementedError(f"URL signing is not supported for '{self.scheme}'")
 
+    def _copy_extra_attributes(self, original_uri: ResourcePath) -> None:
+        # May be overridden by subclasses to transfer attributes when a
+        # ResourcePath is constructed using the "clone" version of the
+        # ResourcePath constructor by passing in a ResourcePath object.
+        pass
+
 
 ResourcePathExpression = str | urllib.parse.ParseResult | ResourcePath | Path
 """Type-annotation alias for objects that can be coerced to ResourcePath.

lsst/resources/http.py

@@ -759,6 +759,42 @@ class HttpResourcePath(ResourcePath):
         a HTTP URL. The value of the variable is not inspected.
     """
 
+    @staticmethod
+    def create_http_resource_path(
+        path: str, *, extra_headers: dict[str, str] | None = None
+    ) -> HttpResourcePath:
+        """Create an instance of `HttpResourcePath` with additional
+        HTTP-specific configuration.
+
+        Parameters
+        ----------
+        path : `str`
+            HTTP URL to be wrapped in a `ResourcePath` instance.
+        extra_headers : `dict` [ `str`, `str` ], optional
+            Additional headers that will be sent with every HTTP request made
+            by this `ResourcePath`.  These override any headers that may be
+            generated internally by `HttpResourcePath` (e.g. authentication
+            headers).
+
+        Return
+        ------
+        instance : `ResourcePath`
+            Newly-created `HttpResourcePath` instance.
+
+        Notes
+        -----
+        Most users should use the `ResourcePath` constructor, instead.
+        """
+        # Make sure we instantiate ResourcePath using a string to guarantee we
+        # get a new ResourcePath.  If we accidentally provided a ResourcePath
+        # instance instead, the ResourcePath constructor sometimes returns
+        # the original object and we would be modifying an object that is
+        # supposed to be immutable.
+        instance = ResourcePath(str(path))
+        assert isinstance(instance, HttpResourcePath)
+        instance._extra_headers = extra_headers
+        return instance
+
     # WebDAV servers known to be able to sign URLs. The values are lowercased
     # server identifiers retrieved from the 'Server' header included in
     # the response to a HTTP OPTIONS request.
@@ -805,39 +841,48 @@ class HttpResourcePath(ResourcePath):
     # and is shared by all instances of this class.
     _tcp_connector: TCPConnector | None = None
 
+    # Additional headers added to every request.
+    _extra_headers: dict[str, str] | None = None
+
     @property
-    def metadata_session(self) -> requests.Session:
+    def metadata_session(self) -> _SessionWrapper:
         """Client session to send requests which do not require upload or
         download of data, i.e. mostly metadata requests.
         """
+        session = None
         if hasattr(self, "_metadata_session"):
             if HttpResourcePath._pid == os.getpid():
-                return self._metadata_session
+                session = self._metadata_session
             else:
                 # The metadata session we have in cache was likely created by
                 # a parent process. Discard all the sessions in that store.
                 self._metadata_session_store.clear()
 
         # Retrieve a new metadata session.
-        HttpResourcePath._pid = os.getpid()
-        self._metadata_session: requests.Session = self._metadata_session_store.get(self)
-        return self._metadata_session
+        if session is None:
+            HttpResourcePath._pid = os.getpid()
+            session = self._metadata_session_store.get(self)
+            self._metadata_session: requests.Session = session
+        return _SessionWrapper(session, extra_headers=self._extra_headers)
 
     @property
-    def data_session(self) -> requests.Session:
+    def data_session(self) -> _SessionWrapper:
         """Client session for uploading and downloading data."""
+        session = None
         if hasattr(self, "_data_session"):
             if HttpResourcePath._pid == os.getpid():
-                return self._data_session
+                session = self._data_session
             else:
                 # The data session we have in cache was likely created by
                 # a parent process. Discard all the sessions in that store.
                 self._data_session_store.clear()
 
         # Retrieve a new data session.
-        HttpResourcePath._pid = os.getpid()
-        self._data_session: requests.Session = self._data_session_store.get(self)
-        return self._data_session
+        if session is None:
+            HttpResourcePath._pid = os.getpid()
+            session = self._data_session_store.get(self)
+            self._data_session: requests.Session = session
+        return _SessionWrapper(session, extra_headers=self._extra_headers)
 
     def _clear_sessions(self) -> None:
         """Close the socket connections that are still open.
@@ -1562,7 +1607,7 @@ class HttpResourcePath(ResourcePath):
         url: str | None = None,
         headers: dict[str, str] | None = None,
         body: str | None = None,
-        session: requests.Session | None = None,
+        session: _SessionWrapper | None = None,
         timeout: tuple[float, float] | None = None,
     ) -> requests.Response:
         """Send a webDAV request and correctly handle redirects.
@@ -1983,6 +2028,10 @@ class HttpResourcePath(ResourcePath):
             with super()._openImpl(mode, encoding=encoding) as http_handle:
                 yield http_handle
 
+    def _copy_extra_attributes(self, original_uri: ResourcePath) -> None:
+        assert isinstance(original_uri, HttpResourcePath)
+        self._extra_headers = original_uri._extra_headers
+
 
 def _dump_response(resp: requests.Response) -> None:
     """Log the contents of a HTTP or webDAV request and its response.
@@ -2193,3 +2242,95 @@ class DavProperty:
     @property
     def href(self) -> str:
         return self._href
+
+
+class _SessionWrapper(contextlib.AbstractContextManager):
+    """Wraps a `requests.Session` to allow header values to be injected with
+    all requests.
+
+    Notes
+    -----
+    `requests.Session` already has a feature for setting headers globally, but
+    our session objects are global and authorization headers can vary for each
+    HttpResourcePath instance.
+    """
+
+    def __init__(self, session: requests.Session, *, extra_headers: dict[str, str] | None) -> None:
+        self._session = session
+        self._extra_headers = extra_headers
+
+    def __enter__(self) -> _SessionWrapper:
+        self._session.__enter__()
+        return self
+
+    def __exit__(
+        self,
+        exc_type: Any,
+        exc_value: Any,
+        traceback: Any,
+    ) -> None:
+        return self._session.__exit__(exc_type, exc_value, traceback)
+
+    def get(
+        self,
+        url: str,
+        *,
+        timeout: tuple[float, float],
+        allow_redirects: bool = True,
+        stream: bool,
+        headers: dict[str, str] | None = None,
+    ) -> requests.Response:
+        return self._session.get(
+            url,
+            timeout=timeout,
+            allow_redirects=allow_redirects,
+            stream=stream,
+            headers=self._augment_headers(headers),
+        )
+
+    def head(
+        self,
+        url: str,
+        *,
+        timeout: tuple[float, float],
+        allow_redirects: bool,
+        stream: bool,
+        headers: dict[str, str] | None = None,
+    ) -> requests.Response:
+        return self._session.head(
+            url,
+            timeout=timeout,
+            allow_redirects=allow_redirects,
+            stream=stream,
+            headers=self._augment_headers(headers),
+        )
+
+    def request(
+        self,
+        method: str,
+        url: str,
+        *,
+        data: str | bytes | BinaryIO | None,
+        timeout: tuple[float, float],
+        allow_redirects: bool,
+        stream: bool,
+        headers: dict[str, str] | None = None,
+    ) -> requests.Response:
+        return self._session.request(
+            method,
+            url,
+            data=data,
+            timeout=timeout,
+            allow_redirects=allow_redirects,
+            stream=stream,
+            headers=self._augment_headers(headers),
+        )
+
+    def _augment_headers(self, headers: dict[str, str] | None) -> dict[str, str]:
+        if headers is None:
+            headers = {}
+
+        if self._extra_headers is not None:
+            headers = headers | self._extra_headers
+
+        return headers

lsst/resources/s3.py

@@ -542,7 +542,7 @@ class S3ResourcePath(ResourcePath):
         try:
             self.client.copy_object(CopySource=copy_source, Bucket=self._bucket, Key=self.relativeToPathRoot)
         except (self.client.exceptions.NoSuchKey, self.client.exceptions.NoSuchBucket) as err:
-            raise FileNotFoundError("No such resource to transfer: {self}") from err
+            raise FileNotFoundError(f"No such resource to transfer: {self}") from err
         except ClientError as err:
             translate_client_error(err, self)
             raise

lsst/resources/tests.py

@@ -61,18 +61,18 @@ def _check_open(
     """
     text_content = "abcdefghijklmnopqrstuvwxyz🙂"
     bytes_content = uuid.uuid4().bytes
-    content_by_mode_suffix = {
+    content_by_mode_suffix: dict[str, str | bytes] = {
         "": text_content,
         "t": text_content,
         "b": bytes_content,
     }
-    empty_content_by_mode_suffix = {
+    empty_content_by_mode_suffix: dict[str, str | bytes] = {
         "": "",
         "t": "",
         "b": b"",
     }
     # To appease mypy
-    double_content_by_mode_suffix = {
+    double_content_by_mode_suffix: dict[str, str | bytes] = {
         "": text_content + text_content,
         "t": text_content + text_content,
         "b": bytes_content + bytes_content,
@@ -143,6 +143,16 @@ def _check_open(
             content_read = read_buffer.read()
             test_case.assertEqual(len(content_read), 0, f"Read: {content_read!r}, expected empty.")
 
+        # Write multiple chunks with flushing to ensure that any handles that
+        # cache without flushing work properly.
+        n = 3
+        with uri.open("w" + mode_suffix, **kwargs) as write_buffer:
+            for _ in range(n):
+                write_buffer.write(content)
+                write_buffer.flush()
+        with uri.open("r" + mode_suffix, **kwargs) as read_buffer:
+            test_case.assertEqual(read_buffer.read(), content * n)
+
         # Write two copies of the content, overwriting the single copy there.
         with uri.open("w" + mode_suffix, **kwargs) as write_buffer:
             write_buffer.write(double_content)

lsst/resources/version.py

@@ -1,2 +1,2 @@
 __all__ = ["__version__"]
-__version__ = "29.2025.2100"
+__version__ = "29.2025.2400"

{lsst_resources-29.2025.2100.dist-info → lsst_resources-29.2025.2400.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lsst-resources
-Version: 29.2025.2100
+Version: 29.2025.2400
 Summary: An abstraction layer for reading and writing from URI file resources.
 Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
 License: BSD 3-Clause License

{lsst_resources-29.2025.2100.dist-info → lsst_resources-29.2025.2400.dist-info}/RECORD

@@ -1,31 +1,31 @@
 lsst/__init__.py,sha256=9I6UQ9gj-ZcPlvsa0OPBo76UujxXVehVzw9yMAOQvyM,466
 lsst/resources/__init__.py,sha256=BDj6uokvd0ZQNGl-Xgz5gZd83Z0L2gFqGSk0KJpylP8,778
-lsst/resources/_resourcePath.py,sha256=neuZ-QNGIqwBavEq4xEBUnQveHeqACIuhVi4COhJI0A,73902
+lsst/resources/_resourcePath.py,sha256=NLjOjz0ARFJxFwE9fw4HowFV0FrdYBX_jiyXODPFSVY,74273
 lsst/resources/dav.py,sha256=XQMAZgthvtJ4hUXZ0avBAHnAYjMYU3m2DPHDyQVT7a8,31547
 lsst/resources/davutils.py,sha256=xALuMRSvYroqn_Jz6bjnj43b4OgOgCJtNW49kyTtuiw,97983
 lsst/resources/file.py,sha256=-jPuoHvTEtx5tnDyNkfwhWAyX0cTwkuMd-JvJn9EGdE,23226
 lsst/resources/gs.py,sha256=Lpo5GAzH7R7HG8E5RMGOdP4j4hjWJn-k6M3OXj0nHQM,12783
-lsst/resources/http.py,sha256=Oa6YlTnHLd0ifkvBNhduI67oiRHE-iQupAL9TEbxyQo,87930
+lsst/resources/http.py,sha256=gQNlmCyx9z_0GwrfBaOQ4WfV3Psz6BTfJYdTyolIntA,92441
 lsst/resources/location.py,sha256=x3Tq0x5o1OXYmZDxYBenUG1N71wtDhnjVAr3s2ZEiu8,7937
 lsst/resources/mem.py,sha256=VOWh7XxJPfqKcFdLZSjKEAfORQ2AHZHpxmjT8LniV60,1008
 lsst/resources/packageresource.py,sha256=vnfeRlpVwpC5cDQZE6Lnh8EH6oZy1sH2vLz9ONYjJ4k,6817
 lsst/resources/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-lsst/resources/s3.py,sha256=IPhNqKzseHsMS9RF9TAl8vhs-sFvXfdFsJCb7VMvuNg,29572
+lsst/resources/s3.py,sha256=u2rZkg12C3JgIehIJOx69NeipdN6fawhvdZKffhdAF8,29573
 lsst/resources/s3utils.py,sha256=ojWf9BPrK9mhGQ8jvs4_8Nsqf9360e79U5FnPTxe24A,14576
 lsst/resources/schemeless.py,sha256=GfJcKzZ0XIeepfQdW4HPZWiZlSp_ej0SEtSiJTrDUQs,10666
-lsst/resources/tests.py,sha256=nTRvfAGrPnh2tsR8jzA-XPuWN1XECOW1f4dDveADmb8,44439
+lsst/resources/tests.py,sha256=G43eaajzIaA6D-zz63vsgYJTeSy9Bso-my_Dh4X34CE,44966
 lsst/resources/utils.py,sha256=6O3Mq7JbPEtqyD2lM77pRpwcPMfV5SxiNMknw-F2vNs,8097
-lsst/resources/version.py,sha256=czd5myijXlfzN1l1OFP7x6hn5ASHoE02wXhI32IqLec,55
+lsst/resources/version.py,sha256=AyivLkwA4FcYO6eiFsfTA_EuqMhydgvzBupNZlQZL-E,55
 lsst/resources/_resourceHandles/__init__.py,sha256=zOcZ8gVEBdAWcHJaZabA8Vdq-wAVcxjbmA_1b1IWM6M,76
 lsst/resources/_resourceHandles/_baseResourceHandle.py,sha256=lQwxDOmFUNJndTxsjpz-HxrQBL0L-z4aXQocHdOEI7c,4676
 lsst/resources/_resourceHandles/_davResourceHandle.py,sha256=12X5-K5KqzG4EV78ZkIIrjcZcFroXy3Y2JQ_N-SDqF0,6616
-lsst/resources/_resourceHandles/_fileResourceHandle.py,sha256=A7_WQPzD0ZlOzNmaI_TPdZybrNxrXPkNHWVla3UFxfs,3676
+lsst/resources/_resourceHandles/_fileResourceHandle.py,sha256=2nC8tfP_ynAfjpzrtkw_1ahx1CuMEFpZ5mLmofSShUk,3676
 lsst/resources/_resourceHandles/_httpResourceHandle.py,sha256=Yami8IVGeru4bLQCag-OvGG0ltz1qyEg57FY4IEB87Y,10995
-lsst/resources/_resourceHandles/_s3ResourceHandle.py,sha256=NkDmPb9bm_zMvr6mMnb-tBmqJDt0yUJrt2gZXR8l7ok,12923
-lsst_resources-29.2025.2100.dist-info/licenses/COPYRIGHT,sha256=yazVsoMmFwhiw5itGrdT4YPmXbpsQyUFjlpOyZIa77M,148
-lsst_resources-29.2025.2100.dist-info/licenses/LICENSE,sha256=7wrtgl8meQ0_RIuv2TjIKpAnNrl-ODH-QLwyHe9citI,1516
-lsst_resources-29.2025.2100.dist-info/METADATA,sha256=h1FqE1jzjXciOQ4whXXfG_UTucO0hZLDLboM-9IV-v8,2237
-lsst_resources-29.2025.2100.dist-info/WHEEL,sha256=zaaOINJESkSfm_4HQVc5ssNzHCPXhJm0kEUakpsEHaU,91
-lsst_resources-29.2025.2100.dist-info/top_level.txt,sha256=eUWiOuVVm9wwTrnAgiJT6tp6HQHXxIhj2QSZ7NYZH80,5
-lsst_resources-29.2025.2100.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-lsst_resources-29.2025.2100.dist-info/RECORD,,
+lsst/resources/_resourceHandles/_s3ResourceHandle.py,sha256=Cp-eBtptskbmthy3DwLKPpYPLvU_lrqtK10X37inHt0,12406
+lsst_resources-29.2025.2400.dist-info/licenses/COPYRIGHT,sha256=yazVsoMmFwhiw5itGrdT4YPmXbpsQyUFjlpOyZIa77M,148
+lsst_resources-29.2025.2400.dist-info/licenses/LICENSE,sha256=7wrtgl8meQ0_RIuv2TjIKpAnNrl-ODH-QLwyHe9citI,1516
+lsst_resources-29.2025.2400.dist-info/METADATA,sha256=ba8Zw1ARVEELnFMBRB-4pn_WOBtDdS-7GKIqlONe-yQ,2237
+lsst_resources-29.2025.2400.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+lsst_resources-29.2025.2400.dist-info/top_level.txt,sha256=eUWiOuVVm9wwTrnAgiJT6tp6HQHXxIhj2QSZ7NYZH80,5
+lsst_resources-29.2025.2400.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
+lsst_resources-29.2025.2400.dist-info/RECORD,,

{lsst_resources-29.2025.2100.dist-info → lsst_resources-29.2025.2400.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (80.8.0)
+Generator: setuptools (80.9.0)
 Root-Is-Purelib: true
 Tag: py3-none-any