lsst-resources 29.2025.2400__tar.gz → 29.2025.2500__tar.gz

{lsst_resources-29.2025.2400/python/lsst_resources.egg-info → lsst_resources-29.2025.2500}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lsst-resources
-Version: 29.2025.2400
+Version: 29.2025.2500
 Summary: An abstraction layer for reading and writing from URI file resources.
 Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
 License: BSD 3-Clause License

{lsst_resources-29.2025.2400 → lsst_resources-29.2025.2500}/doc/lsst.resources/CHANGES.rst

@@ -1,3 +1,34 @@
+Resources v29.1.0 (2025-06-13)
+==============================
+
+Miscellaneous Changes of Minor Interest
+---------------------------------------
+
+New Features
+------------
+
+- * Added ``ResourcePath.mtransfer()`` for doing multiple transfers in parallel.
+    The number of workers can be controlled using the ``$LSST_RESOURCES_NUM_WORKERS`` environment variable.
+  * ``transfer_from`` and ``as_local`` now have an additional parameter that can control whether implicit multithreading should be used for a single download.
+  * ``as_local`` has a new parameter that can be used to explicitly specify the local download location.
+    This can be used for ``transfer_from`` to allow the file to be downloaded to the local destination directory immediately. (`DM-31824 <https://rubinobs.atlassian.net/browse/DM-31824>`_)
+- Added specialized support for schemes ``davs://`` and ``dav://`` hosted by storage endpoints implementing WebDAV protocol as described in `RFC-4918 HTTP Extensions for Web Distributed Authoring and Versioning (WebDAV) <http://www.webdav.org/specs/rfc4918.html>`_. (`DM-49784 <https://rubinobs.atlassian.net/browse/DM-49784>`_)
+- Added new bulk removal API: ``ResourcePath.mremove()``.
+  This can be 10 times faster than calling ``remove()`` in a loop. (`DM-50724 <https://rubinobs.atlassian.net/browse/DM-50724>`_)
+
+
+Miscellaneous Changes of Minor Interest
+---------------------------------------
+
+- It is now possible to control how bulk APIs such as ``mexists()`` and ``mtransfer()`` work.
+  Added ``$LSST_RESOURCES_NUM_WORKERS`` environment variable to specify how many workers should be used.
+  The default is derived from the number of CPUs but capped at 10.
+  Also the ``mexists()`` method has an explicit parameter to allow the number of workers to be specified.
+  Added ``$LSST_RESOURCES_EXECUTOR`` to specify how the jobs should be executed.
+  The default is ``threads`` (which is the same as used previously) but on Linux more performance may be achievable by setting this environment variable to ``process``. (`DM-50074 <https://rubinobs.atlassian.net/browse/DM-50074>`_)
+- * Fixed problem with multiple ``flush()`` calls with S3 resource handle for small chunks.
+  * Fixed bug in File resource handle where ``flush()`` was mistakenly calling ``close()``. (`DM-51087 <https://rubinobs.atlassian.net/browse/DM-51087>`_)
+
 Resources v29.0.0 (2025-03-25)
 ==============================
 

{lsst_resources-29.2025.2400 → lsst_resources-29.2025.2500}/python/lsst/resources/_resourcePath.py

@@ -1255,7 +1255,10 @@ class ResourcePath:  # numpydoc ignore=PR02
         """
         return self
 
-    def _as_local(self, multithreaded: bool = True, tmpdir: ResourcePath | None = None) -> tuple[str, bool]:
+    @contextlib.contextmanager
+    def _as_local(
+        self, multithreaded: bool = True, tmpdir: ResourcePath | None = None
+    ) -> Iterator[ResourcePath]:
         """Return the location of the (possibly remote) resource as local file.
 
         This is a helper function for `as_local` context manager.
@@ -1274,13 +1277,9 @@ class ResourcePath:  # numpydoc ignore=PR02
 
         Returns
         -------
-        path : `str`
-            If this is a remote resource, it will be a copy of the resource
-            on the local file system, probably in a temporary directory.
-            For a local resource this should be the actual path to the
-            resource.
-        is_temporary : `bool`
-            Indicates if the local path is a temporary file or not.
+        local_uri : `ResourcePath`
+            A URI to a local POSIX file. This can either be the same resource
+            or a local downloaded copy of the resource.
         """
         raise NotImplementedError()
 
@@ -1330,18 +1329,8 @@ class ResourcePath:  # numpydoc ignore=PR02
         temp_dir = ResourcePath(tmpdir, forceDirectory=True) if tmpdir is not None else None
         if temp_dir is not None and not temp_dir.isLocal:
             raise ValueError(f"Temporary directory for as_local must be local resource not {temp_dir}")
-        local_src, is_temporary = self._as_local(multithreaded=multithreaded, tmpdir=temp_dir)
-        local_uri = ResourcePath(local_src, isTemporary=is_temporary)
-
-        try:
+        with self._as_local(multithreaded=multithreaded, tmpdir=temp_dir) as local_uri:
             yield local_uri
-        finally:
-            # The caller might have relocated the temporary file.
-            # Do not ever delete if the temporary matches self
-            # (since it may have been that a temporary file was made local
-            # but already was local).
-            if self != local_uri and is_temporary and local_uri.exists():
-                local_uri.remove()
 
     @classmethod
     @contextlib.contextmanager

{lsst_resources-29.2025.2400 → lsst_resources-29.2025.2500}/python/lsst/resources/dav.py

@@ -172,7 +172,21 @@ dav_globals: DavGlobals = DavGlobals()
 
 
 class DavResourcePath(ResourcePath):
-    """WebDAV resource."""
+    """WebDAV resource.
+
+    Parameters
+    ----------
+    uri : `ResourcePathExpression`
+        URI to store in object.
+    root : `str` or `ResourcePath` or `None`, optional
+        Root for relative URIs. Not used in this constructor.
+    forceAbsolute : `bool`
+        Whether to force absolute URI. A WebDAV URI is always absolute.
+    forceDirectory : `bool` or `None`, optional
+        Whether this URI represents a directory.
+    isTemporary : `bool` or `None`, optional
+        Whether this URI represents a temporary resource.
+    """
 
     def __init__(
         self,
@@ -382,7 +396,10 @@ class DavResourcePath(ResourcePath):
         headers.update({"Accept-Encoding": "identity"})
         return self._client.read_range(self._internal_url, start=start, end=end, headers=headers)
 
-    def _as_local(self, multithreaded: bool = True, tmpdir: ResourcePath | None = None) -> tuple[str, bool]:
+    @contextlib.contextmanager
+    def _as_local(
+        self, multithreaded: bool = True, tmpdir: ResourcePath | None = None
+    ) -> Iterator[ResourcePath]:
         """Download object and place in temporary directory.
 
         Parameters
@@ -399,10 +416,9 @@ class DavResourcePath(ResourcePath):
 
         Returns
         -------
-        path : `str`
-            Path to local temporary file.
-        temporary : `bool`
-            Always returns `True`. This is always a temporary file.
+        local_uri : `ResourcePath`
+            A URI to a local POSIX file corresponding to a local temporary
+            downloaded copy of the resource.
         """
         # We need to ensure that this resource is actually a file. dCache
         # responds with a HTML-formatted content to a HTTP GET request to a
@@ -417,9 +433,9 @@ class DavResourcePath(ResourcePath):
         else:
             buffer_size = _calc_tmpdir_buffer_size(tmpdir.ospath)
 
-        with ResourcePath.temporary_uri(suffix=self.getExtension(), prefix=tmpdir, delete=False) as tmp_uri:
+        with ResourcePath.temporary_uri(suffix=self.getExtension(), prefix=tmpdir, delete=True) as tmp_uri:
             self._client.download(self._internal_url, tmp_uri.ospath, buffer_size)
-            return tmp_uri.ospath, True
+            yield tmp_uri
 
     def write(self, data: BinaryIO | bytes, overwrite: bool = True) -> None:
         """Write the supplied bytes to the new resource.
@@ -470,7 +486,7 @@ class DavResourcePath(ResourcePath):
 
         Parameters
         ----------
-        recursive: `bool`
+        recursive : `bool`
             If `True` recursively remove all files and directories under this
             directory.
 

{lsst_resources-29.2025.2400 → lsst_resources-29.2025.2500}/python/lsst/resources/file.py

@@ -79,7 +79,10 @@ class FileResourcePath(ResourcePath):
         """Remove the resource."""
         os.remove(self.ospath)
 
-    def _as_local(self, multithreaded: bool = True, tmpdir: ResourcePath | None = None) -> tuple[str, bool]:
+    @contextlib.contextmanager
+    def _as_local(
+        self, multithreaded: bool = True, tmpdir: ResourcePath | None = None
+    ) -> Iterator[ResourcePath]:
         """Return the local path of the file.
 
         This is an internal helper for ``as_local()``.
@@ -93,12 +96,10 @@ class FileResourcePath(ResourcePath):
 
         Returns
         -------
-        path : `str`
-            The local path to this file.
-        temporary : `bool`
-            Always returns the temporary nature of the input file resource.
+        local_uri : `ResourcePath`
+            A local URI. In this case it will be itself.
         """
-        return self.ospath, self.isTemporary
+        yield self
 
     def read(self, size: int = -1) -> bytes:
         with open(self.ospath, "rb") as fh:

{lsst_resources-29.2025.2400 → lsst_resources-29.2025.2500}/python/lsst/resources/gs.py

@@ -202,17 +202,20 @@ class GSResourcePath(ResourcePath):
         # Should this method do anything at all?
         self.blob.upload_from_string(b"", retry=_RETRY_POLICY)
 
-    def _as_local(self, multithreaded: bool = True, tmpdir: ResourcePath | None = None) -> tuple[str, bool]:
+    @contextlib.contextmanager
+    def _as_local(
+        self, multithreaded: bool = True, tmpdir: ResourcePath | None = None
+    ) -> Iterator[ResourcePath]:
         with (
-            ResourcePath.temporary_uri(prefix=tmpdir, suffix=self.getExtension(), delete=False) as tmp_uri,
+            ResourcePath.temporary_uri(prefix=tmpdir, suffix=self.getExtension(), delete=True) as tmp_uri,
             time_this(log, msg="Downloading %s to local file", args=(self,)),
         ):
             try:
                 with tmp_uri.open("wb") as tmpFile:
                     self.blob.download_to_file(tmpFile, retry=_RETRY_POLICY)
+                yield tmp_uri
             except NotFound as e:
                 raise FileNotFoundError(f"No such resource: {self}") from e
-        return tmp_uri.ospath, True
 
     def transfer_from(
         self,

{lsst_resources-29.2025.2400 → lsst_resources-29.2025.2500}/python/lsst/resources/http.py

@@ -1531,7 +1531,10 @@ class HttpResourcePath(ResourcePath):
         except json.JSONDecodeError:
             raise ValueError(f"could not deserialize response to POST request for URL {self}")
 
-    def _as_local(self, multithreaded: bool = True, tmpdir: ResourcePath | None = None) -> tuple[str, bool]:
+    @contextlib.contextmanager
+    def _as_local(
+        self, multithreaded: bool = True, tmpdir: ResourcePath | None = None
+    ) -> Iterator[ResourcePath]:
         """Download object over HTTP and place in temporary directory.
 
         Parameters
@@ -1548,10 +1551,9 @@ class HttpResourcePath(ResourcePath):
 
         Returns
         -------
-        path : `str`
-            Path to local temporary file.
-        temporary : `bool`
-            Always returns `True`. This is always a temporary file.
+        local_uri : `ResourcePath`
+            A URI to a local POSIX file corresponding to a local temporary
+            downloaded copy of the resource.
         """
         # Use the session as a context manager to ensure that connections
         # to both the front end and back end servers are closed after the
@@ -1570,7 +1572,7 @@ class HttpResourcePath(ResourcePath):
                 buffer_size = _calc_tmpdir_buffer_size(tmpdir.ospath)
 
             with ResourcePath.temporary_uri(
-                suffix=self.getExtension(), prefix=tmpdir, delete=False
+                suffix=self.getExtension(), prefix=tmpdir, delete=True
             ) as tmp_uri:
                 expected_length = int(resp.headers.get("Content-Length", "-1"))
                 with time_this(
@@ -1586,20 +1588,20 @@ class HttpResourcePath(ResourcePath):
                             tmpFile.write(chunk)
                             content_length += len(chunk)
 
-            # Check that the expected and actual content lengths match. Perform
-            # this check only when the contents of the file was not encoded by
-            # the server.
-            if (
-                "Content-Encoding" not in resp.headers
-                and expected_length >= 0
-                and expected_length != content_length
-            ):
-                raise ValueError(
-                    f"Size of downloaded file does not match value in Content-Length header for {self}: "
-                    f"expecting {expected_length} and got {content_length} bytes"
-                )
+                # Check that the expected and actual content lengths match.
+                # Perform this check only when the contents of the file was not
+                # encoded by the server.
+                if (
+                    "Content-Encoding" not in resp.headers
+                    and expected_length >= 0
+                    and expected_length != content_length
+                ):
+                    raise ValueError(
+                        f"Size of downloaded file does not match value in Content-Length header for {self}: "
+                        f"expecting {expected_length} and got {content_length} bytes"
+                    )
 
-            return tmpFile.name, True
+                yield tmp_uri
 
     def _send_webdav_request(
         self,

{lsst_resources-29.2025.2400 → lsst_resources-29.2025.2500}/python/lsst/resources/mem.py

@@ -13,6 +13,9 @@ from __future__ import annotations
 
 __all__ = ("InMemoryResourcePath",)
 
+import contextlib
+from collections.abc import Iterator
+
 from ._resourcePath import ResourcePath
 
 
@@ -27,5 +30,8 @@ class InMemoryResourcePath(ResourcePath):
         """Test for existence and always return False."""
         return True
 
-    def _as_local(self, multithreaded: bool = True, tmpdir: ResourcePath | None = None) -> tuple[str, bool]:
+    @contextlib.contextmanager
+    def _as_local(
+        self, multithreaded: bool = True, tmpdir: ResourcePath | None = None
+    ) -> Iterator[ResourcePath]:
         raise RuntimeError(f"Do not know how to retrieve data for URI '{self}'")

{lsst_resources-29.2025.2400 → lsst_resources-29.2025.2500}/python/lsst/resources/s3.py

@@ -477,7 +477,10 @@ class S3ResourcePath(ResourcePath):
 
         return s3, f"{self._bucket}/{self.relativeToPathRoot}"
 
-    def _as_local(self, multithreaded: bool = True, tmpdir: ResourcePath | None = None) -> tuple[str, bool]:
+    @contextlib.contextmanager
+    def _as_local(
+        self, multithreaded: bool = True, tmpdir: ResourcePath | None = None
+    ) -> Iterator[ResourcePath]:
         """Download object from S3 and place in temporary directory.
 
         Parameters
@@ -494,13 +497,12 @@ class S3ResourcePath(ResourcePath):
 
         Returns
         -------
-        path : `str`
-            Path to local temporary file.
-        temporary : `bool`
-            Always returns `True`. This is always a temporary file.
+        local_uri : `ResourcePath`
+            A URI to a local POSIX file corresponding to a local temporary
+            downloaded copy of the resource.
         """
         with (
-            ResourcePath.temporary_uri(prefix=tmpdir, suffix=self.getExtension(), delete=False) as tmp_uri,
+            ResourcePath.temporary_uri(prefix=tmpdir, suffix=self.getExtension(), delete=True) as tmp_uri,
             self._use_threads_temp_override(multithreaded),
             time_this(log, msg="Downloading %s to local file", args=(self,)),
         ):
@@ -511,7 +513,7 @@ class S3ResourcePath(ResourcePath):
             )
             with tmp_uri.open("wb") as tmpFile:
                 self._download_file(tmpFile, progress)
-        return tmp_uri.ospath, True
+            yield tmp_uri
 
     @backoff.on_exception(backoff.expo, all_retryable_errors, max_time=max_retry_time)
     def _upload_file(self, local_file: ResourcePath, progress: ProgressPercentage | None) -> None:
@@ -542,7 +544,7 @@ class S3ResourcePath(ResourcePath):
         try:
             self.client.copy_object(CopySource=copy_source, Bucket=self._bucket, Key=self.relativeToPathRoot)
         except (self.client.exceptions.NoSuchKey, self.client.exceptions.NoSuchBucket) as err:
-            raise FileNotFoundError(f"No such resource to transfer: {self}") from err
+            raise FileNotFoundError(f"No such resource to transfer: {src} -> {self}") from err
         except ClientError as err:
             translate_client_error(err, self)
             raise
@@ -609,10 +611,10 @@ class S3ResourcePath(ResourcePath):
         timer_msg = "Transfer from %s to %s"
         timer_args = (src, self)
 
-        if isinstance(src, type(self)):
-            # Looks like an S3 remote uri so we can use direct copy
-            # note that boto3.resource.meta.copy is cleverer than the low
-            # level copy_object
+        if isinstance(src, type(self)) and self.client == src.client:
+            # Looks like an S3 remote uri so we can use direct copy.
+            # This only works if the source and destination are using the same
+            # S3 endpoint and profile.
             with time_this(log, msg=timer_msg, args=timer_args):
                 self._copy_from(src)
 

lsst_resources-29.2025.2500/python/lsst/resources/version.py

@@ -0,0 +1,2 @@
+__all__ = ["__version__"]
+__version__ = "29.2025.2500"

{lsst_resources-29.2025.2400 → lsst_resources-29.2025.2500/python/lsst_resources.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lsst-resources
-Version: 29.2025.2400
+Version: 29.2025.2500
 Summary: An abstraction layer for reading and writing from URI file resources.
 Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
 License: BSD 3-Clause License

{lsst_resources-29.2025.2400 → lsst_resources-29.2025.2500}/tests/test_dav.py

@@ -284,12 +284,12 @@ class DavReadWriteTestCase(GenericReadWriteTestCase, unittest.TestCase):
         self.assertTrue(remote_file.exists())
         self.assertEqual(remote_file.size(), len(contents))
 
-        local_path, is_temp = remote_file._as_local()
-        self.assertTrue(is_temp)
-        self.assertTrue(os.path.exists(local_path))
-        self.assertTrue(os.stat(local_path).st_size, len(contents))
-        self.assertEqual(ResourcePath(local_path).read(), contents)
-        os.remove(local_path)
+        with remote_file._as_local() as local_uri:
+            self.assertTrue(local_uri.isTemporary)
+            self.assertTrue(os.path.exists(local_uri.ospath))
+            self.assertTrue(os.stat(local_uri.ospath).st_size, len(contents))
+            self.assertEqual(local_uri.read(), contents)
+        self.assertFalse(local_uri.exists())
 
     def test_dav_size(self):
         # Retrieving the size of a non-existent file must raise.

{lsst_resources-29.2025.2400 → lsst_resources-29.2025.2500}/tests/test_http.py

@@ -315,12 +315,12 @@ class HttpReadWriteWebdavTestCase(GenericReadWriteTestCase, unittest.TestCase):
         remote_file = self.tmpdir.join(self._get_file_name())
         self.assertIsNone(remote_file.write(data=contents, overwrite=True))
 
-        local_path, is_temp = remote_file._as_local()
-        self.assertTrue(is_temp)
-        self.assertTrue(os.path.exists(local_path))
-        self.assertTrue(os.stat(local_path).st_size, len(contents))
-        self.assertEqual(ResourcePath(local_path).read(), contents)
-        os.remove(local_path)
+        with remote_file._as_local() as local_uri:
+            self.assertTrue(local_uri.isTemporary)
+            self.assertTrue(os.path.exists(local_uri.ospath))
+            self.assertTrue(os.stat(local_uri.ospath).st_size, len(contents))
+            self.assertEqual(local_uri.read(), contents)
+        self.assertFalse(local_uri.exists())
 
     def test_dav_size(self):
         # Size of a non-existent file must raise.

{lsst_resources-29.2025.2400 → lsst_resources-29.2025.2500}/tests/test_location.py

@@ -98,7 +98,7 @@ class LocationTestCase(unittest.TestCase):
 
         for uriInfo in uriStrings:
             uri = ResourcePath(uriInfo[0], root=testRoot, forceAbsolute=uriInfo[1], forceDirectory=uriInfo[2])
-            with self.subTest(in_uri=uriInfo[0], out_uri=uri):
+            with self.subTest(in_uri=repr(uriInfo[0]), out_uri=repr(uri)):
                 self.assertEqual(uri.scheme, uriInfo[3], "test scheme")
                 self.assertEqual(uri.netloc, uriInfo[4], "test netloc")
                 self.assertEqual(uri.path, uriInfo[5], "test path")
@@ -115,7 +115,7 @@ class LocationTestCase(unittest.TestCase):
 
         for uriInfo in uriStrings:
             uri = ResourcePath(uriInfo[0], forceAbsolute=uriInfo[1], forceDirectory=uriInfo[2])
-            with self.subTest(in_uri=uriInfo[0], out_uri=uri):
+            with self.subTest(in_uri=repr(uriInfo[0]), out_uri=repr(uri)):
                 self.assertEqual(uri.scheme, uriInfo[3], "test scheme")
                 self.assertEqual(uri.netloc, uriInfo[4], "test netloc")
                 # Use ospath here to ensure that we have unquoted any
@@ -133,7 +133,7 @@ class LocationTestCase(unittest.TestCase):
 
         for uriInfo in uriStrings:
             uri = ResourcePath(uriInfo[0], forceAbsolute=False).updatedFile(uriInfo[1])
-            with self.subTest(in_uri=uriInfo[0], out_uri=uri):
+            with self.subTest(in_uri=repr(uriInfo[0]), out_uri=repr(uri)):
                 self.assertEqual(uri.path, uriInfo[2])
 
         # Check that schemeless can become file scheme.
@@ -333,7 +333,7 @@ class LocationTestCase(unittest.TestCase):
         """Test round tripping of the posix to os.path conversion helpers."""
         testPaths = ("/a/b/c.e", "a/b", "a/b/", "/a/b", "/a/b/", "a/b/c.e")
         for p in testPaths:
-            with self.subTest(path=p):
+            with self.subTest(path=repr(p)):
                 self.assertEqual(os2posix(posix2os(p)), p)
 
     def testSplit(self):
@@ -368,7 +368,7 @@ class LocationTestCase(unittest.TestCase):
         )
 
         for p, e in zip(testPaths, expected, strict=True):
-            with self.subTest(path=p):
+            with self.subTest(path=repr(p)):
                 uri = ResourcePath(p, testRoot)
                 head, tail = uri.split()
                 self.assertEqual((head.geturl(), tail), e)

{lsst_resources-29.2025.2400 → lsst_resources-29.2025.2500}/tests/test_s3.py

@@ -351,6 +351,17 @@ class S3WithProfileReadWriteTestCase(S3ReadWriteTestCaseBase, unittest.TestCase)
         self.assertEqual(path2._bucket, "ceph:bucket2")
         self.assertIsNone(path2._profile)
 
+    def test_transfer_from_different_endpoints(self):
+        # Create a bucket using a different endpoint (the default endpoint.)
+        boto3.resource("s3").create_bucket(Bucket="source-bucket")
+        source_path = ResourcePath("s3://source-bucket/file.txt")
+        source_path.write(b"123")
+        target_path = ResourcePath(f"s3://{self.netloc}/target.txt")
+        # Transfer from default endpoint to custom endpoint with custom
+        # profile.
+        target_path.transfer_from(source_path)
+        self.assertEqual(target_path.read(), b"123")
+
 
 if __name__ == "__main__":
     unittest.main()

lsst_resources-29.2025.2400/python/lsst/resources/version.py

@@ -1,2 +0,0 @@
-__all__ = ["__version__"]
-__version__ = "29.2025.2400"