lsst-resources 29.0.0rc7__py3-none-any.whl → 29.2025.4600__py3-none-any.whl

lsst/resources/file.py

@@ -21,6 +21,7 @@ import os.path
 import posixpath
 import re
 import shutil
+import stat
 import urllib.parse
 from collections.abc import Iterator
 from typing import IO, TYPE_CHECKING
@@ -79,19 +80,27 @@ class FileResourcePath(ResourcePath):
         """Remove the resource."""
         os.remove(self.ospath)
 
-    def _as_local(self) -> tuple[str, bool]:
+    @contextlib.contextmanager
+    def _as_local(
+        self, multithreaded: bool = True, tmpdir: ResourcePath | None = None
+    ) -> Iterator[ResourcePath]:
         """Return the local path of the file.
 
         This is an internal helper for ``as_local()``.
 
+        Parameters
+        ----------
+        multithreaded : `bool`, optional
+            Unused.
+        tmpdir : `ResourcePath` or `None`, optional
+            Unused.
+
         Returns
         -------
-        path : `str`
-            The local path to this file.
-        temporary : `bool`
-            Always returns the temporary nature of the input file resource.
+        local_uri : `ResourcePath`
+            A local URI. In this case it will be itself.
         """
-        return self.ospath, self.isTemporary
+        yield self
 
     def read(self, size: int = -1) -> bytes:
         with open(self.ospath, "rb") as fh:
@@ -99,7 +108,7 @@ class FileResourcePath(ResourcePath):
 
     def write(self, data: bytes, overwrite: bool = True) -> None:
         dir = os.path.dirname(self.ospath)
-        if not os.path.exists(dir):
+        if dir and not os.path.exists(dir):
             _create_directories(dir)
         mode = "wb" if overwrite else "xb"
         with open(self.ospath, mode) as f:
@@ -141,6 +150,7 @@ class FileResourcePath(ResourcePath):
         transfer: str,
         overwrite: bool = False,
         transaction: TransactionProtocol | None = None,
+        multithreaded: bool = True,
     ) -> None:
         """Transfer the current resource to a local file.
 
@@ -155,6 +165,8 @@ class FileResourcePath(ResourcePath):
             Allow an existing file to be overwritten. Defaults to `False`.
         transaction : `~lsst.resources.utils.TransactionProtocol`, optional
             If a transaction is provided, undo actions will be registered.
+        multithreaded : `bool`, optional
+            Whether threads are allowed to be used or not.
         """
         # Fail early to prevent delays if remote resources are requested
         if transfer not in self.transferModes:
@@ -172,9 +184,53 @@ class FileResourcePath(ResourcePath):
                 transfer,
             )
 
+        # The output location should not exist unless overwrite=True.
+        # Rather than use `exists()`, use os.stat since we might need
+        # the full answer later.
+        dest_stat: os.stat_result | None
+        try:
+            # Do not read through links of the file itself.
+            dest_stat = os.lstat(self.ospath)
+        except FileNotFoundError:
+            dest_stat = None
+
+        # It is possible that the source URI and target URI refer
+        # to the same file. This can happen for a number of reasons
+        # (such as soft links in the path, or they really are the same).
+        # In that case log a message and return as if the transfer
+        # completed (it technically did). A temporary file download
+        # can't be the same so the test can be skipped.
+        if dest_stat and src.isLocal and not src.isTemporary:
+            # Be consistent and use lstat here (even though realpath
+            # has been called). It does not harm.
+            local_src_stat = os.lstat(src.ospath)
+            if dest_stat.st_ino == local_src_stat.st_ino and dest_stat.st_dev == local_src_stat.st_dev:
+                log.debug(
+                    "Destination URI %s is the same file as source URI %s, returning immediately."
+                    " No further action required.",
+                    self,
+                    src,
+                )
+                return
+
+        if not overwrite and dest_stat:
+            raise FileExistsError(
+                f"Destination path '{self}' already exists. Transfer from {src} cannot be completed."
+            )
+
+        # Make the destination path absolute (but don't follow links since
+        # that would possibly cause us to end up in the wrong place if the
+        # file existed already as a soft link)
+        newFullPath = os.path.abspath(self.ospath)
+        outputDir = os.path.dirname(newFullPath)
+
         # We do not have to special case FileResourcePath here because
-        # as_local handles that.
-        with src.as_local() as local_uri:
+        # as_local handles that. If remote download, download it to the
+        # destination directory to allow an atomic rename but only if that
+        # directory exists because we do not want to create a directory
+        # but then end up with the download failing.
+        tmpdir = outputDir if os.path.exists(outputDir) else None
+        with src.as_local(multithreaded=multithreaded, tmpdir=tmpdir) as local_uri:
             is_temporary = local_uri.isTemporary
             local_src = local_uri.ospath
 
@@ -228,45 +284,6 @@ class FileResourcePath(ResourcePath):
             if src != local_uri and is_temporary and transfer == "copy":
                 transfer = "move"
 
-            # The output location should not exist unless overwrite=True.
-            # Rather than use `exists()`, use os.stat since we might need
-            # the full answer later.
-            dest_stat: os.stat_result | None
-            try:
-                # Do not read through links of the file itself.
-                dest_stat = os.lstat(self.ospath)
-            except FileNotFoundError:
-                dest_stat = None
-
-            # It is possible that the source URI and target URI refer
-            # to the same file. This can happen for a number of reasons
-            # (such as soft links in the path, or they really are the same).
-            # In that case log a message and return as if the transfer
-            # completed (it technically did). A temporary file download
-            # can't be the same so the test can be skipped.
-            if dest_stat and not is_temporary:
-                # Be consistent and use lstat here (even though realpath
-                # has been called). It does not harm.
-                local_src_stat = os.lstat(local_src)
-                if dest_stat.st_ino == local_src_stat.st_ino and dest_stat.st_dev == local_src_stat.st_dev:
-                    log.debug(
-                        "Destination URI %s is the same file as source URI %s, returning immediately."
-                        " No further action required.",
-                        self,
-                        local_uri,
-                    )
-                    return
-
-            if not overwrite and dest_stat:
-                raise FileExistsError(
-                    f"Destination path '{self}' already exists. Transfer from {src} cannot be completed."
-                )
-
-            # Make the path absolute (but don't follow links since that
-            # would possibly cause us to end up in the wrong place if the
-            # file existed already as a soft link)
-            newFullPath = os.path.abspath(self.ospath)
-            outputDir = os.path.dirname(newFullPath)
             if not os.path.isdir(outputDir):
                 # Must create the directory -- this can not be rolled back
                 # since another transfer running concurrently may
@@ -312,15 +329,38 @@ class FileResourcePath(ResourcePath):
                 # the same output directory. This at least guarantees that
                 # if multiple processes are writing to the same file
                 # simultaneously the file we end up with will not be corrupt.
-                with self.temporary_uri(prefix=self.parent(), suffix=self.getExtension()) as temp_copy:
-                    shutil.copy(local_src, temp_copy.ospath)
-                    with transaction.undoWith(f"copy from {local_src}", os.remove, newFullPath):
-                        # os.rename works even if the file exists.
-                        # It's possible that another process has copied a file
-                        # in whilst this one was copying. If overwrite
-                        # protection is needed then another stat() call should
-                        # happen here.
-                        os.rename(temp_copy.ospath, newFullPath)
+                if overwrite:
+                    with self.temporary_uri(prefix=self.parent(), suffix=self.getExtension()) as temp_copy:
+                        shutil.copy(local_src, temp_copy.ospath)
+                        with transaction.undoWith(f"copy from {local_src}", os.remove, newFullPath):
+                            os.rename(temp_copy.ospath, newFullPath)
+                else:
+                    # Create the file exclusively to ensure that no others are
+                    # trying to write.
+                    temp_path = newFullPath + ".transfer-tmp"
+                    try:
+                        with open(temp_path, "x"):
+                            pass
+                    except FileExistsError:
+                        raise FileExistsError(
+                            f"Another process is writing to '{self}'."
+                            f" Transfer from {src} cannot be completed."
+                        )
+                    with transaction.undoWith(f"copy from {local_src}", os.remove, temp_path):
+                        # Make sure file is writable, no matter the umask.
+                        st = os.stat(temp_path)
+                        os.chmod(temp_path, st.st_mode | stat.S_IWUSR)
+                        shutil.copy(local_src, temp_path)
+                    # Use link/remove to atomically and exclusively move the
+                    # file into place (only one concurrent linker can win).
+                    try:
+                        os.link(temp_path, newFullPath)
+                    except FileExistsError:
+                        raise FileExistsError(
+                            f"Another process wrote to '{self}'. Transfer from {src} cannot be completed."
+                        )
+                    finally:
+                        os.remove(temp_path)
             elif transfer == "link":
                 # Try hard link and if that fails use a symlink
                 with transaction.undoWith(f"link to {local_src}", os.remove, newFullPath):

lsst/resources/gs.py

@@ -202,17 +202,20 @@ class GSResourcePath(ResourcePath):
         # Should this method do anything at all?
         self.blob.upload_from_string(b"", retry=_RETRY_POLICY)
 
-    def _as_local(self) -> tuple[str, bool]:
+    @contextlib.contextmanager
+    def _as_local(
+        self, multithreaded: bool = True, tmpdir: ResourcePath | None = None
+    ) -> Iterator[ResourcePath]:
         with (
-            ResourcePath.temporary_uri(suffix=self.getExtension(), delete=False) as tmp_uri,
+            ResourcePath.temporary_uri(prefix=tmpdir, suffix=self.getExtension(), delete=True) as tmp_uri,
             time_this(log, msg="Downloading %s to local file", args=(self,)),
         ):
             try:
                 with tmp_uri.open("wb") as tmpFile:
                     self.blob.download_to_file(tmpFile, retry=_RETRY_POLICY)
+                yield tmp_uri
             except NotFound as e:
                 raise FileNotFoundError(f"No such resource: {self}") from e
-        return tmp_uri.ospath, True
 
     def transfer_from(
         self,
@@ -220,6 +223,7 @@ class GSResourcePath(ResourcePath):
         transfer: str = "copy",
         overwrite: bool = False,
         transaction: TransactionProtocol | None = None,
+        multithreaded: bool = True,
     ) -> None:
         if transfer not in self.transferModes:
             raise ValueError(f"Transfer mode '{transfer}' not supported by URI scheme {self.scheme}")
@@ -271,7 +275,10 @@ class GSResourcePath(ResourcePath):
                         break
         else:
             # Use local file and upload it
-            with src.as_local() as local_uri, time_this(log, msg=timer_msg, args=timer_args):
+            with (
+                src.as_local(multithreaded=multithreaded) as local_uri,
+                time_this(log, msg=timer_msg, args=timer_args),
+            ):
                 self.blob.upload_from_filename(local_uri.ospath, retry=_RETRY_POLICY)
 
         # This was an explicit move requested from a remote resource