toil 6.1.0a1__py3-none-any.whl → 8.0.0__py3-none-any.whl

toil/cwl/utils.py

@@ -16,21 +16,11 @@
 
 import logging
 import os
-from pathlib import PurePosixPath
 import posixpath
 import stat
-from typing import (
-    Any,
-    Callable,
-    Dict,
-    Iterable,
-    List,
-    MutableMapping,
-    MutableSequence,
-    Type,
-    TypeVar,
-    Union,
-)
+from collections.abc import Iterable, MutableMapping, MutableSequence
+from pathlib import PurePosixPath
+from typing import Any, Callable, TypeVar, Union
 
 from toil.fileStores import FileID
 from toil.fileStores.abstractFileStore import AbstractFileStore
@@ -55,7 +45,7 @@ try:
     import cwltool.errors
 
     CWL_UNSUPPORTED_REQUIREMENT_EXCEPTION: Union[
-        Type[cwltool.errors.UnsupportedRequirement], Type[CWLUnsupportedException]
+        type[cwltool.errors.UnsupportedRequirement], type[CWLUnsupportedException]
     ] = cwltool.errors.UnsupportedRequirement
 except ImportError:
     CWL_UNSUPPORTED_REQUIREMENT_EXCEPTION = CWLUnsupportedException
@@ -92,8 +82,8 @@ def visit_cwl_class_and_reduce(
     rec: Any,
     classes: Iterable[str],
     op_down: Callable[[Any], DownReturnType],
-    op_up: Callable[[Any, DownReturnType, List[UpReturnType]], UpReturnType],
-) -> List[UpReturnType]:
+    op_up: Callable[[Any, DownReturnType, list[UpReturnType]], UpReturnType],
+) -> list[UpReturnType]:
     """
     Apply the given operations to all CWL objects with the given named CWL class.
 
@@ -130,9 +120,12 @@ def visit_cwl_class_and_reduce(
     return results
 
 
-DirectoryStructure = Dict[str, Union[str, "DirectoryStructure"]]
+DirectoryStructure = dict[str, Union[str, "DirectoryStructure"]]
 
-def get_from_structure(dir_dict: DirectoryStructure, path: str) -> Union[str, DirectoryStructure, None]:
+
+def get_from_structure(
+    dir_dict: DirectoryStructure, path: str
+) -> Union[str, DirectoryStructure, None]:
     """
     Given a relative path, follow it in the given directory structure.
 
@@ -144,7 +137,7 @@ def get_from_structure(dir_dict: DirectoryStructure, path: str) -> Union[str, Di
     parts = PurePosixPath(posixpath.normpath(path)).parts
     if len(parts) == 0:
         return dir_dict
-    if parts[0] in ('..', '/'):
+    if parts[0] in ("..", "/"):
         raise RuntimeError(f"Path {path} not resolvable in virtual directory")
     found: Union[str, DirectoryStructure] = dir_dict
     for part in parts:
@@ -161,8 +154,8 @@ def get_from_structure(dir_dict: DirectoryStructure, path: str) -> Union[str, Di
 
 def download_structure(
     file_store: AbstractFileStore,
-    index: Dict[str, str],
-    existing: Dict[str, str],
+    index: dict[str, str],
+    existing: dict[str, str],
     dir_dict: DirectoryStructure,
     into_dir: str,
 ) -> None:
@@ -215,7 +208,9 @@ def download_structure(
                 )
             else:
                 # We need to download from some other kind of URL.
-                size, executable = AbstractJobStore.read_from_url(value, open(dest_path, 'wb'))
+                size, executable = AbstractJobStore.read_from_url(
+                    value, open(dest_path, "wb")
+                )
                 if executable:
                     # Make the written file executable
                     os.chmod(dest_path, os.stat(dest_path).st_mode | stat.S_IXUSR)

toil/deferred.py

@@ -11,7 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-import fcntl
+import errno
 import logging
 import os
 import tempfile
@@ -21,13 +21,16 @@ from contextlib import contextmanager
 import dill
 
 from toil.lib.io import robust_rmtree
+from toil.lib.threading import safe_lock, safe_unlock_and_close
 from toil.realtimeLogger import RealtimeLogger
 from toil.resource import ModuleDescriptor
 
 logger = logging.getLogger(__name__)
 
 
-class DeferredFunction(namedtuple('DeferredFunction', 'function args kwargs name module')):
+class DeferredFunction(
+    namedtuple("DeferredFunction", "function args kwargs name module")
+):
     """
     >>> from collections import defaultdict
     >>> df = DeferredFunction.create(defaultdict, None, {'x':1}, y=2)
@@ -36,6 +39,7 @@ class DeferredFunction(namedtuple('DeferredFunction', 'function args kwargs name
     >>> df.invoke() == defaultdict(None, x=1, y=2)
     True
     """
+
     @classmethod
     def create(cls, function, *args, **kwargs):
         """
@@ -50,21 +54,25 @@ class DeferredFunction(namedtuple('DeferredFunction', 'function args kwargs name
         # concurrently running jobs when the cache state is loaded from disk. By implication we
         # should serialize as early as possible. We need to serialize the function as well as its
         # arguments.
-        return cls(*list(map(dill.dumps, (function, args, kwargs))),
-                   name=function.__name__,
-                   module=ModuleDescriptor.forModule(function.__module__).globalize())
+        return cls(
+            *list(map(dill.dumps, (function, args, kwargs))),
+            name=function.__name__,
+            module=ModuleDescriptor.forModule(function.__module__).globalize(),
+        )
 
     def invoke(self):
         """
         Invoke the captured function with the captured arguments.
         """
-        logger.debug('Running deferred function %s.', self)
+        logger.debug("Running deferred function %s.", self)
         self.module.makeLoadable()
-        function, args, kwargs = list(map(dill.loads, (self.function, self.args, self.kwargs)))
+        function, args, kwargs = list(
+            map(dill.loads, (self.function, self.args, self.kwargs))
+        )
         return function(*args, **kwargs)
 
     def __str__(self):
-        return f'{self.__class__.__name__}({self.name}, ...)'
+        return f"{self.__class__.__name__}({self.name}, ...)"
 
     __repr__ = __str__
 
@@ -93,13 +101,13 @@ class DeferredFunctionManager:
     """
 
     # Define what directory the state directory should actaully be, under the base
-    STATE_DIR_STEM = 'deferred'
+    STATE_DIR_STEM = "deferred"
     # Have a prefix to distinguish our deferred functions from e.g. NFS
     # "silly rename" files, or other garbage that people put in our
     # directory
-    PREFIX = 'func'
+    PREFIX = "func"
     # And a suffix to distinguish in-progress from completed files
-    WIP_SUFFIX = '.tmp'
+    WIP_SUFFIX = ".tmp"
 
     def __init__(self, stateDirBase: str) -> None:
         """
@@ -122,25 +130,31 @@ class DeferredFunctionManager:
 
         # We need to get a state file, locked by us and not somebody scanning for abandoned state files.
         # So we suffix not-yet-ready ones with our suffix
-        self.stateFD, self.stateFileName = tempfile.mkstemp(dir=self.stateDir,
-                                                            prefix=self.PREFIX,
-                                                            suffix=self.WIP_SUFFIX)
+        self.stateFD, self.stateFileName = tempfile.mkstemp(
+            dir=self.stateDir, prefix=self.PREFIX, suffix=self.WIP_SUFFIX
+        )
 
         # Lock the state file. The lock will automatically go away if our process does.
         try:
-            fcntl.lockf(self.stateFD, fcntl.LOCK_EX | fcntl.LOCK_NB)
+            safe_lock(self.stateFD, block=False)
         except OSError as e:
-            # Someone else might have locked it even though they should not have.
-            raise RuntimeError(f"Could not lock deferred function state file {self.stateFileName}: {str(e)}")
+            if e.errno in (errno.EACCES, errno.EAGAIN):
+                # Someone else locked it even though they should not have.
+                raise RuntimeError(
+                    f"Could not lock deferred function state file {self.stateFileName}"
+                ) from e
+            else:
+                # Something else went wrong
+                raise
 
         # Rename it to remove the suffix
-        os.rename(self.stateFileName, self.stateFileName[:-len(self.WIP_SUFFIX)])
-        self.stateFileName = self.stateFileName[:-len(self.WIP_SUFFIX)]
+        os.rename(self.stateFileName, self.stateFileName[: -len(self.WIP_SUFFIX)])
+        self.stateFileName = self.stateFileName[: -len(self.WIP_SUFFIX)]
 
-        # Wrap the FD in a Python file object, which we will use to actually use it.
+        # Get a Python file object for the file, which we will use to actually use it.
         # Problem: we can't be readable and writable at the same time. So we need two file objects.
-        self.stateFileOut = os.fdopen(self.stateFD, 'wb')
-        self.stateFileIn = open(self.stateFileName, 'rb')
+        self.stateFileOut = open(self.stateFileName, "wb")
+        self.stateFileIn = open(self.stateFileName, "rb")
 
         logger.debug("Opened with own state file %s" % self.stateFileName)
 
@@ -157,10 +171,7 @@ class DeferredFunctionManager:
             os.unlink(self.stateFileName)
 
         # Unlock it
-        fcntl.lockf(self.stateFD, fcntl.LOCK_UN)
-
-        # Don't bother with close, destroying will close and it seems to maybe
-        # have been GC'd already anyway.
+        safe_unlock_and_close(self.stateFD)
 
     @contextmanager
     def open(self):
@@ -177,8 +188,9 @@ class DeferredFunctionManager:
         self._runOrphanedDeferredFunctions()
 
         try:
+
             def defer(deferredFunction):
-                # Just serialize defered functions one after the other.
+                # Just serialize deferred functions one after the other.
                 # If serializing later ones fails, eariler ones will still be intact.
                 # We trust dill to protect sufficiently against partial reads later.
                 logger.debug("Deferring function %s" % repr(deferredFunction))
@@ -216,7 +228,6 @@ class DeferredFunctionManager:
             logger.exception(err)
             # we tried, lets move on
 
-
     def _runDeferredFunction(self, deferredFunction):
         """
         Run a deferred function (either our own or someone else's).
@@ -228,9 +239,15 @@ class DeferredFunctionManager:
             deferredFunction.invoke()
         except Exception as err:
             # Report this in real time, if enabled. Otherwise the only place it ends up is the worker log.
-            RealtimeLogger.error("Failed to run deferred function %s: %s", repr(deferredFunction), str(err))
+            RealtimeLogger.error(
+                "Failed to run deferred function %s: %s",
+                repr(deferredFunction),
+                str(err),
+            )
         except:
-            RealtimeLogger.error("Failed to run deferred function %s", repr(deferredFunction))
+            RealtimeLogger.error(
+                "Failed to run deferred function %s", repr(deferredFunction)
+            )
 
     def _runAllDeferredFunctions(self, fileObj):
         """
@@ -318,11 +335,16 @@ class DeferredFunctionManager:
                     continue
 
                 try:
-                    fcntl.lockf(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
-                except OSError:
-                    # File is still locked by someone else.
-                    # Look at the next file instead
-                    continue
+                    safe_lock(fd, block=False)
+                except OSError as e:
+                    os.close(fd)
+                    if e.errno in (errno.EACCES, errno.EAGAIN):
+                        # File is still locked by someone else.
+                        # Look at the next file instead
+                        continue
+                    else:
+                        # Something else went wrong
+                        raise
 
                 logger.debug("Locked file %s" % fullFilename)
 
@@ -330,7 +352,7 @@ class DeferredFunctionManager:
                 foundFiles = True
 
                 # Actually run all the stored deferred functions
-                fileObj = os.fdopen(fd, 'rb')
+                fileObj = open(fullFilename, "rb")
                 self._runAllDeferredFunctions(fileObj)
                 states_handled += 1
 
@@ -342,10 +364,9 @@ class DeferredFunctionManager:
                     pass
 
                 # Unlock it
-                fcntl.lockf(fd, fcntl.LOCK_UN)
-
-                # Now close it. This closes the backing file descriptor. See
-                # <https://stackoverflow.com/a/24984929>
-                fileObj.close()
+                safe_unlock_and_close(fd)
 
-        logger.debug("Ran orphaned deferred functions from %d abandoned state files", states_handled)
+        logger.debug(
+            "Ran orphaned deferred functions from %d abandoned state files",
+            states_handled,
+        )

toil/exceptions.py

@@ -1,7 +1,7 @@
 """Neutral place for exceptions, to break import cycles."""
 
 import logging
-from typing import TYPE_CHECKING, List
+from typing import TYPE_CHECKING
 
 from toil.statsAndLogging import StatsAndLogging
 
@@ -16,7 +16,7 @@ class FailedJobsException(Exception):
     def __init__(
         self,
         job_store: "AbstractJobStore",
-        failed_jobs: List["JobDescription"],
+        failed_jobs: list["JobDescription"],
         exit_code: int = 1,
     ):
         """
@@ -36,7 +36,9 @@ class FailedJobsException(Exception):
             for job_desc in failed_jobs:
                 if job_desc.logJobStoreFileID:
                     with job_desc.getLogFileHandle(job_store) as f:
-                        self.msg += "\n" + StatsAndLogging.formatLogStream(f, job_desc)
+                        self.msg += "\n" + StatsAndLogging.formatLogStream(
+                            f, f'Log from job "{job_desc}"'
+                        )
         # catch failures to prepare more complex details and only return the basics
         except Exception:
             logger.exception("Exception when compiling information about failed jobs")

toil/fileStores/__init__.py

@@ -28,7 +28,7 @@ class FileID(str):
     the job store if unavailable in the ID.
     """
 
-    def __new__(cls, fileStoreID: str, *args: Any) -> 'FileID':
+    def __new__(cls, fileStoreID: str, *args: Any) -> "FileID":
         return super().__new__(cls, fileStoreID)
 
     def __init__(self, fileStoreID: str, size: int, executable: bool = False) -> None:
@@ -43,18 +43,18 @@ class FileID(str):
         return f'{self.size}:{"1" if self.executable else "0"}:{self}'
 
     @classmethod
-    def forPath(cls, fileStoreID: str, filePath: str) -> 'FileID':
+    def forPath(cls, fileStoreID: str, filePath: str) -> "FileID":
         executable = os.stat(filePath).st_mode & stat.S_IXUSR != 0
         return cls(fileStoreID, os.stat(filePath).st_size, executable)
 
     @classmethod
-    def unpack(cls, packedFileStoreID: str) -> 'FileID':
+    def unpack(cls, packedFileStoreID: str) -> "FileID":
         """Unpack the result of pack() into a FileID object."""
         # Only separate twice in case the FileID itself has colons in it
-        vals = packedFileStoreID.split(':', 2)
+        vals = packedFileStoreID.split(":", 2)
         # Break up the packed value
         size = int(vals[0])
-        executable = (vals[1] == "1")
+        executable = vals[1] == "1"
         value = vals[2]
         # Create the FileID
         return cls(value, size, executable)