toil 5.12.0__py3-none-any.whl → 6.1.0__py3-none-any.whl

toil/lib/io.py

@@ -2,6 +2,7 @@ import logging
 import os
 import shutil
 import stat
+import tempfile
 import uuid
 from contextlib import contextmanager
 from io import BytesIO
@@ -9,6 +10,26 @@ from typing import IO, Any, Callable, Iterator, Optional, Union
 
 logger = logging.getLogger(__name__)
 
+def mkdtemp(suffix: Optional[str] = None, prefix: Optional[str] = None, dir: Optional[str] = None) -> str:
+    """
+    Make a temporary directory like tempfile.mkdtemp, but with relaxed permissions.
+
+    The permissions on the directory will be 711 instead of 700, allowing the
+    group and all other users to traverse the directory. This is necessary if
+    the direcotry is on NFS and the Docker daemon would like to mount it or a
+    file inside it into a container, because on NFS even the Docker daemon
+    appears bound by the file permissions.
+
+    See <https://github.com/DataBiosphere/toil/issues/4644>, and
+    <https://stackoverflow.com/a/67928880> which talks about a similar problem
+    but in the context of user namespaces.
+    """
+    # Make the directory
+    result = tempfile.mkdtemp(suffix=suffix, prefix=prefix, dir=dir)
+    # Grant all the permissions: full control for user, and execute for group and other
+    os.chmod(result, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
+    # Return the path created
+    return result
 
 def robust_rmtree(path: Union[str, bytes]) -> None:
     """
@@ -161,17 +182,43 @@ def make_public_dir(in_directory: Optional[str] = None) -> str:
     os.chmod(this_should_never_happen, 0o777)
     return this_should_never_happen
 
-def try_path(path: str) -> Optional[str]:
+def try_path(path: str, min_size: int = 100 * 1024 * 1024) -> Optional[str]:
     """
     Try to use the given path. Return it if it exists or can be made,
     and we can make things within it, or None otherwise.
+
+    :param min_size: Reject paths on filesystems smaller than this many bytes.
     """
+
     try:
         os.makedirs(path, exist_ok=True)
     except OSError:
         # Maybe we lack permissions
         return None
-    return path if os.path.exists(path) and os.access(path, os.W_OK) else None
+
+    if not os.path.exists(path):
+        # We didn't manage to make it
+        return None
+
+    if not os.access(path, os.W_OK):
+        # It doesn't look writable
+        return None
+
+    try:
+        stats = os.statvfs(path)
+    except OSError:
+        # Maybe we lack permissions
+        return None
+
+    # Is the filesystem big enough?
+    # We need to look at the FS size and not the free space so we don't change
+    # over to a different filesystem when this one fills up.
+    fs_size = stats.f_frsize * stats.f_blocks
+    if fs_size < min_size:
+        # Too small
+        return None
+
+    return path
 
 
 class WriteWatchingStream:

toil/lib/misc.py

@@ -9,7 +9,7 @@ import sys
 import time
 import typing
 from contextlib import closing
-from typing import Iterator, List, Optional, Union
+from typing import Iterator, List, Optional
 
 import pytz
 

toil/lib/resources.py

@@ -13,6 +13,8 @@
 # limitations under the License.
 import fnmatch
 import os
+import math
+import sys
 import resource
 from typing import List, Tuple
 
@@ -20,12 +22,17 @@ from typing import List, Tuple
 def get_total_cpu_time_and_memory_usage() -> Tuple[float, int]:
     """
     Gives the total cpu time of itself and all its children, and the maximum RSS memory usage of
-    itself and its single largest child.
+    itself and its single largest child (in kibibytes).
     """
     me = resource.getrusage(resource.RUSAGE_SELF)
     children = resource.getrusage(resource.RUSAGE_CHILDREN)
     total_cpu_time = me.ru_utime + me.ru_stime + children.ru_utime + children.ru_stime
     total_memory_usage = me.ru_maxrss + children.ru_maxrss
+    if sys.platform == "darwin":
+        # On Linux, getrusage works in "kilobytes" (really kibibytes), but on
+        # Mac it works in bytes. See
+        # <https://github.com/python/cpython/issues/74698>
+        total_memory_usage = int(math.ceil(total_memory_usage / 1024))
     return total_cpu_time, total_memory_usage
 
 
@@ -42,7 +49,7 @@ def glob(glob_pattern: str, directoryname: str) -> List[str]:
     the glob_pattern and returns a list=[].
 
     :param directoryname: Any accessible folder name on the filesystem.
-    :param glob_pattern: A string like "*.txt", which would find all text files.
+    :param glob_pattern: A string like ``*.txt``, which would find all text files.
     :return: A list=[] of absolute filepaths matching the glob pattern.
     """
     matches = []

toil/lib/threading.py

@@ -16,6 +16,7 @@
 # Note: renamed from "threading.py" to "threading.py" to avoid conflicting imports
 # from the built-in "threading" from psutil in python3.9
 import atexit
+import errno
 import fcntl
 import logging
 import math
@@ -25,7 +26,7 @@ import tempfile
 import threading
 import traceback
 from contextlib import contextmanager
-from typing import Any, Dict, Iterator, Optional, Union, cast
+from typing import Dict, Iterator, Optional, Union, cast
 
 import psutil  # type: ignore
 
@@ -108,9 +109,12 @@ def cpu_count() -> int:
         return cast(int, cached)
 
     # Get the fallback answer of all the CPUs on the machine
-    total_machine_size = cast(int, psutil.cpu_count(logical=True))
+    psutil_cpu_count = cast(Optional[int], psutil.cpu_count(logical=True))
+    if psutil_cpu_count is None:
+        logger.debug('Could not retrieve the logical CPU count.')
 
-    logger.debug('Total machine size: %d cores', total_machine_size)
+    total_machine_size: Union[float, int] = psutil_cpu_count if psutil_cpu_count is not None else float('inf')
+    logger.debug('Total machine size: %s core(s)', total_machine_size)
 
     # cgroups may limit the size
     cgroup_size: Union[float, int] = float('inf')
@@ -150,13 +154,13 @@ def cpu_count() -> int:
             if quota == -1:
                 # But the quota can be -1 for unset.
                 # Assume we can use the whole machine.
-                return total_machine_size
-
-            # The thread count is how many multiples of a wall clock period we
-            # can burn in that period.
-            cgroup_size = int(math.ceil(float(quota)/float(period)))
+                cgroup_size = float('inf')
+            else:
+                # The thread count is how many multiples of a wall clock period we
+                # can burn in that period.
+                cgroup_size = int(math.ceil(float(quota)/float(period)))
 
-            logger.debug('Control group size in cores: %d', cgroup_size)
+            logger.debug('Control group size in cores: %s', cgroup_size)
     except:
         # We can't actually read these cgroup fields. Maybe we are a mac or something.
         logger.debug('Could not inspect cgroup: %s', traceback.format_exc())
@@ -174,9 +178,16 @@ def cpu_count() -> int:
     else:
         logger.debug('CPU affinity not available')
 
-    # Return the smaller of the actual thread count and the cgroup's limit, minimum 1.
-    result = cast(int, max(1, min(min(affinity_size, cgroup_size), total_machine_size)))
-    logger.debug('cpu_count: %s', str(result))
+    limit: Union[float, int] = float('inf')
+    # Apply all the limits to take the smallest
+    limit = min(limit, total_machine_size)
+    limit = min(limit, cgroup_size)
+    limit = min(limit, affinity_size)
+    if limit < 1 or limit == float('inf'):
+        # Fall back to 1 if we can't get a size
+        limit = 1
+    result = int(limit)
+    logger.debug('cpu_count: %s', result)
     # Make sure to remember it for the next call
     setattr(cpu_count, 'result', result)
     return result
@@ -358,6 +369,9 @@ def global_mutex(base_dir: str, mutex: str) -> Iterator[None]:
     :param str mutex: Mutex to lock. Must be a permissible path component.
     """
 
+    if not os.path.isdir(base_dir):
+        raise RuntimeError(f"Directory {base_dir} for mutex does not exist")
+
     # Define a filename
     lock_filename = os.path.join(base_dir, 'toil-mutex-' + mutex)
 
@@ -368,18 +382,32 @@ def global_mutex(base_dir: str, mutex: str) -> Iterator[None]:
     # get a lock on the deleted file.
 
     while True:
-        fd = -1
-
-        try:
-            # Try to create the file, ignoring if it exists or not.
-            fd = os.open(lock_filename, os.O_CREAT | os.O_WRONLY)
+        # Try to create the file, ignoring if it exists or not.
+        fd = os.open(lock_filename, os.O_CREAT | os.O_WRONLY)
 
-            # Wait until we can exclusively lock it.
-            fcntl.lockf(fd, fcntl.LOCK_EX)
+        # Wait until we can exclusively lock it.
+        fcntl.lockf(fd, fcntl.LOCK_EX)
 
-            # Holding the lock, make sure we are looking at the same file on disk still.
+        # Holding the lock, make sure we are looking at the same file on disk still.
+        try:
+            # So get the stats from the open file
             fd_stats = os.fstat(fd)
+        except OSError as e:
+            if e.errno == errno.ESTALE:
+                # The file handle has gone stale, because somebody removed the file.
+                # Try again.
+                try:
+                    fcntl.lockf(fd, fcntl.LOCK_UN)
+                except OSError:
+                    pass
+                os.close(fd)
+                continue
+            else:
+                # Something else broke
+                raise
 
+        try:
+            # And get the stats for the name in the directory
             path_stats: Optional[os.stat_result] = os.stat(lock_filename)
         except FileNotFoundError:
             path_stats = None
@@ -389,10 +417,9 @@ def global_mutex(base_dir: str, mutex: str) -> Iterator[None]:
             # any). This usually happens, because before someone releases a
             # lock, they delete the file. Go back and contend again. TODO: This
             # allows a lot of queue jumping on our mutex.
-            if fd != -1:
-                fcntl.lockf(fd, fcntl.LOCK_UN)
-                os.close(fd)
-                continue
+            fcntl.lockf(fd, fcntl.LOCK_UN)
+            os.close(fd)
+            continue
         else:
             # We have a lock on the file that the name points to. Since we
             # hold the lock, nobody will be deleting it or can be in the
@@ -407,14 +434,40 @@ def global_mutex(base_dir: str, mutex: str) -> Iterator[None]:
         # Delete it while we still own it, so we can't delete it from out from
         # under someone else who thinks they are holding it.
         logger.debug('PID %d releasing mutex %s', os.getpid(), lock_filename)
-        os.unlink(lock_filename)
-        if fd != -1:
-            fcntl.lockf(fd, fcntl.LOCK_UN)
-            # Note that we are unlinking it and then unlocking it; a lot of people
-            # might have opened it before we unlinked it and will wake up when they
-            # get the worthless lock on the now-unlinked file. We have to do some
-            # stat gymnastics above to work around this.
-            os.close(fd)
+
+        # We have had observations in the wild of the lock file not exisiting
+        # when we go to unlink it, causing a crash on mutex release. See
+        # <https://github.com/DataBiosphere/toil/issues/4654>.
+        #
+        # We want to tolerate this; maybe unlink() interacts with fcntl() locks
+        # on NFS in a way that is actually fine, somehow? But we also want to
+        # complain loudly if something is tampering with our locks or not
+        # really enforcing locks on the filesystem, so we will notice if it is
+        # the cause of further problems.
+        try:
+            path_stats = os.stat(lock_filename)
+        except FileNotFoundError:
+            path_stats = None
+
+        # Check to make sure it still looks locked before we unlink.
+        if path_stats is None:
+            logger.error('PID %d had mutex %s disappear while locked! Mutex system is not working!', os.getpid(), lock_filename)
+        elif fd_stats.st_dev != path_stats.st_dev or fd_stats.st_ino != path_stats.st_ino:
+            logger.error('PID %d had mutex %s get replaced while locked! Mutex system is not working!', os.getpid(), lock_filename)
+
+        if path_stats is not None:
+            try:
+                # Unlink the file
+                os.unlink(lock_filename)
+            except FileNotFoundError:
+                logger.error('PID %d had mutex %s disappear between stat and unlink while unlocking! Mutex system is not working!', os.getpid(), lock_filename)
+
+        # Note that we are unlinking it and then unlocking it; a lot of people
+        # might have opened it before we unlinked it and will wake up when they
+        # get the worthless lock on the now-unlinked file. We have to do some
+        # stat gymnastics above to work around this.
+        fcntl.lockf(fd, fcntl.LOCK_UN)
+        os.close(fd)
 
 
 class LastProcessStandingArena:
@@ -475,8 +528,8 @@ class LastProcessStandingArena:
         logger.debug('Joining arena %s', self.lockfileDir)
 
         # Make sure we're not in it already.
-        assert self.lockfileName is None
-        assert self.lockfileFD is None
+        if self.lockfileName is not None or self.lockfileFD is not None:
+            raise RuntimeError("A process is already in the arena")
 
         with global_mutex(self.base_dir, self.mutex):
             # Now nobody else should also be trying to join or leave.
@@ -486,9 +539,14 @@ class LastProcessStandingArena:
                 os.mkdir(self.lockfileDir)
             except FileExistsError:
                 pass
+            except Exception as e:
+                raise RuntimeError("Could not make lock file directory " + self.lockfileDir) from e
 
             # Make ourselves a file in it and lock it to prove we are alive.
-            self.lockfileFD, self.lockfileName = tempfile.mkstemp(dir=self.lockfileDir) # type: ignore
+            try:
+                self.lockfileFD, self.lockfileName = tempfile.mkstemp(dir=self.lockfileDir) # type: ignore
+            except Exception as e:
+                raise RuntimeError("Could not make lock file in " + self.lockfileDir) from e
             # Nobody can see it yet, so lock it right away
             fcntl.lockf(self.lockfileFD, fcntl.LOCK_EX) # type: ignore
 
@@ -511,8 +569,8 @@ class LastProcessStandingArena:
         """
 
         # Make sure we're in it to start.
-        assert self.lockfileName is not None
-        assert self.lockfileFD is not None
+        if self.lockfileName is None or self.lockfileFD is None:
+            raise RuntimeError("This process is not in the arena.")
 
         logger.debug('Leaving arena %s', self.lockfileDir)
 
@@ -533,7 +591,12 @@ class LastProcessStandingArena:
                 # There is someone claiming to be here. Are they alive?
                 full_path = os.path.join(self.lockfileDir, item)
 
-                fd = os.open(full_path, os.O_RDONLY)
+                try:
+                    fd = os.open(full_path, os.O_RDONLY)
+                except OSError as e:
+                    # suddenly file doesnt exist on network file system?
+                    continue
+
                 try:
                     fcntl.lockf(fd, fcntl.LOCK_SH | fcntl.LOCK_NB)
                 except OSError as e: