toil 6.0.0__py3-none-any.whl → 6.1.0__py3-none-any.whl

toil/lib/io.py

@@ -182,17 +182,43 @@ def make_public_dir(in_directory: Optional[str] = None) -> str:
     os.chmod(this_should_never_happen, 0o777)
     return this_should_never_happen
 
-def try_path(path: str) -> Optional[str]:
+def try_path(path: str, min_size: int = 100 * 1024 * 1024) -> Optional[str]:
     """
     Try to use the given path. Return it if it exists or can be made,
     and we can make things within it, or None otherwise.
+
+    :param min_size: Reject paths on filesystems smaller than this many bytes.
     """
+
     try:
         os.makedirs(path, exist_ok=True)
     except OSError:
         # Maybe we lack permissions
         return None
-    return path if os.path.exists(path) and os.access(path, os.W_OK) else None
+
+    if not os.path.exists(path):
+        # We didn't manage to make it
+        return None
+
+    if not os.access(path, os.W_OK):
+        # It doesn't look writable
+        return None
+
+    try:
+        stats = os.statvfs(path)
+    except OSError:
+        # Maybe we lack permissions
+        return None
+
+    # Is the filesystem big enough?
+    # We need to look at the FS size and not the free space so we don't change
+    # over to a different filesystem when this one fills up.
+    fs_size = stats.f_frsize * stats.f_blocks
+    if fs_size < min_size:
+        # Too small
+        return None
+
+    return path
 
 
 class WriteWatchingStream:

toil/lib/resources.py

@@ -13,6 +13,8 @@
 # limitations under the License.
 import fnmatch
 import os
+import math
+import sys
 import resource
 from typing import List, Tuple
 
@@ -20,12 +22,17 @@ from typing import List, Tuple
 def get_total_cpu_time_and_memory_usage() -> Tuple[float, int]:
     """
     Gives the total cpu time of itself and all its children, and the maximum RSS memory usage of
-    itself and its single largest child.
+    itself and its single largest child (in kibibytes).
     """
     me = resource.getrusage(resource.RUSAGE_SELF)
     children = resource.getrusage(resource.RUSAGE_CHILDREN)
     total_cpu_time = me.ru_utime + me.ru_stime + children.ru_utime + children.ru_stime
     total_memory_usage = me.ru_maxrss + children.ru_maxrss
+    if sys.platform == "darwin":
+        # On Linux, getrusage works in "kilobytes" (really kibibytes), but on
+        # Mac it works in bytes. See
+        # <https://github.com/python/cpython/issues/74698>
+        total_memory_usage = int(math.ceil(total_memory_usage / 1024))
     return total_cpu_time, total_memory_usage
 
 

toil/lib/threading.py

@@ -109,9 +109,12 @@ def cpu_count() -> int:
         return cast(int, cached)
 
     # Get the fallback answer of all the CPUs on the machine
-    total_machine_size = cast(int, psutil.cpu_count(logical=True))
+    psutil_cpu_count = cast(Optional[int], psutil.cpu_count(logical=True))
+    if psutil_cpu_count is None:
+        logger.debug('Could not retrieve the logical CPU count.')
 
-    logger.debug('Total machine size: %d cores', total_machine_size)
+    total_machine_size: Union[float, int] = psutil_cpu_count if psutil_cpu_count is not None else float('inf')
+    logger.debug('Total machine size: %s core(s)', total_machine_size)
 
     # cgroups may limit the size
     cgroup_size: Union[float, int] = float('inf')
@@ -151,13 +154,13 @@ def cpu_count() -> int:
             if quota == -1:
                 # But the quota can be -1 for unset.
                 # Assume we can use the whole machine.
-                return total_machine_size
-
-            # The thread count is how many multiples of a wall clock period we
-            # can burn in that period.
-            cgroup_size = int(math.ceil(float(quota)/float(period)))
+                cgroup_size = float('inf')
+            else:
+                # The thread count is how many multiples of a wall clock period we
+                # can burn in that period.
+                cgroup_size = int(math.ceil(float(quota)/float(period)))
 
-            logger.debug('Control group size in cores: %d', cgroup_size)
+            logger.debug('Control group size in cores: %s', cgroup_size)
     except:
         # We can't actually read these cgroup fields. Maybe we are a mac or something.
         logger.debug('Could not inspect cgroup: %s', traceback.format_exc())
@@ -175,9 +178,16 @@ def cpu_count() -> int:
     else:
         logger.debug('CPU affinity not available')
 
-    # Return the smaller of the actual thread count and the cgroup's limit, minimum 1.
-    result = cast(int, max(1, min(min(affinity_size, cgroup_size), total_machine_size)))
-    logger.debug('cpu_count: %s', str(result))
+    limit: Union[float, int] = float('inf')
+    # Apply all the limits to take the smallest
+    limit = min(limit, total_machine_size)
+    limit = min(limit, cgroup_size)
+    limit = min(limit, affinity_size)
+    if limit < 1 or limit == float('inf'):
+        # Fall back to 1 if we can't get a size
+        limit = 1
+    result = int(limit)
+    logger.debug('cpu_count: %s', result)
     # Make sure to remember it for the next call
     setattr(cpu_count, 'result', result)
     return result
@@ -529,9 +539,14 @@ class LastProcessStandingArena:
                 os.mkdir(self.lockfileDir)
             except FileExistsError:
                 pass
+            except Exception as e:
+                raise RuntimeError("Could not make lock file directory " + self.lockfileDir) from e
 
             # Make ourselves a file in it and lock it to prove we are alive.
-            self.lockfileFD, self.lockfileName = tempfile.mkstemp(dir=self.lockfileDir) # type: ignore
+            try:
+                self.lockfileFD, self.lockfileName = tempfile.mkstemp(dir=self.lockfileDir) # type: ignore
+            except Exception as e:
+                raise RuntimeError("Could not make lock file in " + self.lockfileDir) from e
             # Nobody can see it yet, so lock it right away
             fcntl.lockf(self.lockfileFD, fcntl.LOCK_EX) # type: ignore
 

toil/options/common.py

@@ -2,13 +2,12 @@ import os
 from argparse import ArgumentParser, Action, _AppendAction
 from typing import Any, Optional, Union, Type, Callable, List, Dict, TYPE_CHECKING
 
-from distutils.util import strtobool
 from configargparse import SUPPRESS
 import logging
 
 from ruamel.yaml import YAML
 
-from toil.lib.conversions import bytes2human, human2bytes
+from toil.lib.conversions import bytes2human, human2bytes, strtobool
 
 from toil.batchSystems.options import add_all_batchsystem_options
 from toil.provisioners import parse_node_types
@@ -595,12 +594,12 @@ def add_base_toil_options(parser: ArgumentParser, jobstore_as_flag: bool = False
         title="Toil log management options.",
         description="Options for how Toil should manage its logs."
     )
-    log_options.add_argument("--maxLogFileSize", dest="maxLogFileSize", default=64000, type=h2b,
+    log_options.add_argument("--maxLogFileSize", dest="maxLogFileSize", default=100 * 1024 * 1024, type=h2b,
                              action=make_open_interval_action(1),
                              help=f"The maximum size of a job log file to keep (in bytes), log files larger than "
                                   f"this will be truncated to the last X bytes. Setting this option to zero will "
                                   f"prevent any truncation. Setting this option to a negative value will truncate "
-                                  f"from the beginning.  Default={bytes2human(64000)}")
+                                  f"from the beginning.  Default={bytes2human(100 * 1024 * 1024)}")
     log_options.add_argument("--writeLogs", dest="writeLogs", nargs='?', action='store', default=None,
                              const=os.getcwd(), metavar="OPT_PATH",
                              help="Write worker logs received by the leader into their own files at the specified "
@@ -692,8 +691,7 @@ def add_base_toil_options(parser: ArgumentParser, jobstore_as_flag: bool = False
                                    'TOIL_APPLIANCE_SELF, which Toil uses to provision mesos for autoscaling.')
     misc_options.add_argument('--statusWait', dest='statusWait', type=int, default=3600, metavar="INT",
                               help="Seconds to wait between reports of running jobs.")
-    misc_options.add_argument('--disableProgress', dest='disableProgress', type=convert_bool, default=False,
-                              metavar="BOOL",
+    misc_options.add_argument('--disableProgress', dest='disableProgress', action="store_true", default=False,
                               help="Disables the progress bar shown when standard error is a terminal.")
 
     # Debug options
@@ -735,4 +733,4 @@ def add_base_toil_options(parser: ArgumentParser, jobstore_as_flag: bool = False
 
     # dest is set to enableCaching to not conflict with the current --caching destination
     caching.add_argument('--disableCaching', dest='enableCaching', action='store_false', help=SUPPRESS)
-    caching.set_defaults(disableCaching=None)
+    caching.set_defaults(enableCaching=None)

toil/options/wdl.py

@@ -13,6 +13,8 @@ def add_wdl_options(parser: ArgumentParser, suppress: bool = True) -> None:
     suppress_help = SUPPRESS if suppress else None
     # include arg names without a wdl specifier if suppress is False
     # this is to avoid possible duplicate options in custom toil scripts, ex outputFile can be a common argument name
+    # TODO: Why do we even need them at all in other Toil scripts? Do we have to worry about dest= collisions?
+    # TODO: Can the better option name be first?
     output_dialect_arguments = ["--wdlOutputDialect"] + (["--outputDialect"] if not suppress else [])
     parser.add_argument(*output_dialect_arguments, dest="output_dialect", type=str, default='cromwell',
                         choices=['cromwell', 'miniwdl'],
@@ -30,3 +32,6 @@ def add_wdl_options(parser: ArgumentParser, suppress: bool = True) -> None:
     reference_inputs_arguments = ["--wdlReferenceInputs"] + (["--referenceInputs"] if not suppress else [])
     parser.add_argument(*reference_inputs_arguments, dest="reference_inputs", type=bool, default=False,
                         help=suppress_help or "Pass input files by URL")
+    container_arguments = ["--wdlContainer"] + (["--container"] if not suppress else [])
+    parser.add_argument(*container_arguments, dest="container", type=str, choices=["singularity", "docker", "auto"], default="auto",
+                        help=suppress_help or "Container engine to use to run WDL tasks")

toil/provisioners/abstractProvisioner.py

@@ -812,6 +812,14 @@ class AbstractProvisioner(ABC):
                 -v /opt:/opt \\
                 -v /etc/kubernetes:/etc/kubernetes \\
                 -v /etc/kubernetes/admin.conf:/root/.kube/config \\
+                # Pass in a path to use for singularity image caching into the container
+                -e TOIL_KUBERNETES_HOST_PATH=/var/lib/toil \\
+                -e SINGULARITY_CACHEDIR=/var/lib/toil/singularity \\
+                -e MINIWDL__SINGULARITY__IMAGE_CACHE=/var/lib/toil/miniwdl \\
+                # These rules are necessary in order to get user namespaces working
+                # https://github.com/apptainer/singularity/issues/5806
+                --security-opt seccomp=unconfined \\
+                --security-opt systempaths=unconfined \\
                 --name=toil_{role} \\
                 {applianceSelf()} \\
                 {entryPointArgs}

toil/statsAndLogging.py

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import gzip
+import io
 import json
 import logging
 import os
@@ -49,7 +50,7 @@ class StatsAndLogging:
         self._worker.start()
 
     @classmethod
-    def formatLogStream(cls, stream: Union[IO[str], IO[bytes]], job_name: Optional[str] = None) -> str:
+    def formatLogStream(cls, stream: Union[IO[str], IO[bytes]], stream_name: str) -> str:
         """
         Given a stream of text or bytes, and the job name, job itself, or some
         other optional stringifyable identity info for the job, return a big
@@ -62,7 +63,7 @@ class StatsAndLogging:
 
         :param stream: The stream of text or bytes to print for the user.
         """
-        lines = [f'Log from job "{job_name}" follows:', '=========>']
+        lines = [f'{stream_name} follows:', '=========>']
 
         for line in stream:
             if isinstance(line, bytes):
@@ -75,13 +76,13 @@ class StatsAndLogging:
 
 
     @classmethod
-    def logWithFormatting(cls, jobStoreID: str, jobLogs: Union[IO[str], IO[bytes]], method: Callable[[str], None] = logger.debug,
+    def logWithFormatting(cls, stream_name: str, jobLogs: Union[IO[str], IO[bytes]], method: Callable[[str], None] = logger.debug,
                             message: Optional[str] = None) -> None:
         if message is not None:
             method(message)
 
-        # Format and log the logs, identifying the job with its job store ID.
-        method(cls.formatLogStream(jobLogs, jobStoreID))
+        # Format and log the logs, identifying the stream with the given name.
+        method(cls.formatLogStream(jobLogs, stream_name))
 
     @classmethod
     def writeLogFiles(cls, jobNames: List[str], jobLogList: List[str], config: 'Config', failed: bool = False) -> None:
@@ -95,7 +96,7 @@ class StatsAndLogging:
             logName = ('failed_' if failed else '') + logName
             counter = 0
             while True:
-                suffix = str(counter).zfill(3) + logExtension
+                suffix = '_' + str(counter).zfill(3) + logExtension
                 fullName = os.path.join(logPath, logName + suffix)
                 #  The maximum file name size in the default HFS+ file system is 255 UTF-16 encoding units, so basically 255 characters
                 if len(fullName) >= 255:
@@ -118,6 +119,9 @@ class StatsAndLogging:
             # we don't have anywhere to write the logs, return now
             return
 
+        # Make sure the destination exists
+        os.makedirs(path, exist_ok=True)
+
         fullName = createName(path, mainFileName, extension, failed)
         with writeFn(fullName, 'wb') as f:
             for l in jobLogList:
@@ -150,8 +154,10 @@ class StatsAndLogging:
             stats = json.loads(statsStr, object_hook=Expando)
             if not stats:
                 return
+
             try:
-                logs = stats.workers.logsToMaster
+                # Handle all the log_to_leader messages
+                logs = stats.workers.logs_to_leader
             except AttributeError:
                 # To be expected if there were no calls to log_to_leader()
                 pass
@@ -160,6 +166,28 @@ class StatsAndLogging:
                     logger.log(int(message.level),
                                'Got message from job at time %s: %s',
                                time.strftime('%m-%d-%Y %H:%M:%S'), message.text)
+
+            try:
+                # Handle all the user-level text streams reported back (command output, etc.)
+                user_logs = stats.workers.logging_user_streams
+            except AttributeError:
+                # To be expected if there were no calls to log_user_stream()
+                pass
+            else:
+                for stream_entry in user_logs:
+                    try:
+                        # Unpack the stream name and text.
+                        name, text = stream_entry.name, stream_entry.text
+                    except AttributeError:
+                        # Doesn't have a user-provided stream name and stream
+                        # text, so skip it.
+                        continue
+                    # Since this is sent as inline text we need to pretend to stream it.
+                    # TODO: Save these as individual files if they start to get too big?
+                    cls.logWithFormatting(name, io.StringIO(text), logger.info)
+                    # Save it as a log file, as if it were a Toil-level job.
+                    cls.writeLogFiles([name], [text], config=config)
+
             try:
                 logs = stats.logs
             except AttributeError:
@@ -168,7 +196,7 @@ class StatsAndLogging:
                 # we may have multiple jobs per worker
                 jobNames = logs.names
                 messages = logs.messages
-                cls.logWithFormatting(jobNames[0], messages,
+                cls.logWithFormatting(f'Log from job "{jobNames[0]}"', messages,
                                       message='Received Toil worker log. Disable debug level logging to hide this output')
                 cls.writeLogFiles(jobNames, messages, config=config)
 

toil/test/batchSystems/test_slurm.py

@@ -4,6 +4,7 @@ from queue import Queue
 import pytest
 
 import toil.batchSystems.slurm
+from toil.batchSystems.abstractBatchSystem import BatchJobExitReason, EXIT_STATUS_UNAVAILABLE_VALUE
 from toil.common import Config
 from toil.lib.misc import CalledProcessErrorStderr
 from toil.test import ToilTest
@@ -284,7 +285,7 @@ class SlurmTest(ToilTest):
     def test_getJobExitCode_job_exists(self):
         self.monkeypatch.setattr(toil.batchSystems.slurm, "call_command", call_sacct)
         job_id = '785023'  # FAILED
-        expected_result = 127
+        expected_result = (127, BatchJobExitReason.FAILED)
         result = self.worker.getJobExitCode(job_id)
         assert result == expected_result, f"{result} != {expected_result}"
 
@@ -303,7 +304,7 @@ class SlurmTest(ToilTest):
         self.monkeypatch.setattr(self.worker, "_getJobDetailsFromSacct", call_sacct_raises)
         self.monkeypatch.setattr(toil.batchSystems.slurm, "call_command", call_scontrol)
         job_id = '787204'  # COMPLETED
-        expected_result = 0
+        expected_result = (0,  BatchJobExitReason.FINISHED)
         result = self.worker.getJobExitCode(job_id)
         assert result == expected_result, f"{result} != {expected_result}"
 
@@ -329,7 +330,7 @@ class SlurmTest(ToilTest):
     def test_coalesce_job_exit_codes_one_exists(self):
         self.monkeypatch.setattr(toil.batchSystems.slurm, "call_command", call_sacct)
         job_ids = ['785023']  # FAILED
-        expected_result = [127]
+        expected_result = [(127,  BatchJobExitReason.FAILED)]
         result = self.worker.coalesce_job_exit_codes(job_ids)
         assert result == expected_result, f"{result} != {expected_result}"
 
@@ -347,7 +348,14 @@ class SlurmTest(ToilTest):
                    '789724',  # RUNNING,
                    '789868',  # PENDING,
                    '789869']  # COMPLETED
-        expected_result = [0, 1, None, None, 0]  # RUNNING and PENDING jobs should return None
+        # RUNNING and PENDING jobs should return None
+        expected_result = [
+            (EXIT_STATUS_UNAVAILABLE_VALUE, BatchJobExitReason.KILLED),
+            (1, BatchJobExitReason.FAILED),
+            None,
+            None,
+            (0, BatchJobExitReason.FINISHED)
+        ]
         result = self.worker.coalesce_job_exit_codes(job_ids)
         assert result == expected_result, f"{result} != {expected_result}"
 
@@ -358,7 +366,14 @@ class SlurmTest(ToilTest):
                    '789724',  # RUNNING,
                    '999999',  # Non-existent,
                    '789869']  # COMPLETED
-        expected_result = [130, 2, None, None, 0]    # RUNNING job should return None
+        # RUNNING job should return None
+        expected_result = [
+            (130, BatchJobExitReason.FAILED),
+            (2, BatchJobExitReason.FAILED),
+            None,
+            None,
+            (0, BatchJobExitReason.FINISHED)
+        ]
         result = self.worker.coalesce_job_exit_codes(job_ids)
         assert result == expected_result, f"{result} != {expected_result}"
 
@@ -370,7 +385,7 @@ class SlurmTest(ToilTest):
         self.monkeypatch.setattr(self.worker, "_getJobDetailsFromSacct", call_sacct_raises)
         self.monkeypatch.setattr(toil.batchSystems.slurm, "call_command", call_scontrol)
         job_ids = ['787204']  # COMPLETED
-        expected_result = [0]
+        expected_result = [(0, BatchJobExitReason.FINISHED)]
         result = self.worker.coalesce_job_exit_codes(job_ids)
         assert result == expected_result, f"{result} != {expected_result}"
 

toil/test/cactus/test_cactus_integration.py

@@ -0,0 +1,58 @@
+import os
+import uuid
+
+from toil.provisioners import cluster_factory
+from toil.test.provisioners.clusterTest import AbstractClusterTest
+
+
+class CactusIntegrationTest(AbstractClusterTest):
+    """
+    Run the Cactus Integration test on a Kubernetes AWS cluster
+    """
+
+    def __init__(self, methodName):
+        super().__init__(methodName=methodName)
+        self.clusterName = "cactus-test-" + str(uuid.uuid4())
+        self.leaderNodeType = "t2.medium"
+        self.clusterType = "kubernetes"
+
+    def setUp(self):
+        super().setUp()
+        self.jobStore = f"aws:{self.awsRegion()}:cluster-{uuid.uuid4()}"
+
+    def test_cactus_integration(self):
+        # Make a cluster with worker nodes
+        self.createClusterUtil(args=["--nodeTypes=t2.xlarge", "-w=1-3"])
+        # get the leader so we know the IP address - we don't need to wait since create cluster
+        # already ensures the leader is running
+        self.cluster = cluster_factory(
+            provisioner="aws", zone=self.zone, clusterName=self.clusterName
+        )
+        self.leader = self.cluster.getLeader()
+
+        CACTUS_COMMIT_SHA = os.environ["CACTUS_COMMIT_SHA"] or "f5adf4013326322ae58ef1eccb8409b71d761583"  # default cactus commit
+
+        # command to install and run cactus on the cluster
+        cactus_command = ("python -m virtualenv --system-site-packages venv && "
+                          ". venv/bin/activate && "
+                          "git clone https://github.com/ComparativeGenomicsToolkit/cactus.git --recursive && "
+                          "cd cactus && "
+                          "git fetch origin && "
+                          f"git checkout {CACTUS_COMMIT_SHA} && "
+                          "git submodule update --init --recursive && "
+                          "pip install --upgrade 'setuptools<66' pip && "
+                          "pip install --upgrade . && "
+                          "pip install --upgrade numpy psutil && "
+                          "time cactus --batchSystem kubernetes --retryCount=3 "
+                          f"--consCores 2 --binariesMode singularity --clean always {self.jobStore} "
+                          "examples/evolverMammals.txt examples/evolverMammals.hal --root mr --defaultDisk 8G --logDebug")
+
+        # run cactus
+        self.sshUtil(
+            [
+                "bash",
+                "-c",
+                cactus_command
+            ]
+        )
+