toil 6.0.0__tar.gz → 6.1.0__tar.gz

toil-6.1.0/PKG-INFO

@@ -0,0 +1,123 @@
+Metadata-Version: 2.1
+Name: toil
+Version: 6.1.0
+Summary: Pipeline management software for clusters.
+Home-page: https://github.com/DataBiosphere/toil
+Author: Benedict Paten and the Toil community
+Author-email: toil-community@googlegroups.com
+License: Apache License v2.0
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: Intended Audience :: Healthcare Industry
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Natural Language :: English
+Classifier: Operating System :: MacOS :: MacOS X
+Classifier: Operating System :: POSIX
+Classifier: Operating System :: POSIX :: Linux
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Topic :: Scientific/Engineering
+Classifier: Topic :: Scientific/Engineering :: Bio-Informatics
+Classifier: Topic :: Scientific/Engineering :: Astronomy
+Classifier: Topic :: Scientific/Engineering :: Atmospheric Science
+Classifier: Topic :: Scientific/Engineering :: Information Analysis
+Classifier: Topic :: Scientific/Engineering :: Medical Science Apps.
+Classifier: Topic :: System :: Distributed Computing
+Classifier: Topic :: Utilities
+Requires-Python: >=3.8
+License-File: LICENSE
+Requires-Dist: dill<0.4,>=0.3.2
+Requires-Dist: requests<3,>=2
+Requires-Dist: docker<8,>=6.1.0
+Requires-Dist: urllib3<3,>=1.26.0
+Requires-Dist: python-dateutil
+Requires-Dist: psutil<6,>=3.0.1
+Requires-Dist: PyPubSub<5,>=4.0.3
+Requires-Dist: addict<2.5,>=2.2.1
+Requires-Dist: pytz>=2012
+Requires-Dist: enlighten<2,>=1.5.2
+Requires-Dist: configargparse<2,>=1.7
+Requires-Dist: ruamel.yaml>=0.15
+Requires-Dist: pyyaml<7,>=6
+Requires-Dist: typing-extensions<5,>=4.6.2
+Provides-Extra: aws
+Requires-Dist: boto<3,>=2.48.0; extra == "aws"
+Requires-Dist: boto3-stubs[boto3,iam,s3,sdb,sts]<2,>=1.28.3.post2; extra == "aws"
+Requires-Dist: mypy-boto3-iam<2,>=1.28.3.post2; extra == "aws"
+Requires-Dist: moto<5,>=4.1.11; extra == "aws"
+Provides-Extra: cwl
+Requires-Dist: cwltool==3.1.20240112164112; extra == "cwl"
+Requires-Dist: schema-salad<9,>=8.4.20230128170514; extra == "cwl"
+Requires-Dist: galaxy-tool-util<23; extra == "cwl"
+Requires-Dist: galaxy-util<23; extra == "cwl"
+Requires-Dist: ruamel.yaml<=0.19,>=0.15; extra == "cwl"
+Requires-Dist: ruamel.yaml.clib>=0.2.6; extra == "cwl"
+Requires-Dist: networkx!=2.8.1,<4; extra == "cwl"
+Requires-Dist: CacheControl[filecache]; extra == "cwl"
+Provides-Extra: encryption
+Requires-Dist: pynacl<2,>=1.4.0; extra == "encryption"
+Provides-Extra: google
+Requires-Dist: apache-libcloud<3,>=2.2.1; extra == "google"
+Requires-Dist: google-cloud-storage<=2.8.0,>=2; extra == "google"
+Requires-Dist: google-auth<3,>=2.18.1; extra == "google"
+Provides-Extra: kubernetes
+Requires-Dist: kubernetes<22,>=12.0.1; extra == "kubernetes"
+Requires-Dist: kubernetes-stubs==v22.6.0post1; extra == "kubernetes"
+Requires-Dist: types-urllib3; extra == "kubernetes"
+Requires-Dist: types-PyYAML; extra == "kubernetes"
+Requires-Dist: idna>=2; extra == "kubernetes"
+Provides-Extra: wdl
+Requires-Dist: miniwdl==1.11.1; extra == "wdl"
+Requires-Dist: wdlparse==0.1.0; extra == "wdl"
+Requires-Dist: graphlib-backport==1.0; python_version < "3.9" and extra == "wdl"
+Provides-Extra: server
+Requires-Dist: connexion[swagger-ui]<3,>=2.10.0; extra == "server"
+Requires-Dist: flask<3,>=2.0; extra == "server"
+Requires-Dist: werkzeug<3,>=2.0; extra == "server"
+Requires-Dist: flask-cors==4.0.0; extra == "server"
+Requires-Dist: gunicorn==21.2.0; extra == "server"
+Requires-Dist: celery<6,>=5.1.0; extra == "server"
+Requires-Dist: wes-service<5,>=4.0.0; extra == "server"
+Requires-Dist: ruamel.yaml<0.19,>=0.15; extra == "server"
+Provides-Extra: htcondor
+Requires-Dist: htcondor<11,>=10.2.0.post1; sys_platform != "darwin" and extra == "htcondor"
+Provides-Extra: mesos
+Requires-Dist: pymesos<0.4,>=0.3.15; python_version < "3.11" and extra == "mesos"
+Provides-Extra: all
+Requires-Dist: boto<3,>=2.48.0; extra == "all"
+Requires-Dist: boto3-stubs[boto3,iam,s3,sdb,sts]<2,>=1.28.3.post2; extra == "all"
+Requires-Dist: mypy-boto3-iam<2,>=1.28.3.post2; extra == "all"
+Requires-Dist: moto<5,>=4.1.11; extra == "all"
+Requires-Dist: cwltool==3.1.20240112164112; extra == "all"
+Requires-Dist: schema-salad<9,>=8.4.20230128170514; extra == "all"
+Requires-Dist: galaxy-tool-util<23; extra == "all"
+Requires-Dist: galaxy-util<23; extra == "all"
+Requires-Dist: ruamel.yaml<=0.19,>=0.15; extra == "all"
+Requires-Dist: ruamel.yaml.clib>=0.2.6; extra == "all"
+Requires-Dist: networkx!=2.8.1,<4; extra == "all"
+Requires-Dist: CacheControl[filecache]; extra == "all"
+Requires-Dist: pynacl<2,>=1.4.0; extra == "all"
+Requires-Dist: apache-libcloud<3,>=2.2.1; extra == "all"
+Requires-Dist: google-cloud-storage<=2.8.0,>=2; extra == "all"
+Requires-Dist: google-auth<3,>=2.18.1; extra == "all"
+Requires-Dist: kubernetes<22,>=12.0.1; extra == "all"
+Requires-Dist: kubernetes-stubs==v22.6.0post1; extra == "all"
+Requires-Dist: types-urllib3; extra == "all"
+Requires-Dist: types-PyYAML; extra == "all"
+Requires-Dist: idna>=2; extra == "all"
+Requires-Dist: miniwdl==1.11.1; extra == "all"
+Requires-Dist: wdlparse==0.1.0; extra == "all"
+Requires-Dist: graphlib-backport==1.0; python_version < "3.9" and extra == "all"
+Requires-Dist: connexion[swagger-ui]<3,>=2.10.0; extra == "all"
+Requires-Dist: flask<3,>=2.0; extra == "all"
+Requires-Dist: werkzeug<3,>=2.0; extra == "all"
+Requires-Dist: flask-cors==4.0.0; extra == "all"
+Requires-Dist: gunicorn==21.2.0; extra == "all"
+Requires-Dist: celery<6,>=5.1.0; extra == "all"
+Requires-Dist: wes-service<5,>=4.0.0; extra == "all"
+Requires-Dist: ruamel.yaml<0.19,>=0.15; extra == "all"
+Requires-Dist: pymesos<0.4,>=0.3.15; python_version < "3.11" and extra == "all"

{toil-6.0.0 → toil-6.1.0}/requirements-cwl.txt

@@ -2,7 +2,7 @@ cwltool==3.1.20240112164112
 schema-salad>=8.4.20230128170514,<9
 galaxy-tool-util<23
 galaxy-util<23
-ruamel.yaml>=0.15,<=0.18.3
+ruamel.yaml>=0.15,<=0.19
 ruamel.yaml.clib>=0.2.6
 networkx!=2.8.1,<4
 CacheControl[filecache]

{toil-6.0.0 → toil-6.1.0}/requirements-dev.txt

@@ -1,19 +1,19 @@
-pytest>=6.2.1,<8
+pytest>=6.2.1,<9
 pytest-cov>=2.12.1,<5
 pytest-timeout>=1.4.2,<3
 stubserver>=1.1,<2
-setuptools>=65.5.1,<69
+setuptools>=65.5.1,<70
 sphinx>=7,<8
 sphinx-autoapi>=2.1.1,<3
 # astroid 3 won't work until some sphinx-autoapi release after 2.1.1
 astroid>=2.15,<3
-sphinx-autodoc-typehints>=1.24.0,<2
+sphinx-autodoc-typehints>=1.24.0,<3
 sphinxcontrib-autoprogram==0.1.8
 cwltest>=2.2.20211116163652
 mypy==1.8.0
 types-requests
 types-setuptools
-types-boto
+types-boto<2.49.18.20240205
 types-pytz
 flake8>=3.8.4,<8
 flake8-bugbear>=20.11.1,<25

{toil-6.0.0 → toil-6.1.0}/requirements-server.txt

@@ -5,4 +5,4 @@ flask-cors==4.0.0
 gunicorn==21.2.0
 celery>=5.1.0, <6
 wes-service>=4.0.0, <5
-ruamel.yaml>=0.15,<0.18.4
+ruamel.yaml>=0.15,<0.19

{toil-6.0.0 → toil-6.1.0}/requirements.txt

@@ -1,6 +1,6 @@
 dill>=0.3.2, <0.4
 requests>=2, <3
-docker>=3.7.2, <8
+docker>=6.1.0, <8
 # Work around https://github.com/docker/docker-py/issues/3113
 urllib3>=1.26.0,<3
 python-dateutil
@@ -11,6 +11,5 @@ pytz>=2012
 enlighten>=1.5.2, <2
 configargparse>=1.7,<2
 ruamel.yaml>=0.15
-ruamel.yaml.string>=0.1.1
 pyyaml>=6,<7
 typing-extensions>=4.6.2, <5

{toil-6.0.0 → toil-6.1.0}/src/toil/batchSystems/abstractBatchSystem.py

@@ -59,13 +59,28 @@ class BatchJobExitReason(enum.IntEnum):
     MEMLIMIT: int = 6
     """Job hit batch system imposed memory limit."""
 
+    @classmethod
+    def to_string(cls, value: int) -> str:
+        """
+        Convert to human-readable string.
+
+        Given an int that may be or may be equal to a value from the enum,
+        produce the string value of its matching enum entry, or a stringified
+        int.
+        """
+        try:
+            return cls(value).name
+        except ValueError:
+            return str(value)
+
 class UpdatedBatchJobInfo(NamedTuple):
     jobID: int
     exitStatus: int
     """
     The exit status (integer value) of the job. 0 implies successful.
 
-    EXIT_STATUS_UNAVAILABLE_VALUE is used when the exit status is not available (e.g. job is lost).
+    EXIT_STATUS_UNAVAILABLE_VALUE is used when the exit status is not available
+    (e.g. job is lost, or otherwise died but actual exit code was not reported).
     """
 
     exitReason: Optional[BatchJobExitReason]
@@ -437,7 +452,7 @@ class BatchSystemSupport(AbstractBatchSystem):
         file_name: str = f'toil_{self.config.workflowID}.{toil_job_id}.{cluster_job_id}.{std}.log'
         logs_dir: str = self.get_batch_logs_dir()
         return os.path.join(logs_dir, file_name)
-    
+
     def format_std_out_err_glob(self, toil_job_id: int) -> str:
         """
         Get a glob string that will match all file paths generated by format_std_out_err_path for a job.
@@ -445,7 +460,7 @@ class BatchSystemSupport(AbstractBatchSystem):
         file_glob: str = f'toil_{self.config.workflowID}.{toil_job_id}.*.log'
         logs_dir: str = self.get_batch_logs_dir()
         return os.path.join(logs_dir, file_glob)
-        
+
     @staticmethod
     def workerCleanup(info: WorkerCleanupInfo) -> None:
         """
@@ -509,7 +524,7 @@ class AbstractScalableBatchSystem(AbstractBatchSystem):
     """
     A batch system that supports a variable number of worker nodes.
 
-    Used by :class:`toil.provisioners.clusterScaler.ClusterScaler` 
+    Used by :class:`toil.provisioners.clusterScaler.ClusterScaler`
     to scale the number of worker nodes in the cluster
     up or down depending on overall load.
     """

{toil-6.0.0 → toil-6.1.0}/src/toil/batchSystems/abstractGridEngineBatchSystem.py

@@ -22,7 +22,7 @@ from typing import Dict, List, Optional, Tuple, Union
 from toil.batchSystems.abstractBatchSystem import (BatchJobExitReason,
                                                    UpdatedBatchJobInfo)
 from toil.batchSystems.cleanup_support import BatchSystemCleanupSupport
-from toil.bus import ExternalBatchIdMessage
+from toil.bus import ExternalBatchIdMessage, get_job_kind
 from toil.job import AcceleratorRequirement
 from toil.lib.misc import CalledProcessErrorStderr
 
@@ -225,23 +225,20 @@ class AbstractGridEngineBatchSystem(BatchSystemCleanupSupport):
             return activity
 
         def _handle_job_status(
-            self, job_id: int, status: Union[int, None], activity: bool
+            self, job_id: int, status: Union[int, Tuple[int, Optional[BatchJobExitReason]], None], activity: bool
         ) -> bool:
             """
             Helper method for checkOnJobs to handle job statuses
             """
             if status is not None:
+                if isinstance(status, int):
+                    code = status
+                    reason = None
+                else:
+                    code, reason = status
                 self.updatedJobsQueue.put(
                     UpdatedBatchJobInfo(
-                        jobID=job_id, exitStatus=status, exitReason=None, wallTime=None
-                    )
-                )
-                self.forgetJob(job_id)
-                return True
-            if status is not None and isinstance(status, BatchJobExitReason):
-                self.updatedJobsQueue.put(
-                    UpdatedBatchJobInfo(
-                        jobID=job_id, exitStatus=1, exitReason=status, wallTime=None
+                        jobID=job_id, exitStatus=code, exitReason=reason, wallTime=None
                     )
                 )
                 self.forgetJob(job_id)
@@ -279,9 +276,9 @@ class AbstractGridEngineBatchSystem(BatchSystemCleanupSupport):
                 logger.error("GridEngine like batch system failure", exc_info=ex)
                 raise
 
-        def coalesce_job_exit_codes(self, batch_job_id_list: list) -> list:
+        def coalesce_job_exit_codes(self, batch_job_id_list: list) -> List[Union[int, Tuple[int, Optional[BatchJobExitReason]], None]]:
             """
-            Returns exit codes for a list of jobs.
+            Returns exit codes and possibly exit reasons for a list of jobs, or None if they are running.
 
             Called by AbstractGridEngineWorker.checkOnJobs().
 
@@ -351,16 +348,19 @@ class AbstractGridEngineBatchSystem(BatchSystemCleanupSupport):
             raise NotImplementedError()
 
         @abstractmethod
-        def getJobExitCode(self, batchJobID):
+        def getJobExitCode(self, batchJobID) -> Union[int, Tuple[int, Optional[BatchJobExitReason]], None]:
             """
-            Returns job exit code or an instance of abstractBatchSystem.BatchJobExitReason.
-            if something else happened other than the job exiting.
-            Implementation-specific; called by AbstractGridEngineWorker.checkOnJobs()
+            Returns job exit code and possibly an instance of abstractBatchSystem.BatchJobExitReason.
 
-            :param string batchjobID: batch system job ID
+            Returns None if the job is still running.
+
+            If the job is not running but the exit code is not available, it
+            will be EXIT_STATUS_UNAVAILABLE_VALUE. Implementation-specific;
+            called by AbstractGridEngineWorker.checkOnJobs().
 
-            :rtype: int|toil.batchSystems.abstractBatchSystem.BatchJobExitReason: exit code int
-                    or BatchJobExitReason if something else happened other than job exiting.
+            The exit code will only be 0 if the job affirmatively succeeded.
+
+            :param string batchjobID: batch system job ID
             """
             raise NotImplementedError()
 
@@ -407,10 +407,10 @@ class AbstractGridEngineBatchSystem(BatchSystemCleanupSupport):
             else:
                 gpus = jobDesc.accelerators
 
-            self.newJobsQueue.put((jobID, jobDesc.cores, jobDesc.memory, jobDesc.command, jobDesc.get_job_kind(),
+            self.newJobsQueue.put((jobID, jobDesc.cores, jobDesc.memory, jobDesc.command, get_job_kind(jobDesc.get_names()),
                                    job_environment, gpus))
             logger.debug("Issued the job command: %s with job id: %s and job name %s", jobDesc.command, str(jobID),
-                         jobDesc.get_job_kind())
+                         get_job_kind(jobDesc.get_names()))
         return jobID
 
     def killBatchJobs(self, jobIDs):

{toil-6.0.0 → toil-6.1.0}/src/toil/batchSystems/cleanup_support.py

@@ -69,8 +69,13 @@ class WorkerCleanupContext:
 
     def __enter__(self) -> None:
         # Set up an arena so we know who is the last worker to leave
-        self.arena = LastProcessStandingArena(Toil.get_toil_coordination_dir(self.workerCleanupInfo.work_dir, self.workerCleanupInfo.coordination_dir),
-                                              self.workerCleanupInfo.workflow_id + '-cleanup')
+        self.arena = LastProcessStandingArena(
+            Toil.get_toil_coordination_dir(
+                self.workerCleanupInfo.work_dir,
+                self.workerCleanupInfo.coordination_dir
+            ),
+            Toil.get_workflow_path_component(self.workerCleanupInfo.workflow_id) + "-cleanup"
+        )
         logger.debug('Entering cleanup arena')
         self.arena.enter()
         logger.debug('Cleanup arena entered')
@@ -90,4 +95,3 @@ class WorkerCleanupContext:
             # Now the coordination_dir is allowed to no longer exist on the node.
         logger.debug('Cleanup arena left')
 
-

{toil-6.0.0 → toil-6.1.0}/src/toil/batchSystems/lsf.py

@@ -25,12 +25,12 @@ import re
 import subprocess
 from datetime import datetime
 from random import randint
-from typing import Dict, List, Optional, Union
+from typing import Dict, List, Optional, Tuple, Union
 
 from dateutil.parser import parse
 from dateutil.tz import tzlocal
 
-from toil.batchSystems.abstractBatchSystem import BatchJobExitReason
+from toil.batchSystems.abstractBatchSystem import BatchJobExitReason, EXIT_STATUS_UNAVAILABLE_VALUE
 from toil.batchSystems.abstractGridEngineBatchSystem import \
     AbstractGridEngineBatchSystem
 from toil.batchSystems.lsfHelper import (check_lsf_json_output_supported,
@@ -161,7 +161,7 @@ class LSFBatchSystem(AbstractGridEngineBatchSystem):
                         status_resonse.append(None)
             return status_resonse
 
-        def getJobExitCode(self, lsfJobID):
+        def getJobExitCode(self, lsfJobID) -> Union[int, Tuple[int, Optional[BatchJobExitReason]], None]:
             # the task is set as part of the job ID if using getBatchSystemID()
             if "NOT_SUBMITTED" in lsfJobID:
                 logger.error("bjobs detected job failed to submit")
@@ -186,7 +186,7 @@ class LSFBatchSystem(AbstractGridEngineBatchSystem):
 
             return self.fallbackGetJobExitCode(job)
 
-        def parse_bjobs_record(self, bjobs_record: dict, job: int) -> Union[int, None]:
+        def parse_bjobs_record(self, bjobs_record: dict, job: int) -> Union[int, Tuple[int, Optional[BatchJobExitReason]], None]:
             """
             Helper functions for getJobExitCode and  to parse the bjobs status record
             """
@@ -224,7 +224,7 @@ class LSFBatchSystem(AbstractGridEngineBatchSystem):
                         exit_info, job
                     )
                     if "TERM_MEMLIMIT" in exit_reason:
-                        return BatchJobExitReason.MEMLIMIT
+                        return (exit_code if exit_code != 0 else EXIT_STATUS_UNAVAILABLE_VALUE, BatchJobExitReason.MEMLIMIT)
                     return exit_code
                 if process_status == "RUN":
                     logger.debug(
@@ -237,7 +237,7 @@ class LSFBatchSystem(AbstractGridEngineBatchSystem):
 
                 return self.getJobExitCodeBACCT(job)
 
-        def getJobExitCodeBACCT(self,job):
+        def getJobExitCodeBACCT(self,job) -> Union[int, Tuple[int, Optional[BatchJobExitReason]], None]:
             # if not found in bjobs, then try bacct (slower than bjobs)
             logger.debug("bjobs failed to detect job - trying bacct: "
                          "{}".format(job))
@@ -258,7 +258,7 @@ class LSFBatchSystem(AbstractGridEngineBatchSystem):
                          "running: {}".format(job))
             return None
 
-        def fallbackGetJobExitCode(self, job):
+        def fallbackGetJobExitCode(self, job) -> Union[int, Tuple[int, Optional[BatchJobExitReason]], None]:
             args = ["bjobs", "-l", str(job)]
             logger.debug(f"Checking job exit code for job via bjobs (fallback): {job}")
             stdout = call_command(args)

{toil-6.0.0 → toil-6.1.0}/src/toil/batchSystems/slurm.py

@@ -16,8 +16,9 @@ import math
 import os
 from argparse import ArgumentParser, _ArgumentGroup
 from shlex import quote
-from typing import Dict, List, Optional, TypeVar, Union
+from typing import Dict, List, Optional, Tuple, TypeVar, Union
 
+from toil.batchSystems.abstractBatchSystem import BatchJobExitReason, EXIT_STATUS_UNAVAILABLE_VALUE
 from toil.batchSystems.abstractGridEngineBatchSystem import \
     AbstractGridEngineBatchSystem
 from toil.batchSystems.options import OptionSetter
@@ -64,7 +65,9 @@ class SlurmBatchSystem(AbstractGridEngineBatchSystem):
                               jobName: str,
                               job_environment: Optional[Dict[str, str]] = None,
                               gpus: Optional[int] = None) -> List[str]:
-            return self.prepareSbatch(cpu, memory, jobID, jobName, job_environment, gpus) + [f'--wrap={command}']
+            # Make sure to use exec so we can get Slurm's signals in the Toil
+            # worker instead of having an intervening Bash
+            return self.prepareSbatch(cpu, memory, jobID, jobName, job_environment, gpus) + [f'--wrap=exec {command}']
 
         def submitJob(self, subLine):
             try:
@@ -95,12 +98,12 @@ class SlurmBatchSystem(AbstractGridEngineBatchSystem):
                 logger.error("sbatch command failed")
                 raise e
 
-        def coalesce_job_exit_codes(self, batch_job_id_list: list) -> list:
+        def coalesce_job_exit_codes(self, batch_job_id_list: list) -> List[Union[int, Tuple[int, Optional[BatchJobExitReason]], None]]:
             """
             Collect all job exit codes in a single call.
             :param batch_job_id_list: list of Job ID strings, where each string has the form
             "<job>[.<task>]".
-            :return: list of job exit codes, associated with the list of job IDs.
+            :return: list of job exit codes or exit code, exit reason pairs associated with the list of job IDs.
             """
             logger.debug("Getting exit codes for slurm jobs: %s", batch_job_id_list)
             # Convert batch_job_id_list to list of integer job IDs.
@@ -111,7 +114,7 @@ class SlurmBatchSystem(AbstractGridEngineBatchSystem):
                 exit_codes.append(self._get_job_return_code(status))
             return exit_codes
 
-        def getJobExitCode(self, batchJobID: str) -> int:
+        def getJobExitCode(self, batchJobID: str) -> Union[int, Tuple[int, Optional[BatchJobExitReason]], None]:
             """
             Get job exit code for given batch job ID.
             :param batchJobID: string of the form "<job>[.<task>]".
@@ -138,18 +141,68 @@ class SlurmBatchSystem(AbstractGridEngineBatchSystem):
                 status_dict = self._getJobDetailsFromScontrol(job_id_list)
             return status_dict
 
-        def _get_job_return_code(self, status: tuple) -> list:
+        def _get_job_return_code(self, status: tuple) -> Union[int, Tuple[int, Optional[BatchJobExitReason]], None]:
             """
+            Given a Slurm return code, status pair, summarize them into a Toil return code, exit reason pair.
+
+            The return code may have already been OR'd with the 128-offset
+            Slurm-reported signal.
+
+            Slurm will report return codes of 0 even if jobs time out instead
+            of succeeding:
+
+                2093597|TIMEOUT|0:0
+                2093597.batch|CANCELLED|0:15
+
+            So we guarantee here that, if the Slurm status string is not a
+            successful one as defined in
+            <https://slurm.schedmd.com/squeue.html#SECTION_JOB-STATE-CODES>, we
+            will not return a successful return code.
+
             Helper function for `getJobExitCode` and `coalesce_job_exit_codes`.
-            :param status: tuple containing the job's state and it's return code.
-            :return: the job's return code if it's completed, otherwise None.
+            :param status: tuple containing the job's state and it's return code from Slurm.
+            :return: the job's return code for Toil if it's completed, otherwise None.
             """
             state, rc = status
-            # If job is in a running state, set return code to None to indicate we don't have
-            # an update.
-            if state in ('PENDING', 'RUNNING', 'CONFIGURING', 'COMPLETING', 'RESIZING', 'SUSPENDED'):
-                rc = None
-            return rc
+
+            # If a job is in one of these states, Slurm can't run it anymore.
+            # We don't include states where the job is held or paused here;
+            # those mean it could run and needs to wait for someone to un-hold
+            # it, so Toil should wait for it.
+            #
+            # We map from each terminal state to the Toil-ontology exit reason.
+            TERMINAL_STATES: Dict[str, BatchJobExitReason] = {
+                "BOOT_FAIL": BatchJobExitReason.LOST,
+                "CANCELLED": BatchJobExitReason.KILLED,
+                "COMPLETED": BatchJobExitReason.FINISHED,
+                "DEADLINE": BatchJobExitReason.KILLED,
+                "FAILED": BatchJobExitReason.FAILED,
+                "NODE_FAIL": BatchJobExitReason.LOST,
+                "OUT_OF_MEMORY": BatchJobExitReason.MEMLIMIT,
+                "PREEMPTED": BatchJobExitReason.KILLED,
+                "TIMEOUT": BatchJobExitReason.KILLED
+            }
+
+            if state not in TERMINAL_STATES:
+                # Don't treat the job as exited yet
+                return None
+
+            exit_reason = TERMINAL_STATES[state]
+
+            if exit_reason == BatchJobExitReason.FINISHED:
+                # The only state that should produce a 0 ever is COMPLETED. So
+                # if the job is COMPLETED and the exit reason is thus FINISHED,
+                # pass along the code it has.
+                return (rc, exit_reason)
+
+            if rc == 0:
+                # The job claims to be in a state other than COMPLETED, but
+                # also to have not encountered a problem. Say the exit status
+                # is unavailable.
+                return (EXIT_STATUS_UNAVAILABLE_VALUE, exit_reason)
+
+            # If the code is nonzero, pass it along.
+            return (rc, exit_reason)
 
         def _getJobDetailsFromSacct(self, job_id_list: list) -> dict:
             """
@@ -283,8 +336,26 @@ class SlurmBatchSystem(AbstractGridEngineBatchSystem):
                           job_environment: Optional[Dict[str, str]],
                           gpus: Optional[int]) -> List[str]:
 
-            #  Returns the sbatch command line before the script to run
+            """
+            Returns the sbatch command line to run to queue the job.
+            """
+
+            # Start by naming the job
             sbatch_line = ['sbatch', '-J', f'toil_job_{jobID}_{jobName}']
+
+            # Make sure the job gets a signal before it disappears so that e.g.
+            # container cleanup finally blocks can run. Ask for SIGINT so we
+            # can get the default Python KeyboardInterrupt which third-party
+            # code is likely to plan for. Make sure to send it to the batch
+            # shell process with "B:", not to all the srun steps it launches
+            # (because there shouldn't be any). We cunningly replaced the batch
+            # shell process with the Toil worker process, so Toil should be
+            # able to get the signal.
+            #
+            # TODO: Add a way to detect when the job failed because it
+            # responded to this signal and use the right exit reason for it.
+            sbatch_line.append("--signal=B:INT@30")
+
             if gpus:
                 sbatch_line = sbatch_line[:1] + [f'--gres=gpu:{gpus}'] + sbatch_line[1:]
             environment = {}

{toil-6.0.0 → toil-6.1.0}/src/toil/bus.py

@@ -87,6 +87,43 @@ from pubsub.core.topicutils import ALL_TOPICS
 
 logger = logging.getLogger( __name__ )
 
+# We define some ways to talk about jobs.
+
+class Names(NamedTuple):
+    """
+    Stores all the kinds of name a job can have.
+    """
+    # Name of the kind of job this is
+    job_name: str
+    # Name of this particular work unit
+    unit_name: str
+    # Human-readable name for the job
+    display_name: str
+    # What the job prints as, used for stats-and-logging log management
+    stats_name: str
+    # Job store ID of the job for the work unit
+    job_store_id: str
+
+def get_job_kind(names: Names) -> str:
+    """
+    Return an identifying string for the job.
+
+    The result may contain spaces.
+
+    Returns: Either the unit name, job name, or display name, which identifies
+             the kind of job it is to toil.
+             Otherwise "Unknown Job" in case no identifier is available
+    """
+    if names.unit_name:
+        return names.unit_name
+    elif names.job_name:
+        return names.job_name
+    elif names.display_name:
+        return names.display_name
+    else:
+        return "Unknown Job"
+
+
 # We define a bunch of named tuple message types.
 # These all need to be plain data: only hold ints, strings, etc.
 
@@ -648,6 +685,7 @@ class JobStatus:
 
     def __repr__(self) -> str:
         return json.dumps(self, default= lambda o: o.__dict__, indent=4)
+
 def replay_message_bus(path: str) -> Dict[str, JobStatus]:
     """
     Replay all the messages and work out what they mean for jobs.