toil 7.0.0__py3-none-any.whl → 8.1.0b1__py3-none-any.whl

toil/test/batchSystems/test_slurm.py

@@ -1,17 +1,28 @@
+import errno
 import textwrap
 from queue import Queue
 
+import logging
 import pytest
+import sys
 
 import toil.batchSystems.slurm
-from toil.batchSystems.abstractBatchSystem import BatchJobExitReason, EXIT_STATUS_UNAVAILABLE_VALUE
+from toil.batchSystems.abstractBatchSystem import (
+    EXIT_STATUS_UNAVAILABLE_VALUE,
+    BatchJobExitReason,
+    BatchSystemSupport,
+)
 from toil.common import Config
 from toil.lib.misc import CalledProcessErrorStderr
 from toil.test import ToilTest
 
+logger = logging.getLogger(__name__)
+
+
 # TODO: Come up with a better way to mock the commands then monkey-patching the
 # command-calling functions.
 
+
 def call_sacct(args, **_) -> str:
     """
     The arguments passed to `call_command` when executing `sacct` are:
@@ -25,6 +36,9 @@ def call_sacct(args, **_) -> str:
         1236|FAILED|0:2
         1236.extern|COMPLETED|0:0
     """
+    if sum(len(a) for a in args) > 1000:
+        # Simulate if the argument list is too long
+        raise OSError(errno.E2BIG, "Argument list is too long")
     # Fake output per fake job-id.
     sacct_info = {
         609663: "609663|FAILED|0:2\n609663.extern|COMPLETED|0:0\n",
@@ -37,7 +51,7 @@ def call_sacct(args, **_) -> str:
         789868: "789868|PENDING|0:0\n",
         789869: "789869|COMPLETED|0:0\n789869.batch|COMPLETED|0:0\n789869.extern|COMPLETED|0:0\n",
     }
-    job_ids = [int(job_id) for job_id in args[3].split(',')]
+    job_ids = [int(job_id) for job_id in args[3].split(",")]
     stdout = ""
     # Glue the fake outputs for the request job-ids together in a single string
     for job_id in job_ids:
@@ -53,7 +67,8 @@ def call_scontrol(args, **_) -> str:
     job_id = int(args[3]) if len(args) > 3 else None
     # Fake output per fake job-id.
     scontrol_info = {
-        787204: textwrap.dedent("""\
+        787204: textwrap.dedent(
+            """\
             JobId=787204 JobName=toil_job_6_CWLJob
                UserId=rapthor-mloose(54386) GroupId=rapthor-mloose(54038) MCS_label=N/A
                Priority=11067 Nice=0 Account=rapthor QOS=normal
@@ -81,8 +96,10 @@ def call_scontrol(args, **_) -> str:
                StdOut=/home/rapthor-mloose/code/toil/cwl-v1.2/tmp/toil_19512746-a9f4-4b99-b9ff-48ca5c1b661c.6.787204.out.log
                Power=
                NtasksPerTRES:0
-            """),
-        789724: textwrap.dedent("""\
+            """
+        ),
+        789724: textwrap.dedent(
+            """\
             JobId=789724 JobName=run_prefactor-cwltool.sh
                UserId=rapthor-mloose(54386) GroupId=rapthor-mloose(54038) MCS_label=N/A
                Priority=7905 Nice=0 Account=rapthor QOS=normal
@@ -110,8 +127,10 @@ def call_scontrol(args, **_) -> str:
                StdOut=/project/rapthor/Share/prefactor/L721962/slurm-789724.out
                Power=
                NtasksPerTRES:0
-            """),
-        789728: textwrap.dedent("""\
+            """
+        ),
+        789728: textwrap.dedent(
+            """\
             JobId=789728 JobName=sleep.sh
                UserId=rapthor-mloose(54386) GroupId=rapthor-mloose(54038) MCS_label=N/A
                Priority=8005 Nice=0 Account=rapthor QOS=normal
@@ -138,38 +157,63 @@ def call_scontrol(args, **_) -> str:
                StdOut=/home/rapthor-mloose/tmp/slurm-789728.out
                Power=
                NtasksPerTRES:0
-            """),
+            """
+        ),
     }
     if job_id is not None:
         try:
             stdout = scontrol_info[job_id]
         except KeyError:
-            raise CalledProcessErrorStderr(1, "slurm_load_jobs error: Invalid job id specified")
+            raise CalledProcessErrorStderr(
+                1, "slurm_load_jobs error: Invalid job id specified"
+            )
     else:
         # Glue the fake outputs for the request job-ids together in a single string
         stdout = ""
         for value in scontrol_info.values():
-            stdout += value + '\n'
+            stdout += value + "\n"
     return stdout
 
+
 def call_sacct_raises(*_):
     """
     Fake that the `sacct` command fails by raising a `CalledProcessErrorStderr`
     """
-    raise CalledProcessErrorStderr(1, "sacct: error: Problem talking to the database: "
-                                      "Connection timed out")
+    raise CalledProcessErrorStderr(
+        1, "sacct: error: Problem talking to the database: " "Connection timed out"
+    )
 
+def call_sinfo(*_) -> str:
+    """
+    Simulate asking for partition info from Slurm
+    """
+    stdout = textwrap.dedent(
+        """\
+        PARTITION GRES TIMELIMIT PRIO_TIER CPUS MEMORY
+        short* (null) 1:00:00 500 256+ 1996800+
+        medium (null) 12:00:00 500 256+ 1996800+
+        long (null) 14-00:00:00 500 256+ 1996800+
+        gpu gpu:A100:8 7-00:00:00 5000 256 996800
+        gpu gpu:A5500:8 7-00:00:00 5000 256 1996800
+        high_priority gpu:A5500:8 7-00:00:00 65000 256 1996800
+        high_priority (null) 7-00:00:00 65000 256+ 1996800+
+        simple_nodelist gpu:A100:8 1:00 65000 256 996800
+        simple_nodelist gpu:A5500:8 1:00 65000 256 1996800
+        simple_nodelist (null) 1:00 65000 256+ 1996800+
+        """
+    )
+    return stdout
 
-class FakeBatchSystem:
+class FakeBatchSystem(BatchSystemSupport):
     """
     Class that implements a minimal Batch System, needed to create a Worker (see below).
     """
 
     def __init__(self):
-        self.config = self.__fake_config()
+        super().__init__(self.__fake_config(), float("inf"), sys.maxsize, sys.maxsize)
 
     def getWaitDuration(self):
-        return 10;
+        return 10
 
     def __fake_config(self):
         """
@@ -181,10 +225,15 @@ class FakeBatchSystem:
         """
         config = Config()
         from uuid import uuid4
+
         config.workflowID = str(uuid4())
-        config.cleanWorkDir = 'always'
+        config.cleanWorkDir = "always"
+        toil.batchSystems.slurm.SlurmBatchSystem.setOptions(lambda o: setattr(config, o, None))
         return config
 
+# Make the mock class not have abstract methods anymore, even though we don't
+# implement them. See <https://stackoverflow.com/a/17345619>.
+FakeBatchSystem.__abstractmethods__ = set()
 
 class SlurmTest(ToilTest):
     """
@@ -198,7 +247,8 @@ class SlurmTest(ToilTest):
             updatedJobsQueue=Queue(),
             killQueue=Queue(),
             killedJobsQueue=Queue(),
-            boss=FakeBatchSystem())
+            boss=FakeBatchSystem(),
+        )
 
     ####
     #### tests for _getJobDetailsFromSacct()
@@ -218,15 +268,25 @@ class SlurmTest(ToilTest):
 
     def test_getJobDetailsFromSacct_many_all_exist(self):
         self.monkeypatch.setattr(toil.batchSystems.slurm, "call_command", call_sacct)
-        expected_result = {754725: ("TIMEOUT", 0), 789456: ("FAILED", 1), 789724: ("RUNNING", 0),
-                           789868: ("PENDING", 0), 789869: ("COMPLETED", 0)}
+        expected_result = {
+            754725: ("TIMEOUT", 0),
+            789456: ("FAILED", 1),
+            789724: ("RUNNING", 0),
+            789868: ("PENDING", 0),
+            789869: ("COMPLETED", 0),
+        }
         result = self.worker._getJobDetailsFromSacct(list(expected_result))
         assert result == expected_result, f"{result} != {expected_result}"
 
     def test_getJobDetailsFromSacct_many_some_exist(self):
         self.monkeypatch.setattr(toil.batchSystems.slurm, "call_command", call_sacct)
-        expected_result = {609663: ("FAILED", 130), 767925: ("FAILED", 2), 1234: (None, None),
-                           1235: (None, None), 765096: ("FAILED", 137)}
+        expected_result = {
+            609663: ("FAILED", 130),
+            767925: ("FAILED", 2),
+            1234: (None, None),
+            1235: (None, None),
+            765096: ("FAILED", 137),
+        }
         result = self.worker._getJobDetailsFromSacct(list(expected_result))
         assert result == expected_result, f"{result} != {expected_result}"
 
@@ -236,6 +296,13 @@ class SlurmTest(ToilTest):
         result = self.worker._getJobDetailsFromSacct(list(expected_result))
         assert result == expected_result, f"{result} != {expected_result}"
 
+    def test_getJobDetailsFromSacct_argument_list_too_big(self):
+        self.monkeypatch.setattr(toil.batchSystems.slurm, "call_command", call_sacct)
+        expected_result = {i: (None, None) for i in range(2000)}
+        result = self.worker._getJobDetailsFromSacct(list(expected_result))
+        assert result == expected_result, f"{result} != {expected_result}"
+
+
     ####
     #### tests for _getJobDetailsFromScontrol()
     ####
@@ -262,13 +329,21 @@ class SlurmTest(ToilTest):
 
     def test_getJobDetailsFromScontrol_many_all_exist(self):
         self.monkeypatch.setattr(toil.batchSystems.slurm, "call_command", call_scontrol)
-        expected_result = {787204: ("COMPLETED", 0), 789724: ("RUNNING", 0), 789728: ("PENDING", 0)}
+        expected_result = {
+            787204: ("COMPLETED", 0),
+            789724: ("RUNNING", 0),
+            789728: ("PENDING", 0),
+        }
         result = self.worker._getJobDetailsFromScontrol(list(expected_result))
         assert result == expected_result, f"{result} != {expected_result}"
 
     def test_getJobDetailsFromScontrol_many_some_exist(self):
         self.monkeypatch.setattr(toil.batchSystems.slurm, "call_command", call_scontrol)
-        expected_result = {787204: ("COMPLETED", 0), 789724: ("RUNNING", 0), 1234: (None, None)}
+        expected_result = {
+            787204: ("COMPLETED", 0),
+            789724: ("RUNNING", 0),
+            1234: (None, None),
+        }
         result = self.worker._getJobDetailsFromScontrol(list(expected_result))
         assert result == expected_result, f"{result} != {expected_result}"
 
@@ -284,14 +359,14 @@ class SlurmTest(ToilTest):
 
     def test_getJobExitCode_job_exists(self):
         self.monkeypatch.setattr(toil.batchSystems.slurm, "call_command", call_sacct)
-        job_id = '785023'  # FAILED
+        job_id = "785023"  # FAILED
         expected_result = (127, BatchJobExitReason.FAILED)
         result = self.worker.getJobExitCode(job_id)
         assert result == expected_result, f"{result} != {expected_result}"
 
     def test_getJobExitCode_job_not_exists(self):
         self.monkeypatch.setattr(toil.batchSystems.slurm, "call_command", call_sacct)
-        job_id = '1234'  # Non-existent
+        job_id = "1234"  # Non-existent
         expected_result = None
         result = self.worker.getJobExitCode(job_id)
         assert result == expected_result, f"{result} != {expected_result}"
@@ -301,10 +376,12 @@ class SlurmTest(ToilTest):
         This test forces the use of `scontrol` to get job information, by letting `sacct`
         raise an exception.
         """
-        self.monkeypatch.setattr(self.worker, "_getJobDetailsFromSacct", call_sacct_raises)
+        self.monkeypatch.setattr(
+            self.worker, "_getJobDetailsFromSacct", call_sacct_raises
+        )
         self.monkeypatch.setattr(toil.batchSystems.slurm, "call_command", call_scontrol)
-        job_id = '787204'  # COMPLETED
-        expected_result = (0,  BatchJobExitReason.FINISHED)
+        job_id = "787204"  # COMPLETED
+        expected_result = (0, BatchJobExitReason.FINISHED)
         result = self.worker.getJobExitCode(job_id)
         assert result == expected_result, f"{result} != {expected_result}"
 
@@ -313,9 +390,11 @@ class SlurmTest(ToilTest):
         This test forces the use of `scontrol` to get job information, by letting `sacct`
         raise an exception. Next, `scontrol` should also raise because it doesn't know the job.
         """
-        self.monkeypatch.setattr(self.worker, "_getJobDetailsFromSacct", call_sacct_raises)
+        self.monkeypatch.setattr(
+            self.worker, "_getJobDetailsFromSacct", call_sacct_raises
+        )
         self.monkeypatch.setattr(toil.batchSystems.slurm, "call_command", call_scontrol)
-        job_id = '1234'  # Non-existent
+        job_id = "1234"  # Non-existent
         try:
             _ = self.worker.getJobExitCode(job_id)
         except CalledProcessErrorStderr:
@@ -329,50 +408,54 @@ class SlurmTest(ToilTest):
 
     def test_coalesce_job_exit_codes_one_exists(self):
         self.monkeypatch.setattr(toil.batchSystems.slurm, "call_command", call_sacct)
-        job_ids = ['785023']  # FAILED
-        expected_result = [(127,  BatchJobExitReason.FAILED)]
+        job_ids = ["785023"]  # FAILED
+        expected_result = [(127, BatchJobExitReason.FAILED)]
         result = self.worker.coalesce_job_exit_codes(job_ids)
         assert result == expected_result, f"{result} != {expected_result}"
 
     def test_coalesce_job_exit_codes_one_not_exists(self):
         self.monkeypatch.setattr(toil.batchSystems.slurm, "call_command", call_sacct)
-        job_ids = ['1234']  # Non-existent
+        job_ids = ["1234"]  # Non-existent
         expected_result = [None]
         result = self.worker.coalesce_job_exit_codes(job_ids)
         assert result == expected_result, f"{result} != {expected_result}"
 
     def test_coalesce_job_exit_codes_many_all_exist(self):
         self.monkeypatch.setattr(toil.batchSystems.slurm, "call_command", call_sacct)
-        job_ids = ['754725',  # TIMEOUT,
-                   '789456',  # FAILED,
-                   '789724',  # RUNNING,
-                   '789868',  # PENDING,
-                   '789869']  # COMPLETED
+        job_ids = [
+            "754725",  # TIMEOUT,
+            "789456",  # FAILED,
+            "789724",  # RUNNING,
+            "789868",  # PENDING,
+            "789869",
+        ]  # COMPLETED
         # RUNNING and PENDING jobs should return None
         expected_result = [
             (EXIT_STATUS_UNAVAILABLE_VALUE, BatchJobExitReason.KILLED),
             (1, BatchJobExitReason.FAILED),
             None,
             None,
-            (0, BatchJobExitReason.FINISHED)
+            (0, BatchJobExitReason.FINISHED),
         ]
         result = self.worker.coalesce_job_exit_codes(job_ids)
         assert result == expected_result, f"{result} != {expected_result}"
 
     def test_coalesce_job_exit_codes_some_exists(self):
         self.monkeypatch.setattr(toil.batchSystems.slurm, "call_command", call_sacct)
-        job_ids = ['609663',  # FAILED (SIGINT)
-                   '767925',  # FAILED,
-                   '789724',  # RUNNING,
-                   '999999',  # Non-existent,
-                   '789869']  # COMPLETED
+        job_ids = [
+            "609663",  # FAILED (SIGINT)
+            "767925",  # FAILED,
+            "789724",  # RUNNING,
+            "999999",  # Non-existent,
+            "789869",
+        ]  # COMPLETED
         # RUNNING job should return None
         expected_result = [
             (130, BatchJobExitReason.FAILED),
             (2, BatchJobExitReason.FAILED),
             None,
             None,
-            (0, BatchJobExitReason.FINISHED)
+            (0, BatchJobExitReason.FINISHED),
         ]
         result = self.worker.coalesce_job_exit_codes(job_ids)
         assert result == expected_result, f"{result} != {expected_result}"
@@ -382,9 +465,11 @@ class SlurmTest(ToilTest):
         This test forces the use of `scontrol` to get job information, by letting `sacct`
         raise an exception.
         """
-        self.monkeypatch.setattr(self.worker, "_getJobDetailsFromSacct", call_sacct_raises)
+        self.monkeypatch.setattr(
+            self.worker, "_getJobDetailsFromSacct", call_sacct_raises
+        )
         self.monkeypatch.setattr(toil.batchSystems.slurm, "call_command", call_scontrol)
-        job_ids = ['787204']  # COMPLETED
+        job_ids = ["787204"]  # COMPLETED
         expected_result = [(0, BatchJobExitReason.FINISHED)]
         result = self.worker.coalesce_job_exit_codes(job_ids)
         assert result == expected_result, f"{result} != {expected_result}"
@@ -394,12 +479,143 @@ class SlurmTest(ToilTest):
         This test forces the use of `scontrol` to get job information, by letting `sacct`
         raise an exception. Next, `scontrol` should also raise because it doesn't know the job.
         """
-        self.monkeypatch.setattr(self.worker, "_getJobDetailsFromSacct", call_sacct_raises)
+        self.monkeypatch.setattr(
+            self.worker, "_getJobDetailsFromSacct", call_sacct_raises
+        )
         self.monkeypatch.setattr(toil.batchSystems.slurm, "call_command", call_scontrol)
-        job_ids = ['1234']  # Non-existent
+        job_ids = ["1234"]  # Non-existent
         try:
             _ = self.worker.coalesce_job_exit_codes(job_ids)
         except CalledProcessErrorStderr:
             pass
         else:
             assert False, "Exception CalledProcessErrorStderr not raised"
+
+    ###
+    ### Tests for partition selection
+    ##
+
+    def test_PartitionSet_get_partition(self):
+        self.monkeypatch.setattr(toil.batchSystems.slurm, "call_command", call_sinfo)
+        ps = toil.batchSystems.slurm.SlurmBatchSystem.PartitionSet()
+
+        # At zero. short will win because simple_nodelist has higher priority.
+        self.assertEqual(ps.get_partition(0), "short")
+        # Easily within the partition
+        self.assertEqual(ps.get_partition(10 * 60), "short")
+        # Exactly on the boundary
+        self.assertEqual(ps.get_partition(60 * 60), "short")
+        # Well within the next partition
+        self.assertEqual(ps.get_partition(2 * 60 * 60), "medium")
+        # Can only fit in long
+        self.assertEqual(ps.get_partition(8 * 24 * 60 * 60), "long")
+        # Could fit in gpu or long
+        self.assertEqual(ps.get_partition(6 * 24 * 60 * 60), "long")
+        # Can't fit in anything
+        with self.assertRaises(Exception):
+            ps.get_partition(365 * 24 * 60 * 60)
+
+    def test_PartitionSet_default_gpu_partition(self):
+        self.monkeypatch.setattr(toil.batchSystems.slurm, "call_command", call_sinfo)
+        ps = toil.batchSystems.slurm.SlurmBatchSystem.PartitionSet()
+
+        # Make sure we picked the useful-length GPU partition and not the super
+        # short one.
+        self.assertEqual(ps.default_gpu_partition.partition_name, "gpu")
+
+    def test_prepareSbatch_partition(self):
+        self.monkeypatch.setattr(toil.batchSystems.slurm, "call_command", call_sinfo)
+        ps = toil.batchSystems.slurm.SlurmBatchSystem.PartitionSet()
+        self.worker.boss.partitions = ps
+        # This is in seconds
+        self.worker.boss.config.slurm_time = 30
+
+        # Without a partition override in the environment, we should get the
+        # "short" partition for this job
+        command = self.worker.prepareSbatch(1, 100, 5, "job5", None, None)
+        assert "--partition=short" in command
+
+        # With a partition override, we should not. But the override will be rewritten.
+        self.worker.boss.config.slurm_args = "--something --partition foo --somethingElse"
+        command = self.worker.prepareSbatch(1, 100, 5, "job5", None, None)
+        assert "--partition=short" not in command
+        assert "--partition=foo" in command
+
+        # All ways of setting partition should work, including =
+        self.worker.boss.config.slurm_args = "--something --partition=foo --somethingElse"
+        command = self.worker.prepareSbatch(1, 100, 5, "job5", None, None)
+        assert "--partition=short" not in command
+        assert "--partition=foo" in command
+
+        # And short options
+        self.worker.boss.config.slurm_args = "--something -p foo --somethingElse"
+        command = self.worker.prepareSbatch(1, 100, 5, "job5", None, None)
+        assert "--partition=short" not in command
+        assert "--partition=foo" in command
+
+        # Partition settings from the config should override automatic selection
+        self.worker.boss.config.slurm_partition = "foobar"
+        self.worker.boss.config.slurm_args = "--something --somethingElse"
+        command = self.worker.prepareSbatch(1, 100, 5, "job5", None, None)
+        assert "--partition=foobar" in command
+
+        # But they should be overridden by the argument overrides
+        self.worker.boss.config.slurm_args = "--something --partition=baz --somethingElse"
+        command = self.worker.prepareSbatch(1, 100, 5, "job5", None, None)
+        assert "--partition=baz" in command
+
+    def test_prepareSbatch_time(self):
+        self.monkeypatch.setattr(toil.batchSystems.slurm, "call_command", call_sinfo)
+        ps = toil.batchSystems.slurm.SlurmBatchSystem.PartitionSet()
+        self.worker.boss.partitions = ps
+        # This is in seconds
+        self.worker.boss.config.slurm_time = 30
+
+        # Without a time override in the environment, we should use the normal
+        # time and the "short" partition
+        command = self.worker.prepareSbatch(1, 100, 5, "job5", None, None)
+        logger.debug("Command: %s", command)
+        assert "--time=0:30" in command
+        assert "--partition=short" in command
+
+        # With a time override, we should use it, slightly translated, and it
+        # should change the selected partition.
+        self.worker.boss.config.slurm_args = "--something --time 10:00:00 --somethingElse"
+        command = self.worker.prepareSbatch(1, 100, 5, "job5", None, None)
+        logger.debug("Command: %s", command)
+        assert "--partition=medium" in command
+        assert "--time=0:36000" in command
+
+        # All ways of setting time should work, including =
+        self.worker.boss.config.slurm_args = "--something --time=10:00:00 --somethingElse"
+        command = self.worker.prepareSbatch(1, 100, 5, "job5", None, None)
+        logger.debug("Command: %s", command)
+        assert "--partition=medium" in command
+        assert "--time=0:36000" in command
+
+        # And short options
+        self.worker.boss.config.slurm_args = "--something -t 10:00:00 --somethingElse"
+        command = self.worker.prepareSbatch(1, 100, 5, "job5", None, None)
+        logger.debug("Command: %s", command)
+        assert "--partition=medium" in command
+        assert "--time=0:36000" in command
+
+    def test_prepareSbatch_export(self):
+        self.monkeypatch.setattr(toil.batchSystems.slurm, "call_command", call_sinfo)
+        ps = toil.batchSystems.slurm.SlurmBatchSystem.PartitionSet()
+        self.worker.boss.partitions = ps
+
+        # Without any overrides, we need --export=ALL
+        command = self.worker.prepareSbatch(1, 100, 5, "job5", None, None)
+        assert "--export=ALL" in command
+
+        # With overrides, we don't get --export=ALL
+        self.worker.boss.config.slurm_args = "--export=foo"
+        command = self.worker.prepareSbatch(1, 100, 5, "job5", None, None)
+        assert "--export=ALL" not in command
+
+        # With --export-file, we don't get --export=ALL as documented.
+        self.worker.boss.config.slurm_args = "--export-file=./thefile.txt"
+        command = self.worker.prepareSbatch(1, 100, 5, "job5", None, None)
+        assert "--export=ALL" not in command
+

toil/test/cactus/test_cactus_integration.py

@@ -30,29 +30,27 @@ class CactusIntegrationTest(AbstractClusterTest):
         )
         self.leader = self.cluster.getLeader()
 
-        CACTUS_COMMIT_SHA = os.environ["CACTUS_COMMIT_SHA"] or "f5adf4013326322ae58ef1eccb8409b71d761583"  # default cactus commit
+        CACTUS_COMMIT_SHA = (
+            os.environ["CACTUS_COMMIT_SHA"]
+            or "f5adf4013326322ae58ef1eccb8409b71d761583"
+        )  # default cactus commit
 
         # command to install and run cactus on the cluster
-        cactus_command = ("python -m virtualenv --system-site-packages venv && "
-                          ". venv/bin/activate && "
-                          "git clone https://github.com/ComparativeGenomicsToolkit/cactus.git --recursive && "
-                          "cd cactus && "
-                          "git fetch origin && "
-                          f"git checkout {CACTUS_COMMIT_SHA} && "
-                          "git submodule update --init --recursive && "
-                          "pip install --upgrade 'setuptools<66' pip && "
-                          "pip install --upgrade . && "
-                          "pip install --upgrade numpy psutil && "
-                          "time cactus --batchSystem kubernetes --retryCount=3 "
-                          f"--consCores 2 --binariesMode singularity --clean always {self.jobStore} "
-                          "examples/evolverMammals.txt examples/evolverMammals.hal --root mr --defaultDisk 8G --logDebug")
-
-        # run cactus
-        self.sshUtil(
-            [
-                "bash",
-                "-c",
-                cactus_command
-            ]
+        cactus_command = (
+            "python -m virtualenv --system-site-packages venv && "
+            ". venv/bin/activate && "
+            "git clone https://github.com/ComparativeGenomicsToolkit/cactus.git --recursive && "
+            "cd cactus && "
+            "git fetch origin && "
+            f"git checkout {CACTUS_COMMIT_SHA} && "
+            "git submodule update --init --recursive && "
+            "pip install --upgrade 'setuptools' pip && "
+            "pip install --upgrade . && "
+            "pip install --upgrade numpy psutil && "
+            "time cactus --batchSystem kubernetes --retryCount=3 "
+            f"--consCores 2 --binariesMode singularity --clean always {self.jobStore} "
+            "examples/evolverMammals.txt examples/evolverMammals.hal --root mr --defaultDisk 8G --logDebug"
         )
 
+        # run cactus
+        self.sshUtil(["bash", "-c", cactus_command])

toil/test/cwl/conftest.py

@@ -14,4 +14,43 @@
 
 # https://pytest.org/latest/example/pythoncollection.html
 
+import json
+import logging
+from io import StringIO
+from typing import Any, Dict, List, Optional, Tuple
+
+from cwltest import utils
+logger = logging.getLogger(__name__)
+
 collect_ignore = ["spec"]
+
+
+# Hook into Pytest for testing CWL conformance with Toil
+# https://pytest.org/en/6.2.x/writing_plugins.html?highlight=conftest#conftest-py-local-per-directory-plugins
+# See cwltool's reference implementation:
+# https://github.com/common-workflow-language/cwltool/blob/05af6c1357c327b3146e9f5da40e7c0aa3e6d976/tests/cwl-conformance/cwltool-conftest.py
+def pytest_cwl_execute_test(
+        config: utils.CWLTestConfig,
+        processfile: str,
+        jobfile: Optional[str]
+) -> Tuple[int, Optional[Dict[str, Any]]]:
+    """Use the CWL reference runner (cwltool) to execute tests."""
+    from toil.cwl.cwltoil import main
+
+    stdout = StringIO()
+    argsl: List[str] = [f"--outdir={config.outdir}"]
+    if config.runner_quiet:
+        argsl.append("--quiet")
+    elif config.verbose:
+        argsl.append("--debug")
+    argsl.extend(config.args)
+    argsl.append(processfile)
+    if jobfile:
+        argsl.append(jobfile)
+    try:
+        result = main(args=argsl, stdout=stdout)
+    except Exception as e:
+        logger.error(e)
+        return 1, {}
+    out = stdout.getvalue()
+    return result, json.loads(out) if out else {}