toil 5.12.0__py3-none-any.whl → 6.1.0__py3-none-any.whl

toil/test/batchSystems/batchSystemTest.py

@@ -31,10 +31,9 @@ from toil.batchSystems.abstractBatchSystem import (AbstractBatchSystem,
 # in order to import properly. Import them later, in tests
 # protected by annotations.
 from toil.batchSystems.mesos.test import MesosTestSupport
-from toil.batchSystems.parasol import ParasolBatchSystem
-from toil.batchSystems.registry import (BATCH_SYSTEM_FACTORY_REGISTRY,
-                                        BATCH_SYSTEMS,
-                                        addBatchSystemFactory,
+from toil.batchSystems.registry import (add_batch_system_factory,
+                                        get_batch_system,
+                                        get_batch_systems,
                                         restore_batch_system_plugin_state,
                                         save_batch_system_plugin_state)
 from toil.batchSystems.singleMachine import SingleMachineBatchSystem
@@ -52,12 +51,9 @@ from toil.test import (ToilTest,
                        needs_kubernetes_installed,
                        needs_lsf,
                        needs_mesos,
-                       needs_parasol,
                        needs_slurm,
-                       needs_tes,
                        needs_torque,
                        slow)
-from toil.test.batchSystems.parasolTestSupport import ParasolTestSupport
 
 logger = logging.getLogger(__name__)
 
@@ -88,16 +84,16 @@ class BatchSystemPluginTest(ToilTest):
         restore_batch_system_plugin_state(self.__state)
         super().tearDown()
 
-    def testAddBatchSystemFactory(self):
+    def test_add_batch_system_factory(self):
         def test_batch_system_factory():
             # TODO: Adding the same batch system under multiple names means we
             # can't actually create Toil options, because each version tries to
             # add its arguments.
             return SingleMachineBatchSystem
 
-        addBatchSystemFactory('testBatchSystem', test_batch_system_factory)
-        assert ('testBatchSystem', test_batch_system_factory) in BATCH_SYSTEM_FACTORY_REGISTRY.items()
-        assert 'testBatchSystem' in BATCH_SYSTEMS
+        add_batch_system_factory('testBatchSystem', test_batch_system_factory)
+        assert 'testBatchSystem' in get_batch_systems()
+        assert get_batch_system('testBatchSystem') == SingleMachineBatchSystem
 
 class hidden:
     """
@@ -248,10 +244,6 @@ class hidden:
             self.batchSystem.killBatchJobs([10])
 
         def test_set_env(self):
-            # Parasol disobeys shell rules and splits the command at the space
-            # character into arguments before exec'ing it, whether the space is
-            # quoted, escaped or not.
-
             # Start with a relatively safe script
             script_shell = 'if [ "x${FOO}" == "xbar" ] ; then exit 23 ; else exit 42 ; fi'
 
@@ -575,23 +567,6 @@ class KubernetesBatchSystemBenchTest(ToilTest):
         self.assertEqual(str(spec.tolerations), "None")
 
 
-@needs_tes
-@needs_fetchable_appliance
-class TESBatchSystemTest(hidden.AbstractBatchSystemTest):
-    """
-    Tests against the TES batch system
-    """
-
-    def supportsWallTime(self):
-        return True
-
-    def createBatchSystem(self):
-        # Import the batch system when we know we have it.
-        # Doesn't really matter for TES right now, but someday it might.
-        from toil.batchSystems.tes import TESBatchSystem
-        return TESBatchSystem(config=self.config,
-                              maxCores=numCores, maxMemory=1e9, maxDisk=2001)
-
 @needs_aws_batch
 @needs_fetchable_appliance
 class AWSBatchBatchSystemTest(hidden.AbstractBatchSystemTest):
@@ -849,7 +824,7 @@ class MaxCoresSingleMachineBatchSystemTest(ToilTest):
             if len(sys.argv) < 3:
                 count(1)
                 try:
-                    time.sleep(1)
+                    time.sleep(0.5)
                 finally:
                     count(-1)
             else:
@@ -910,9 +885,10 @@ class MaxCoresSingleMachineBatchSystemTest(ToilTest):
                         logger.info(f'maxCores: {maxCores}, '
                                  f'coresPerJob: {coresPerJob}, '
                                  f'load: {load}')
-                        # This is the key assertion:
+                        # This is the key assertion: we shouldn't run too many jobs.
+                        # Because of nondeterminism we can't guarantee hitting the limit.
                         expectedMaxConcurrentTasks = min(maxCores // coresPerJob, jobs)
-                        self.assertEqual(maxConcurrentTasks, expectedMaxConcurrentTasks)
+                        self.assertLessEqual(maxConcurrentTasks, expectedMaxConcurrentTasks)
                         resetCounters(self.counterPath)
 
     @skipIf(SingleMachineBatchSystem.numCores < 3, 'Need at least three cores to run this test')
@@ -965,82 +941,6 @@ class Service(Job.Service):
         subprocess.check_call(self.cmd + ' -1', shell=True)
 
 
-@slow
-@needs_parasol
-class ParasolBatchSystemTest(hidden.AbstractBatchSystemTest, ParasolTestSupport):
-    """
-    Tests the Parasol batch system
-    """
-
-    def supportsWallTime(self):
-        return True
-
-    def _createConfig(self):
-        config = super()._createConfig()
-        # can't use _getTestJobStorePath since that method removes the directory
-        config.jobStore = self._createTempDir('jobStore')
-        return config
-
-    def createBatchSystem(self) -> AbstractBatchSystem:
-        memory = int(3e9)
-        self._startParasol(numCores=numCores, memory=memory)
-
-        return ParasolBatchSystem(config=self.config,
-                                  maxCores=numCores,
-                                  maxMemory=memory,
-                                  maxDisk=1001)
-
-    def tearDown(self):
-        super().tearDown()
-        self._stopParasol()
-
-    def testBatchResourceLimits(self):
-        jobDesc1 = JobDescription(command="sleep 1000",
-                                  requirements=dict(memory=1 << 30, cores=1,
-                                                    disk=1000, accelerators=[],
-                                                    preemptible=preemptible),
-                                  jobName='testResourceLimits')
-        job1 = self.batchSystem.issueBatchJob(jobDesc1)
-        self.assertIsNotNone(job1)
-        jobDesc2 = JobDescription(command="sleep 1000",
-                                  requirements=dict(memory=2 << 30, cores=1,
-                                                    disk=1000, accelerators=[],
-                                                    preemptible=preemptible),
-                                  jobName='testResourceLimits')
-        job2 = self.batchSystem.issueBatchJob(jobDesc2)
-        self.assertIsNotNone(job2)
-        batches = self._getBatchList()
-        self.assertEqual(len(batches), 2)
-        # It would be better to directly check that the batches have the correct memory and cpu
-        # values, but Parasol seems to slightly change the values sometimes.
-        self.assertNotEqual(batches[0]['ram'], batches[1]['ram'])
-        # Need to kill one of the jobs because there are only two cores available
-        self.batchSystem.killBatchJobs([job2])
-        job3 = self.batchSystem.issueBatchJob(jobDesc1)
-        self.assertIsNotNone(job3)
-        batches = self._getBatchList()
-        self.assertEqual(len(batches), 1)
-
-    def _parseBatchString(self, batchString):
-        import re
-        batchInfo = dict()
-        memPattern = re.compile(r"(\d+\.\d+)([kgmbt])")
-        items = batchString.split()
-        batchInfo["cores"] = int(items[7])
-        memMatch = memPattern.match(items[8])
-        ramValue = float(memMatch.group(1))
-        ramUnits = memMatch.group(2)
-        ramConversion = {'b': 1e0, 'k': 1e3, 'm': 1e6, 'g': 1e9, 't': 1e12}
-        batchInfo["ram"] = ramValue * ramConversion[ramUnits]
-        return batchInfo
-
-    def _getBatchList(self):
-        # noinspection PyUnresolvedReferences
-        exitStatus, batchLines = self.batchSystem._runParasol(['list', 'batches'])
-        self.assertEqual(exitStatus, 0)
-        return [self._parseBatchString(line) for line in batchLines[1:] if line]
-
-
 @slow
 @needs_gridengine
 class GridEngineBatchSystemTest(hidden.AbstractGridEngineBatchSystemTest):

toil/test/batchSystems/test_slurm.py

@@ -4,6 +4,7 @@ from queue import Queue
 import pytest
 
 import toil.batchSystems.slurm
+from toil.batchSystems.abstractBatchSystem import BatchJobExitReason, EXIT_STATUS_UNAVAILABLE_VALUE
 from toil.common import Config
 from toil.lib.misc import CalledProcessErrorStderr
 from toil.test import ToilTest
@@ -16,7 +17,8 @@ def call_sacct(args, **_) -> str:
     The arguments passed to `call_command` when executing `sacct` are:
     ['sacct', '-n', '-j', '<comma-separated list of job-ids>', '--format',
     'JobIDRaw,State,ExitCode', '-P', '-S', '1970-01-01']
-    The multi-line output is something like:
+    The multi-line output is something like::
+
         1234|COMPLETED|0:0
         1234.batch|COMPLETED|0:0
         1235|PENDING|0:0
@@ -46,7 +48,7 @@ def call_sacct(args, **_) -> str:
 def call_scontrol(args, **_) -> str:
     """
     The arguments passed to `call_command` when executing `scontrol` are:
-    ['scontrol', 'show', 'job'] or ['scontrol', 'show', 'job', '<job-id>']
+    ``['scontrol', 'show', 'job']`` or ``['scontrol', 'show', 'job', '<job-id>']``
     """
     job_id = int(args[3]) if len(args) > 3 else None
     # Fake output per fake job-id.
@@ -283,7 +285,7 @@ class SlurmTest(ToilTest):
     def test_getJobExitCode_job_exists(self):
         self.monkeypatch.setattr(toil.batchSystems.slurm, "call_command", call_sacct)
         job_id = '785023'  # FAILED
-        expected_result = 127
+        expected_result = (127, BatchJobExitReason.FAILED)
         result = self.worker.getJobExitCode(job_id)
         assert result == expected_result, f"{result} != {expected_result}"
 
@@ -302,7 +304,7 @@ class SlurmTest(ToilTest):
         self.monkeypatch.setattr(self.worker, "_getJobDetailsFromSacct", call_sacct_raises)
         self.monkeypatch.setattr(toil.batchSystems.slurm, "call_command", call_scontrol)
         job_id = '787204'  # COMPLETED
-        expected_result = 0
+        expected_result = (0,  BatchJobExitReason.FINISHED)
         result = self.worker.getJobExitCode(job_id)
         assert result == expected_result, f"{result} != {expected_result}"
 
@@ -328,7 +330,7 @@ class SlurmTest(ToilTest):
     def test_coalesce_job_exit_codes_one_exists(self):
         self.monkeypatch.setattr(toil.batchSystems.slurm, "call_command", call_sacct)
         job_ids = ['785023']  # FAILED
-        expected_result = [127]
+        expected_result = [(127,  BatchJobExitReason.FAILED)]
         result = self.worker.coalesce_job_exit_codes(job_ids)
         assert result == expected_result, f"{result} != {expected_result}"
 
@@ -346,7 +348,14 @@ class SlurmTest(ToilTest):
                    '789724',  # RUNNING,
                    '789868',  # PENDING,
                    '789869']  # COMPLETED
-        expected_result = [0, 1, None, None, 0]  # RUNNING and PENDING jobs should return None
+        # RUNNING and PENDING jobs should return None
+        expected_result = [
+            (EXIT_STATUS_UNAVAILABLE_VALUE, BatchJobExitReason.KILLED),
+            (1, BatchJobExitReason.FAILED),
+            None,
+            None,
+            (0, BatchJobExitReason.FINISHED)
+        ]
         result = self.worker.coalesce_job_exit_codes(job_ids)
         assert result == expected_result, f"{result} != {expected_result}"
 
@@ -357,7 +366,14 @@ class SlurmTest(ToilTest):
                    '789724',  # RUNNING,
                    '999999',  # Non-existent,
                    '789869']  # COMPLETED
-        expected_result = [130, 2, None, None, 0]    # RUNNING job should return None
+        # RUNNING job should return None
+        expected_result = [
+            (130, BatchJobExitReason.FAILED),
+            (2, BatchJobExitReason.FAILED),
+            None,
+            None,
+            (0, BatchJobExitReason.FINISHED)
+        ]
         result = self.worker.coalesce_job_exit_codes(job_ids)
         assert result == expected_result, f"{result} != {expected_result}"
 
@@ -369,7 +385,7 @@ class SlurmTest(ToilTest):
         self.monkeypatch.setattr(self.worker, "_getJobDetailsFromSacct", call_sacct_raises)
         self.monkeypatch.setattr(toil.batchSystems.slurm, "call_command", call_scontrol)
         job_ids = ['787204']  # COMPLETED
-        expected_result = [0]
+        expected_result = [(0, BatchJobExitReason.FINISHED)]
         result = self.worker.coalesce_job_exit_codes(job_ids)
         assert result == expected_result, f"{result} != {expected_result}"
 

toil/test/cactus/test_cactus_integration.py

@@ -0,0 +1,58 @@
+import os
+import uuid
+
+from toil.provisioners import cluster_factory
+from toil.test.provisioners.clusterTest import AbstractClusterTest
+
+
+class CactusIntegrationTest(AbstractClusterTest):
+    """
+    Run the Cactus Integration test on a Kubernetes AWS cluster
+    """
+
+    def __init__(self, methodName):
+        super().__init__(methodName=methodName)
+        self.clusterName = "cactus-test-" + str(uuid.uuid4())
+        self.leaderNodeType = "t2.medium"
+        self.clusterType = "kubernetes"
+
+    def setUp(self):
+        super().setUp()
+        self.jobStore = f"aws:{self.awsRegion()}:cluster-{uuid.uuid4()}"
+
+    def test_cactus_integration(self):
+        # Make a cluster with worker nodes
+        self.createClusterUtil(args=["--nodeTypes=t2.xlarge", "-w=1-3"])
+        # get the leader so we know the IP address - we don't need to wait since create cluster
+        # already ensures the leader is running
+        self.cluster = cluster_factory(
+            provisioner="aws", zone=self.zone, clusterName=self.clusterName
+        )
+        self.leader = self.cluster.getLeader()
+
+        CACTUS_COMMIT_SHA = os.environ["CACTUS_COMMIT_SHA"] or "f5adf4013326322ae58ef1eccb8409b71d761583"  # default cactus commit
+
+        # command to install and run cactus on the cluster
+        cactus_command = ("python -m virtualenv --system-site-packages venv && "
+                          ". venv/bin/activate && "
+                          "git clone https://github.com/ComparativeGenomicsToolkit/cactus.git --recursive && "
+                          "cd cactus && "
+                          "git fetch origin && "
+                          f"git checkout {CACTUS_COMMIT_SHA} && "
+                          "git submodule update --init --recursive && "
+                          "pip install --upgrade 'setuptools<66' pip && "
+                          "pip install --upgrade . && "
+                          "pip install --upgrade numpy psutil && "
+                          "time cactus --batchSystem kubernetes --retryCount=3 "
+                          f"--consCores 2 --binariesMode singularity --clean always {self.jobStore} "
+                          "examples/evolverMammals.txt examples/evolverMammals.hal --root mr --defaultDisk 8G --logDebug")
+
+        # run cactus
+        self.sshUtil(
+            [
+                "bash",
+                "-c",
+                cactus_command
+            ]
+        )
+