toil 6.1.0a1__py3-none-any.whl → 7.0.0__py3-none-any.whl

toil/test/batchSystems/test_slurm.py

@@ -4,6 +4,7 @@ from queue import Queue
 import pytest
 
 import toil.batchSystems.slurm
+from toil.batchSystems.abstractBatchSystem import BatchJobExitReason, EXIT_STATUS_UNAVAILABLE_VALUE
 from toil.common import Config
 from toil.lib.misc import CalledProcessErrorStderr
 from toil.test import ToilTest
@@ -192,7 +193,7 @@ class SlurmTest(ToilTest):
 
     def setUp(self):
         self.monkeypatch = pytest.MonkeyPatch()
-        self.worker = toil.batchSystems.slurm.SlurmBatchSystem.Worker(
+        self.worker = toil.batchSystems.slurm.SlurmBatchSystem.GridEngineThread(
             newJobsQueue=Queue(),
             updatedJobsQueue=Queue(),
             killQueue=Queue(),
@@ -284,7 +285,7 @@ class SlurmTest(ToilTest):
     def test_getJobExitCode_job_exists(self):
         self.monkeypatch.setattr(toil.batchSystems.slurm, "call_command", call_sacct)
         job_id = '785023'  # FAILED
-        expected_result = 127
+        expected_result = (127, BatchJobExitReason.FAILED)
         result = self.worker.getJobExitCode(job_id)
         assert result == expected_result, f"{result} != {expected_result}"
 
@@ -303,7 +304,7 @@ class SlurmTest(ToilTest):
         self.monkeypatch.setattr(self.worker, "_getJobDetailsFromSacct", call_sacct_raises)
         self.monkeypatch.setattr(toil.batchSystems.slurm, "call_command", call_scontrol)
         job_id = '787204'  # COMPLETED
-        expected_result = 0
+        expected_result = (0,  BatchJobExitReason.FINISHED)
         result = self.worker.getJobExitCode(job_id)
         assert result == expected_result, f"{result} != {expected_result}"
 
@@ -329,7 +330,7 @@ class SlurmTest(ToilTest):
     def test_coalesce_job_exit_codes_one_exists(self):
         self.monkeypatch.setattr(toil.batchSystems.slurm, "call_command", call_sacct)
         job_ids = ['785023']  # FAILED
-        expected_result = [127]
+        expected_result = [(127,  BatchJobExitReason.FAILED)]
         result = self.worker.coalesce_job_exit_codes(job_ids)
         assert result == expected_result, f"{result} != {expected_result}"
 
@@ -347,7 +348,14 @@ class SlurmTest(ToilTest):
                    '789724',  # RUNNING,
                    '789868',  # PENDING,
                    '789869']  # COMPLETED
-        expected_result = [0, 1, None, None, 0]  # RUNNING and PENDING jobs should return None
+        # RUNNING and PENDING jobs should return None
+        expected_result = [
+            (EXIT_STATUS_UNAVAILABLE_VALUE, BatchJobExitReason.KILLED),
+            (1, BatchJobExitReason.FAILED),
+            None,
+            None,
+            (0, BatchJobExitReason.FINISHED)
+        ]
         result = self.worker.coalesce_job_exit_codes(job_ids)
         assert result == expected_result, f"{result} != {expected_result}"
 
@@ -358,7 +366,14 @@ class SlurmTest(ToilTest):
                    '789724',  # RUNNING,
                    '999999',  # Non-existent,
                    '789869']  # COMPLETED
-        expected_result = [130, 2, None, None, 0]    # RUNNING job should return None
+        # RUNNING job should return None
+        expected_result = [
+            (130, BatchJobExitReason.FAILED),
+            (2, BatchJobExitReason.FAILED),
+            None,
+            None,
+            (0, BatchJobExitReason.FINISHED)
+        ]
         result = self.worker.coalesce_job_exit_codes(job_ids)
         assert result == expected_result, f"{result} != {expected_result}"
 
@@ -370,7 +385,7 @@ class SlurmTest(ToilTest):
         self.monkeypatch.setattr(self.worker, "_getJobDetailsFromSacct", call_sacct_raises)
         self.monkeypatch.setattr(toil.batchSystems.slurm, "call_command", call_scontrol)
         job_ids = ['787204']  # COMPLETED
-        expected_result = [0]
+        expected_result = [(0, BatchJobExitReason.FINISHED)]
         result = self.worker.coalesce_job_exit_codes(job_ids)
         assert result == expected_result, f"{result} != {expected_result}"
 

toil/test/cactus/test_cactus_integration.py

@@ -0,0 +1,58 @@
+import os
+import uuid
+
+from toil.provisioners import cluster_factory
+from toil.test.provisioners.clusterTest import AbstractClusterTest
+
+
+class CactusIntegrationTest(AbstractClusterTest):
+    """
+    Run the Cactus Integration test on a Kubernetes AWS cluster
+    """
+
+    def __init__(self, methodName):
+        super().__init__(methodName=methodName)
+        self.clusterName = "cactus-test-" + str(uuid.uuid4())
+        self.leaderNodeType = "t2.medium"
+        self.clusterType = "kubernetes"
+
+    def setUp(self):
+        super().setUp()
+        self.jobStore = f"aws:{self.awsRegion()}:cluster-{uuid.uuid4()}"
+
+    def test_cactus_integration(self):
+        # Make a cluster with worker nodes
+        self.createClusterUtil(args=["--nodeTypes=t2.xlarge", "-w=1-3"])
+        # get the leader so we know the IP address - we don't need to wait since create cluster
+        # already ensures the leader is running
+        self.cluster = cluster_factory(
+            provisioner="aws", zone=self.zone, clusterName=self.clusterName
+        )
+        self.leader = self.cluster.getLeader()
+
+        CACTUS_COMMIT_SHA = os.environ["CACTUS_COMMIT_SHA"] or "f5adf4013326322ae58ef1eccb8409b71d761583"  # default cactus commit
+
+        # command to install and run cactus on the cluster
+        cactus_command = ("python -m virtualenv --system-site-packages venv && "
+                          ". venv/bin/activate && "
+                          "git clone https://github.com/ComparativeGenomicsToolkit/cactus.git --recursive && "
+                          "cd cactus && "
+                          "git fetch origin && "
+                          f"git checkout {CACTUS_COMMIT_SHA} && "
+                          "git submodule update --init --recursive && "
+                          "pip install --upgrade 'setuptools<66' pip && "
+                          "pip install --upgrade . && "
+                          "pip install --upgrade numpy psutil && "
+                          "time cactus --batchSystem kubernetes --retryCount=3 "
+                          f"--consCores 2 --binariesMode singularity --clean always {self.jobStore} "
+                          "examples/evolverMammals.txt examples/evolverMammals.hal --root mr --defaultDisk 8G --logDebug")
+
+        # run cactus
+        self.sshUtil(
+            [
+                "bash",
+                "-c",
+                cactus_command
+            ]
+        )
+