toil 6.1.0__py3-none-any.whl → 7.0.0__py3-none-any.whl

toil/test/utils/utilsTest.py

@@ -56,6 +56,7 @@ class UtilsTest(ToilTest):
         super().setUp()
         self.tempDir = self._createTempDir()
         self.tempFile = get_temp_file(rootDir=self.tempDir)
+        self.outputFile = get_temp_file(rootDir=self.tempDir)
         self.outputFile = 'someSortedStuff.txt'
         self.toilDir = os.path.join(self.tempDir, "jobstore")
         self.assertFalse(os.path.exists(self.toilDir))
@@ -73,9 +74,9 @@ class UtilsTest(ToilTest):
             '-m',
             'toil.test.sort.sort',
             f'file:{self.toilDir}',
+            f'--fileToSort={self.tempFile}',
+            f'--outputFile={self.outputFile}',
             '--clean=never',
-            '--numLines=1',
-            '--lineLength=1'
         ]
 
         self.restart_sort_workflow_cmd = [
@@ -91,7 +92,7 @@ class UtilsTest(ToilTest):
         if os.path.exists(self.toilDir):
             shutil.rmtree(self.toilDir)
 
-        for f in ['fileToSort.txt', 'sortedFile.txt', 'output.txt']:
+        for f in [self.tempFile, self.outputFile, os.path.join(self.tempDir, "output.txt")]:
             if os.path.exists(f):
                 os.remove(f)
 
@@ -314,14 +315,14 @@ class UtilsTest(ToilTest):
     def testGetPIDStatus(self):
         """Test that ToilStatus.getPIDStatus() behaves as expected."""
         wf = subprocess.Popen(self.sort_workflow_cmd)
-        self.check_status('RUNNING', status_fn=ToilStatus.getPIDStatus, seconds=20)
+        self.check_status('RUNNING', status_fn=ToilStatus.getPIDStatus, seconds=60)
         wf.wait()
-        self.check_status('COMPLETED', status_fn=ToilStatus.getPIDStatus)
+        self.check_status('COMPLETED', status_fn=ToilStatus.getPIDStatus, seconds=60)
 
         # TODO: we need to reach into the FileJobStore's files and delete this
         #  shared file. We assume we know its internal layout.
         os.remove(os.path.join(self.toilDir, 'files/shared/pid.log'))
-        self.check_status('QUEUED', status_fn=ToilStatus.getPIDStatus)
+        self.check_status('QUEUED', status_fn=ToilStatus.getPIDStatus, seconds=60)
 
     def testGetStatusFailedToilWF(self):
         """
@@ -331,9 +332,9 @@ class UtilsTest(ToilTest):
         """
         # --badWorker is set to force failure.
         wf = subprocess.Popen(self.sort_workflow_cmd + ['--badWorker=1'])
-        self.check_status('RUNNING', status_fn=ToilStatus.getStatus)
+        self.check_status('RUNNING', status_fn=ToilStatus.getStatus, seconds=60)
         wf.wait()
-        self.check_status('ERROR', status_fn=ToilStatus.getStatus)
+        self.check_status('ERROR', status_fn=ToilStatus.getStatus, seconds=60)
 
     @needs_cwl
     @needs_docker
@@ -341,22 +342,22 @@ class UtilsTest(ToilTest):
         """Test that ToilStatus.getStatus() behaves as expected with a failing CWL workflow."""
         # --badWorker is set to force failure.
         cmd = ['toil-cwl-runner', '--jobStore', self.toilDir, '--clean=never', '--badWorker=1',
-               'src/toil/test/cwl/sorttool.cwl', '--reverse', '--input', 'src/toil/test/cwl/whale.txt']
+               'src/toil/test/cwl/sorttool.cwl', '--reverse', '--input', 'src/toil/test/cwl/whale.txt', f'--outdir={self.tempDir}']
         wf = subprocess.Popen(cmd)
-        self.check_status('RUNNING', status_fn=ToilStatus.getStatus)
+        self.check_status('RUNNING', status_fn=ToilStatus.getStatus, seconds=60)
         wf.wait()
-        self.check_status('ERROR', status_fn=ToilStatus.getStatus)
+        self.check_status('ERROR', status_fn=ToilStatus.getStatus, seconds=60)
 
     @needs_cwl
     @needs_docker
     def testGetStatusSuccessfulCWLWF(self):
         """Test that ToilStatus.getStatus() behaves as expected with a successful CWL workflow."""
         cmd = ['toil-cwl-runner', '--jobStore', self.toilDir, '--clean=never',
-               'src/toil/test/cwl/sorttool.cwl', '--reverse', '--input', 'src/toil/test/cwl/whale.txt']
+               'src/toil/test/cwl/sorttool.cwl', '--reverse', '--input', 'src/toil/test/cwl/whale.txt', f'--outdir={self.tempDir}']
         wf = subprocess.Popen(cmd)
-        self.check_status('RUNNING', status_fn=ToilStatus.getStatus, seconds=20)
+        self.check_status('RUNNING', status_fn=ToilStatus.getStatus, seconds=60)
         wf.wait()
-        self.check_status('COMPLETED', status_fn=ToilStatus.getStatus)
+        self.check_status('COMPLETED', status_fn=ToilStatus.getStatus, seconds=60)
 
     @needs_cwl
     @patch('builtins.print')
@@ -375,23 +376,24 @@ class UtilsTest(ToilTest):
         args, kwargs = mock_print.call_args
         self.assertIn('invalidcommand', args[0])
 
+    @pytest.mark.timeout(1200)
     def testRestartAttribute(self):
         """
-        Test that the job store is only destroyed when we observe a succcessful workflow run.
+        Test that the job store is only destroyed when we observe a successful workflow run.
         The following simulates a failing workflow that attempts to resume without restart().
         In this case, the job store should not be destroyed until restart() is called.
         """
         # Run a workflow that will always fail
-        cmd = self.restart_sort_workflow_cmd + ['--badWorker=1']
+        cmd = self.restart_sort_workflow_cmd + ['--badWorker=1', '--logDebug']
         subprocess.run(cmd)
 
-        restart_cmd = self.restart_sort_workflow_cmd + ['--badWorker=0', '--restart']
+        restart_cmd = self.restart_sort_workflow_cmd + ['--badWorker=0', '--logDebug', '--restart']
         subprocess.run(restart_cmd)
 
         # Check the job store exists after restart attempt
         self.assertTrue(os.path.exists(self.toilDir))
 
-        successful_cmd = [python, '-m', 'toil.test.sort.sort', 'file:' + self.toilDir,
+        successful_cmd = [python, '-m', 'toil.test.sort.sort', '--logDebug', 'file:' + self.toilDir,
                                   '--restart']
         subprocess.run(successful_cmd)
 

toil/test/wdl/wdltoil_test.py

@@ -16,12 +16,12 @@ from toil.test import (ToilTest,
                        needs_docker_cuda,
                        needs_google_storage,
                        needs_singularity_or_docker,
+                       needs_wdl,
                        slow, integrative)
-from toil.test.provisioners.clusterTest import AbstractClusterTest
 from toil.version import exactPython
 from toil.wdl.wdltoil import WDLSectionJob, WDLWorkflowGraph
 
-
+@needs_wdl
 class BaseWDLTest(ToilTest):
     """Base test class for WDL tests."""
 
@@ -45,7 +45,7 @@ class WDLConformanceTests(BaseWDLTest):
     def setUpClass(cls) -> None:
 
         url = "https://github.com/DataBiosphere/wdl-conformance-tests.git"
-        commit = "032fb99a1458d456b6d5f17d27928469ec1a1c68"
+        commit = "c87b62b4f460e009fd42edec13669c4db14cf90c"
 
         p = subprocess.Popen(
             f"git clone {url} {cls.wdl_dir} && cd {cls.wdl_dir} && git checkout {commit}",
@@ -64,7 +64,7 @@ class WDLConformanceTests(BaseWDLTest):
     # estimated running time: 2 minutes
     @slow
     def test_conformance_tests_v10(self):
-        tests_to_run = "0,1,5-7,9-15,17,22-24,26,28-30,32-40,53,57-59,62,67-69"
+        tests_to_run = "0-15,17-20,22-71,73-77"
         p = subprocess.run(self.base_command + ["-v", "1.0", "-n", tests_to_run], capture_output=True)
 
         if p.returncode != 0:
@@ -75,7 +75,7 @@ class WDLConformanceTests(BaseWDLTest):
     # estimated running time: 2 minutes
     @slow
     def test_conformance_tests_v11(self):
-        tests_to_run = "2-11,13-15,17-20,22-24,26,29,30,32-40,53,57-59,62,67-69"
+        tests_to_run = "1-63,65-71,73-75,77"
         p = subprocess.run(self.base_command + ["-v", "1.1", "-n", tests_to_run], capture_output=True)
 
         if p.returncode != 0:
@@ -83,6 +83,16 @@ class WDLConformanceTests(BaseWDLTest):
 
         p.check_returncode()
 
+    @slow
+    def test_conformance_tests_integration(self):
+        ids_to_run = "encode,tut01,tut02,tut03,tut04"
+        p = subprocess.run(self.base_command + ["-v", "1.0", "--id", ids_to_run], capture_output=True)
+        
+        if p.returncode != 0:
+            print(p.stdout.decode('utf-8', errors='replace'))
+
+        p.check_returncode()
+
     @classmethod
     def tearDownClass(cls) -> None:
         upper_dir = os.path.dirname(os.getcwd())
@@ -116,6 +126,14 @@ class WDLTests(BaseWDLTest):
         assert os.path.exists(result['ga4ghMd5.value'])
         assert os.path.basename(result['ga4ghMd5.value']) == 'md5sum.txt'
 
+    def test_missing_output_directory(self):
+        """
+        Test if Toil can run a WDL workflow into a new directory.
+        """
+        wdl = os.path.abspath('src/toil/test/wdl/md5sum/md5sum.1.0.wdl')
+        json_file = os.path.abspath('src/toil/test/wdl/md5sum/md5sum.json')
+        subprocess.check_call(self.base_command + [wdl, json_file, '-o', os.path.join(self.output_dir, "does", "not", "exist"), '--logDebug', '--retryCount=0'])
+
     @needs_singularity_or_docker
     def test_miniwdl_self_test(self, extra_args: Optional[List[str]] = None) -> None:
         """Test if the MiniWDL self test runs and produces the expected output."""
@@ -141,7 +159,7 @@ class WDLTests(BaseWDLTest):
         assert isinstance(outputs['hello_caller.message_files'], list)
         assert len(outputs['hello_caller.message_files']) == 2
         for item in outputs['hello_caller.message_files']:
-            # All the files should be strings in the "out" direcotry
+            # All the files should be strings in the "out" directory
             assert isinstance(item, str)
             assert item.startswith(out_dir)
 
@@ -349,70 +367,5 @@ class WDLTests(BaseWDLTest):
                     assert "successor" in result[1]
 
 
-@integrative
-@slow
-@pytest.mark.timeout(600)
-class WDLKubernetesClusterTest(AbstractClusterTest):
-    """
-    Ensure WDL works on the Kubernetes batchsystem.
-    """
-
-    def __init__(self, name):
-        super().__init__(name)
-        self.clusterName = 'wdl-integration-test-' + str(uuid4())
-        # t2.medium is the minimum t2 instance that permits Kubernetes
-        self.leaderNodeType = "t2.medium"
-        self.instanceTypes = ["t2.medium"]
-        self.clusterType = "kubernetes"
-
-    def setUp(self) -> None:
-        super().setUp()
-        self.jobStore = f'aws:{self.awsRegion()}:wdl-test-{uuid4()}'
-
-    def launchCluster(self) -> None:
-        self.createClusterUtil(args=['--leaderStorage', str(self.requestedLeaderStorage),
-                                     '--nodeTypes', ",".join(self.instanceTypes),
-                                     '-w', ",".join(self.numWorkers),
-                                     '--nodeStorage', str(self.requestedLeaderStorage)])
-
-    def test_wdl_kubernetes_cluster(self):
-        """
-        Test that a wdl workflow works on a kubernetes cluster. Launches a cluster with 1 worker. This runs a wdl
-        workflow that performs an image pull on the worker.
-        :return:
-        """
-        self.numWorkers = "1"
-        self.requestedLeaderStorage = 30
-        # create the cluster
-        self.launchCluster()
-        # get leader
-        self.cluster = cluster_factory(
-            provisioner="aws", zone=self.zone, clusterName=self.clusterName
-        )
-        self.leader = self.cluster.getLeader()
-
-        url = "https://github.com/DataBiosphere/wdl-conformance-tests.git"
-        commit = "09b9659cd01473e836738a2e0dd205df0adb49c5"
-        wdl_dir = "wdl_conformance_tests"
-
-        # get the wdl-conformance-tests repo to get WDL tasks to run
-        self.sshUtil([
-            "bash",
-            "-c",
-            f"git clone {url} {wdl_dir} && cd {wdl_dir} && git checkout {commit}"
-        ])
-
-        # run on kubernetes batchsystem
-        toil_options = ['--batchSystem=kubernetes',
-                        f"--jobstore={self.jobStore}"]
-
-        # run WDL workflow that will run singularity
-        test_options = [f"tests/md5sum/md5sum.wdl", f"tests/md5sum/md5sum.json"]
-        self.sshUtil([
-            "bash",
-            "-c",
-            f"cd {wdl_dir} && toil-wdl-runner {' '.join(test_options)} {' '.join(toil_options)}"])
-
-
 if __name__ == "__main__":
     unittest.main()  # run all tests

toil/test/wdl/wdltoil_test_kubernetes.py

@@ -0,0 +1,77 @@
+import unittest
+
+from toil.test.provisioners.clusterTest import AbstractClusterTest
+from uuid import uuid4
+
+import pytest
+
+from toil.provisioners import cluster_factory
+from toil.test import (slow, integrative)
+
+@integrative
+@slow
+@pytest.mark.timeout(600)
+class WDLKubernetesClusterTest(AbstractClusterTest):
+    """
+    Ensure WDL works on the Kubernetes batchsystem.
+    """
+
+    def __init__(self, name):
+        super().__init__(name)
+        self.clusterName = 'wdl-integration-test-' + str(uuid4())
+        # t2.medium is the minimum t2 instance that permits Kubernetes
+        self.leaderNodeType = "t2.medium"
+        self.instanceTypes = ["t2.medium"]
+        self.clusterType = "kubernetes"
+
+    def setUp(self) -> None:
+        super().setUp()
+        self.jobStore = f'aws:{self.awsRegion()}:wdl-test-{uuid4()}'
+
+    def launchCluster(self) -> None:
+        self.createClusterUtil(args=['--leaderStorage', str(self.requestedLeaderStorage),
+                                     '--nodeTypes', ",".join(self.instanceTypes),
+                                     '-w', ",".join(self.numWorkers),
+                                     '--nodeStorage', str(self.requestedLeaderStorage)])
+
+    def test_wdl_kubernetes_cluster(self):
+        """
+        Test that a wdl workflow works on a kubernetes cluster. Launches a cluster with 1 worker. This runs a wdl
+        workflow that performs an image pull on the worker.
+        :return:
+        """
+        self.numWorkers = "1"
+        self.requestedLeaderStorage = 30
+        # create the cluster
+        self.launchCluster()
+        # get leader
+        self.cluster = cluster_factory(
+            provisioner="aws", zone=self.zone, clusterName=self.clusterName
+        )
+        self.leader = self.cluster.getLeader()
+
+        url = "https://github.com/DataBiosphere/wdl-conformance-tests.git"
+        commit = "09b9659cd01473e836738a2e0dd205df0adb49c5"
+        wdl_dir = "wdl_conformance_tests"
+
+        # get the wdl-conformance-tests repo to get WDL tasks to run
+        self.sshUtil([
+            "bash",
+            "-c",
+            f"git clone {url} {wdl_dir} && cd {wdl_dir} && git checkout {commit}"
+        ])
+
+        # run on kubernetes batchsystem
+        toil_options = ['--batchSystem=kubernetes',
+                        f"--jobstore={self.jobStore}"]
+
+        # run WDL workflow that will run singularity
+        test_options = [f"tests/md5sum/md5sum.wdl", f"tests/md5sum/md5sum.json"]
+        self.sshUtil([
+            "bash",
+            "-c",
+            f"cd {wdl_dir} && toil-wdl-runner {' '.join(test_options)} {' '.join(toil_options)}"])
+
+
+if __name__ == "__main__":
+    unittest.main()  # run all tests

toil/toilState.py

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import logging
+import time
 from typing import Dict, Optional, Set
 
 from toil.bus import JobUpdatedMessage, MessageBus
@@ -183,12 +184,70 @@ class ToilState:
         if job_id in self.__job_database:
             # Update the one true copy in place
             old_truth = self.__job_database[job_id]
-            old_truth.check_new_version(new_truth)
+            old_truth.assert_is_not_newer_than(new_truth)
             old_truth.__dict__.update(new_truth.__dict__)
         else:
             # Just keep the new one
             self.__job_database[job_id] = new_truth
 
+    def reset_job_expecting_change(self, job_id: str, timeout: float) -> bool:
+        """
+        Discard any local modifications to a JobDescription.
+
+        Will make modifications from other hosts visible.
+
+        Will wait for up to timeout seconds for a modification (or deletion)
+        from another host to actually be visible.
+
+        Always replaces the JobDescription with what is stored in the job
+        store, even if no modification ends up being visible.
+
+        Returns True if an update was detected in time, and False otherwise.
+        """
+
+        start_time = time.time()
+        wait_time = 0.1
+        initially_known = job_id in self.__job_database
+        new_truth: Optional[JobDescription] = None
+        while True:
+            try:
+                new_truth = self.__job_store.load_job(job_id)
+            except NoSuchJobException:
+                # The job is gone now.
+                if job_id in self.__job_database:
+                    # So forget about it
+                    del self.__job_database[job_id]
+                    # TODO: Other collections may still reference it.
+                if initially_known:
+                    # Job was deleted, that's an update
+                    return True
+            else:
+                if job_id in self.__job_database:
+                    # We have an old version to compare against
+                    old_truth = self.__job_database[job_id]
+                    old_truth.assert_is_not_newer_than(new_truth)
+                    if old_truth.is_updated_by(new_truth):
+                        # Do the update
+                        old_truth.__dict__.update(new_truth.__dict__)
+                        return True
+                else:
+                    # Just keep the new one. That's an update.
+                    self.__job_database[job_id] = new_truth
+                    return True
+            # We looked but didn't get a good update
+            time_elapsed = time.time() - start_time
+            if time_elapsed >= timeout:
+                # We're out of time to check.
+                if new_truth is not None:
+                    # Commit whatever we managed to load to accomplish a real
+                    # reset.
+                    old_truth.__dict__.update(new_truth.__dict__)
+                return False
+            # Wait a little and poll again
+            time.sleep(min(timeout - time_elapsed, wait_time))
+            # Using exponential backoff
+            wait_time *= 2
+
     # The next 3 functions provide tracking of how many successor jobs a given job
     # is waiting on, exposing only legit operations.
     # TODO: turn these into messages?
@@ -247,10 +306,10 @@ class ToilState:
 
         :param jobDesc: The description for the root job of the workflow being run.
         """
-        # If the job description has a command, is a checkpoint, has services
+        # If the job description has a body, is a checkpoint, has services
         # or is ready to be deleted it is ready to be processed (i.e. it is updated)
         if (
-            jobDesc.command is not None
+            jobDesc.has_body()
             or (
                 isinstance(jobDesc, CheckpointJobDescription)
                 and jobDesc.checkpoint is not None
@@ -259,10 +318,10 @@ class ToilState:
             or jobDesc.nextSuccessors() is None
         ):
             logger.debug(
-                "Found job to run: %s, with command: %s, with checkpoint: %s, with "
+                "Found job to run: %s, with body: %s, with checkpoint: %s, with "
                 "services: %s, with no next successors: %s",
                 jobDesc.jobStoreID,
-                jobDesc.command is not None,
+                jobDesc.has_body(),
                 isinstance(jobDesc, CheckpointJobDescription)
                 and jobDesc.checkpoint is not None,
                 len(jobDesc.services) > 0,
@@ -272,18 +331,18 @@ class ToilState:
             self.bus.publish(JobUpdatedMessage(str(jobDesc.jobStoreID), 0))
 
             if isinstance(jobDesc, CheckpointJobDescription) and jobDesc.checkpoint is not None:
-                jobDesc.command = jobDesc.checkpoint
+                jobDesc.restore_checkpoint()
 
         else:  # There exist successors
             logger.debug(
                 "Adding job: %s to the state with %s successors",
                 jobDesc.jobStoreID,
-                len(jobDesc.nextSuccessors()),
+                len(jobDesc.nextSuccessors() or set()),
             )
 
             # Record the number of successors
             self.successorCounts[str(jobDesc.jobStoreID)] = len(
-                jobDesc.nextSuccessors()
+                jobDesc.nextSuccessors() or set()
             )
 
             def processSuccessorWithMultiplePredecessors(successor: JobDescription) -> None:
@@ -305,7 +364,7 @@ class ToilState:
                     self._buildToilState(successor)
 
             # For each successor
-            for successorJobStoreID in jobDesc.nextSuccessors():
+            for successorJobStoreID in jobDesc.nextSuccessors() or set():
 
                 # If the successor does not yet point back at a
                 # predecessor we have not yet considered it