toil 6.1.0a1__py3-none-any.whl → 8.0.0__py3-none-any.whl

toil/test/wdl/wdltoil_test_kubernetes.py

@@ -0,0 +1,91 @@
+import unittest
+from uuid import uuid4
+
+import pytest
+
+from toil.provisioners import cluster_factory
+from toil.test import integrative, slow
+from toil.test.provisioners.clusterTest import AbstractClusterTest
+from toil.test.wdl.wdltoil_test import (
+    WDL_CONFORMANCE_TEST_COMMIT,
+    WDL_CONFORMANCE_TEST_REPO,
+)
+
+
+@integrative
+@slow
+@pytest.mark.timeout(1800)
+class WDLKubernetesClusterTest(AbstractClusterTest):
+    """
+    Ensure WDL works on the Kubernetes batchsystem.
+    """
+
+    def __init__(self, name):
+        super().__init__(name)
+        self.clusterName = "wdl-integration-test-" + str(uuid4())
+        # t2.medium is the minimum t2 instance that permits Kubernetes
+        self.leaderNodeType = "t2.medium"
+        self.instanceTypes = ["t2.medium"]
+        self.clusterType = "kubernetes"
+
+    def setUp(self) -> None:
+        super().setUp()
+        self.jobStore = f"aws:{self.awsRegion()}:wdl-test-{uuid4()}"
+
+    def launchCluster(self) -> None:
+        self.createClusterUtil(
+            args=[
+                "--leaderStorage",
+                str(self.requestedLeaderStorage),
+                "--nodeTypes",
+                ",".join(self.instanceTypes),
+                "-w",
+                ",".join(self.numWorkers),
+                "--nodeStorage",
+                str(self.requestedLeaderStorage),
+            ]
+        )
+
+    def test_wdl_kubernetes_cluster(self):
+        """
+        Test that a wdl workflow works on a kubernetes cluster. Launches a cluster with 1 worker. This runs a wdl
+        workflow that performs an image pull on the worker.
+        :return:
+        """
+        self.numWorkers = "1"
+        self.requestedLeaderStorage = 30
+        # create the cluster
+        self.launchCluster()
+        # get leader
+        self.cluster = cluster_factory(
+            provisioner="aws", zone=self.zone, clusterName=self.clusterName
+        )
+        self.leader = self.cluster.getLeader()
+
+        wdl_dir = "wdl_conformance_tests"
+
+        # get the wdl-conformance-tests repo to get WDL tasks to run
+        self.sshUtil(
+            [
+                "bash",
+                "-c",
+                f"git clone {WDL_CONFORMANCE_TEST_REPO} {wdl_dir} && cd {wdl_dir} && git checkout {WDL_CONFORMANCE_TEST_COMMIT}",
+            ]
+        )
+
+        # run on kubernetes batchsystem
+        toil_options = ["--batchSystem=kubernetes", f"--jobstore={self.jobStore}"]
+
+        # run WDL workflow that will run singularity
+        test_options = [f"tests/md5sum/md5sum.wdl", f"tests/md5sum/md5sum.json"]
+        self.sshUtil(
+            [
+                "bash",
+                "-c",
+                f"cd {wdl_dir} && toil-wdl-runner {' '.join(test_options)} {' '.join(toil_options)}",
+            ]
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()  # run all tests

toil/toilState.py

@@ -12,12 +12,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import logging
-from typing import Dict, Optional, Set
+import time
+from typing import Optional
 
 from toil.bus import JobUpdatedMessage, MessageBus
 from toil.job import CheckpointJobDescription, JobDescription
-from toil.jobStores.abstractJobStore import (AbstractJobStore,
-                                             NoSuchJobException)
+from toil.jobStores.abstractJobStore import AbstractJobStore, NoSuchJobException
 
 logger = logging.getLogger(__name__)
 
@@ -62,47 +62,47 @@ class ToilState:
         # This holds the one true copy of every JobDescription in the leader.
         # TODO: Do in-place update instead of assignment when we load so we
         # can't let any non-true copies escape.
-        self.__job_database: Dict[str, JobDescription] = {}
+        self.__job_database: dict[str, JobDescription] = {}
 
         # Maps from successor (child or follow-on) jobStoreID to predecessor jobStoreIDs
-        self.successor_to_predecessors: Dict[str, Set[str]] = {}
+        self.successor_to_predecessors: dict[str, set[str]] = {}
 
         # Hash of jobStoreIDs to counts of numbers of successors issued.
         # There are no entries for jobs without successors in this map.
-        self.successorCounts: Dict[str, int] = {}
+        self.successorCounts: dict[str, int] = {}
 
         # This is a hash of service jobs, referenced by jobStoreID, to their client's ID
-        self.service_to_client: Dict[str, str] = {}
+        self.service_to_client: dict[str, str] = {}
 
         # Holds, for each client job ID, the job IDs of its services that are
         # possibly currently issued. Includes every service host that has been
         # given to the service manager by the leader, and hasn't been seen by
         # the leader as stopped yet.
-        self.servicesIssued: Dict[str, Set[str]] = {}
+        self.servicesIssued: dict[str, set[str]] = {}
 
         # Holds the IDs of jobs that are currently issued to the batch system
         # and haven't come back yet.
         # TODO: a bit redundant with leader's issued_jobs_by_batch_system_id
-        self.jobs_issued: Set[str] = set()
+        self.jobs_issued: set[str] = set()
 
         # The set of totally failed jobs - this needs to be filtered at the
         # end to remove jobs that were removed by checkpoints
-        self.totalFailedJobs: Set[str] = set()
+        self.totalFailedJobs: set[str] = set()
 
         # Jobs (as jobStoreIDs) with successors that have totally failed
-        self.hasFailedSuccessors: Set[str] = set()
+        self.hasFailedSuccessors: set[str] = set()
 
         # The set of successors of failed jobs as a set of jobStoreIds
-        self.failedSuccessors: Set[str] = set()
+        self.failedSuccessors: set[str] = set()
 
         # Set of jobs that have multiple predecessors that have one or more predecessors
         # finished, but not all of them.
-        self.jobsToBeScheduledWithMultiplePredecessors: Set[str] = set()
+        self.jobsToBeScheduledWithMultiplePredecessors: set[str] = set()
 
     def load_workflow(
         self,
         rootJob: JobDescription,
-        jobCache: Optional[Dict[str, JobDescription]] = None
+        jobCache: Optional[dict[str, JobDescription]] = None,
     ) -> None:
         """
         Load the workflow rooted at the given job.
@@ -183,12 +183,70 @@ class ToilState:
         if job_id in self.__job_database:
             # Update the one true copy in place
             old_truth = self.__job_database[job_id]
-            old_truth.check_new_version(new_truth)
+            old_truth.assert_is_not_newer_than(new_truth)
             old_truth.__dict__.update(new_truth.__dict__)
         else:
             # Just keep the new one
             self.__job_database[job_id] = new_truth
 
+    def reset_job_expecting_change(self, job_id: str, timeout: float) -> bool:
+        """
+        Discard any local modifications to a JobDescription.
+
+        Will make modifications from other hosts visible.
+
+        Will wait for up to timeout seconds for a modification (or deletion)
+        from another host to actually be visible.
+
+        Always replaces the JobDescription with what is stored in the job
+        store, even if no modification ends up being visible.
+
+        Returns True if an update was detected in time, and False otherwise.
+        """
+
+        start_time = time.time()
+        wait_time = 0.1
+        initially_known = job_id in self.__job_database
+        new_truth: Optional[JobDescription] = None
+        while True:
+            try:
+                new_truth = self.__job_store.load_job(job_id)
+            except NoSuchJobException:
+                # The job is gone now.
+                if job_id in self.__job_database:
+                    # So forget about it
+                    del self.__job_database[job_id]
+                    # TODO: Other collections may still reference it.
+                if initially_known:
+                    # Job was deleted, that's an update
+                    return True
+            else:
+                if job_id in self.__job_database:
+                    # We have an old version to compare against
+                    old_truth = self.__job_database[job_id]
+                    old_truth.assert_is_not_newer_than(new_truth)
+                    if old_truth.is_updated_by(new_truth):
+                        # Do the update
+                        old_truth.__dict__.update(new_truth.__dict__)
+                        return True
+                else:
+                    # Just keep the new one. That's an update.
+                    self.__job_database[job_id] = new_truth
+                    return True
+            # We looked but didn't get a good update
+            time_elapsed = time.time() - start_time
+            if time_elapsed >= timeout:
+                # We're out of time to check.
+                if new_truth is not None:
+                    # Commit whatever we managed to load to accomplish a real
+                    # reset.
+                    old_truth.__dict__.update(new_truth.__dict__)
+                return False
+            # Wait a little and poll again
+            time.sleep(min(timeout - time_elapsed, wait_time))
+            # Using exponential backoff
+            wait_time *= 2
+
     # The next 3 functions provide tracking of how many successor jobs a given job
     # is waiting on, exposing only legit operations.
     # TODO: turn these into messages?
@@ -222,7 +280,11 @@ class ToilState:
             )
         else:
             self.successorCounts[predecessor_id] -= 1
-            logger.debug("Successors: one fewer for %s, now have %d", predecessor_id, self.successorCounts[predecessor_id])
+            logger.debug(
+                "Successors: one fewer for %s, now have %d",
+                predecessor_id,
+                self.successorCounts[predecessor_id],
+            )
             if self.successorCounts[predecessor_id] == 0:
                 del self.successorCounts[predecessor_id]
 
@@ -237,7 +299,6 @@ class ToilState:
         else:
             return self.successorCounts[predecessor_id]
 
-
     def _buildToilState(self, jobDesc: JobDescription) -> None:
         """
         Build the ToilState class from the subtree root JobDescription.
@@ -247,10 +308,10 @@ class ToilState:
 
         :param jobDesc: The description for the root job of the workflow being run.
         """
-        # If the job description has a command, is a checkpoint, has services
+        # If the job description has a body, is a checkpoint, has services
         # or is ready to be deleted it is ready to be processed (i.e. it is updated)
         if (
-            jobDesc.command is not None
+            jobDesc.has_body()
             or (
                 isinstance(jobDesc, CheckpointJobDescription)
                 and jobDesc.checkpoint is not None
@@ -259,10 +320,10 @@ class ToilState:
             or jobDesc.nextSuccessors() is None
         ):
             logger.debug(
-                "Found job to run: %s, with command: %s, with checkpoint: %s, with "
+                "Found job to run: %s, with body: %s, with checkpoint: %s, with "
                 "services: %s, with no next successors: %s",
                 jobDesc.jobStoreID,
-                jobDesc.command is not None,
+                jobDesc.has_body(),
                 isinstance(jobDesc, CheckpointJobDescription)
                 and jobDesc.checkpoint is not None,
                 len(jobDesc.services) > 0,
@@ -271,22 +332,27 @@ class ToilState:
             # Set the job updated because we should be able to make progress on it.
             self.bus.publish(JobUpdatedMessage(str(jobDesc.jobStoreID), 0))
 
-            if isinstance(jobDesc, CheckpointJobDescription) and jobDesc.checkpoint is not None:
-                jobDesc.command = jobDesc.checkpoint
+            if (
+                isinstance(jobDesc, CheckpointJobDescription)
+                and jobDesc.checkpoint is not None
+            ):
+                jobDesc.restore_checkpoint()
 
         else:  # There exist successors
             logger.debug(
                 "Adding job: %s to the state with %s successors",
                 jobDesc.jobStoreID,
-                len(jobDesc.nextSuccessors()),
+                len(jobDesc.nextSuccessors() or set()),
             )
 
             # Record the number of successors
             self.successorCounts[str(jobDesc.jobStoreID)] = len(
-                jobDesc.nextSuccessors()
+                jobDesc.nextSuccessors() or set()
             )
 
-            def processSuccessorWithMultiplePredecessors(successor: JobDescription) -> None:
+            def processSuccessorWithMultiplePredecessors(
+                successor: JobDescription,
+            ) -> None:
                 # If jobDesc is not reported as complete by the successor
                 if jobDesc.jobStoreID not in successor.predecessorsFinished:
 
@@ -295,17 +361,21 @@ class ToilState:
 
                 # If the successor has no predecessors to finish
                 if len(successor.predecessorsFinished) > successor.predecessorNumber:
-                    raise RuntimeError("There are more finished predecessors than possible.")
+                    raise RuntimeError(
+                        "There are more finished predecessors than possible."
+                    )
                 if len(successor.predecessorsFinished) == successor.predecessorNumber:
 
                     # It is ready to be run, so remove it from the set of waiting jobs
-                    self.jobsToBeScheduledWithMultiplePredecessors.remove(successorJobStoreID)
+                    self.jobsToBeScheduledWithMultiplePredecessors.remove(
+                        successorJobStoreID
+                    )
 
                     # Recursively consider the successor
                     self._buildToilState(successor)
 
             # For each successor
-            for successorJobStoreID in jobDesc.nextSuccessors():
+            for successorJobStoreID in jobDesc.nextSuccessors() or set():
 
                 # If the successor does not yet point back at a
                 # predecessor we have not yet considered it
@@ -324,9 +394,16 @@ class ToilState:
 
                         # We put the successor job in the set of waiting successor
                         # jobs with multiple predecessors
-                        if successorJobStoreID in self.jobsToBeScheduledWithMultiplePredecessors:
-                            raise RuntimeError("Failed to schedule the successor job. The successor job is already scheduled.")
-                        self.jobsToBeScheduledWithMultiplePredecessors.add(successorJobStoreID)
+                        if (
+                            successorJobStoreID
+                            in self.jobsToBeScheduledWithMultiplePredecessors
+                        ):
+                            raise RuntimeError(
+                                "Failed to schedule the successor job. The successor job is already scheduled."
+                            )
+                        self.jobsToBeScheduledWithMultiplePredecessors.add(
+                            successorJobStoreID
+                        )
 
                         # Process successor
                         processSuccessorWithMultiplePredecessors(successor)
@@ -340,14 +417,22 @@ class ToilState:
                     # We've already seen the successor
 
                     # Add the job as a predecessor
-                    if jobDesc.jobStoreID in self.successor_to_predecessors[successorJobStoreID]:
-                        raise RuntimeError("Failed to add the job as a predecessor. The job is already added as a predecessor.")
+                    if (
+                        jobDesc.jobStoreID
+                        in self.successor_to_predecessors[successorJobStoreID]
+                    ):
+                        raise RuntimeError(
+                            "Failed to add the job as a predecessor. The job is already added as a predecessor."
+                        )
                     self.successor_to_predecessors[successorJobStoreID].add(
                         str(jobDesc.jobStoreID)
                     )
 
                     # If the successor has multiple predecessors
-                    if successorJobStoreID in self.jobsToBeScheduledWithMultiplePredecessors:
+                    if (
+                        successorJobStoreID
+                        in self.jobsToBeScheduledWithMultiplePredecessors
+                    ):
 
                         # Get the successor from cache
                         successor = self.get_job(successorJobStoreID)

toil/utils/toilConfig.py

@@ -26,11 +26,20 @@ logger = logging.getLogger(__name__)
 def main() -> None:
     parser = ArgParser()
 
-    parser.add_argument("output", default="config.yaml", help="Filepath to write the config file too. Default=%("
-                                                              "default)s")
+    parser.add_argument(
+        "output",
+        default="config.yaml",
+        help="Filepath to write the config file too. Default=%(" "default)s",
+    )
     add_logging_options(parser)
     options = parser.parse_args()
     set_logging_from_options(options)
-    logger.debug("Attempting to write a default config file to %s.", os.path.abspath(options.output))
+    logger.debug(
+        "Attempting to write a default config file to %s.",
+        os.path.abspath(options.output),
+    )
     generate_config(os.path.abspath(options.output))
-    logger.info("Successfully wrote a default config file to %s.", os.path.abspath(options.output))
+    logger.info(
+        "Successfully wrote a default config file to %s.",
+        os.path.abspath(options.output),
+    )

toil/utils/toilDebugFile.py

@@ -17,10 +17,10 @@ import logging
 import os.path
 import sys
 from typing import Optional
-from distutils.util import strtobool
 
 from toil.common import Config, Toil, parser_with_common_options
 from toil.jobStores.fileJobStore import FileJobStore
+from toil.lib.conversions import strtobool
 from toil.lib.resources import glob
 from toil.statsAndLogging import set_logging_from_options
 
@@ -44,17 +44,23 @@ def fetchJobStoreFiles(jobStore: FileJobStore, options: argparse.Namespace) -> N
     # globbing around inside it. Does this even work?
 
     for jobStoreFile in options.fetch:
-        jobStoreHits = glob(directoryname=options.jobStore,
-                            glob_pattern=jobStoreFile)
+        jobStoreHits = glob(directoryname=options.jobStore, glob_pattern=jobStoreFile)
         for jobStoreFileID in jobStoreHits:
-            logger.debug(f"Copying job store file: {jobStoreFileID} to {options.localFilePath[0]}")
-            jobStore.read_file(jobStoreFileID,
-                               os.path.join(options.localFilePath[0],
-                                           os.path.basename(jobStoreFileID)),
-                               symlink=options.useSymlinks)
-
-
-def printContentsOfJobStore(job_store: FileJobStore, job_id: Optional[str] = None) -> None:
+            logger.debug(
+                f"Copying job store file: {jobStoreFileID} to {options.localFilePath[0]}"
+            )
+            jobStore.read_file(
+                jobStoreFileID,
+                os.path.join(
+                    options.localFilePath[0], os.path.basename(jobStoreFileID)
+                ),
+                symlink=options.useSymlinks,
+            )
+
+
+def printContentsOfJobStore(
+    job_store: FileJobStore, job_id: Optional[str] = None
+) -> None:
     """
     Fetch a list of all files contained in the job store if nameOfJob is not
     declared, otherwise it only prints out the names of files for that specific
@@ -90,22 +96,33 @@ def printContentsOfJobStore(job_store: FileJobStore, job_id: Optional[str] = Non
 
 def main() -> None:
     parser = parser_with_common_options(jobstore_option=True, prog="toil debug-file")
-    parser.add_argument("--localFilePath",
-                        nargs=1,
-                        help="Location to which to copy job store files.")
-    parser.add_argument("--fetch",
-                        nargs="+",
-                        help="List of job-store files to be copied locally."
-                             "Use either explicit names (i.e. 'data.txt'), or "
-                             "specify glob patterns (i.e. '*.txt')")
-    parser.add_argument("--listFilesInJobStore", type=strtobool,
-                        help="Prints a list of the current files in the jobStore.")
-    parser.add_argument("--fetchEntireJobStore", type=strtobool,
-                        help="Copy all job store files into a local directory.")
-    parser.add_argument("--useSymlinks", type=strtobool,
-                        help="Creates symlink 'shortcuts' of files in the localFilePath"
-                             " instead of hardlinking or copying, where possible.  If this is"
-                             " not possible, it will copy the files (shutil.copyfile()).")
+    parser.add_argument(
+        "--localFilePath", nargs=1, help="Location to which to copy job store files."
+    )
+    parser.add_argument(
+        "--fetch",
+        nargs="+",
+        help="List of job-store files to be copied locally."
+        "Use either explicit names (i.e. 'data.txt'), or "
+        "specify glob patterns (i.e. '*.txt')",
+    )
+    parser.add_argument(
+        "--listFilesInJobStore",
+        type=strtobool,
+        help="Prints a list of the current files in the jobStore.",
+    )
+    parser.add_argument(
+        "--fetchEntireJobStore",
+        type=strtobool,
+        help="Copy all job store files into a local directory.",
+    )
+    parser.add_argument(
+        "--useSymlinks",
+        type=strtobool,
+        help="Creates symlink 'shortcuts' of files in the localFilePath"
+        " instead of hardlinking or copying, where possible.  If this is"
+        " not possible, it will copy the files (shutil.copyfile()).",
+    )
 
     # Load the jobStore
     options = parser.parse_args()