toil 5.12.0__py3-none-any.whl → 6.1.0a1__py3-none-any.whl

toil/test/utils/toilDebugTest.py

@@ -1,4 +1,3 @@
-"""A set of test cases for toilwdl.py"""
 # Copyright (C) 2015-2021 Regents of the University of California
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -15,32 +14,33 @@
 import logging
 import os
 import subprocess
-from pathlib import Path
+import tempfile
+
+import pytest
+
+from toil.test import ToilTest
 
 from toil.lib.resources import glob
 from toil.test import slow
 from toil.version import python
 
-import pytest
-
 logger = logging.getLogger(__name__)
 
 
-@pytest.fixture
-def workflow_debug_jobstore(tmp_path: Path) -> str:
-    jobStorePath = str(tmp_path / "toilWorkflowRun")
+def workflow_debug_jobstore() -> str:
+    job_store_path = os.path.join(tempfile.mkdtemp(), "toilWorkflowRun")
     subprocess.check_call(
         [
             python,
             os.path.abspath("src/toil/test/utils/ABCWorkflowDebug/debugWorkflow.py"),
-            jobStorePath,
+            job_store_path,
         ]
     )
-    return jobStorePath
+    return job_store_path
 
 
 @slow
-def testJobStoreContents(workflow_debug_jobstore: str):
+def testJobStoreContents():
     """
     Test toilDebugFile.printContentsOfJobStore().
 
@@ -48,14 +48,13 @@ def testJobStoreContents(workflow_debug_jobstore: str):
     jobStore.  'A.txt', 'C.txt', 'ABC.txt' are then created.  This checks to
     make sure these contents are found in the jobStore and printed.
     """
-    jobStoreDir = workflow_debug_jobstore
     contents = ["A.txt", "B.txt", "C.txt", "ABC.txt", "mkFile.py"]
 
     subprocess.check_call(
         [
             python,
             os.path.abspath("src/toil/utils/toilDebugFile.py"),
-            jobStoreDir,
+            workflow_debug_jobstore(),
             "--logDebug",
             "--listFilesInJobStore=True",
         ]
@@ -78,7 +77,7 @@ def testJobStoreContents(workflow_debug_jobstore: str):
     os.remove(jobstoreFileContents)
 
 
-def fetchFiles(symLink, jobStoreDir: str, outputDir):
+def fetchFiles(symLink: bool, jobStoreDir: str, outputDir: str):
     """
     Fn for testFetchJobStoreFiles() and testFetchJobStoreFilesWSymlinks().
 
@@ -99,8 +98,8 @@ def fetchFiles(symLink, jobStoreDir: str, outputDir):
         "*C.txt",
         "*ABC.txt",
         "*mkFile.py",
-        "--localFilePath=" + outputDir,
-        "--useSymlinks=" + str(symLink),
+        f"--localFilePath={outputDir}",
+        f"--useSymlinks={symLink}",
     ]
     print(cmd)
     subprocess.check_call(cmd)
@@ -114,22 +113,89 @@ def fetchFiles(symLink, jobStoreDir: str, outputDir):
 
 
 # expected run time = 4s
-def testFetchJobStoreFiles(tmp_path: Path, workflow_debug_jobstore: str) -> None:
-    """Test toilDebugFile.fetchJobStoreFiles() without using symlinks."""
-    outputDir = tmp_path / "testoutput"
-    outputDir.mkdir()
-    fetchFiles(
-        symLink=False, jobStoreDir=workflow_debug_jobstore, outputDir=str(outputDir)
-    )
+def testFetchJobStoreFiles() -> None:
+    """Test toilDebugFile.fetchJobStoreFiles() symlinks."""
+    job_store_dir = workflow_debug_jobstore()
+    output_dir = os.path.join(os.path.dirname(job_store_dir), "testoutput")
+    os.makedirs(output_dir, exist_ok=True)
+    for symlink in (True, False):
+        fetchFiles(symLink=symlink, jobStoreDir=job_store_dir, outputDir=output_dir)
+
+class DebugJobTest(ToilTest):
+    """
+    Test the toil debug-job command.
+    """
+
+    def _get_job_store_and_job_id(self):
+        """
+        Get a job store and the ID of a failing job within it.
+        """
+
+        # First make a job store.
+        job_store = os.path.join(self._createTempDir(), "tree")
+
+        logger.info("Running workflow that always fails")
+        try:
+            # Run an always-failign workflow
+            subprocess.check_call([
+                python,
+                os.path.abspath("src/toil/test/docs/scripts/example_alwaysfail.py"),
+                "--retryCount=0",
+                "--logCritical",
+                "--disableProgress=True",
+                job_store
+            ], stderr=subprocess.DEVNULL)
+            raise RuntimeError("Failing workflow succeeded!")
+        except subprocess.CalledProcessError:
+            # Should fail to run
+            logger.info("Task failed successfully")
+            pass
+
+        # Get the job ID.
+        # TODO: This assumes a lot about the FileJobStore. Use the MessageBus instead?
+        job_id = "kind-explode/" + os.listdir(os.path.join(job_store, "jobs/kind-explode"))[0]
+
+        return job_store, job_id
+
+    def test_run_job(self):
+        """
+        Make sure that we can use toil debug-job to try and run a job in-process.
+        """
+
+        job_store, job_id = self._get_job_store_and_job_id()
+
+        logger.info("Trying to rerun job %s", job_id)
+
+        # Rerun the job, which should fail again
+        output = subprocess.check_output([
+            "toil",
+            "debug-job",
+            "--logDebug",
+            job_store,
+            job_id
+        ], stderr=subprocess.STDOUT)
+        # Even if the job fails, the attempt to run it will succeed.
+        log = output.decode('utf-8')
+        assert "Boom!" in log, f"Did not find the expected exception message in: {log}"
+
+
+    def test_print_job_info(self):
+        """
+        Make sure that we can use --printJobInfo to get information on a job from a job store.
+        """
+
+        job_store, job_id = self._get_job_store_and_job_id()
+
+        logger.info("Trying to print job info for job %s", job_id)
+
+        # Print the job info and make sure that doesn't crash.
+        subprocess.check_call([
+            "toil",
+            "debug-job",
+            "--logDebug",
+            job_store,
+            "--printJobInfo",
+            job_id
+        ])
 
 
-# expected run time = 4s
-def testFetchJobStoreFilesWSymlinks(
-    tmp_path: Path, workflow_debug_jobstore: str
-) -> None:
-    """Test toilDebugFile.fetchJobStoreFiles() using symlinks."""
-    outputDir = tmp_path / "testoutput"
-    outputDir.mkdir()
-    fetchFiles(
-        symLink=True, jobStoreDir=workflow_debug_jobstore, outputDir=str(outputDir)
-    )

toil/test/utils/toilKillTest.py

@@ -21,7 +21,8 @@ import time
 import unittest
 
 from toil.common import Toil
-from toil.jobStores.abstractJobStore import NoSuchFileException, NoSuchJobStoreException
+from toil.jobStores.abstractJobStore import (NoSuchFileException,
+                                             NoSuchJobStoreException)
 from toil.jobStores.utils import generate_locator
 from toil.test import ToilTest, needs_aws_s3, needs_cwl
 
@@ -77,7 +78,6 @@ class ToilKillTest(ToilTest):
                     logger.info('Waiting for kill flag...')
             except (NoSuchJobStoreException, NoSuchFileException):
                 logger.info('Waiting for job store to be openable...')
-                pass
             time.sleep(2)
 
         # run toil kill

toil/test/utils/utilsTest.py

@@ -115,6 +115,26 @@ class UtilsTest(ToilTest):
             commandTokens.append('--failIfNotComplete')
         return commandTokens
 
+    def test_config_functionality(self):
+        """Ensure that creating and reading back the config file works"""
+        config_file = os.path.abspath("config.yaml")
+        config_command = [self.toilMain, 'config', config_file]
+        # make sure the command `toil config file_path` works
+        try:
+            subprocess.check_call(config_command)
+        except subprocess.CalledProcessError:
+            self.fail("The toil config utility failed!")
+
+        parser = Job.Runner.getDefaultArgumentParser()
+        # make sure that toil can read from the generated config file
+        try:
+            parser.parse_args(["random_jobstore", "--config", config_file])
+        except SystemExit:
+            self.fail("Failed to parse the default generated config file!")
+        finally:
+            os.remove(config_file)
+
+
     @needs_rsync3
     @pytest.mark.timeout(1200)
     @needs_aws_ec2

toil/test/wdl/wdltoil_test.py

@@ -4,20 +4,36 @@ import shutil
 import subprocess
 import unittest
 import uuid
-import zipfile
-from urllib.request import urlretrieve
+from typing import Any, Dict, List, Optional, Set
+from unittest.mock import patch
 
-import pytest
+from unittest.mock import patch
+from typing import Any, Dict, List, Set
 
-from toil.test import ToilTest, needs_docker, needs_docker_cuda, needs_google_storage, needs_java, needs_singularity_or_docker, slow
+from toil.test import (ToilTest,
+                       needs_docker_cuda,
+                       needs_google_storage,
+                       needs_singularity_or_docker,
+                       slow)
 from toil.version import exactPython
-# Don't import the test case directly or pytest will test it again.
-import toil.test.wdl.toilwdlTest
+from toil.wdl.wdltoil import WDLSectionJob, WDLWorkflowGraph
 
 
-class ToilConformanceTests(toil.test.wdl.toilwdlTest.BaseToilWdlTest):
+class BaseWDLTest(ToilTest):
+    """Base test class for WDL tests."""
+    def setUp(self) -> None:
+        """Runs anew before each test to create farm fresh temp dirs."""
+        self.output_dir = os.path.join('/tmp/', 'toil-wdl-test-' + str(uuid.uuid4()))
+        os.makedirs(self.output_dir)
+
+    def tearDown(self) -> None:
+        if os.path.exists(self.output_dir):
+            shutil.rmtree(self.output_dir)
+
+
+class WDLConformanceTests(BaseWDLTest):
     """
-    New WDL conformance tests for Toil
+    WDL conformance tests for Toil.
     """
     wdl_dir = "wdl-conformance-tests"
     @classmethod
@@ -47,7 +63,7 @@ class ToilConformanceTests(toil.test.wdl.toilwdlTest.BaseToilWdlTest):
         p = subprocess.run(self.base_command + ["-v", "1.0", "-n", tests_to_run], capture_output=True)
 
         if p.returncode != 0:
-            print(p.stdout)
+            print(p.stdout.decode('utf-8', errors='replace'))
 
         p.check_returncode()
 
@@ -58,7 +74,7 @@ class ToilConformanceTests(toil.test.wdl.toilwdlTest.BaseToilWdlTest):
         p = subprocess.run(self.base_command + ["-v", "1.1", "-n", tests_to_run], capture_output=True)
 
         if p.returncode != 0:
-            print(p.stdout)
+            print(p.stdout.decode('utf-8', errors='replace'))
 
         p.check_returncode()
 
@@ -69,26 +85,23 @@ class ToilConformanceTests(toil.test.wdl.toilwdlTest.BaseToilWdlTest):
         shutil.rmtree("wdl-conformance-tests")
 
 
-class WdlToilTest(toil.test.wdl.toilwdlTest.ToilWdlTest):
-    """
-    Version of the old Toil WDL tests that tests the new MiniWDL-based implementation.
-    """
-
+class WDLTests(BaseWDLTest):
+    """Tests for Toil's MiniWDL-based implementation."""
     @classmethod
     def setUpClass(cls) -> None:
         """Runs once for all tests."""
         cls.base_command = [exactPython, '-m', 'toil.wdl.wdltoil']
 
-    # We inherit a testMD5sum but it is going to need Singularity and not
-    # Docker now. And also needs to have a WDL 1.0+ WDL file. So we replace it.
+    # We inherit a testMD5sum but it is going to need Singularity or Docker
+    # now. And also needs to have a WDL 1.0+ WDL file. So we replace it.
     @needs_singularity_or_docker
-    def testMD5sum(self):
+    def test_MD5sum(self):
         """Test if Toil produces the same outputs as known good outputs for WDL's
         GATK tutorial #1."""
         wdl = os.path.abspath('src/toil/test/wdl/md5sum/md5sum.1.0.wdl')
         json_file = os.path.abspath('src/toil/test/wdl/md5sum/md5sum.json')
 
-        result_json = subprocess.check_output(self.base_command + [wdl, json_file, '-o', self.output_dir, '--logDebug'])
+        result_json = subprocess.check_output(self.base_command + [wdl, json_file, '-o', self.output_dir, '--logDebug', '--retryCount=0'])
         result = json.loads(result_json)
 
         assert 'ga4ghMd5.value' in result
@@ -96,25 +109,13 @@ class WdlToilTest(toil.test.wdl.toilwdlTest.ToilWdlTest):
         assert os.path.exists(result['ga4ghMd5.value'])
         assert os.path.basename(result['ga4ghMd5.value']) == 'md5sum.txt'
 
-    def test_empty_file_path(self):
-        """Test if empty File type inputs are protected against"""
-        wdl = os.path.abspath('src/toil/test/wdl/md5sum/md5sum.1.0.wdl')
-        json_file = os.path.abspath('src/toil/test/wdl/md5sum/empty_file.json')
-
-        p = subprocess.Popen(self.base_command + [wdl, json_file, '-o', self.output_dir, '--logDebug'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-        stdout, stderr = p.communicate()
-        retval = p.wait()
-
-        assert retval != 0
-        assert b'Could not find' in stderr
-
     @needs_singularity_or_docker
-    def test_miniwdl_self_test(self):
+    def test_miniwdl_self_test(self, extra_args: Optional[List[str]] = None) -> None:
         """Test if the MiniWDL self test runs and produces the expected output."""
         wdl_file = os.path.abspath('src/toil/test/wdl/miniwdl_self_test/self_test.wdl')
         json_file = os.path.abspath('src/toil/test/wdl/miniwdl_self_test/inputs.json')
 
-        result_json = subprocess.check_output(self.base_command + [wdl_file, json_file, '-o', self.output_dir, '--outputDialect', 'miniwdl'])
+        result_json = subprocess.check_output(self.base_command + [wdl_file, json_file, '--logDebug', '-o', self.output_dir, '--outputDialect', 'miniwdl'] + (extra_args or []))
         result = json.loads(result_json)
 
         # Expect MiniWDL-style output with a designated "dir"
@@ -138,10 +139,17 @@ class WdlToilTest(toil.test.wdl.toilwdlTest.ToilWdlTest):
         assert 'hello_caller.messages' in outputs
         assert outputs['hello_caller.messages'] == ["Hello, Alyssa P. Hacker!", "Hello, Ben Bitdiddle!"]
 
+    @needs_singularity_or_docker
+    def test_miniwdl_self_test_by_reference(self) -> None:
+        """
+        Test if the MiniWDL self test works when passing input files by URL reference.
+        """
+        self.test_miniwdl_self_test(extra_args=["--referenceInputs=True"])
+
     @slow
     @needs_docker_cuda
     def test_giraffe_deepvariant(self):
-        """Test if Giraffe and CPU DeepVariant run. This could take 25 minutes."""
+        """Test if Giraffe and GPU DeepVariant run. This could take 25 minutes."""
         # TODO: enable test if nvidia-container-runtime and Singularity are installed but Docker isn't.
 
         json_dir = self._createTempDir()
@@ -184,7 +192,7 @@ class WdlToilTest(toil.test.wdl.toilwdlTest.ToilWdlTest):
     @slow
     @needs_singularity_or_docker
     def test_giraffe(self):
-        """Test if Giraffe runs. This could take 12 minutes. Also we scale it down."""
+        """Test if Giraffe runs. This could take 12 minutes. Also we scale it down but it still demands lots of memory."""
         # TODO: enable test if nvidia-container-runtime and Singularity are installed but Docker isn't.
 
         json_dir = self._createTempDir()
@@ -224,17 +232,112 @@ class WdlToilTest(toil.test.wdl.toilwdlTest.ToilWdlTest):
         assert os.path.exists(result['ga4ghMd5.value'])
         assert os.path.basename(result['ga4ghMd5.value']) == 'md5sum.txt'
 
-    def test_empty_file_path(self):
-        """Test if empty File type inputs are protected against"""
-        wdl = os.path.abspath('src/toil/test/wdl/md5sum/md5sum.1.0.wdl')
-        json_file = os.path.abspath('src/toil/test/wdl/md5sum/empty_file.json')
-
-        p = subprocess.Popen(self.base_command + [wdl, json_file, '-o', self.output_dir, '--logDebug'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-        stdout, stderr = p.communicate()
-        retval = p.wait()
+    def test_coalesce(self):
+        """
+        Test if WDLSectionJob can coalesce WDL decls.
+
+        White box test; will need to be changed or removed if the WDL interpreter changes.
+        """
+
+        # Set up data structures for our fake workflow graph to pull from.
+        # This has all decl-type nodes
+        all_decls: Set[str] = set()
+        # And this has all transitive dependencies for all nodes.
+        all_deps: Dict[str, Set[str]] = {}
+
+        def mock_is_decl(self: Any, node_id: str) -> bool:
+            """
+            Replacement function to determine if a node is a decl or not.
+            """
+            return node_id in all_decls
+
+        def mock_get_transitive_dependencies(self: Any, node_id: str) -> Set[str]:
+            """
+            Replacement function to get all the transitive dependencies of a node.
+            """
+            return all_deps[node_id]
+
+        # These are the only two methods coalesce_nodes calls, so we can
+        # replace them to ensure our graph is used without actually needing to
+        # make any WDL objects for it.
+        #
+        # If that changes, the test will need to change! Maybe then it will be
+        # worth extracting a base type for this interface.
+        with patch.object(WDLWorkflowGraph, 'is_decl', mock_is_decl):
+            with patch.object(WDLWorkflowGraph, 'get_transitive_dependencies', mock_get_transitive_dependencies):
+
+                with self.subTest(msg="Two unrelated decls can coalesce"):
+                    # Set up two unrelated decls
+                    all_decls = {"decl1", "decl2"}
+                    all_deps = {
+                        "decl1": set(),
+                        "decl2": set()
+                    }
+
+                    result = WDLSectionJob.coalesce_nodes(["decl1", "decl2"], WDLWorkflowGraph([]))
+                    
+                    # Make sure they coalesced
+                    assert len(result) == 1
+                    assert "decl1" in result[0]
+                    assert "decl2" in result[0]
+
+                with self.subTest(msg="A decl will not coalesce with a non-decl"):
+                    all_decls = {"decl"}
+                    all_deps = {
+                        "decl": set(),
+                        "nondecl": set()
+                    }
+
+                    result = WDLSectionJob.coalesce_nodes(["decl", "nondecl"], WDLWorkflowGraph([]))
+                    
+                    assert len(result) == 2
+                    assert len(result[0]) == 1
+                    assert len(result[1]) == 1
+
+
+                with self.subTest(msg="Two adjacent decls with a common dependency can coalesce"):
+                    all_decls = {"decl1", "decl2"}
+                    all_deps = {
+                        "decl1": {"base"},
+                        "decl2": {"base"},
+                        "base": set()
+                    }
+
+                    result = WDLSectionJob.coalesce_nodes(["base", "decl1", "decl2"], WDLWorkflowGraph([]))
+                    
+                    assert len(result) == 2
+                    assert "base" in result[0]
+                    assert "decl1" in result[1]
+                    assert "decl2" in result[1]
+
+                with self.subTest(msg="Two adjacent decls with different dependencies will not coalesce"):
+                    all_decls = {"decl1", "decl2"}
+                    all_deps = {
+                        "decl1": {"base"},
+                        "decl2": set(),
+                        "base": set()
+                    }
+
+                    result = WDLSectionJob.coalesce_nodes(["base", "decl1", "decl2"], WDLWorkflowGraph([]))
+                    
+                    assert len(result) == 3
+                    assert "base" in result[0]
+
+                with self.subTest(msg="Two adjacent decls with different successors will coalesce"):
+                    all_decls = {"decl1", "decl2"}
+                    all_deps = {
+                        "decl1": set(),
+                        "decl2": set(),
+                        "successor": {"decl2"}
+                    }
+
+                    result = WDLSectionJob.coalesce_nodes(["decl1", "decl2", "successor"], WDLWorkflowGraph([]))
+                    
+                    assert len(result) == 2
+                    assert "decl1" in result[0]
+                    assert "decl2" in result[0]
+                    assert "successor" in result[1]
 
-        assert retval != 0
-        assert b'Could not find' in stderr
 
 if __name__ == "__main__":
     unittest.main()  # run all tests

toil/toilState.py

@@ -183,6 +183,7 @@ class ToilState:
         if job_id in self.__job_database:
             # Update the one true copy in place
             old_truth = self.__job_database[job_id]
+            old_truth.check_new_version(new_truth)
             old_truth.__dict__.update(new_truth.__dict__)
         else:
             # Just keep the new one
@@ -293,7 +294,8 @@ class ToilState:
                     successor.predecessorsFinished.add(jobDesc.jobStoreID)
 
                 # If the successor has no predecessors to finish
-                assert len(successor.predecessorsFinished) <= successor.predecessorNumber
+                if len(successor.predecessorsFinished) > successor.predecessorNumber:
+                    raise RuntimeError("There are more finished predecessors than possible.")
                 if len(successor.predecessorsFinished) == successor.predecessorNumber:
 
                     # It is ready to be run, so remove it from the set of waiting jobs
@@ -322,7 +324,8 @@ class ToilState:
 
                         # We put the successor job in the set of waiting successor
                         # jobs with multiple predecessors
-                        assert successorJobStoreID not in self.jobsToBeScheduledWithMultiplePredecessors
+                        if successorJobStoreID in self.jobsToBeScheduledWithMultiplePredecessors:
+                            raise RuntimeError("Failed to schedule the successor job. The successor job is already scheduled.")
                         self.jobsToBeScheduledWithMultiplePredecessors.add(successorJobStoreID)
 
                         # Process successor
@@ -337,10 +340,8 @@ class ToilState:
                     # We've already seen the successor
 
                     # Add the job as a predecessor
-                    assert (
-                        jobDesc.jobStoreID
-                        not in self.successor_to_predecessors[successorJobStoreID]
-                    )
+                    if jobDesc.jobStoreID in self.successor_to_predecessors[successorJobStoreID]:
+                        raise RuntimeError("Failed to add the job as a predecessor. The job is already added as a predecessor.")
                     self.successor_to_predecessors[successorJobStoreID].add(
                         str(jobDesc.jobStoreID)
                     )

toil/utils/toilClean.py

@@ -22,7 +22,7 @@ logger = logging.getLogger(__name__)
 
 
 def main() -> None:
-    parser = parser_with_common_options(jobstore_option=True)
+    parser = parser_with_common_options(jobstore_option=True, prog="toil clean")
 
     options = parser.parse_args()
     set_logging_from_options(options)

toil/utils/toilConfig.py

@@ -0,0 +1,36 @@
+# Copyright (C) 2015-2021 Regents of the University of California
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Create a config file with all default Toil options."""
+import logging
+import os
+
+from configargparse import ArgParser
+
+from toil.common import generate_config
+from toil.statsAndLogging import add_logging_options, set_logging_from_options
+
+logger = logging.getLogger(__name__)
+
+
+def main() -> None:
+    parser = ArgParser()
+
+    parser.add_argument("output", default="config.yaml", help="Filepath to write the config file too. Default=%("
+                                                              "default)s")
+    add_logging_options(parser)
+    options = parser.parse_args()
+    set_logging_from_options(options)
+    logger.debug("Attempting to write a default config file to %s.", os.path.abspath(options.output))
+    generate_config(os.path.abspath(options.output))
+    logger.info("Successfully wrote a default config file to %s.", os.path.abspath(options.output))