toil 5.12.0__py3-none-any.whl → 6.1.0__py3-none-any.whl

toil/test/src/miscTests.py

@@ -16,12 +16,14 @@ import logging
 import os
 import random
 import sys
-import tempfile
 from uuid import uuid4
 
 from toil.common import getNodeID
 from toil.lib.exceptions import panic, raise_
-from toil.lib.io import AtomicFileCreate, atomic_install, atomic_tmp_file
+from toil.lib.io import (AtomicFileCreate,
+                         atomic_install,
+                         atomic_tmp_file,
+                         mkdtemp)
 from toil.lib.misc import CalledProcessErrorStderr, call_command
 from toil.test import ToilTest, slow
 
@@ -63,7 +65,7 @@ class MiscTests(ToilTest):
         files = {}
         # Create a random directory structure
         for i in range(0,10):
-            directories.append(tempfile.mkdtemp(dir=random.choice(directories), prefix='test'))
+            directories.append(mkdtemp(dir=random.choice(directories), prefix='test'))
         # Create 50 random file entries in different locations in the directories. 75% of the time
         # these are fresh files of size [1, 10] MB and 25% of the time they are hard links to old
         # files.

toil/test/src/promisedRequirementTest.py

@@ -48,7 +48,7 @@ class hidden:
                                      cores=1, memory='1M', disk='1M')
                 values = Job.Runner.startToil(root, self.getOptions(tempDir))
                 maxValue = max(values)
-                self.assertEqual(maxValue, self.cpuCount // coresPerJob)
+                self.assertLessEqual(maxValue, self.cpuCount // coresPerJob)
 
         @slow
         @retry_flaky_test(prepare=[batchSystemTest.hidden.AbstractBatchSystemJobTest.tearDown,
@@ -74,7 +74,7 @@ class hidden:
                                                 disk='1M'))
                 Job.Runner.startToil(root, self.getOptions(tempDir))
                 _, maxValue = batchSystemTest.getCounters(counterPath)
-                self.assertEqual(maxValue, self.cpuCount // coresPerJob)
+                self.assertLessEqual(maxValue, self.cpuCount // coresPerJob)
 
         def getOptions(self, tempDir, caching=True):
             options = super().getOptions(tempDir)
@@ -194,7 +194,7 @@ def logDiskUsage(job, funcName, sleep=0):
     :return: job function's disk usage
     """
     diskUsage = job.disk
-    job.fileStore.logToMaster(f'{funcName}: {diskUsage}')
+    job.fileStore.log_to_leader(f'{funcName}: {diskUsage}')
     time.sleep(sleep)
     return diskUsage
 

toil/test/src/realtimeLoggerTest.py

@@ -57,7 +57,7 @@ class MessageDetector(logging.StreamHandler):
 
 class LogTest(Job):
     def __init__(self):
-        Job.__init__(self, memory=100000, cores=2, disk='3G')
+        Job.__init__(self, memory=100000, cores=1, disk='3G')
 
     def run(self, fileStore):
         RealtimeLogger.info('This should be logged at info level')

toil/test/src/resourceTest.py

@@ -209,7 +209,7 @@ class ResourceTest(ToilTest):
         """
 
         def script():
-            import argparse
+            from configargparse import ArgumentParser
 
             from toil.common import Toil
             from toil.job import Job
@@ -218,7 +218,7 @@ class ResourceTest(ToilTest):
                 pass
 
             if __name__ == '__main__':
-                parser = argparse.ArgumentParser()
+                parser = ArgumentParser()
                 Job.Runner.addToilOptions(parser)
                 options = parser.parse_args()
                 job = Job.wrapFn(fn, memory='10M', cores=0.1, disk='10M')

toil/test/src/restartDAGTest.py

@@ -19,8 +19,8 @@ import shutil
 import signal
 
 from toil.common import Toil
-from toil.job import Job
 from toil.exceptions import FailedJobsException
+from toil.job import Job
 from toil.test import ToilTest, slow
 
 logger = logging.getLogger(__name__)

toil/test/src/resumabilityTest.py

@@ -14,17 +14,17 @@
 
 import os
 
+from toil.exceptions import FailedJobsException
 from toil.job import Job
 from toil.jobStores.abstractJobStore import NoSuchFileException
-from toil.exceptions import FailedJobsException
 from toil.test import ToilTest, slow
 
 
-@slow
 class ResumabilityTest(ToilTest):
     """
     https://github.com/BD2KGenomics/toil/issues/808
     """
+    @slow
     def test(self):
         """
         Tests that a toil workflow that fails once can be resumed without a NoSuchJobException.
@@ -53,6 +53,34 @@ class ResumabilityTest(ToilTest):
             # store ID: n/t/jobwbijqL failed with exit value 1"
             self.assertTrue("failed with exit value" not in logString)
 
+    def test_chaining(self):
+        """
+        Tests that a job which is chained to and fails can resume and succeed.
+        """
+
+        options = Job.Runner.getDefaultOptions(self._getTestJobStorePath())
+        options.logLevel = "DEBUG"
+        options.retryCount = 0
+        tempDir = self._createTempDir()
+        options.logFile = os.path.join(tempDir, "log.txt")
+
+        root = Job.wrapJobFn(chaining_parent)
+
+        with self.assertRaises(FailedJobsException):
+            # This one is intended to fail.
+            Job.Runner.startToil(root, options)
+
+        with open(options.logFile, 'r') as f:
+            log_content = f.read()
+            # Make sure we actually did do chaining
+            assert "Chaining from" in log_content
+
+        # Because of the chaining, the problem we are looking for is the job
+        # with the root ID not being able to load the body of a job with a
+        # different ID. That doesn't look like a job deleted despite failure.
+        options.restart = True
+        Job.Runner.startToil(root, options)
+
 def parent(job):
     """
     Set up a bunch of dummy child jobs, and a bad job that needs to be
@@ -62,6 +90,12 @@ def parent(job):
         job.addChildJobFn(goodChild)
     job.addFollowOnJobFn(badChild)
 
+def chaining_parent(job):
+    """
+    Set up a failing job to chain to.
+    """
+    job.addFollowOnJobFn(badChild)
+
 def goodChild(job):
     """
     Does nothing.

toil/test/src/retainTempDirTest.py

@@ -14,8 +14,8 @@
 import os
 import shutil
 
-from toil.job import Job
 from toil.exceptions import FailedJobsException
+from toil.job import Job
 from toil.test import ToilTest
 
 

toil/test/src/systemTest.py

@@ -1,9 +1,9 @@
 import errno
 import multiprocessing
 import os
-import tempfile
 from functools import partial
 
+from toil.lib.io import mkdtemp
 from toil.lib.threading import cpu_count
 from toil.test import ToilTest
 
@@ -37,7 +37,7 @@ class SystemTest(ToilTest):
 
 
 def _testAtomicityOfNonEmptyDirectoryRenamesTask(parent, child, _):
-    tmpChildDir = tempfile.mkdtemp(dir=parent, prefix='child', suffix='.tmp')
+    tmpChildDir = mkdtemp(dir=parent, prefix='child', suffix='.tmp')
     grandChild = os.path.join(tmpChildDir, 'grandChild')
     open(grandChild, 'w').close()
     grandChildId = os.stat(grandChild).st_ino

toil/test/src/toilContextManagerTest.py

@@ -61,10 +61,10 @@ class ToilContextManagerTest(ToilTest):
 
 class HelloWorld(Job):
     def __init__(self):
-        Job.__init__(self, memory=100000, cores=2, disk='1M')
+        Job.__init__(self, memory=100000, disk='1M')
 
     def run(self, fileStore):
-        fileID = self.addChildJobFn(childFn, cores=1, memory='1M', disk='1M').rv()
+        fileID = self.addChildJobFn(childFn, memory='1M', disk='1M').rv()
         return self.addFollowOn(FollowOn(fileID)).rv()
 
 

toil/test/src/userDefinedJobArgTypeTest.py

@@ -59,7 +59,7 @@ class UserDefinedJobArgTypeTest(ToilTest):
 
 class JobClass(Job):
     def __init__(self, level, foo):
-        Job.__init__(self, memory=100000, cores=2, disk="300M")
+        Job.__init__(self, memory=100000, cores=1, disk="300M")
         self.level = level
         self.foo = foo
 

toil/test/utils/toilDebugTest.py

@@ -1,4 +1,3 @@
-"""A set of test cases for toilwdl.py"""
 # Copyright (C) 2015-2021 Regents of the University of California
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -15,32 +14,33 @@
 import logging
 import os
 import subprocess
-from pathlib import Path
+import tempfile
+
+import pytest
+
+from toil.test import ToilTest
 
 from toil.lib.resources import glob
 from toil.test import slow
 from toil.version import python
 
-import pytest
-
 logger = logging.getLogger(__name__)
 
 
-@pytest.fixture
-def workflow_debug_jobstore(tmp_path: Path) -> str:
-    jobStorePath = str(tmp_path / "toilWorkflowRun")
+def workflow_debug_jobstore() -> str:
+    job_store_path = os.path.join(tempfile.mkdtemp(), "toilWorkflowRun")
     subprocess.check_call(
         [
             python,
             os.path.abspath("src/toil/test/utils/ABCWorkflowDebug/debugWorkflow.py"),
-            jobStorePath,
+            job_store_path,
         ]
     )
-    return jobStorePath
+    return job_store_path
 
 
 @slow
-def testJobStoreContents(workflow_debug_jobstore: str):
+def testJobStoreContents():
     """
     Test toilDebugFile.printContentsOfJobStore().
 
@@ -48,14 +48,13 @@ def testJobStoreContents(workflow_debug_jobstore: str):
     jobStore.  'A.txt', 'C.txt', 'ABC.txt' are then created.  This checks to
     make sure these contents are found in the jobStore and printed.
     """
-    jobStoreDir = workflow_debug_jobstore
     contents = ["A.txt", "B.txt", "C.txt", "ABC.txt", "mkFile.py"]
 
     subprocess.check_call(
         [
             python,
             os.path.abspath("src/toil/utils/toilDebugFile.py"),
-            jobStoreDir,
+            workflow_debug_jobstore(),
             "--logDebug",
             "--listFilesInJobStore=True",
         ]
@@ -78,7 +77,7 @@ def testJobStoreContents(workflow_debug_jobstore: str):
     os.remove(jobstoreFileContents)
 
 
-def fetchFiles(symLink, jobStoreDir: str, outputDir):
+def fetchFiles(symLink: bool, jobStoreDir: str, outputDir: str):
     """
     Fn for testFetchJobStoreFiles() and testFetchJobStoreFilesWSymlinks().
 
@@ -99,8 +98,8 @@ def fetchFiles(symLink, jobStoreDir: str, outputDir):
         "*C.txt",
         "*ABC.txt",
         "*mkFile.py",
-        "--localFilePath=" + outputDir,
-        "--useSymlinks=" + str(symLink),
+        f"--localFilePath={outputDir}",
+        f"--useSymlinks={symLink}",
     ]
     print(cmd)
     subprocess.check_call(cmd)
@@ -114,22 +113,89 @@ def fetchFiles(symLink, jobStoreDir: str, outputDir):
 
 
 # expected run time = 4s
-def testFetchJobStoreFiles(tmp_path: Path, workflow_debug_jobstore: str) -> None:
-    """Test toilDebugFile.fetchJobStoreFiles() without using symlinks."""
-    outputDir = tmp_path / "testoutput"
-    outputDir.mkdir()
-    fetchFiles(
-        symLink=False, jobStoreDir=workflow_debug_jobstore, outputDir=str(outputDir)
-    )
+def testFetchJobStoreFiles() -> None:
+    """Test toilDebugFile.fetchJobStoreFiles() symlinks."""
+    job_store_dir = workflow_debug_jobstore()
+    output_dir = os.path.join(os.path.dirname(job_store_dir), "testoutput")
+    os.makedirs(output_dir, exist_ok=True)
+    for symlink in (True, False):
+        fetchFiles(symLink=symlink, jobStoreDir=job_store_dir, outputDir=output_dir)
+
+class DebugJobTest(ToilTest):
+    """
+    Test the toil debug-job command.
+    """
+
+    def _get_job_store_and_job_id(self):
+        """
+        Get a job store and the ID of a failing job within it.
+        """
+
+        # First make a job store.
+        job_store = os.path.join(self._createTempDir(), "tree")
+
+        logger.info("Running workflow that always fails")
+        try:
+            # Run an always-failign workflow
+            subprocess.check_call([
+                python,
+                os.path.abspath("src/toil/test/docs/scripts/example_alwaysfail.py"),
+                "--retryCount=0",
+                "--logCritical",
+                "--disableProgress",
+                job_store
+            ], stderr=subprocess.DEVNULL)
+            raise RuntimeError("Failing workflow succeeded!")
+        except subprocess.CalledProcessError:
+            # Should fail to run
+            logger.info("Task failed successfully")
+            pass
+
+        # Get the job ID.
+        # TODO: This assumes a lot about the FileJobStore. Use the MessageBus instead?
+        job_id = "kind-explode/" + os.listdir(os.path.join(job_store, "jobs/kind-explode"))[0]
+
+        return job_store, job_id
+
+    def test_run_job(self):
+        """
+        Make sure that we can use toil debug-job to try and run a job in-process.
+        """
+
+        job_store, job_id = self._get_job_store_and_job_id()
+
+        logger.info("Trying to rerun job %s", job_id)
+
+        # Rerun the job, which should fail again
+        output = subprocess.check_output([
+            "toil",
+            "debug-job",
+            "--logDebug",
+            job_store,
+            job_id
+        ], stderr=subprocess.STDOUT)
+        # Even if the job fails, the attempt to run it will succeed.
+        log = output.decode('utf-8')
+        assert "Boom!" in log, f"Did not find the expected exception message in: {log}"
+
+
+    def test_print_job_info(self):
+        """
+        Make sure that we can use --printJobInfo to get information on a job from a job store.
+        """
+
+        job_store, job_id = self._get_job_store_and_job_id()
+
+        logger.info("Trying to print job info for job %s", job_id)
+
+        # Print the job info and make sure that doesn't crash.
+        subprocess.check_call([
+            "toil",
+            "debug-job",
+            "--logDebug",
+            job_store,
+            "--printJobInfo",
+            job_id
+        ])
 
 
-# expected run time = 4s
-def testFetchJobStoreFilesWSymlinks(
-    tmp_path: Path, workflow_debug_jobstore: str
-) -> None:
-    """Test toilDebugFile.fetchJobStoreFiles() using symlinks."""
-    outputDir = tmp_path / "testoutput"
-    outputDir.mkdir()
-    fetchFiles(
-        symLink=True, jobStoreDir=workflow_debug_jobstore, outputDir=str(outputDir)
-    )

toil/test/utils/toilKillTest.py

@@ -21,7 +21,8 @@ import time
 import unittest
 
 from toil.common import Toil
-from toil.jobStores.abstractJobStore import NoSuchFileException, NoSuchJobStoreException
+from toil.jobStores.abstractJobStore import (NoSuchFileException,
+                                             NoSuchJobStoreException)
 from toil.jobStores.utils import generate_locator
 from toil.test import ToilTest, needs_aws_s3, needs_cwl
 
@@ -77,7 +78,6 @@ class ToilKillTest(ToilTest):
                     logger.info('Waiting for kill flag...')
             except (NoSuchJobStoreException, NoSuchFileException):
                 logger.info('Waiting for job store to be openable...')
-                pass
             time.sleep(2)
 
         # run toil kill

toil/test/utils/utilsTest.py

@@ -39,7 +39,7 @@ from toil.test import (ToilTest,
                        needs_rsync3,
                        slow)
 from toil.test.sort.sortTest import makeFileToSort
-from toil.utils.toilStats import getStats, processData
+from toil.utils.toilStats import get_stats, process_data
 from toil.utils.toilStatus import ToilStatus
 from toil.version import python
 
@@ -115,6 +115,26 @@ class UtilsTest(ToilTest):
             commandTokens.append('--failIfNotComplete')
         return commandTokens
 
+    def test_config_functionality(self):
+        """Ensure that creating and reading back the config file works"""
+        config_file = os.path.abspath("config.yaml")
+        config_command = [self.toilMain, 'config', config_file]
+        # make sure the command `toil config file_path` works
+        try:
+            subprocess.check_call(config_command)
+        except subprocess.CalledProcessError:
+            self.fail("The toil config utility failed!")
+
+        parser = Job.Runner.getDefaultArgumentParser()
+        # make sure that toil can read from the generated config file
+        try:
+            parser.parse_args(["random_jobstore", "--config", config_file])
+        except SystemExit:
+            self.fail("Failed to parse the default generated config file!")
+        finally:
+            os.remove(config_file)
+
+
     @needs_rsync3
     @pytest.mark.timeout(1200)
     @needs_aws_ec2
@@ -278,8 +298,8 @@ class UtilsTest(ToilTest):
         config = Config()
         config.setOptions(options)
         jobStore = Toil.resumeJobStore(config.jobStore)
-        stats = getStats(jobStore)
-        collatedStats = processData(jobStore.config, stats)
+        stats = get_stats(jobStore)
+        collatedStats = process_data(jobStore.config, stats)
         self.assertTrue(len(collatedStats.job_types) == 2, "Some jobs are not represented in the stats.")
 
     def check_status(self, status, status_fn, seconds=20):