toil 6.1.0a1__py3-none-any.whl → 8.0.0__py3-none-any.whl

toil/test/src/threadingTest.py

@@ -6,28 +6,29 @@ import time
 import traceback
 from functools import partial
 
-from toil.lib.threading import (LastProcessStandingArena,
-                                cpu_count,
-                                global_mutex)
+from toil.lib.threading import LastProcessStandingArena, cpu_count, global_mutex
 from toil.test import ToilTest
 
 log = logging.getLogger(__name__)
 
+
 class ThreadingTest(ToilTest):
     """Test Toil threading/synchronization tools."""
+
     def testGlobalMutexOrdering(self):
         for it in range(10):
-            log.info('Iteration %d', it)
+            log.info("Iteration %d", it)
 
             scope = self._createTempDir()
-            mutex = 'mutex'
+            mutex = "mutex"
             # Use processes (as opposed to threads) to prevent GIL from ordering things artificially
             pool = multiprocessing.Pool(processes=cpu_count())
             try:
                 numTasks = 100
                 results = pool.map_async(
                     func=partial(_testGlobalMutexOrderingTask, scope, mutex),
-                    iterable=list(range(numTasks)))
+                    iterable=list(range(numTasks)),
+                )
                 results = results.get()
             finally:
                 pool.close()
@@ -40,17 +41,18 @@ class ThreadingTest(ToilTest):
 
     def testLastProcessStanding(self):
         for it in range(10):
-            log.info('Iteration %d', it)
+            log.info("Iteration %d", it)
 
             scope = self._createTempDir()
-            arena_name = 'thunderdome'
+            arena_name = "thunderdome"
             # Use processes (as opposed to threads) to prevent GIL from ordering things artificially
             pool = multiprocessing.Pool(processes=cpu_count())
             try:
                 numTasks = 100
                 results = pool.map_async(
                     func=partial(_testLastProcessStandingTask, scope, arena_name),
-                    iterable=list(range(numTasks)))
+                    iterable=list(range(numTasks)),
+                )
                 results = results.get()
             finally:
                 pool.close()
@@ -61,19 +63,24 @@ class ThreadingTest(ToilTest):
                 # Make sure all workers say they succeeded
                 self.assertEqual(item, True)
             for filename in os.listdir(scope):
-                assert not filename.startswith('precious'), f"File {filename} still exists"
+                assert not filename.startswith(
+                    "precious"
+                ), f"File {filename} still exists"
+
 
 def _testGlobalMutexOrderingTask(scope, mutex, number):
     try:
         # We will all fight over the potato
-        potato = os.path.join(scope, 'potato')
+        potato = os.path.join(scope, "potato")
 
         with global_mutex(scope, mutex):
-            log.info('PID %d = num %d running', os.getpid(), number)
-            assert not os.path.exists(potato), "We see someone else holding the potato file"
+            log.info("PID %d = num %d running", os.getpid(), number)
+            assert not os.path.exists(
+                potato
+            ), "We see someone else holding the potato file"
 
             # Put our name there
-            with open(potato, 'w') as out_stream:
+            with open(potato, "w") as out_stream:
                 out_stream.write(str(number))
 
             # Wait
@@ -82,51 +89,63 @@ def _testGlobalMutexOrderingTask(scope, mutex, number):
             # Make sure our name is still there
             with open(potato) as in_stream:
                 seen = in_stream.read().rstrip()
-                assert seen == str(number), f"We are {number} but {seen} stole our potato!"
+                assert seen == str(
+                    number
+                ), f"We are {number} but {seen} stole our potato!"
 
             os.unlink(potato)
             assert not os.path.exists(potato), "We left the potato behind"
-            log.info('PID %d = num %d dropped potato', os.getpid(), number)
+            log.info("PID %d = num %d dropped potato", os.getpid(), number)
         return True
     except:
         traceback.print_exc()
         return False
 
+
 def _testLastProcessStandingTask(scope, arena_name, number):
     try:
         arena = LastProcessStandingArena(scope, arena_name)
 
         arena.enter()
-        log.info('PID %d = num %d entered arena', os.getpid(), number)
+        log.info("PID %d = num %d entered arena", os.getpid(), number)
         try:
             # We all make files
-            my_precious = os.path.join(scope, 'precious' + str(number))
+            my_precious = os.path.join(scope, "precious" + str(number))
 
             # Put our name there
-            with open(my_precious, 'w') as out_stream:
+            with open(my_precious, "w") as out_stream:
                 out_stream.write(str(number))
 
             # Wait
             time.sleep(random.random() * 0.01)
 
             # Make sure our file is still there unmodified
-            assert os.path.exists(my_precious), f"Precious file {my_precious} has been stolen!"
+            assert os.path.exists(
+                my_precious
+            ), f"Precious file {my_precious} has been stolen!"
             with open(my_precious) as in_stream:
                 seen = in_stream.read().rstrip()
-                assert seen == str(number), f"We are {number} but saw {seen} in our precious file!"
+                assert seen == str(
+                    number
+                ), f"We are {number} but saw {seen} in our precious file!"
         finally:
             was_last = False
             for _ in arena.leave():
                 was_last = True
-                log.info('PID %d = num %d is last standing', os.getpid(), number)
+                log.info("PID %d = num %d is last standing", os.getpid(), number)
 
                 # Clean up all the files
                 for filename in os.listdir(scope):
-                    if filename.startswith('precious'):
-                        log.info('PID %d = num %d cleaning up %s', os.getpid(), number, filename)
+                    if filename.startswith("precious"):
+                        log.info(
+                            "PID %d = num %d cleaning up %s",
+                            os.getpid(),
+                            number,
+                            filename,
+                        )
                         os.unlink(os.path.join(scope, filename))
 
-            log.info('PID %d = num %d left arena', os.getpid(), number)
+            log.info("PID %d = num %d left arena", os.getpid(), number)
 
         return True
     except:

toil/test/src/toilContextManagerTest.py

@@ -29,13 +29,13 @@ class ToilContextManagerTest(ToilTest):
 
     def testContextManger(self):
         options = Job.Runner.getDefaultOptions(self._getTestJobStorePath())
-        options.logLevel = 'INFO'
+        options.logLevel = "INFO"
         with Toil(options) as toil:
             toil.start(HelloWorld())
 
     def testNoContextManger(self):
         options = Job.Runner.getDefaultOptions(self._getTestJobStorePath())
-        options.logLevel = 'INFO'
+        options.logLevel = "INFO"
         toil = Toil(options)
         self.assertRaises(ToilContextManagerException, toil.start, HelloWorld())
 
@@ -45,7 +45,9 @@ class ToilContextManagerTest(ToilTest):
             with Toil(options) as toil:
                 _ = toil.start(HelloWorld())
                 # oh no, an error! :(
-                raise RuntimeError("we died after workflow completion but before our export finished")
+                raise RuntimeError(
+                    "we died after workflow completion but before our export finished"
+                )
         except RuntimeError:
             pass
 
@@ -54,17 +56,18 @@ class ToilContextManagerTest(ToilTest):
             fileID = toil.restart()
             print(fileID)
             # Hopefully the error didn't cause us to lose all our work!
-            toil.exportFile(fileID, 'file://' + self.exportPath)
+            toil.exportFile(fileID, "file://" + self.exportPath)
         with open(self.exportPath) as f:
             # The file should have all our content
             self.assertEqual(f.read(), "Hello, World!")
 
+
 class HelloWorld(Job):
     def __init__(self):
-        Job.__init__(self, memory=100000, disk='1M')
+        Job.__init__(self, memory=100000, disk="1M")
 
     def run(self, fileStore):
-        fileID = self.addChildJobFn(childFn, memory='1M', disk='1M').rv()
+        fileID = self.addChildJobFn(childFn, memory="1M", disk="1M").rv()
         return self.addFollowOn(FollowOn(fileID)).rv()
 
 
@@ -81,7 +84,7 @@ class FollowOn(Job):
 
     def run(self, fileStore):
         tempDir = fileStore.getLocalTempDir()
-        tempFilePath = "/".join([tempDir, 'LocalCopy'])
+        tempFilePath = "/".join([tempDir, "LocalCopy"])
         with fileStore.readGlobalFileStream(self.fileId) as globalFile:
             with open(tempFilePath, "wb") as localFile:
                 localFile.write(globalFile.read())

toil/test/src/userDefinedJobArgTypeTest.py

@@ -52,9 +52,11 @@ class UserDefinedJobArgTypeTest(ToilTest):
         self._testFromMain()
 
     def _testFromMain(self):
-        testMethodName = self.id().split('.')[-1]
-        self.assertTrue(testMethodName.endswith('FromMain'))
-        subprocess.check_call([sys.executable, '-m', self.__module__, testMethodName[:-8]])
+        testMethodName = self.id().split(".")[-1]
+        self.assertTrue(testMethodName.endswith("FromMain"))
+        subprocess.check_call(
+            [sys.executable, "-m", self.__module__, testMethodName[:-8]]
+        )
 
 
 class JobClass(Job):
@@ -66,8 +68,9 @@ class JobClass(Job):
     def run(self, fileStore):
         self.foo.assertIsCopy()
         if self.level < 2:
-            self.addChildJobFn(jobFunction, self.level + 1, Foo(), cores=1, memory="1M",
-                               disk="300M")
+            self.addChildJobFn(
+                jobFunction, self.level + 1, Foo(), cores=1, memory="1M", disk="300M"
+            )
 
 
 def jobFunction(job, level, foo):

toil/test/src/workerTest.py

@@ -12,6 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from typing import Optional
+
 from toil.common import Config
 from toil.job import CheckpointJobDescription, JobDescription
 from toil.jobStores.fileJobStore import FileJobStore
@@ -21,27 +23,45 @@ from toil.worker import nextChainable
 
 class WorkerTests(ToilTest):
     """Test miscellaneous units of the worker."""
+
     def setUp(self):
         super().setUp()
         path = self._getTestJobStorePath()
         self.jobStore = FileJobStore(path)
         self.config = Config()
-        self.config.jobStore = 'file:%s' % path
+        self.config.jobStore = "file:%s" % path
         self.jobStore.initialize(self.config)
         self.jobNumber = 0
 
     def testNextChainable(self):
         """Make sure chainable/non-chainable jobs are identified correctly."""
-        def createTestJobDesc(memory, cores, disk, preemptible, checkpoint):
+
+        def createTestJobDesc(
+            memory,
+            cores,
+            disk,
+            preemptible: bool = True,
+            checkpoint: bool = False,
+            local: Optional[bool] = None,
+        ):
             """
             Create a JobDescription with no command (representing a Job that
             has already run) and return the JobDescription.
             """
-            name = 'job%d' % self.jobNumber
+            name = "job%d" % self.jobNumber
             self.jobNumber += 1
 
             descClass = CheckpointJobDescription if checkpoint else JobDescription
-            jobDesc = descClass(requirements={'memory': memory, 'cores': cores, 'disk': disk, 'preemptible': preemptible}, jobName=name)
+            jobDesc = descClass(
+                requirements={
+                    "memory": memory,
+                    "cores": cores,
+                    "disk": disk,
+                    "preemptible": preemptible,
+                },
+                jobName=name,
+                local=local,
+            )
 
             # Assign an ID
             self.jobStore.assign_job_id(jobDesc)
@@ -49,46 +69,76 @@ class WorkerTests(ToilTest):
             # Save and return the JobDescription
             return self.jobStore.create_job(jobDesc)
 
-        for successorType in ['addChild', 'addFollowOn']:
+        for successorType in ["addChild", "addFollowOn"]:
             # Try with the branch point at both child and follow-on stages
 
             # Identical non-checkpoint jobs should be chainable.
-            jobDesc1 = createTestJobDesc(1, 2, 3, True, False)
-            jobDesc2 = createTestJobDesc(1, 2, 3, True, False)
+            jobDesc1 = createTestJobDesc(1, 2, 3)
+            jobDesc2 = createTestJobDesc(1, 2, 3)
             getattr(jobDesc1, successorType)(jobDesc2.jobStoreID)
             chainable = nextChainable(jobDesc1, self.jobStore, self.config)
             self.assertNotEqual(chainable, None)
-            self.assertEqual(jobDesc2.jobStoreID, chainable.jobStoreID)
+            self.assertEqual(chainable.jobStoreID, jobDesc2.jobStoreID)
 
             # Identical checkpoint jobs should not be chainable.
-            jobDesc1 = createTestJobDesc(1, 2, 3, True, False)
-            jobDesc2 = createTestJobDesc(1, 2, 3, True, True)
+            jobDesc1 = createTestJobDesc(1, 2, 3, checkpoint=True)
+            jobDesc2 = createTestJobDesc(1, 2, 3, checkpoint=True)
             getattr(jobDesc1, successorType)(jobDesc2.jobStoreID)
-            self.assertEqual(None, nextChainable(jobDesc1, self.jobStore, self.config))
+            self.assertEqual(nextChainable(jobDesc1, self.jobStore, self.config), None)
+
+            # Changing checkpoint from false to true should make it not chainable.
+            jobDesc1 = createTestJobDesc(1, 2, 3, checkpoint=False)
+            jobDesc2 = createTestJobDesc(1, 2, 3, checkpoint=True)
+            getattr(jobDesc1, successorType)(jobDesc2.jobStoreID)
+            self.assertEqual(nextChainable(jobDesc1, self.jobStore, self.config), None)
 
             # If there is no child we should get nothing to chain.
-            jobDesc1 = createTestJobDesc(1, 2, 3, True, False)
-            self.assertEqual(None, nextChainable(jobDesc1, self.jobStore, self.config))
+            jobDesc1 = createTestJobDesc(1, 2, 3)
+            self.assertEqual(nextChainable(jobDesc1, self.jobStore, self.config), None)
 
             # If there are 2 or more children we should get nothing to chain.
-            jobDesc1 = createTestJobDesc(1, 2, 3, True, False)
-            jobDesc2 = createTestJobDesc(1, 2, 3, True, False)
-            jobDesc3 = createTestJobDesc(1, 2, 3, True, False)
+            jobDesc1 = createTestJobDesc(1, 2, 3)
+            jobDesc2 = createTestJobDesc(1, 2, 3)
+            jobDesc3 = createTestJobDesc(1, 2, 3)
             getattr(jobDesc1, successorType)(jobDesc2.jobStoreID)
             getattr(jobDesc1, successorType)(jobDesc3.jobStoreID)
-            self.assertEqual(None, nextChainable(jobDesc1, self.jobStore, self.config))
+            self.assertEqual(nextChainable(jobDesc1, self.jobStore, self.config), None)
 
             # If there is an increase in resource requirements we should get nothing to chain.
-            reqs = {'memory': 1, 'cores': 2, 'disk': 3, 'preemptible': True, 'checkpoint': False}
-            for increased_attribute in ('memory', 'cores', 'disk'):
+            base_reqs = {
+                "memory": 1,
+                "cores": 2,
+                "disk": 3,
+                "preemptible": True,
+                "checkpoint": False,
+            }
+            for increased_attribute in ("memory", "cores", "disk"):
+                reqs = dict(base_reqs)
                 jobDesc1 = createTestJobDesc(**reqs)
                 reqs[increased_attribute] += 1
                 jobDesc2 = createTestJobDesc(**reqs)
                 getattr(jobDesc1, successorType)(jobDesc2.jobStoreID)
-                self.assertEqual(None, nextChainable(jobDesc1, self.jobStore, self.config))
+                self.assertEqual(
+                    nextChainable(jobDesc1, self.jobStore, self.config), None
+                )
 
             # A change in preemptability from True to False should be disallowed.
-            jobDesc1 = createTestJobDesc(1, 2, 3, True, False)
-            jobDesc2 = createTestJobDesc(1, 2, 3, False, True)
+            jobDesc1 = createTestJobDesc(1, 2, 3, preemptible=True)
+            jobDesc2 = createTestJobDesc(1, 2, 3, preemptible=False)
             getattr(jobDesc1, successorType)(jobDesc2.jobStoreID)
-            self.assertEqual(None, nextChainable(jobDesc1, self.jobStore, self.config))
+            self.assertEqual(nextChainable(jobDesc1, self.jobStore, self.config), None)
+
+            # A change in local-ness from True to False should be disallowed.
+            jobDesc1 = createTestJobDesc(1, 2, 3, local=True)
+            jobDesc2 = createTestJobDesc(1, 2, 3, local=False)
+            getattr(jobDesc1, successorType)(jobDesc2.jobStoreID)
+            self.assertEqual(nextChainable(jobDesc1, self.jobStore, self.config), None)
+
+            # A change in local-ness from False to True should be allowed,
+            # since running locally is an optional optimization.
+            jobDesc1 = createTestJobDesc(1, 2, 3, local=False)
+            jobDesc2 = createTestJobDesc(1, 2, 3, local=True)
+            getattr(jobDesc1, successorType)(jobDesc2.jobStoreID)
+            chainable = nextChainable(jobDesc1, self.jobStore, self.config)
+            self.assertNotEqual(chainable, None)
+            self.assertEqual(chainable.jobStoreID, jobDesc2.jobStoreID)

toil/test/utils/toilDebugTest.py

@@ -16,12 +16,8 @@ import os
 import subprocess
 import tempfile
 
-import pytest
-
-from toil.test import ToilTest
-
 from toil.lib.resources import glob
-from toil.test import slow
+from toil.test import ToilTest, needs_wdl, slow
 from toil.version import python
 
 logger = logging.getLogger(__name__)
@@ -121,6 +117,7 @@ def testFetchJobStoreFiles() -> None:
     for symlink in (True, False):
         fetchFiles(symLink=symlink, jobStoreDir=job_store_dir, outputDir=output_dir)
 
+
 class DebugJobTest(ToilTest):
     """
     Test the toil debug-job command.
@@ -136,27 +133,77 @@ class DebugJobTest(ToilTest):
 
         logger.info("Running workflow that always fails")
         try:
-            # Run an always-failign workflow
-            subprocess.check_call([
-                python,
-                os.path.abspath("src/toil/test/docs/scripts/example_alwaysfail.py"),
-                "--retryCount=0",
-                "--logCritical",
-                "--disableProgress=True",
-                job_store
-            ], stderr=subprocess.DEVNULL)
+            # Run an always-failing workflow
+            subprocess.check_call(
+                [
+                    python,
+                    os.path.abspath("src/toil/test/docs/scripts/example_alwaysfail.py"),
+                    "--retryCount=0",
+                    "--logCritical",
+                    "--disableProgress",
+                    job_store,
+                ],
+                stderr=subprocess.DEVNULL,
+            )
             raise RuntimeError("Failing workflow succeeded!")
         except subprocess.CalledProcessError:
             # Should fail to run
             logger.info("Task failed successfully")
-            pass
 
         # Get the job ID.
         # TODO: This assumes a lot about the FileJobStore. Use the MessageBus instead?
-        job_id = "kind-explode/" + os.listdir(os.path.join(job_store, "jobs/kind-explode"))[0]
+        job_id = (
+            "kind-explode/"
+            + os.listdir(os.path.join(job_store, "jobs/kind-explode"))[0]
+        )
 
         return job_store, job_id
 
+    def _get_wdl_job_store_and_job_name(self):
+        """
+        Get a job store and the name of a failed job in it that actually wanted to use some files.
+        """
+
+        # First make a job store.
+        job_store = os.path.join(self._createTempDir(), "tree")
+
+        logger.info("Running workflow that always fails")
+        # Run an always-failing workflow
+        wf_result = subprocess.run(
+            [
+                "toil-wdl-runner",
+                os.path.abspath(
+                    "src/toil/test/docs/scripts/example_alwaysfail_with_files.wdl"
+                ),
+                "--retryCount=0",
+                "--logDebug",
+                "--disableProgress",
+                "--jobStore",
+                job_store,
+            ],
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
+            encoding="utf-8",
+            errors="replace",
+        )
+        logger.debug("Always-failing workflow output: %s", wf_result.stdout)
+        if wf_result.returncode == 0:
+            raise RuntimeError("Failing workflow succeeded!")
+        else:
+            logger.info("Task failed successfully")
+
+        # Make sure that the job store we created actually has its job store
+        # root job ID file. If it doesn't, we failed during workflow setup and
+        # not because of a real failing job.
+        assert os.path.exists(
+            os.path.join(job_store, "files/shared/rootJobStoreID")
+        ), "Failed workflow still needs a root job"
+
+        # Get a job name for a job that fails
+        job_name = "WDLTaskJob"
+
+        return job_store, job_name
+
     def test_run_job(self):
         """
         Make sure that we can use toil debug-job to try and run a job in-process.
@@ -167,18 +214,14 @@ class DebugJobTest(ToilTest):
         logger.info("Trying to rerun job %s", job_id)
 
         # Rerun the job, which should fail again
-        output = subprocess.check_output([
-            "toil",
-            "debug-job",
-            "--logDebug",
-            job_store,
-            job_id
-        ], stderr=subprocess.STDOUT)
+        output = subprocess.check_output(
+            ["toil", "debug-job", "--logDebug", job_store, job_id],
+            stderr=subprocess.STDOUT,
+        )
         # Even if the job fails, the attempt to run it will succeed.
-        log = output.decode('utf-8')
+        log = output.decode("utf-8")
         assert "Boom!" in log, f"Did not find the expected exception message in: {log}"
 
-
     def test_print_job_info(self):
         """
         Make sure that we can use --printJobInfo to get information on a job from a job store.
@@ -189,13 +232,40 @@ class DebugJobTest(ToilTest):
         logger.info("Trying to print job info for job %s", job_id)
 
         # Print the job info and make sure that doesn't crash.
-        subprocess.check_call([
-            "toil",
-            "debug-job",
-            "--logDebug",
-            job_store,
-            "--printJobInfo",
-            job_id
-        ])
+        subprocess.check_call(
+            ["toil", "debug-job", "--logDebug", job_store, "--printJobInfo", job_id]
+        )
 
+    @needs_wdl
+    def test_retrieve_task_directory(self):
+        """
+        Make sure that we can use --retrieveTaskDirectory to get the input files for a job.
+        """
+
+        job_store, job_name = self._get_wdl_job_store_and_job_name()
 
+        logger.info("Trying to retrieve task dorectory for job %s", job_name)
+
+        dest_dir = os.path.join(self._createTempDir(), "dump")
+
+        # Print the job info and make sure that doesn't crash.
+        subprocess.check_call(
+            [
+                "toil",
+                "debug-job",
+                "--logDebug",
+                job_store,
+                job_name,
+                "--retrieveTaskDirectory",
+                dest_dir,
+            ]
+        )
+
+        first_file = os.path.join(
+            dest_dir,
+            "inside/mnt/miniwdl_task_container/work/_miniwdl_inputs/0/test.txt",
+        )
+        assert os.path.exists(
+            first_file
+        ), "Input file not found in fake container environment"
+        self.assertEqual(open(first_file).read(), "These are the contents\n")

toil/test/utils/toilKillTest.py

@@ -21,8 +21,7 @@ import time
 import unittest
 
 from toil.common import Toil
-from toil.jobStores.abstractJobStore import (NoSuchFileException,
-                                             NoSuchJobStoreException)
+from toil.jobStores.abstractJobStore import NoSuchFileException, NoSuchJobStoreException
 from toil.jobStores.utils import generate_locator
 from toil.test import ToilTest, needs_aws_s3, needs_cwl
 
@@ -61,7 +60,7 @@ class ToilKillTest(ToilTest):
         kill_cmd = ["toil", "kill", self.job_store]
 
         # run the sleep workflow
-        logger.info('Running workflow: %s', ' '.join(run_cmd))
+        logger.info("Running workflow: %s", " ".join(run_cmd))
         cwl_process = subprocess.Popen(run_cmd)
 
         # wait until workflow starts running
@@ -75,9 +74,9 @@ class ToilKillTest(ToilTest):
                     # kill flag exists to be deleted to kill the leader
                     break
                 else:
-                    logger.info('Waiting for kill flag...')
+                    logger.info("Waiting for kill flag...")
             except (NoSuchJobStoreException, NoSuchFileException):
-                logger.info('Waiting for job store to be openable...')
+                logger.info("Waiting for job store to be openable...")
             time.sleep(2)
 
         # run toil kill