toil 6.1.0a1__py3-none-any.whl → 8.0.0__py3-none-any.whl

toil/test/src/helloWorldTest.py

@@ -22,27 +22,30 @@ class HelloWorldTest(ToilTest):
         options.logLevel = "INFO"
         Job.Runner.startToil(HelloWorld(), options)
 
+
 class HelloWorld(Job):
     def __init__(self):
-        Job.__init__(self,  memory=100000, cores=1, disk="3G")
+        Job.__init__(self, memory=100000, cores=1, disk="3G")
 
     def run(self, fileStore):
         fileID = self.addChildJobFn(childFn, cores=1, memory="1M", disk="3G").rv()
         self.addFollowOn(FollowOn(fileID))
 
+
 def childFn(job):
     with job.fileStore.writeGlobalFileStream() as (fH, fileID):
         fH.write(b"Hello, World!")
         return fileID
 
+
 class FollowOn(Job):
-    def __init__(self,fileId):
+    def __init__(self, fileId):
         Job.__init__(self)
-        self.fileId=fileId
+        self.fileId = fileId
 
     def run(self, fileStore):
         tempDir = fileStore.getLocalTempDir()
-        tempFilePath = "/".join([tempDir,"LocalCopy"])
+        tempFilePath = "/".join([tempDir, "LocalCopy"])
         with fileStore.readGlobalFileStream(self.fileId) as globalFile:
             with open(tempFilePath, "wb") as localFile:
                 localFile.write(globalFile.read())

toil/test/src/importExportFileTest.py

@@ -28,14 +28,16 @@ class ImportExportFileTest(ToilTest):
     def setUp(self):
         super().setUp()
         self.tmp_dir = self._createTempDir()
-        self.output_file_path = f'{self.tmp_dir}/out'
-        self.message_portion_1 = 'What do you get when you cross a seal and a polar bear?'
-        self.message_portion_2 = '  A polar bear.'
+        self.output_file_path = f"{self.tmp_dir}/out"
+        self.message_portion_1 = (
+            "What do you get when you cross a seal and a polar bear?"
+        )
+        self.message_portion_2 = "  A polar bear."
 
     def create_file(self, content, executable=False):
-        file_path = f'{self.tmp_dir}/{uuid.uuid4()}'
+        file_path = f"{self.tmp_dir}/{uuid.uuid4()}"
 
-        with open(file_path, 'w') as f:
+        with open(file_path, "w") as f:
             f.write(content)
 
         if executable:
@@ -48,26 +50,44 @@ class ImportExportFileTest(ToilTest):
         with Toil(options) as toil:
             if not options.restart:
                 msg_portion_file_path = self.create_file(content=self.message_portion_1)
-                msg_portion_file_id = toil.importFile(f'file://{msg_portion_file_path}')
+                msg_portion_file_id = toil.importFile(f"file://{msg_portion_file_path}")
                 self.assertIsInstance(msg_portion_file_id, FileID)
-                self.assertEqual(os.stat(msg_portion_file_path).st_size, msg_portion_file_id.size)
+                self.assertEqual(
+                    os.stat(msg_portion_file_path).st_size, msg_portion_file_id.size
+                )
 
                 file_that_can_trigger_failure_when_job_starts = self.create_file(
-                    content='Time to freak out!' if fail else 'Keep calm and carry on.')
-                self.trigger_file_id = toil.importFile(f'file://{file_that_can_trigger_failure_when_job_starts}')
+                    content="Time to freak out!" if fail else "Keep calm and carry on."
+                )
+                self.trigger_file_id = toil.importFile(
+                    f"file://{file_that_can_trigger_failure_when_job_starts}"
+                )
                 workflow_final_output_file_id = toil.start(
-                    RestartingJob(msg_portion_file_id, self.trigger_file_id, self.message_portion_2))
+                    RestartingJob(
+                        msg_portion_file_id,
+                        self.trigger_file_id,
+                        self.message_portion_2,
+                    )
+                )
             else:
                 # TODO: We're hackily updating this file without using the
                 #  correct FileStore interface. User code should not do this!
                 with toil._jobStore.update_file_stream(self.trigger_file_id) as f:
-                    f.write(('Time to freak out!' if fail else 'Keep calm and carry on.').encode('utf-8'))
+                    f.write(
+                        (
+                            "Time to freak out!" if fail else "Keep calm and carry on."
+                        ).encode("utf-8")
+                    )
 
                 workflow_final_output_file_id = toil.restart()
 
-            toil.exportFile(workflow_final_output_file_id, f'file://{self.output_file_path}')
+            toil.exportFile(
+                workflow_final_output_file_id, f"file://{self.output_file_path}"
+            )
             with open(self.output_file_path) as f:
-                self.assertEqual(f.read(), f'{self.message_portion_1}{self.message_portion_2}')
+                self.assertEqual(
+                    f.read(), f"{self.message_portion_1}{self.message_portion_2}"
+                )
 
     def _run_import_export_workflow(self, restart):
         options = Job.Runner.getDefaultOptions(self._getTestJobStorePath())
@@ -100,17 +120,21 @@ class ImportExportFileTest(ToilTest):
         with Toil(options) as toil:
             # TODO: test this with non-local (AWS, Google)
             #  Note: this is somewhat done in src/toil/test/src/fileStoreTest.py
-            with self.subTest('Testing permissions are preserved for local importFile/exportFile'):
+            with self.subTest(
+                "Testing permissions are preserved for local importFile/exportFile"
+            ):
                 for executable in True, False:
-                    file_path = self.create_file(content='Hello', executable=executable)
+                    file_path = self.create_file(content="Hello", executable=executable)
                     initial_permissions = os.stat(file_path).st_mode & stat.S_IXUSR
-                    file_id = toil.importFile(f'file://{file_path}')
-                    toil.exportFile(file_id, f'file://{self.output_file_path}')
-                    current_permissions = os.stat(self.output_file_path).st_mode & stat.S_IXUSR
+                    file_id = toil.importFile(f"file://{file_path}")
+                    toil.exportFile(file_id, f"file://{self.output_file_path}")
+                    current_permissions = (
+                        os.stat(self.output_file_path).st_mode & stat.S_IXUSR
+                    )
                     assert initial_permissions == current_permissions
 
-            with self.subTest('Testing relative paths without the file:// schema.'):
-                relative_path_data = 'Everything is relative.'
+            with self.subTest("Testing relative paths without the file:// schema."):
+                relative_path_data = "Everything is relative."
                 file_path = self.create_file(content=relative_path_data)
 
                 file_id = toil.importFile(os.path.relpath(file_path))
@@ -118,31 +142,37 @@ class ImportExportFileTest(ToilTest):
                 with open(self.output_file_path) as f:
                     self.assertEqual(f.read(), relative_path_data)
 
-            with self.subTest('Test local importFile accepts a shared_file_name.'):
+            with self.subTest("Test local importFile accepts a shared_file_name."):
                 # TODO: whyyyy do we allow this?  shared file names are not unique and can overwrite each other
                 #  ...not only that... we can't use exportFile on them afterwards!?
-                file_path = self.create_file(content='why')
-                shared_file_name = 'users_should_probably_not_be_allowed_to_make_shared_files.bad'
-                toil.importFile(f'file://{file_path}', sharedFileName=shared_file_name)
-                with toil._jobStore.read_shared_file_stream(shared_file_name, encoding='utf-8') as f:
-                    self.assertEqual(f.read(), 'why')
+                file_path = self.create_file(content="why")
+                shared_file_name = (
+                    "users_should_probably_not_be_allowed_to_make_shared_files.bad"
+                )
+                toil.importFile(f"file://{file_path}", sharedFileName=shared_file_name)
+                with toil._jobStore.read_shared_file_stream(
+                    shared_file_name, encoding="utf-8"
+                ) as f:
+                    self.assertEqual(f.read(), "why")
 
 
 class RestartingJob(Job):
     def __init__(self, msg_portion_file_id, trigger_file_id, message_portion_2):
-        Job.__init__(self,  memory=100000, cores=1, disk="1M")
+        Job.__init__(self, memory=100000, cores=1, disk="1M")
         self.msg_portion_file_id = msg_portion_file_id
         self.trigger_file_id = trigger_file_id
         self.message_portion_2 = message_portion_2
 
     def run(self, file_store):
         with file_store.readGlobalFileStream(self.trigger_file_id) as readable:
-            if readable.read() == b'Time to freak out!':
-                raise RuntimeError('D:')
+            if readable.read() == b"Time to freak out!":
+                raise RuntimeError("D:")
 
         with file_store.writeGlobalFileStream() as (writable, output_file_id):
-            with file_store.readGlobalFileStream(self.msg_portion_file_id, encoding='utf-8') as readable:
+            with file_store.readGlobalFileStream(
+                self.msg_portion_file_id, encoding="utf-8"
+            ) as readable:
                 # combine readable.read() (the original message 1) with message 2
                 # this will be the final output of the workflow
-                writable.write(f'{readable.read()}{self.message_portion_2}'.encode())
+                writable.write(f"{readable.read()}{self.message_portion_2}".encode())
                 return output_file_id

toil/test/src/jobDescriptionTest.py

@@ -18,6 +18,7 @@ from configargparse import ArgumentParser
 
 from toil.common import Toil
 from toil.job import Job, JobDescription, TemporaryID
+from toil.resource import ModuleDescriptor
 from toil.test import ToilTest
 
 
@@ -30,7 +31,7 @@ class JobDescriptionTest(ToilTest):
         Job.Runner.addToilOptions(parser)
         options = parser.parse_args(args=[self.jobStorePath])
         self.toil = Toil(options)
-        self.assertEqual( self.toil, self.toil.__enter__() )
+        self.assertEqual(self.toil, self.toil.__enter__())
 
     def tearDown(self):
         self.toil.__exit__(None, None, None)
@@ -43,17 +44,31 @@ class JobDescriptionTest(ToilTest):
         Tests the public interface of a JobDescription.
         """
 
-        command = "by your command"
-        memory = 2^32
-        disk = 2^32
+        memory = 2 ^ 32
+        disk = 2 ^ 32
         cores = "1"
         preemptible = 1
 
-        j = JobDescription(command=command, requirements={"memory": memory, "cores": cores, "disk": disk, "preemptible": preemptible},
-                           jobName='testJobGraph', unitName='noName')
+        j = JobDescription(
+            requirements={
+                "memory": memory,
+                "cores": cores,
+                "disk": disk,
+                "preemptible": preemptible,
+            },
+            jobName="testJobGraph",
+            unitName="noName",
+        )
+
+        # Without a body, and with nothing to run, nextSuccessors will be None
+        self.assertEqual(j.has_body(), False)
+        self.assertEqual(j.nextSuccessors(), None)
+
+        # Attach a body so the job has something to do itself.
+        j.attach_body("fake", ModuleDescriptor.forModule("toil"))
+        self.assertEqual(j.has_body(), True)
 
-        #Check attributes
-        self.assertEqual(j.command, command)
+        # Check attributes
         self.assertEqual(j.memory, memory)
         self.assertEqual(j.disk, disk)
         self.assertEqual(j.cores, int(cores))
@@ -67,30 +82,40 @@ class JobDescriptionTest(ToilTest):
         self.assertEqual(j.predecessorsFinished, set())
         self.assertEqual(j.logJobStoreFileID, None)
 
-        #Check equals function (should be based on object identity and not contents)
-        j2 = JobDescription(command=command, requirements={"memory": memory, "cores": cores, "disk": disk, "preemptible": preemptible},
-                            jobName='testJobGraph', unitName='noName')
+        # Check equals function (should be based on object identity and not contents)
+        j2 = JobDescription(
+            requirements={
+                "memory": memory,
+                "cores": cores,
+                "disk": disk,
+                "preemptible": preemptible,
+            },
+            jobName="testJobGraph",
+            unitName="noName",
+        )
+        j2.attach_body("fake", ModuleDescriptor.forModule("toil"))
         self.assertNotEqual(j, j2)
         ###TODO test other functionality
 
     def testJobDescriptionSequencing(self):
-        j = JobDescription(command='command', requirements={},  jobName='unimportant')
+        j = JobDescription(requirements={}, jobName="unimportant")
 
-        j.addChild('child')
-        j.addFollowOn('followOn')
+        j.addChild("child")
+        j.addFollowOn("followOn")
 
-        # With a command, nothing should be ready to run
+        # With a body, nothing should be ready to run
+        j.attach_body("fake", ModuleDescriptor.forModule("toil"))
         self.assertEqual(list(j.nextSuccessors()), [])
 
-        # With command cleared, child should be ready to run
-        j.command = None
-        self.assertEqual(list(j.nextSuccessors()), ['child'])
+        # With body cleared, child should be ready to run
+        j.detach_body()
+        self.assertEqual(list(j.nextSuccessors()), ["child"])
 
         # Without the child, the follow-on should be ready to run
-        j.filterSuccessors(lambda jID: jID != 'child')
-        self.assertEqual(list(j.nextSuccessors()), ['followOn'])
+        j.filterSuccessors(lambda jID: jID != "child")
+        self.assertEqual(list(j.nextSuccessors()), ["followOn"])
 
         # Without the follow-on, we should return None, to be distinct from an
         # empty list. Nothing left to do!
-        j.filterSuccessors(lambda jID: jID != 'followOn')
+        j.filterSuccessors(lambda jID: jID != "followOn")
         self.assertEqual(j.nextSuccessors(), None)

toil/test/src/jobEncapsulationTest.py

@@ -20,6 +20,7 @@ from toil.test.src.jobTest import fn1Test
 
 class JobEncapsulationTest(ToilTest):
     """Tests testing the EncapsulationJob class."""
+
     def testEncapsulation(self):
         """
         Tests the Job.encapsulation method, which uses the EncapsulationJob
@@ -61,6 +62,7 @@ class JobEncapsulationTest(ToilTest):
 def noOp():
     pass
 
+
 def encapsulatedJobFn(job, string, outFile):
     a = job.addChildFn(fn1Test, string, outFile, name="inner-a")
     b = a.addFollowOnFn(fn1Test, a.rv(), outFile, name="inner-b")

toil/test/src/jobFileStoreTest.py

@@ -23,7 +23,7 @@ from toil.test import ToilTest, slow
 
 logger = logging.getLogger(__name__)
 
-PREFIX_LENGTH=200
+PREFIX_LENGTH = 200
 
 
 # TODO: This test is ancient and while similar tests exist in `fileStoreTest.py`, none of them look
@@ -32,6 +32,7 @@ class JobFileStoreTest(ToilTest):
     """
     Tests testing the methods defined in :class:toil.fileStores.abstractFileStore.AbstractFileStore.
     """
+
     def testCachingFileStore(self):
         options = Job.Runner.getDefaultOptions(self._getTestJobStorePath())
         with Toil(options) as workflow:
@@ -43,38 +44,42 @@ class JobFileStoreTest(ToilTest):
         with Toil(options) as workflow:
             workflow.start(Job.wrapJobFn(simpleFileStoreJob))
 
-    def _testJobFileStore(self, retryCount=0, badWorker=0.0, stringNo=1, stringLength=1000000,
-                          testNo=2):
+    def _testJobFileStore(
+        self, retryCount=0, badWorker=0.0, stringNo=1, stringLength=1000000, testNo=2
+    ):
         """
         Creates a chain of jobs, each reading and writing files using the
         toil.fileStores.abstractFileStore.AbstractFileStore interface. Verifies the files written are always what we
         expect.
         """
         for test in range(testNo):
-            #Make a list of random strings, each of 100k chars and hash the first 200
-            #base prefix to the string
+            # Make a list of random strings, each of 100k chars and hash the first 200
+            # base prefix to the string
             def randomString():
                 chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
                 s = "".join([random.choice(chars) for i in range(stringLength)])
                 return s[:PREFIX_LENGTH], s
-            #Total length is 2 million characters (20 strings of length 100K each)
+
+            # Total length is 2 million characters (20 strings of length 100K each)
             testStrings = dict([randomString() for i in range(stringNo)])
             options = Job.Runner.getDefaultOptions(self._getTestJobStorePath())
             options.logLevel = "DEBUG"
-            options.retryCount=retryCount
-            options.badWorker=badWorker
+            options.retryCount = retryCount
+            options.badWorker = badWorker
             options.badWorkerFailInterval = 1.0
             chainLength = 10
             # Run the workflow, the return value being the number of failed jobs
-            Job.Runner.startToil(Job.wrapJobFn(fileTestJob, [],
-                                               testStrings, chainLength),
-                                 options)
+            Job.Runner.startToil(
+                Job.wrapJobFn(fileTestJob, [], testStrings, chainLength), options
+            )
 
     def testJobFileStore(self):
         """
         Tests case that about half the files are cached
         """
-        self._testJobFileStore(retryCount=0, badWorker=0.0,  stringNo=5, stringLength=1000000)
+        self._testJobFileStore(
+            retryCount=0, badWorker=0.0, stringNo=5, stringLength=1000000
+        )
 
     @slow
     def testJobFileStoreWithBadWorker(self):
@@ -82,79 +87,98 @@ class JobFileStoreTest(ToilTest):
         Tests case that about half the files are cached and the worker is randomly
         failing.
         """
-        self._testJobFileStore(retryCount=100, badWorker=0.5,  stringNo=5, stringLength=1000000)
+        self._testJobFileStore(
+            retryCount=100, badWorker=0.5, stringNo=5, stringLength=1000000
+        )
 
 
 def fileTestJob(job, inputFileStoreIDs, testStrings, chainLength):
     """
     Test job exercises toil.fileStores.abstractFileStore.AbstractFileStore functions
     """
-    outputFileStoreIds = [] #Strings passed to the next job in the chain
+    outputFileStoreIds = []  # Strings passed to the next job in the chain
 
-    #Load the input jobStoreFileIDs and check that they map to the
-    #same set of random input strings, exercising the different functions in the fileStore interface
+    # Load the input jobStoreFileIDs and check that they map to the
+    # same set of random input strings, exercising the different functions in the fileStore interface
     for fileStoreID in inputFileStoreIDs:
         if random.random() > 0.5:
-            #Read the file for the fileStoreID, randomly picking a way to invoke readGlobalFile
+            # Read the file for the fileStoreID, randomly picking a way to invoke readGlobalFile
             if random.random() > 0.5:
-                local_path = job.fileStore.getLocalTempFileName() if random.random() > 0.5 else None
+                local_path = (
+                    job.fileStore.getLocalTempFileName()
+                    if random.random() > 0.5
+                    else None
+                )
                 cache = random.random() > 0.5
 
-                tempFile = job.fileStore.readGlobalFile(fileStoreID,
-                                                        local_path,
-                                                        cache=cache)
+                tempFile = job.fileStore.readGlobalFile(
+                    fileStoreID, local_path, cache=cache
+                )
                 with open(tempFile) as fH:
                     string = fH.readline()
-                logging.info("Downloaded %s to local path %s with cache %s and got %s with %d letters",
-                              fileStoreID, local_path, cache, tempFile, len(string))
+                logging.info(
+                    "Downloaded %s to local path %s with cache %s and got %s with %d letters",
+                    fileStoreID,
+                    local_path,
+                    cache,
+                    tempFile,
+                    len(string),
+                )
             else:
-                #Check the local file is as we expect
-                with job.fileStore.readGlobalFileStream(fileStoreID, 'utf-8') as fH:
+                # Check the local file is as we expect
+                with job.fileStore.readGlobalFileStream(fileStoreID, "utf-8") as fH:
                     string = fH.readline()
                 logging.info("Streamed %s and got %d letters", fileStoreID, len(string))
-            #Check the string we get back is what we expect
-            assert string[:PREFIX_LENGTH] in testStrings, f"Could not find string: {string[:PREFIX_LENGTH]}"
-            assert testStrings[string[:PREFIX_LENGTH]] == string, f"Mismatch in string: {string[:PREFIX_LENGTH]}"
-
-            #This allows the file to be passed to the next job
+            # Check the string we get back is what we expect
+            assert (
+                string[:PREFIX_LENGTH] in testStrings
+            ), f"Could not find string: {string[:PREFIX_LENGTH]}"
+            assert (
+                testStrings[string[:PREFIX_LENGTH]] == string
+            ), f"Mismatch in string: {string[:PREFIX_LENGTH]}"
+
+            # This allows the file to be passed to the next job
             outputFileStoreIds.append(fileStoreID)
         else:
-            #This tests deletion
+            # This tests deletion
             logging.info("Deleted %s", fileStoreID)
             job.fileStore.deleteGlobalFile(fileStoreID)
 
-    #Fill out the output strings until we have the same number as the input strings
-    #exercising different ways of writing files to the file store
+    # Fill out the output strings until we have the same number as the input strings
+    # exercising different ways of writing files to the file store
     while len(outputFileStoreIds) < len(testStrings):
-        #Pick a string and write it into a file
+        # Pick a string and write it into a file
         testString = random.choice(list(testStrings.values()))
         if random.random() > 0.5:
-            #Make a local copy of the file
-            tempFile = job.fileStore.getLocalTempFile() if random.random() > 0.5 \
-            else os.path.join(job.fileStore.getLocalTempDir(), "temp.txt")
-            with open(tempFile, 'w') as fH:
+            # Make a local copy of the file
+            tempFile = (
+                job.fileStore.getLocalTempFile()
+                if random.random() > 0.5
+                else os.path.join(job.fileStore.getLocalTempDir(), "temp.txt")
+            )
+            with open(tempFile, "w") as fH:
                 fH.write(testString)
-            #Write a local copy of the file using the local file
+            # Write a local copy of the file using the local file
             fileStoreID = job.fileStore.writeGlobalFile(tempFile)
 
             # Make sure it returned a valid and correct FileID with the right size
             assert isinstance(fileStoreID, FileID)
-            assert fileStoreID.size == len(testString.encode('utf-8'))
+            assert fileStoreID.size == len(testString.encode("utf-8"))
 
             outputFileStoreIds.append(fileStoreID)
         else:
-            #Use the writeGlobalFileStream method to write the file
+            # Use the writeGlobalFileStream method to write the file
             with job.fileStore.writeGlobalFileStream() as (fH, fileStoreID):
-                fH.write(testString.encode('utf-8'))
+                fH.write(testString.encode("utf-8"))
                 outputFileStoreIds.append(fileStoreID)
 
-            #Make sure it returned a valid and correct FileID with the right size
+            # Make sure it returned a valid and correct FileID with the right size
             assert isinstance(fileStoreID, FileID)
-            assert fileStoreID.size == len(testString.encode('utf-8'))
+            assert fileStoreID.size == len(testString.encode("utf-8"))
 
     if chainLength > 0:
-        #Make a child that will read these files and check it gets the same results
-        job.addChildJobFn(fileTestJob, outputFileStoreIds, testStrings, chainLength-1)
+        # Make a child that will read these files and check it gets the same results
+        job.addChildJobFn(fileTestJob, outputFileStoreIds, testStrings, chainLength - 1)
 
 
 fileStoreString = "Testing writeGlobalFile"
@@ -163,13 +187,13 @@ streamingFileStoreString = "Testing writeGlobalFileStream"
 
 def simpleFileStoreJob(job):
     localFilePath = os.path.join(job.fileStore.getLocalTempDir(), "parentTemp.txt")
-    with open(localFilePath, 'w') as f:
+    with open(localFilePath, "w") as f:
         f.write(fileStoreString)
     testID1 = job.fileStore.writeGlobalFile(localFilePath)
 
     testID2 = None
     with job.fileStore.writeGlobalFileStream() as (f, fileID):
-        f.write(streamingFileStoreString.encode('utf-8'))
+        f.write(streamingFileStoreString.encode("utf-8"))
         testID2 = fileID
 
     job.addChildJobFn(fileStoreChild, testID1, testID2)
@@ -177,12 +201,12 @@ def simpleFileStoreJob(job):
 
 def fileStoreChild(job, testID1, testID2):
     with job.fileStore.readGlobalFileStream(testID1) as f:
-        assert(f.read().decode('utf-8') == fileStoreString)
+        assert f.read().decode("utf-8") == fileStoreString
 
     localFilePath = os.path.join(job.fileStore.getLocalTempDir(), "childTemp.txt")
     job.fileStore.readGlobalFile(testID2, localFilePath)
     with open(localFilePath) as f:
-        assert(f.read() == streamingFileStoreString)
+        assert f.read() == streamingFileStoreString
 
     job.fileStore.deleteLocalFile(testID2)
     try: