toil 6.1.0a1__py3-none-any.whl → 8.0.0__py3-none-any.whl

toil/test/src/jobTest.py

@@ -29,6 +29,7 @@ logger = logging.getLogger(__name__)
 
 class JobTest(ToilTest):
     """Tests the job class."""
+
     @classmethod
     def setUpClass(cls):
         super().setUpClass()
@@ -121,9 +122,9 @@ class JobTest(ToilTest):
 
     @slow
     def testTrivialDAGConsistency(self):
-        options = Job.Runner.getDefaultOptions(self._createTempDir() + '/jobStore')
-        options.clean = 'always'
-        options.logLevel = 'debug'
+        options = Job.Runner.getDefaultOptions(self._createTempDir() + "/jobStore")
+        options.clean = "always"
+        options.logLevel = "debug"
         i = Job.wrapJobFn(trivialParent)
         with Toil(options) as toil:
             try:
@@ -136,9 +137,9 @@ class JobTest(ToilTest):
 
     @pytest.mark.timeout(300)
     def testDAGConsistency(self):
-        options = Job.Runner.getDefaultOptions(self._createTempDir() + '/jobStore')
-        options.clean = 'always'
-        options.logLevel = 'debug'
+        options = Job.Runner.getDefaultOptions(self._createTempDir() + "/jobStore")
+        options.clean = "always"
+        options.logLevel = "debug"
         i = Job.wrapJobFn(parent)
         with Toil(options) as toil:
             try:
@@ -155,9 +156,9 @@ class JobTest(ToilTest):
         Slightly more complex case. The stranded job's predecessors are siblings instead of
         parent/child.
         """
-        options = Job.Runner.getDefaultOptions(self._createTempDir() + '/jobStore')
-        options.clean = 'always'
-        options.logLevel = 'debug'
+        options = Job.Runner.getDefaultOptions(self._createTempDir() + "/jobStore")
+        options.clean = "always"
+        options.logLevel = "debug"
         i = Job.wrapJobFn(diamond)
         with Toil(options) as toil:
             try:
@@ -196,8 +197,12 @@ class JobTest(ToilTest):
 
             # Test making multiple roots
             childEdges2 = childEdges.copy()
-            childEdges2.add((nodeNumber, 1))  # This creates an extra root at "nodeNumber"
-            rootJob2 = self.makeJobGraph(nodeNumber + 1, childEdges2, followOnEdges, None, False)
+            childEdges2.add(
+                (nodeNumber, 1)
+            )  # This creates an extra root at "nodeNumber"
+            rootJob2 = self.makeJobGraph(
+                nodeNumber + 1, childEdges2, followOnEdges, None, False
+            )
             try:
                 rootJob2.checkJobGraphConnected()
                 self.assertTrue(False)  # Multiple roots were not detected
@@ -209,8 +214,9 @@ class JobTest(ToilTest):
                 adjacencyList[fNode].add(tNode)
                 self.assertTrue(not self.isAcyclic(adjacencyList))
                 try:
-                    self.makeJobGraph(nodeNumber, childEdges,
-                                      followOnEdges, None).checkJobGraphAcylic()
+                    self.makeJobGraph(
+                        nodeNumber, childEdges, followOnEdges, None
+                    ).checkJobGraphAcylic()
                     self.assertTrue(False)  # A cycle was not detected
                 except JobGraphDeadlockException:
                     pass  # This is the expected behaviour
@@ -218,22 +224,25 @@ class JobTest(ToilTest):
                 childEdges.remove((fNode, tNode))
                 adjacencyList[fNode].remove(tNode)
                 # Check is now acyclic again
-                self.makeJobGraph(nodeNumber, childEdges,
-                                  followOnEdges, None, False).checkJobGraphAcylic()
+                self.makeJobGraph(
+                    nodeNumber, childEdges, followOnEdges, None, False
+                ).checkJobGraphAcylic()
 
             def checkFollowOnEdgeCycleDetection(fNode, tNode):
                 followOnEdges.add((fNode, tNode))  # Create a cycle
                 try:
-                    self.makeJobGraph(nodeNumber, childEdges,
-                                      followOnEdges, None, False).checkJobGraphAcylic()
+                    self.makeJobGraph(
+                        nodeNumber, childEdges, followOnEdges, None, False
+                    ).checkJobGraphAcylic()
                     # self.assertTrue(False) #The cycle was not detected
                 except JobGraphDeadlockException:
                     pass  # This is the expected behaviour
                 # Remove the edges
                 followOnEdges.remove((fNode, tNode))
                 # Check is now acyclic again
-                self.makeJobGraph(nodeNumber, childEdges,
-                                  followOnEdges, None, False).checkJobGraphAcylic()
+                self.makeJobGraph(
+                    nodeNumber, childEdges, followOnEdges, None, False
+                ).checkJobGraphAcylic()
 
             # Now try adding edges that create a cycle
 
@@ -257,9 +266,16 @@ class JobTest(ToilTest):
 
             # Try adding a follow on edge between two nodes with shared descendants
             fNode, tNode = self.getRandomEdge(nodeNumber)
-            if (len(self.reachable(tNode, adjacencyList)
-                        .intersection(self.reachable(fNode, adjacencyList))) > 0
-                and (fNode, tNode) not in childEdges and (fNode, tNode) not in followOnEdges):
+            if (
+                len(
+                    self.reachable(tNode, adjacencyList).intersection(
+                        self.reachable(fNode, adjacencyList)
+                    )
+                )
+                > 0
+                and (fNode, tNode) not in childEdges
+                and (fNode, tNode) not in followOnEdges
+            ):
                 checkFollowOnEdgeCycleDetection(fNode, tNode)
 
     @slow
@@ -268,7 +284,7 @@ class JobTest(ToilTest):
         Test for issue #1465: Detection of checkpoint jobs that are not leaf vertices
         identifies leaf vertices incorrectly
 
-        Test verification of new checkpoint jobs being leaf verticies,
+        Test verification of new checkpoint jobs being leaf vertices,
         starting with the following baseline workflow::
 
             Parent
@@ -279,7 +295,9 @@ class JobTest(ToilTest):
 
         def createWorkflow():
             rootJob = Job.wrapJobFn(simpleJobFn, "Parent")
-            childCheckpointJob = rootJob.addChildJobFn(simpleJobFn, "Child", checkpoint=True)
+            childCheckpointJob = rootJob.addChildJobFn(
+                simpleJobFn, "Child", checkpoint=True
+            )
             return rootJob, childCheckpointJob
 
         self.runNewCheckpointIsLeafVertexTest(createWorkflow)
@@ -315,35 +333,45 @@ class JobTest(ToilTest):
 
         """
 
-        logger.debug('Test checkpoint job that is a leaf vertex')
-        self.runCheckpointVertexTest(*createWorkflowFn(),
-                                     expectedException=None)
-
-        logger.debug('Test checkpoint job that is not a leaf vertex due to the presence of a service')
-        self.runCheckpointVertexTest(*createWorkflowFn(),
-                                     checkpointJobService=TrivialService("LeafTestService"),
-                                     expectedException=JobGraphDeadlockException)
-
-        logger.debug('Test checkpoint job that is not a leaf vertex due to the presence of a child job')
-        self.runCheckpointVertexTest(*createWorkflowFn(),
-                                     checkpointJobChild=Job.wrapJobFn(
-                                         simpleJobFn, "LeafTestChild"),
-                                     expectedException=JobGraphDeadlockException)
-
-        logger.debug('Test checkpoint job that is not a leaf vertex due to the presence of a follow-on job')
-        self.runCheckpointVertexTest(*createWorkflowFn(),
-                                     checkpointJobFollowOn=Job.wrapJobFn(
-                                         simpleJobFn,
-                                         "LeafTestFollowOn"),
-                                     expectedException=JobGraphDeadlockException)
-
-    def runCheckpointVertexTest(self,
-                                workflowRootJob,
-                                checkpointJob,
-                                checkpointJobService=None,
-                                checkpointJobChild=None,
-                                checkpointJobFollowOn=None,
-                                expectedException=None):
+        logger.debug("Test checkpoint job that is a leaf vertex")
+        self.runCheckpointVertexTest(*createWorkflowFn(), expectedException=None)
+
+        logger.debug(
+            "Test checkpoint job that is not a leaf vertex due to the presence of a service"
+        )
+        self.runCheckpointVertexTest(
+            *createWorkflowFn(),
+            checkpointJobService=TrivialService("LeafTestService"),
+            expectedException=JobGraphDeadlockException
+        )
+
+        logger.debug(
+            "Test checkpoint job that is not a leaf vertex due to the presence of a child job"
+        )
+        self.runCheckpointVertexTest(
+            *createWorkflowFn(),
+            checkpointJobChild=Job.wrapJobFn(simpleJobFn, "LeafTestChild"),
+            expectedException=JobGraphDeadlockException
+        )
+
+        logger.debug(
+            "Test checkpoint job that is not a leaf vertex due to the presence of a follow-on job"
+        )
+        self.runCheckpointVertexTest(
+            *createWorkflowFn(),
+            checkpointJobFollowOn=Job.wrapJobFn(simpleJobFn, "LeafTestFollowOn"),
+            expectedException=JobGraphDeadlockException
+        )
+
+    def runCheckpointVertexTest(
+        self,
+        workflowRootJob,
+        checkpointJob,
+        checkpointJobService=None,
+        checkpointJobChild=None,
+        checkpointJobFollowOn=None,
+        expectedException=None,
+    ):
         """
         Modifies the checkpoint job according to the given parameters
         then runs the workflow, checking for the expected exception, if any.
@@ -380,7 +408,7 @@ class JobTest(ToilTest):
         jobStore = self._getTestJobStorePath()
         for test in range(5):
             # Temporary file
-            tempDir = self._createTempDir(purpose='tempDir')
+            tempDir = self._createTempDir(purpose="tempDir")
             # Make a random DAG for the set of child edges
             nodeNumber = random.choice(range(2, 8))
             childEdges = self.makeRandomDAG(nodeNumber)
@@ -421,8 +449,8 @@ class JobTest(ToilTest):
                     numberOfFailedJobs = 0
                 except FailedJobsException as e:
                     numberOfFailedJobs = e.numberOfFailedJobs
-                    if totalTrys > 32: #p(fail after this many restarts) ~= 0.5**32
-                        self.fail() #Exceeded a reasonable number of restarts
+                    if totalTrys > 32:  # p(fail after this many restarts) ~= 0.5**32
+                        self.fail()  # Exceeded a reasonable number of restarts
                     totalTrys += 1
 
             # For each job check it created a valid output file and add the ordering
@@ -447,7 +475,7 @@ class JobTest(ToilTest):
     def getRandomEdge(nodeNumber):
         assert nodeNumber > 1
         fNode = random.choice(range(nodeNumber - 1))
-        return fNode, random.choice(range(fNode+1, nodeNumber))
+        return fNode, random.choice(range(fNode + 1, nodeNumber))
 
     @staticmethod
     def makeRandomDAG(nodeNumber):
@@ -457,7 +485,9 @@ class JobTest(ToilTest):
         referring to nodes and the edge is from a to b.
         """
         # Pick number of total edges to create
-        edgeNumber = random.choice(range(nodeNumber - 1, 1 + (nodeNumber * (nodeNumber - 1) // 2)))
+        edgeNumber = random.choice(
+            range(nodeNumber - 1, 1 + (nodeNumber * (nodeNumber - 1) // 2))
+        )
         # Make a spanning tree of edges so that nodes are connected
         edges = {(random.choice(range(i)), i) for i in range(1, nodeNumber)}
         # Add extra random edges until there are edgeNumber edges
@@ -499,8 +529,10 @@ class JobTest(ToilTest):
         """
 
         def makeAugmentedAdjacencyList():
-            augmentedAdjacencyList = [childAdjacencyList[i].union(followOnAdjacencyList[i])
-                                      for i in range(len(childAdjacencyList))]
+            augmentedAdjacencyList = [
+                childAdjacencyList[i].union(followOnAdjacencyList[i])
+                for i in range(len(childAdjacencyList))
+            ]
 
             def addImpliedEdges(node, followOnEdges):
                 # Let node2 be a child of node or a successor of a child of node.
@@ -550,7 +582,9 @@ class JobTest(ToilTest):
 
         return followOnEdges
 
-    def makeJobGraph(self, nodeNumber, childEdges, followOnEdges, outPath, addServices=True):
+    def makeJobGraph(
+        self, nodeNumber, childEdges, followOnEdges, outPath, addServices=True
+    ):
         """
         Converts a DAG into a job graph. childEdges and followOnEdges are the lists of child and
         followOn edges.
@@ -560,9 +594,15 @@ class JobTest(ToilTest):
 
         def makeJob(string):
             promises = []
-            job = Job.wrapFn(fn2Test, promises, string,
-                             None if outPath is None else os.path.join(outPath, string),
-                             cores=0.1, memory="0.5G", disk="0.1G")
+            job = Job.wrapFn(
+                fn2Test,
+                promises,
+                string,
+                None if outPath is None else os.path.join(outPath, string),
+                cores=0.1,
+                memory="0.5G",
+                disk="0.1G",
+            )
             jobsToPromisesMap[job] = promises
             return job
 
@@ -581,7 +621,16 @@ class JobTest(ToilTest):
             predecessors[jobs[tNode]].append(jobs[fNode])
 
         # Map of jobs to return values
-        jobsToRvs = {job: job.addService(TrivialService(job.rv(), cores=0.1, memory="0.5G", disk="0.1G")) if addServices else job.rv() for job in jobs}
+        jobsToRvs = {
+            job: (
+                job.addService(
+                    TrivialService(job.rv(), cores=0.1, memory="0.5G", disk="0.1G")
+                )
+                if addServices
+                else job.rv()
+            )
+            for job in jobs
+        }
 
         def getRandomPredecessor(job):
             predecessor = random.choice(list(predecessors[job]))
@@ -622,9 +671,11 @@ class JobTest(ToilTest):
                 return False
         return True
 
+
 def simpleJobFn(job, value):
     job.fileStore.log_to_leader(value)
 
+
 def fn1Test(string, outputFile):
     """
     Function appends the next character after the last character in the given
@@ -633,7 +684,7 @@ def fn1Test(string, outputFile):
     """
 
     rV = string + chr(ord(string[-1]) + 1)
-    with open(outputFile, 'w') as fH:
+    with open(outputFile, "w") as fH:
         fH.write(rV)
     return rV
 
@@ -643,7 +694,7 @@ def fn2Test(pStrings, s, outputFile):
     Function concatenates the strings in pStrings and s, in that order, and writes the result to
     the output file. Returns s.
     """
-    with open(outputFile, 'w') as fH:
+    with open(outputFile, "w") as fH:
         fH.write(" ".join(pStrings) + " " + s)
     return s
 
@@ -687,13 +738,12 @@ def child(job):
 
 
 def errorChild(job):
-    raise RuntimeError('Child failure')
+    raise RuntimeError("Child failure")
 
 
 class TrivialService(Job.Service):
     def __init__(self, message, *args, **kwargs):
-        """ Service that does nothing, used to check for deadlocks
-        """
+        """Service that does nothing, used to check for deadlocks"""
         Job.Service.__init__(self, *args, **kwargs)
         self.message = message
 
@@ -707,5 +757,5 @@ class TrivialService(Job.Service):
         pass
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     unittest.main()

toil/test/src/miscTests.py

@@ -20,10 +20,7 @@ from uuid import uuid4
 
 from toil.common import getNodeID
 from toil.lib.exceptions import panic, raise_
-from toil.lib.io import (AtomicFileCreate,
-                         atomic_install,
-                         atomic_tmp_file,
-                         mkdtemp)
+from toil.lib.io import AtomicFileCreate, atomic_install, atomic_tmp_file, mkdtemp
 from toil.lib.misc import CalledProcessErrorStderr, call_command
 from toil.test import ToilTest, slow
 
@@ -36,6 +33,7 @@ class MiscTests(ToilTest):
     This class contains miscellaneous tests that don't have enough content to be their own test
     file, and that don't logically fit in with any of the other test suites.
     """
+
     def setUp(self):
         super().setUp()
         self.testDir = self._createTempDir()
@@ -49,14 +47,14 @@ class MiscTests(ToilTest):
 
     @slow
     def testGetSizeOfDirectoryWorks(self):
-        '''A test to make sure toil.common.getDirSizeRecursively does not
+        """A test to make sure toil.common.getDirSizeRecursively does not
         underestimate the amount of disk space needed.
 
         Disk space allocation varies from system to system.  The computed value
         should always be equal to or slightly greater than the creation value.
         This test generates a number of random directories and randomly sized
         files to test this using getDirSizeRecursively.
-        '''
+        """
         from toil.common import getDirSizeRecursively
 
         # a list of the directories used in the test
@@ -64,17 +62,17 @@ class MiscTests(ToilTest):
         # A dict of {FILENAME: FILESIZE} for all files used in the test
         files = {}
         # Create a random directory structure
-        for i in range(0,10):
-            directories.append(mkdtemp(dir=random.choice(directories), prefix='test'))
+        for i in range(0, 10):
+            directories.append(mkdtemp(dir=random.choice(directories), prefix="test"))
         # Create 50 random file entries in different locations in the directories. 75% of the time
         # these are fresh files of size [1, 10] MB and 25% of the time they are hard links to old
         # files.
         while len(files) <= 50:
             fileName = os.path.join(random.choice(directories), self._getRandomName())
-            if random.randint(0,100) < 75:
+            if random.randint(0, 100) < 75:
                 # Create a fresh file in the range of 1-10 MB
                 fileSize = int(round(random.random(), 2) * 10 * 1024 * 1024)
-                with open(fileName, 'wb') as fileHandle:
+                with open(fileName, "wb") as fileHandle:
                     fileHandle.write(os.urandom(fileSize))
                 files[fileName] = fileSize
             else:
@@ -83,7 +81,7 @@ class MiscTests(ToilTest):
                     continue
                 linkSrc = random.choice(list(files.keys()))
                 os.link(linkSrc, fileName)
-                files[fileName] = 'Link to %s' % linkSrc
+                files[fileName] = "Link to %s" % linkSrc
 
         computedDirectorySize = getDirSizeRecursively(self.testDir)
         totalExpectedSize = sum(x for x in list(files.values()) if isinstance(x, int))
@@ -101,7 +99,7 @@ class MiscTests(ToilTest):
 
     def _write_test_file(self, outf_tmp):
         with open(outf_tmp, "w") as fh:
-            fh.write(self.id() + '\n')
+            fh.write(self.id() + "\n")
 
     def test_atomic_install(self):
         outf = self._get_test_out_file(".foo.gz")
@@ -111,7 +109,7 @@ class MiscTests(ToilTest):
         self.assertTrue(os.path.exists(outf))
 
     def test_atomic_install_dev(self):
-        devn = '/dev/null'
+        devn = "/dev/null"
         tmp = atomic_tmp_file(devn)
         self.assertEqual(tmp, devn)
         atomic_install(tmp, devn)
@@ -138,10 +136,13 @@ class MiscTests(ToilTest):
         self.assertTrue(isinstance(o, str), str(type(o)))
 
     def test_call_command_err(self):
-        with self.assertRaisesRegex(CalledProcessErrorStderr,
-                                     "^Command '\\['cat', '/dev/Frankenheimer']' exit status 1: cat: /dev/Frankenheimer: No such file or directory\n$"):
+        with self.assertRaisesRegex(
+            CalledProcessErrorStderr,
+            "^Command '\\['cat', '/dev/Frankenheimer']' exit status 1: cat: /dev/Frankenheimer: No such file or directory\n$",
+        ):
             call_command(["cat", "/dev/Frankenheimer"])
 
+
 class TestPanic(ToilTest):
     def test_panic_by_hand(self):
         try:
@@ -192,7 +193,7 @@ class TestPanic(ToilTest):
             self.line_of_primary_exc = inspect.currentframe().f_lineno + 1
             raise ValueError("primary")
         except:
-            with panic( log ):
+            with panic(log):
                 raise RuntimeError("secondary")
 
     def try_and_nested_panic_with_secondary(self):
@@ -200,8 +201,8 @@ class TestPanic(ToilTest):
             self.line_of_primary_exc = inspect.currentframe().f_lineno + 1
             raise ValueError("primary")
         except:
-            with panic( log ):
-                with panic( log ):
+            with panic(log):
+                with panic(log):
                     raise RuntimeError("secondary")
 
     def __assert_raised_exception_is_primary(self):