toil 6.1.0a1__py3-none-any.whl → 8.0.0__py3-none-any.whl

toil/test/src/resourceTest.py

@@ -30,7 +30,7 @@ from toil.version import exactPython
 
 
 @contextmanager
-def tempFileContaining(content, suffix=''):
+def tempFileContaining(content, suffix=""):
     """
     Write a file with the given contents, and keep it on disk as long as the context is active.
     :param str content: The contents of the file.
@@ -38,7 +38,7 @@ def tempFileContaining(content, suffix=''):
     """
     fd, path = tempfile.mkstemp(suffix=suffix)
     try:
-        encoded = content.encode('utf-8')
+        encoded = content.encode("utf-8")
         assert os.write(fd, encoded) == len(encoded)
     except:
         os.close(fd)
@@ -52,41 +52,55 @@ def tempFileContaining(content, suffix=''):
 
 class ResourceTest(ToilTest):
     """Test module descriptors and resources derived from them."""
+
     def testStandAlone(self):
-        self._testExternal(moduleName='userScript', pyFiles=('userScript.py', 'helper.py'))
+        self._testExternal(
+            moduleName="userScript", pyFiles=("userScript.py", "helper.py")
+        )
 
     def testPackage(self):
-        self._testExternal(moduleName='foo.userScript', pyFiles=('foo/__init__.py',
-                                                                 'foo/userScript.py',
-                                                                 'foo/bar/__init__.py',
-                                                                 'foo/bar/helper.py'))
+        self._testExternal(
+            moduleName="foo.userScript",
+            pyFiles=(
+                "foo/__init__.py",
+                "foo/userScript.py",
+                "foo/bar/__init__.py",
+                "foo/bar/helper.py",
+            ),
+        )
 
     def testVirtualEnv(self):
-        self._testExternal(moduleName='foo.userScript',
-                           virtualenv=True,
-                           pyFiles=('foo/__init__.py',
-                                    'foo/userScript.py',
-                                    'foo/bar/__init__.py',
-                                    'foo/bar/helper.py',
-                                    'de/pen/dency.py',
-                                    'de/__init__.py',
-                                    'de/pen/__init__.py'))
+        self._testExternal(
+            moduleName="foo.userScript",
+            virtualenv=True,
+            pyFiles=(
+                "foo/__init__.py",
+                "foo/userScript.py",
+                "foo/bar/__init__.py",
+                "foo/bar/helper.py",
+                "de/pen/dency.py",
+                "de/__init__.py",
+                "de/pen/__init__.py",
+            ),
+        )
 
     def testStandAloneInPackage(self):
-        self.assertRaises(ResourceException,
-                          self._testExternal,
-                          moduleName='userScript',
-                          pyFiles=('__init__.py', 'userScript.py', 'helper.py'))
+        self.assertRaises(
+            ResourceException,
+            self._testExternal,
+            moduleName="userScript",
+            pyFiles=("__init__.py", "userScript.py", "helper.py"),
+        )
 
     def _testExternal(self, moduleName, pyFiles, virtualenv=False):
         dirPath = self._createTempDir()
         if virtualenv:
             self.assertTrue(inVirtualEnv())
             # --never-download prevents silent upgrades to pip, wheel and setuptools
-            subprocess.check_call(['virtualenv', '--never-download', '--python', exactPython, dirPath])
-            sitePackages = os.path.join(dirPath, 'lib',
-                                        exactPython,
-                                        'site-packages')
+            subprocess.check_call(
+                ["virtualenv", "--never-download", "--python", exactPython, dirPath]
+            )
+            sitePackages = os.path.join(dirPath, "lib", exactPython, "site-packages")
             # tuple assignment is necessary to make this line immediately precede the try:
             oldPrefix, sys.prefix, dirPath = sys.prefix, dirPath, sitePackages
         else:
@@ -95,21 +109,23 @@ class ResourceTest(ToilTest):
             for relPath in pyFiles:
                 path = os.path.join(dirPath, relPath)
                 os.makedirs(os.path.dirname(path), exist_ok=True)
-                with open(path, 'w') as f:
-                    f.write('pass\n')
+                with open(path, "w") as f:
+                    f.write("pass\n")
             sys.path.append(dirPath)
             try:
                 userScript = importlib.import_module(moduleName)
                 try:
-                    self._test(userScript.__name__,
-                               expectedContents=pyFiles,
-                               allowExtraContents=True)
+                    self._test(
+                        userScript.__name__,
+                        expectedContents=pyFiles,
+                        allowExtraContents=True,
+                    )
                 finally:
                     del userScript
                     while moduleName:
                         del sys.modules[moduleName]
                         self.assertFalse(moduleName in sys.modules)
-                        moduleName = '.'.join(moduleName.split('.')[:-1])
+                        moduleName = ".".join(moduleName.split(".")[:-1])
 
             finally:
                 sys.path.remove(dirPath)
@@ -120,17 +136,22 @@ class ResourceTest(ToilTest):
     def testBuiltIn(self):
         # Create a ModuleDescriptor for the module containing ModuleDescriptor, i.e. toil.resource
         module_name = ModuleDescriptor.__module__
-        self.assertEqual(module_name, 'toil.resource')
+        self.assertEqual(module_name, "toil.resource")
         self._test(module_name, shouldBelongToToil=True)
 
-    def _test(self, module_name,
-              shouldBelongToToil=False, expectedContents=None, allowExtraContents=True):
+    def _test(
+        self,
+        module_name,
+        shouldBelongToToil=False,
+        expectedContents=None,
+        allowExtraContents=True,
+    ):
         module = ModuleDescriptor.forModule(module_name)
         # Assert basic attributes and properties
         self.assertEqual(module.belongsToToil, shouldBelongToToil)
         self.assertEqual(module.name, module_name)
         if shouldBelongToToil:
-            self.assertTrue(module.dirPath.endswith('/src'))
+            self.assertTrue(module.dirPath.endswith("/src"))
 
         # Before the module is saved as a resource, localize() and globalize() are identity
         # methods. This should log.warnings.
@@ -139,27 +160,32 @@ class ResourceTest(ToilTest):
         # Create a mock job store ...
         jobStore = MagicMock()
         # ... to generate a fake URL for the resource ...
-        url = 'file://foo.zip'
+        url = "file://foo.zip"
         jobStore.getSharedPublicUrl.return_value = url
         # ... and save the resource to it.
         resource = module.saveAsResourceTo(jobStore)
         # Ensure that the URL generation method is actually called, ...
-        jobStore.getSharedPublicUrl.assert_called_once_with(sharedFileName=resource.pathHash)
+        jobStore.getSharedPublicUrl.assert_called_once_with(
+            sharedFileName=resource.pathHash
+        )
         # ... and that ensure that write_shared_file_stream is called.
-        jobStore.write_shared_file_stream.assert_called_once_with(shared_file_name=resource.pathHash,
-                                                                  encrypted=False)
+        jobStore.write_shared_file_stream.assert_called_once_with(
+            shared_file_name=resource.pathHash, encrypted=False
+        )
         # Now it gets a bit complicated: Ensure that the context manager returned by the
         # jobStore's write_shared_file_stream() method is entered and that the file handle yielded
         # by the context manager is written to once with the zipped source tree from which
         # 'toil.resource' was originally imported. Keep the zipped tree around such that we can
         # mock the download later.
-        file_handle = jobStore.write_shared_file_stream.return_value.__enter__.return_value
+        file_handle = (
+            jobStore.write_shared_file_stream.return_value.__enter__.return_value
+        )
         # The first 0 index selects the first call of write(), the second 0 selects positional
         # instead of keyword arguments, and the third 0 selects the first positional, i.e. the
         # contents. This is a bit brittle since it assumes that all the data is written in a
         # single call to write(). If more calls are made we can easily concatenate them.
         zipFile = file_handle.write.call_args_list[0][0][0]
-        self.assertTrue(zipFile.startswith(b'PK'))  # the magic header for ZIP files
+        self.assertTrue(zipFile.startswith(b"PK"))  # the magic header for ZIP files
 
         # Check contents if requested
         if expectedContents is not None:
@@ -186,7 +212,7 @@ class ResourceTest(ToilTest):
             # urlopen() that yields the zipped tree ...
             mock_urlopen = MagicMock()
             mock_urlopen.return_value.read.return_value = zipFile
-            with patch('toil.resource.urlopen', mock_urlopen):
+            with patch("toil.resource.urlopen", mock_urlopen):
                 # ... and use it to download and unpack the resource
                 localModule = module.localize()
             # The name should be equal between original and localized resource ...
@@ -217,22 +243,27 @@ class ResourceTest(ToilTest):
             def fn():
                 pass
 
-            if __name__ == '__main__':
+            if __name__ == "__main__":
                 parser = ArgumentParser()
                 Job.Runner.addToilOptions(parser)
                 options = parser.parse_args()
-                job = Job.wrapFn(fn, memory='10M', cores=0.1, disk='10M')
+                job = Job.wrapFn(fn, memory="10M", cores=0.1, disk="10M")
                 with Toil(options) as toil:
                     toil.start(job)
 
-        scriptBody = dedent('\n'.join(getsource(script).split('\n')[1:]))
-        shebang = '#! %s\n' % sys.executable
+        scriptBody = dedent("\n".join(getsource(script).split("\n")[1:]))
+        shebang = "#! %s\n" % sys.executable
         with tempFileContaining(shebang + scriptBody) as scriptPath:
-            self.assertFalse(scriptPath.endswith(('.py', '.pyc')))
+            self.assertFalse(scriptPath.endswith((".py", ".pyc")))
             os.chmod(scriptPath, 0o755)
-            jobStorePath = scriptPath + '.jobStore'
-            process = subprocess.Popen([scriptPath, jobStorePath], stderr=subprocess.PIPE)
+            jobStorePath = scriptPath + ".jobStore"
+            process = subprocess.Popen(
+                [scriptPath, jobStorePath], stderr=subprocess.PIPE
+            )
             stdout, stderr = process.communicate()
-            self.assertTrue('The name of a user script/module must end in .py or .pyc.' in stderr.decode('utf-8'))
+            self.assertTrue(
+                "The name of a user script/module must end in .py or .pyc."
+                in stderr.decode("utf-8")
+            )
             self.assertNotEqual(0, process.returncode)
             self.assertFalse(os.path.exists(jobStorePath))

toil/test/src/restartDAGTest.py

@@ -31,9 +31,10 @@ class RestartDAGTest(ToilTest):
     Tests that restarted job DAGs don't run children of jobs that failed in the first run till the
     parent completes successfully in the restart.
     """
+
     def setUp(self):
         super().setUp()
-        self.tempDir = self._createTempDir(purpose='tempDir')
+        self.tempDir = self._createTempDir(purpose="tempDir")
         self.testJobStore = self._getTestJobStorePath()
 
     def tearDown(self):
@@ -42,11 +43,11 @@ class RestartDAGTest(ToilTest):
 
     @slow
     def testRestartedWorkflowSchedulesCorrectJobsOnFailedParent(self):
-        self._testRestartedWorkflowSchedulesCorrectJobs('raise')
+        self._testRestartedWorkflowSchedulesCorrectJobs("raise")
 
     @slow
     def testRestartedWorkflowSchedulesCorrectJobsOnKilledParent(self):
-        self._testRestartedWorkflowSchedulesCorrectJobs('kill')
+        self._testRestartedWorkflowSchedulesCorrectJobs("kill")
 
     def _testRestartedWorkflowSchedulesCorrectJobs(self, failType):
         """
@@ -63,12 +64,12 @@ class RestartDAGTest(ToilTest):
         """
         # Specify options
         options = Job.Runner.getDefaultOptions(self.testJobStore)
-        options.logLevel = 'DEBUG'
+        options.logLevel = "DEBUG"
         options.retryCount = 0
         options.clean = "never"
 
-        parentFile = os.path.join(self.tempDir, 'parent')
-        childFile = os.path.join(self.tempDir, 'child')
+        parentFile = os.path.join(self.tempDir, "parent")
+        childFile = os.path.join(self.tempDir, "child")
 
         # Make the first job
         root = Job.wrapJobFn(passingFn)
@@ -87,11 +88,11 @@ class RestartDAGTest(ToilTest):
         assert not os.path.exists(childFile)
 
         # Run the test
-        for runMode in 'start', 'restart':
+        for runMode in "start", "restart":
             self.errorRaised = None
             try:
                 with Toil(options) as toil:
-                    if runMode == 'start':
+                    if runMode == "start":
                         toil.start(root)
                     else:
                         toil.restart()
@@ -103,19 +104,27 @@ class RestartDAGTest(ToilTest):
                 # it together in this finally clause.
                 if self.errorRaised is not None:
                     if not os.path.exists(parentFile):
-                        failReasons.append('The failing parent file did not exist on toil "%s".'
-                                           % runMode)
+                        failReasons.append(
+                            'The failing parent file did not exist on toil "%s".'
+                            % runMode
+                        )
                     if os.path.exists(childFile):
-                        failReasons.append('The child file existed.  i.e. the child was run on '
-                                           'toil "%s".' % runMode)
+                        failReasons.append(
+                            "The child file existed.  i.e. the child was run on "
+                            'toil "%s".' % runMode
+                        )
                     if isinstance(self.errorRaised, FailedJobsException):
                         if self.errorRaised.numberOfFailedJobs != 3:
-                            failReasons.append('FailedJobsException was raised on toil "%s" but '
-                                               'the number of failed jobs (%s) was not 3.'
-                                               % (runMode, self.errorRaised.numberOfFailedJobs))
+                            failReasons.append(
+                                'FailedJobsException was raised on toil "%s" but '
+                                "the number of failed jobs (%s) was not 3."
+                                % (runMode, self.errorRaised.numberOfFailedJobs)
+                            )
                     elif isinstance(self.errorRaised, AssertionError):
-                        failReasons.append('Toil raised an AssertionError instead of a '
-                                           'FailedJobsException on toil "%s".' % runMode)
+                        failReasons.append(
+                            "Toil raised an AssertionError instead of a "
+                            'FailedJobsException on toil "%s".' % runMode
+                        )
                     else:
                         failReasons.append("Toil raised error: %s" % self.errorRaised)
                     self.errorRaised = None
@@ -123,8 +132,12 @@ class RestartDAGTest(ToilTest):
                 else:
                     self.fail('No errors were raised on toil "%s".' % runMode)
         if failReasons:
-            self.fail('Test failed for ({}) reasons:\n\t{}'.format(len(failReasons),
-                                                               '\n\t'.join(failReasons)))
+            self.fail(
+                "Test failed for ({}) reasons:\n\t{}".format(
+                    len(failReasons), "\n\t".join(failReasons)
+                )
+            )
+
 
 def passingFn(job, fileName=None):
     """
@@ -135,7 +148,8 @@ def passingFn(job, fileName=None):
     """
     if fileName is not None:
         # Emulates system touch.
-        open(fileName, 'w').close()
+        open(fileName, "w").close()
+
 
 def failingFn(job, failType, fileName):
     """
@@ -145,11 +159,11 @@ def failingFn(job, failType, fileName):
     :param str failType: 'raise' or 'kill
     :param str fileName: The name of a file that must be created.
     """
-    assert failType in ('raise', 'kill')
+    assert failType in ("raise", "kill")
     # Use that function to avoid code redundancy
     passingFn(job, fileName)
 
-    if failType == 'raise':
+    if failType == "raise":
         assert False
     else:
         os.kill(os.getpid(), signal.SIGKILL)

toil/test/src/resumabilityTest.py

@@ -24,6 +24,7 @@ class ResumabilityTest(ToilTest):
     """
     https://github.com/BD2KGenomics/toil/issues/808
     """
+
     @slow
     def test(self):
         """
@@ -70,7 +71,7 @@ class ResumabilityTest(ToilTest):
             # This one is intended to fail.
             Job.Runner.startToil(root, options)
 
-        with open(options.logFile, 'r') as f:
+        with open(options.logFile) as f:
             log_content = f.read()
             # Make sure we actually did do chaining
             assert "Chaining from" in log_content
@@ -81,6 +82,7 @@ class ResumabilityTest(ToilTest):
         options.restart = True
         Job.Runner.startToil(root, options)
 
+
 def parent(job):
     """
     Set up a bunch of dummy child jobs, and a bad job that needs to be
@@ -90,18 +92,21 @@ def parent(job):
         job.addChildJobFn(goodChild)
     job.addFollowOnJobFn(badChild)
 
+
 def chaining_parent(job):
     """
     Set up a failing job to chain to.
     """
     job.addFollowOnJobFn(badChild)
 
+
 def goodChild(job):
     """
     Does nothing.
     """
     return
 
+
 def badChild(job):
     """
     Fails the first time it's run, succeeds the second time.
@@ -110,6 +115,8 @@ def badChild(job):
         with job.fileStore.jobStore.read_shared_file_stream("alreadyRun") as fileHandle:
             fileHandle.read()
     except NoSuchFileException as ex:
-        with job.fileStore.jobStore.write_shared_file_stream("alreadyRun", encrypted=False) as fileHandle:
+        with job.fileStore.jobStore.write_shared_file_stream(
+            "alreadyRun", encrypted=False
+        ) as fileHandle:
             fileHandle.write(b"failed once\n")
         raise RuntimeError(f"this is an expected error: {str(ex)}")

toil/test/src/retainTempDirTest.py

@@ -23,6 +23,7 @@ class CleanWorkDirTest(ToilTest):
     """
     Tests testing :class:toil.fileStores.abstractFileStore.AbstractFileStore
     """
+
     def setUp(self):
         super().setUp()
         self.testDir = self._createTempDir()
@@ -33,33 +34,61 @@ class CleanWorkDirTest(ToilTest):
 
     def testNever(self):
         retainedTempData = self._runAndReturnWorkDir("never", job=tempFileTestJob)
-        self.assertNotEqual(retainedTempData, [], "The worker's temporary workspace was deleted despite "
-                                                  "cleanWorkDir being set to 'never'")
+        self.assertNotEqual(
+            retainedTempData,
+            [],
+            "The worker's temporary workspace was deleted despite "
+            "cleanWorkDir being set to 'never'",
+        )
 
     def testAlways(self):
         retainedTempData = self._runAndReturnWorkDir("always", job=tempFileTestJob)
-        self.assertEqual(retainedTempData, [], "The worker's temporary workspace was not deleted despite "
-                                               "cleanWorkDir being set to 'always'")
+        self.assertEqual(
+            retainedTempData,
+            [],
+            "The worker's temporary workspace was not deleted despite "
+            "cleanWorkDir being set to 'always'",
+        )
 
     def testOnErrorWithError(self):
-        retainedTempData = self._runAndReturnWorkDir("onError", job=tempFileTestErrorJob, expectError=True)
-        self.assertEqual(retainedTempData, [], "The worker's temporary workspace was not deleted despite "
-                                               "an error occurring and cleanWorkDir being set to 'onError'")
+        retainedTempData = self._runAndReturnWorkDir(
+            "onError", job=tempFileTestErrorJob, expectError=True
+        )
+        self.assertEqual(
+            retainedTempData,
+            [],
+            "The worker's temporary workspace was not deleted despite "
+            "an error occurring and cleanWorkDir being set to 'onError'",
+        )
 
     def testOnErrorWithNoError(self):
         retainedTempData = self._runAndReturnWorkDir("onError", job=tempFileTestJob)
-        self.assertNotEqual(retainedTempData, [], "The worker's temporary workspace was deleted despite "
-                                                  "no error occurring and cleanWorkDir being set to 'onError'")
+        self.assertNotEqual(
+            retainedTempData,
+            [],
+            "The worker's temporary workspace was deleted despite "
+            "no error occurring and cleanWorkDir being set to 'onError'",
+        )
 
     def testOnSuccessWithError(self):
-        retainedTempData = self._runAndReturnWorkDir("onSuccess", job=tempFileTestErrorJob, expectError=True)
-        self.assertNotEqual(retainedTempData, [], "The worker's temporary workspace was deleted despite "
-                                                  "an error occurring and cleanWorkDir being set to 'onSuccesss'")
+        retainedTempData = self._runAndReturnWorkDir(
+            "onSuccess", job=tempFileTestErrorJob, expectError=True
+        )
+        self.assertNotEqual(
+            retainedTempData,
+            [],
+            "The worker's temporary workspace was deleted despite "
+            "an error occurring and cleanWorkDir being set to 'onSuccesss'",
+        )
 
     def testOnSuccessWithSuccess(self):
         retainedTempData = self._runAndReturnWorkDir("onSuccess", job=tempFileTestJob)
-        self.assertEqual(retainedTempData, [], "The worker's temporary workspace was not deleted despite "
-                                               "a successful job execution and cleanWorkDir being set to 'onSuccesss'")
+        self.assertEqual(
+            retainedTempData,
+            [],
+            "The worker's temporary workspace was not deleted despite "
+            "a successful job execution and cleanWorkDir being set to 'onSuccesss'",
+        )
 
     def _runAndReturnWorkDir(self, cleanWorkDir, job, expectError=False):
         """
@@ -89,10 +118,12 @@ class CleanWorkDirTest(ToilTest):
         else:
             self.fail("Toil run succeeded unexpectedly")
 
+
 def tempFileTestJob(job):
     with open(job.fileStore.getLocalTempFile(), "w") as f:
         f.write("test file retention")
 
+
 def tempFileTestErrorJob(job):
     with open(job.fileStore.getLocalTempFile(), "w") as f:
         f.write("test file retention")

toil/test/src/systemTest.py

@@ -10,17 +10,21 @@ from toil.test import ToilTest
 
 class SystemTest(ToilTest):
     """Test various assumptions about the operating system's behavior."""
+
     def testAtomicityOfNonEmptyDirectoryRenames(self):
         for _ in range(100):
-            parent = self._createTempDir(purpose='parent')
-            child = os.path.join(parent, 'child')
+            parent = self._createTempDir(purpose="parent")
+            child = os.path.join(parent, "child")
             # Use processes (as opposed to threads) to prevent GIL from ordering things artificially
             pool = multiprocessing.Pool(processes=cpu_count())
             try:
                 numTasks = cpu_count() * 10
                 grandChildIds = pool.map_async(
-                    func=partial(_testAtomicityOfNonEmptyDirectoryRenamesTask, parent, child),
-                    iterable=list(range(numTasks)))
+                    func=partial(
+                        _testAtomicityOfNonEmptyDirectoryRenamesTask, parent, child
+                    ),
+                    iterable=list(range(numTasks)),
+                )
                 grandChildIds = grandChildIds.get()
             finally:
                 pool.close()
@@ -31,15 +35,15 @@ class SystemTest(ToilTest):
             self.assertEqual(len(grandChildIds), 1)
             # Assert that the winner's grandChild wasn't silently overwritten by a looser
             expectedGrandChildId = grandChildIds[0]
-            actualGrandChild = os.path.join(child, 'grandChild')
+            actualGrandChild = os.path.join(child, "grandChild")
             actualGrandChildId = os.stat(actualGrandChild).st_ino
             self.assertEqual(actualGrandChildId, expectedGrandChildId)
 
 
 def _testAtomicityOfNonEmptyDirectoryRenamesTask(parent, child, _):
-    tmpChildDir = mkdtemp(dir=parent, prefix='child', suffix='.tmp')
-    grandChild = os.path.join(tmpChildDir, 'grandChild')
-    open(grandChild, 'w').close()
+    tmpChildDir = mkdtemp(dir=parent, prefix="child", suffix=".tmp")
+    grandChild = os.path.join(tmpChildDir, "grandChild")
+    open(grandChild, "w").close()
     grandChildId = os.stat(grandChild).st_ino
     try:
         os.rename(tmpChildDir, child)