toil 8.1.0b1__py3-none-any.whl → 8.2.0__py3-none-any.whl

toil/test/src/resumabilityTest.py

@@ -12,29 +12,32 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import os
+from pathlib import Path
 
 from toil.exceptions import FailedJobsException
-from toil.job import Job
+from toil.job import Job, JobFunctionWrappingJob
 from toil.jobStores.abstractJobStore import NoSuchFileException
-from toil.test import ToilTest, slow
+from toil.test import pslow as slow
 
+import pytest
 
-class ResumabilityTest(ToilTest):
+
+class TestResumability:
     """
     https://github.com/BD2KGenomics/toil/issues/808
     """
 
     @slow
-    def test(self):
+    @pytest.mark.slow
+    def test(self, tmp_path: Path) -> None:
         """
         Tests that a toil workflow that fails once can be resumed without a NoSuchJobException.
         """
-        options = Job.Runner.getDefaultOptions(self._getTestJobStorePath())
+        options = Job.Runner.getDefaultOptions(tmp_path / "jobstore")
         options.logLevel = "INFO"
         options.retryCount = 0
         root = Job.wrapJobFn(parent)
-        with self.assertRaises(FailedJobsException):
+        with pytest.raises(FailedJobsException):
             # This one is intended to fail.
             Job.Runner.startToil(root, options)
 
@@ -44,30 +47,32 @@ class ResumabilityTest(ToilTest):
         # system code notices that the job has been deleted despite
         # the failure and avoids the failure.
         options.restart = True
-        tempDir = self._createTempDir()
-        options.logFile = os.path.join(tempDir, "log.txt")
+        tempDir = tmp_path / "tempdir"
+        tempDir.mkdir()
+        options.logFile = str(tempDir / "log.txt")
         Job.Runner.startToil(root, options)
         with open(options.logFile) as f:
             logString = f.read()
             # We are looking for e.g. "Batch system is reporting that
             # the jobGraph with batch system ID: 1 and jobGraph
             # store ID: n/t/jobwbijqL failed with exit value 1"
-            self.assertTrue("failed with exit value" not in logString)
+            assert "failed with exit value" not in logString
 
-    def test_chaining(self):
+    def test_chaining(self, tmp_path: Path) -> None:
         """
         Tests that a job which is chained to and fails can resume and succeed.
         """
 
-        options = Job.Runner.getDefaultOptions(self._getTestJobStorePath())
+        options = Job.Runner.getDefaultOptions(tmp_path / "jobstore")
         options.logLevel = "DEBUG"
         options.retryCount = 0
-        tempDir = self._createTempDir()
-        options.logFile = os.path.join(tempDir, "log.txt")
+        tempDir = tmp_path / "tempdir"
+        tempDir.mkdir()
+        options.logFile = str(tempDir / "log.txt")
 
         root = Job.wrapJobFn(chaining_parent)
 
-        with self.assertRaises(FailedJobsException):
+        with pytest.raises(FailedJobsException):
             # This one is intended to fail.
             Job.Runner.startToil(root, options)
 
@@ -83,7 +88,7 @@ class ResumabilityTest(ToilTest):
         Job.Runner.startToil(root, options)
 
 
-def parent(job):
+def parent(job: Job) -> None:
     """
     Set up a bunch of dummy child jobs, and a bad job that needs to be
     restarted as the follow on.
@@ -93,21 +98,21 @@ def parent(job):
     job.addFollowOnJobFn(badChild)
 
 
-def chaining_parent(job):
+def chaining_parent(job: Job) -> None:
     """
     Set up a failing job to chain to.
     """
     job.addFollowOnJobFn(badChild)
 
 
-def goodChild(job):
+def goodChild(job: Job) -> None:
     """
     Does nothing.
     """
     return
 
 
-def badChild(job):
+def badChild(job: JobFunctionWrappingJob) -> None:
     """
     Fails the first time it's run, succeeds the second time.
     """

toil/test/src/retainTempDirTest.py

@@ -11,93 +11,92 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import argparse
+from collections.abc import Callable
 import os
-import shutil
+from pathlib import Path
 
 from toil.exceptions import FailedJobsException
-from toil.job import Job
-from toil.test import ToilTest
+from toil.job import Job, JobFunctionWrappingJob
 
+import pytest
 
-class CleanWorkDirTest(ToilTest):
+
+class TestCleanWorkDir:
     """
     Tests testing :class:toil.fileStores.abstractFileStore.AbstractFileStore
     """
 
-    def setUp(self):
-        super().setUp()
-        self.testDir = self._createTempDir()
-
-    def tearDown(self):
-        super().tearDown()
-        shutil.rmtree(self.testDir)
-
-    def testNever(self):
-        retainedTempData = self._runAndReturnWorkDir("never", job=tempFileTestJob)
-        self.assertNotEqual(
-            retainedTempData,
-            [],
+    def testNever(self, tmp_path: Path) -> None:
+        retainedTempData = self._runAndReturnWorkDir(
+            tmp_path, "never", job=tempFileTestJob
+        )
+        assert retainedTempData != [], (
             "The worker's temporary workspace was deleted despite "
-            "cleanWorkDir being set to 'never'",
+            "cleanWorkDir being set to 'never'"
         )
 
-    def testAlways(self):
-        retainedTempData = self._runAndReturnWorkDir("always", job=tempFileTestJob)
-        self.assertEqual(
-            retainedTempData,
-            [],
+    def testAlways(self, tmp_path: Path) -> None:
+        retainedTempData = self._runAndReturnWorkDir(
+            tmp_path, "always", job=tempFileTestJob
+        )
+        assert retainedTempData == [], (
             "The worker's temporary workspace was not deleted despite "
-            "cleanWorkDir being set to 'always'",
+            "cleanWorkDir being set to 'always'"
         )
 
-    def testOnErrorWithError(self):
+    def testOnErrorWithError(self, tmp_path: Path) -> None:
         retainedTempData = self._runAndReturnWorkDir(
-            "onError", job=tempFileTestErrorJob, expectError=True
+            tmp_path, "onError", job=tempFileTestErrorJob, expectError=True
         )
-        self.assertEqual(
-            retainedTempData,
-            [],
+        assert retainedTempData == [], (
             "The worker's temporary workspace was not deleted despite "
-            "an error occurring and cleanWorkDir being set to 'onError'",
+            "an error occurring and cleanWorkDir being set to 'onError'"
         )
 
-    def testOnErrorWithNoError(self):
-        retainedTempData = self._runAndReturnWorkDir("onError", job=tempFileTestJob)
-        self.assertNotEqual(
-            retainedTempData,
-            [],
+    def testOnErrorWithNoError(self, tmp_path: Path) -> None:
+        retainedTempData = self._runAndReturnWorkDir(
+            tmp_path, "onError", job=tempFileTestJob
+        )
+        assert retainedTempData != [], (
             "The worker's temporary workspace was deleted despite "
-            "no error occurring and cleanWorkDir being set to 'onError'",
+            "no error occurring and cleanWorkDir being set to 'onError'"
         )
 
-    def testOnSuccessWithError(self):
+    def testOnSuccessWithError(self, tmp_path: Path) -> None:
         retainedTempData = self._runAndReturnWorkDir(
-            "onSuccess", job=tempFileTestErrorJob, expectError=True
+            tmp_path, "onSuccess", job=tempFileTestErrorJob, expectError=True
         )
-        self.assertNotEqual(
-            retainedTempData,
-            [],
+        assert retainedTempData != [], (
             "The worker's temporary workspace was deleted despite "
-            "an error occurring and cleanWorkDir being set to 'onSuccesss'",
+            "an error occurring and cleanWorkDir being set to 'onSuccesss'"
         )
 
-    def testOnSuccessWithSuccess(self):
-        retainedTempData = self._runAndReturnWorkDir("onSuccess", job=tempFileTestJob)
-        self.assertEqual(
-            retainedTempData,
-            [],
+    def testOnSuccessWithSuccess(self, tmp_path: Path) -> None:
+        retainedTempData = self._runAndReturnWorkDir(
+            tmp_path, "onSuccess", job=tempFileTestJob
+        )
+        assert retainedTempData == [], (
             "The worker's temporary workspace was not deleted despite "
-            "a successful job execution and cleanWorkDir being set to 'onSuccesss'",
+            "a successful job execution and cleanWorkDir being set to 'onSuccesss'"
         )
 
-    def _runAndReturnWorkDir(self, cleanWorkDir, job, expectError=False):
+    def _runAndReturnWorkDir(
+        self,
+        tmp_path: Path,
+        cleanWorkDir: str,
+        job: Callable[[JobFunctionWrappingJob], None],
+        expectError: bool = False,
+    ) -> list[str]:
         """
         Runs toil with the specified job and cleanWorkDir setting. expectError determines whether the test's toil
         run is expected to succeed, and the test will fail if that expectation is not met. returns the contents of
         the workDir after completion of the run
         """
-        options = Job.Runner.getDefaultOptions(self._getTestJobStorePath())
-        options.workDir = self.testDir
+        workdir = tmp_path / "testDir"
+        workdir.mkdir()
+        options = Job.Runner.getDefaultOptions(tmp_path / "jobstore")
+        options.workDir = str(workdir)
         options.clean = "always"
         options.cleanWorkDir = cleanWorkDir
         A = Job.wrapJobFn(job)
@@ -105,26 +104,30 @@ class CleanWorkDirTest(ToilTest):
             self._launchError(A, options)
         else:
             self._launchRegular(A, options)
-        return os.listdir(self.testDir)
+        return os.listdir(workdir)
 
-    def _launchRegular(self, A, options):
+    def _launchRegular(
+        self, A: JobFunctionWrappingJob, options: argparse.Namespace
+    ) -> None:
         Job.Runner.startToil(A, options)
 
-    def _launchError(self, A, options):
+    def _launchError(
+        self, A: JobFunctionWrappingJob, options: argparse.Namespace
+    ) -> None:
         try:
             Job.Runner.startToil(A, options)
         except FailedJobsException:
             pass  # we expect a job to fail here
         else:
-            self.fail("Toil run succeeded unexpectedly")
+            pytest.fail("Toil run succeeded unexpectedly")
 
 
-def tempFileTestJob(job):
+def tempFileTestJob(job: JobFunctionWrappingJob) -> None:
     with open(job.fileStore.getLocalTempFile(), "w") as f:
         f.write("test file retention")
 
 
-def tempFileTestErrorJob(job):
+def tempFileTestErrorJob(job: JobFunctionWrappingJob) -> None:
     with open(job.fileStore.getLocalTempFile(), "w") as f:
         f.write("test file retention")
     raise RuntimeError()  # test failure

toil/test/src/systemTest.py

@@ -1,46 +1,50 @@
 import errno
 import multiprocessing
 import os
+from pathlib import Path
 from functools import partial
+from typing import Any, Optional
 
 from toil.lib.io import mkdtemp
 from toil.lib.threading import cpu_count
-from toil.test import ToilTest
 
 
-class SystemTest(ToilTest):
+class TestSystem:
     """Test various assumptions about the operating system's behavior."""
 
-    def testAtomicityOfNonEmptyDirectoryRenames(self):
+    def testAtomicityOfNonEmptyDirectoryRenames(self, tmp_path: Path) -> None:
         for _ in range(100):
-            parent = self._createTempDir(purpose="parent")
-            child = os.path.join(parent, "child")
+            parent = tmp_path / f"parent{_}"
+            parent.mkdir()
+            child = parent / "child"
             # Use processes (as opposed to threads) to prevent GIL from ordering things artificially
             pool = multiprocessing.Pool(processes=cpu_count())
             try:
                 numTasks = cpu_count() * 10
-                grandChildIds = pool.map_async(
+                temp_grandChildIds = pool.map_async(
                     func=partial(
                         _testAtomicityOfNonEmptyDirectoryRenamesTask, parent, child
                     ),
                     iterable=list(range(numTasks)),
                 )
-                grandChildIds = grandChildIds.get()
+                grandChildIds = temp_grandChildIds.get()
             finally:
                 pool.close()
                 pool.join()
-            self.assertEqual(len(grandChildIds), numTasks)
+            assert len(grandChildIds) == numTasks
             # Assert that we only had one winner
             grandChildIds = [n for n in grandChildIds if n is not None]
-            self.assertEqual(len(grandChildIds), 1)
+            assert len(grandChildIds) == 1
             # Assert that the winner's grandChild wasn't silently overwritten by a looser
             expectedGrandChildId = grandChildIds[0]
-            actualGrandChild = os.path.join(child, "grandChild")
-            actualGrandChildId = os.stat(actualGrandChild).st_ino
-            self.assertEqual(actualGrandChildId, expectedGrandChildId)
+            actualGrandChild = child / "grandChild"
+            actualGrandChildId = actualGrandChild.stat().st_ino
+            assert actualGrandChildId == expectedGrandChildId
 
 
-def _testAtomicityOfNonEmptyDirectoryRenamesTask(parent, child, _):
+def _testAtomicityOfNonEmptyDirectoryRenamesTask(
+    parent: Path, child: Path, _: Any
+) -> Optional[int]:
     tmpChildDir = mkdtemp(dir=parent, prefix="child", suffix=".tmp")
     grandChild = os.path.join(tmpChildDir, "grandChild")
     open(grandChild, "w").close()

toil/test/src/threadingTest.py

@@ -1,100 +1,100 @@
 import logging
 import multiprocessing
 import os
+from pathlib import Path
 import random
 import time
 import traceback
 from functools import partial
 
 from toil.lib.threading import LastProcessStandingArena, cpu_count, global_mutex
-from toil.test import ToilTest
 
 log = logging.getLogger(__name__)
 
 
-class ThreadingTest(ToilTest):
+class TestThreading:
     """Test Toil threading/synchronization tools."""
 
-    def testGlobalMutexOrdering(self):
+    def testGlobalMutexOrdering(self, tmp_path: Path) -> None:
         for it in range(10):
             log.info("Iteration %d", it)
 
-            scope = self._createTempDir()
+            scope = tmp_path / f"tempDir{it}"
+            scope.mkdir()
             mutex = "mutex"
             # Use processes (as opposed to threads) to prevent GIL from ordering things artificially
             pool = multiprocessing.Pool(processes=cpu_count())
             try:
                 numTasks = 100
-                results = pool.map_async(
+                temp_results = pool.map_async(
                     func=partial(_testGlobalMutexOrderingTask, scope, mutex),
                     iterable=list(range(numTasks)),
                 )
-                results = results.get()
+                results = temp_results.get()
             finally:
                 pool.close()
                 pool.join()
 
-            self.assertEqual(len(results), numTasks)
+            assert len(results) == numTasks
             for item in results:
                 # Make sure all workers say they succeeded
-                self.assertEqual(item, True)
+                assert item is True
 
-    def testLastProcessStanding(self):
+    def testLastProcessStanding(self, tmp_path: Path) -> None:
         for it in range(10):
             log.info("Iteration %d", it)
 
-            scope = self._createTempDir()
+            scope = tmp_path / f"tempDir{it}"
+            scope.mkdir()
             arena_name = "thunderdome"
             # Use processes (as opposed to threads) to prevent GIL from ordering things artificially
             pool = multiprocessing.Pool(processes=cpu_count())
             try:
                 numTasks = 100
-                results = pool.map_async(
+                temp_results = pool.map_async(
                     func=partial(_testLastProcessStandingTask, scope, arena_name),
                     iterable=list(range(numTasks)),
                 )
-                results = results.get()
+                results = temp_results.get()
             finally:
                 pool.close()
                 pool.join()
 
-            self.assertEqual(len(results), numTasks)
+            assert len(results) == numTasks
             for item in results:
                 # Make sure all workers say they succeeded
-                self.assertEqual(item, True)
+                assert item is True
             for filename in os.listdir(scope):
                 assert not filename.startswith(
                     "precious"
                 ), f"File {filename} still exists"
 
 
-def _testGlobalMutexOrderingTask(scope, mutex, number):
+def _testGlobalMutexOrderingTask(scope: Path, mutex: str, number: int) -> bool:
     try:
         # We will all fight over the potato
-        potato = os.path.join(scope, "potato")
+        potato = scope / "potato"
 
         with global_mutex(scope, mutex):
             log.info("PID %d = num %d running", os.getpid(), number)
-            assert not os.path.exists(
-                potato
-            ), "We see someone else holding the potato file"
+            assert not potato.exists(), "We see someone else holding the potato file"
 
             # Put our name there
-            with open(potato, "w") as out_stream:
+            with potato.open("w") as out_stream:
                 out_stream.write(str(number))
 
             # Wait
             time.sleep(random.random() * 0.01)
 
             # Make sure our name is still there
-            with open(potato) as in_stream:
+            with potato.open() as in_stream:
                 seen = in_stream.read().rstrip()
                 assert seen == str(
                     number
                 ), f"We are {number} but {seen} stole our potato!"
 
-            os.unlink(potato)
-            assert not os.path.exists(potato), "We left the potato behind"
+            potato.unlink()
+            assert not potato.exists(), "We left the potato behind"
             log.info("PID %d = num %d dropped potato", os.getpid(), number)
         return True
     except:
@@ -102,7 +102,7 @@ def _testGlobalMutexOrderingTask(scope, mutex, number):
         return False
 
 
-def _testLastProcessStandingTask(scope, arena_name, number):
+def _testLastProcessStandingTask(scope: Path, arena_name: str, number: int) -> bool:
     try:
         arena = LastProcessStandingArena(scope, arena_name)
 
@@ -110,20 +110,18 @@ def _testLastProcessStandingTask(scope, arena_name, number):
         log.info("PID %d = num %d entered arena", os.getpid(), number)
         try:
             # We all make files
-            my_precious = os.path.join(scope, "precious" + str(number))
+            my_precious = scope / f"precious{number}"
 
             # Put our name there
-            with open(my_precious, "w") as out_stream:
+            with my_precious.open("w") as out_stream:
                 out_stream.write(str(number))
 
             # Wait
             time.sleep(random.random() * 0.01)
 
             # Make sure our file is still there unmodified
-            assert os.path.exists(
-                my_precious
-            ), f"Precious file {my_precious} has been stolen!"
-            with open(my_precious) as in_stream:
+            assert my_precious.exists(), f"Precious file {my_precious} has been stolen!"
+            with my_precious.open() as in_stream:
                 seen = in_stream.read().rstrip()
                 assert seen == str(
                     number
@@ -131,7 +129,6 @@ def _testLastProcessStandingTask(scope, arena_name, number):
         finally:
             was_last = False
             for _ in arena.leave():
-                was_last = True
                 log.info("PID %d = num %d is last standing", os.getpid(), number)
 
                 # Clean up all the files
@@ -143,7 +140,7 @@ def _testLastProcessStandingTask(scope, arena_name, number):
                             number,
                             filename,
                         )
-                        os.unlink(os.path.join(scope, filename))
+                        (scope / filename).unlink()
 
             log.info("PID %d = num %d left arena", os.getpid(), number)
 

toil/test/utils/ABCWorkflowDebug/B_file.txt

@@ -0,0 +1 @@
+B