toil 8.1.0b1__py3-none-any.whl → 8.2.0__py3-none-any.whl

toil/test/src/regularLogTest.py

@@ -14,34 +14,34 @@
 import logging
 import mimetypes
 import os
+from pathlib import Path
 import subprocess
 import sys
+from typing import Optional
 
-from toil.test import ToilTest, slow
+from toil.test import pslow as slow
 from toil.test.mesos import helloWorld
 
 logging.basicConfig(level=logging.DEBUG)
 logger = logging.getLogger(__name__)
 
 
-class RegularLogTest(ToilTest):
+class RegularLogTest:
 
-    def setUp(self) -> None:
-        super().setUp()
-        self.tempDir = self._createTempDir(purpose="tempDir")
-
-    def _getFiles(self, dir):
+    def _getFiles(self, dirpath: Path) -> list[str]:
         return [
-            os.path.join(dir, f)
-            for f in os.listdir(dir)
-            if os.path.isfile(os.path.join(dir, f))
+            os.path.join(dirpath, f)
+            for f in os.listdir(dirpath)
+            if os.path.isfile(os.path.join(dirpath, f))
         ]
 
-    def _assertFileTypeExists(self, dir, extension, encoding=None):
+    def _assertFileTypeExists(
+        self, dirpath: Path, extension: str, encoding: Optional[str] = None
+    ) -> None:
         # an encoding of None implies no compression
-        logger.info("Checking for %s file in %s", extension, dir)
-        onlyFiles = self._getFiles(dir)
-        logger.info("Found: %s", str(os.listdir(dir)))
+        logger.info("Checking for %s file in %s", extension, dirpath)
+        onlyFiles = self._getFiles(dirpath)
+        logger.info("Found: %s", str(os.listdir(dirpath)))
         onlyLogs = [f for f in onlyFiles if f.endswith(extension)]
         logger.info("Found matching: %s", str(onlyLogs))
         assert onlyLogs
@@ -57,10 +57,10 @@ class RegularLogTest(ToilTest):
                         assert f.read().startswith(b"\x1f\x8b")
                     else:
                         mime = mimetypes.guess_type(log)
-                        self.assertEqual(mime[1], encoding)
+                        assert mime[1] == encoding
 
     @slow
-    def testLogToMaster(self):
+    def testLogToMaster(self) -> None:
         toilOutput = subprocess.check_output(
             [
                 sys.executable,
@@ -74,7 +74,7 @@ class RegularLogTest(ToilTest):
         )
         assert helloWorld.childMessage in toilOutput.decode("utf-8")
 
-    def testWriteLogs(self):
+    def testWriteLogs(self, tmp_path: Path) -> None:
         subprocess.check_call(
             [
                 sys.executable,
@@ -83,13 +83,13 @@ class RegularLogTest(ToilTest):
                 "./toilTest",
                 "--clean=always",
                 "--logLevel=debug",
-                "--writeLogs=%s" % self.tempDir,
+                f"--writeLogs={tmp_path}",
             ]
         )
-        self._assertFileTypeExists(self.tempDir, ".log")
+        self._assertFileTypeExists(tmp_path, ".log")
 
     @slow
-    def testWriteGzipLogs(self):
+    def testWriteGzipLogs(self, tmp_path: Path) -> None:
         subprocess.check_call(
             [
                 sys.executable,
@@ -98,13 +98,13 @@ class RegularLogTest(ToilTest):
                 "./toilTest",
                 "--clean=always",
                 "--logLevel=debug",
-                "--writeLogsGzip=%s" % self.tempDir,
+                f"--writeLogsGzip={tmp_path}",
             ]
         )
-        self._assertFileTypeExists(self.tempDir, ".log.gz", "gzip")
+        self._assertFileTypeExists(tmp_path, ".log.gz", "gzip")
 
     @slow
-    def testMultipleLogToMaster(self):
+    def testMultipleLogToMaster(self) -> None:
         toilOutput = subprocess.check_output(
             [
                 sys.executable,
@@ -118,7 +118,7 @@ class RegularLogTest(ToilTest):
         )
         assert helloWorld.parentMessage in toilOutput.decode("utf-8")
 
-    def testRegularLog(self):
+    def testRegularLog(self) -> None:
         toilOutput = subprocess.check_output(
             [
                 sys.executable,

toil/test/src/resourceTest.py

@@ -11,8 +11,10 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+from collections.abc import Iterable
 import importlib
 import os
+from pathlib import Path
 import subprocess
 import sys
 import tempfile
@@ -22,44 +24,38 @@ from io import BytesIO
 from textwrap import dedent
 from unittest.mock import MagicMock, patch
 from zipfile import ZipFile
+from typing import Optional
 
 from toil import inVirtualEnv
 from toil.resource import ModuleDescriptor, Resource, ResourceException
-from toil.test import ToilTest
 from toil.version import exactPython
 
+import pytest
 
-@contextmanager
-def tempFileContaining(content, suffix=""):
+
+def tempFileContaining(directory: Path, content: str, suffix: str = "") -> str:
     """
     Write a file with the given contents, and keep it on disk as long as the context is active.
     :param str content: The contents of the file.
     :param str suffix: The extension to use for the temporary file.
     """
-    fd, path = tempfile.mkstemp(suffix=suffix)
-    try:
+    with (directory / "temp").open("wb") as fd:
         encoded = content.encode("utf-8")
-        assert os.write(fd, encoded) == len(encoded)
-    except:
-        os.close(fd)
-        raise
-    else:
-        os.close(fd)
-        yield path
-    finally:
-        os.unlink(path)
+        assert fd.write(encoded) == len(encoded)
+    return str(directory / "temp")
 
 
-class ResourceTest(ToilTest):
+class TestResource:
     """Test module descriptors and resources derived from them."""
 
-    def testStandAlone(self):
+    def testStandAlone(self, tmp_path: Path) -> None:
         self._testExternal(
-            moduleName="userScript", pyFiles=("userScript.py", "helper.py")
+            tmp_path, moduleName="userScript", pyFiles=("userScript.py", "helper.py")
         )
 
-    def testPackage(self):
+    def testPackage(self, tmp_path: Path) -> None:
         self._testExternal(
+            tmp_path,
             moduleName="foo.userScript",
             pyFiles=(
                 "foo/__init__.py",
@@ -69,8 +65,9 @@ class ResourceTest(ToilTest):
             ),
         )
 
-    def testVirtualEnv(self):
+    def testVirtualEnv(self, tmp_path: Path) -> None:
         self._testExternal(
+            tmp_path,
             moduleName="foo.userScript",
             virtualenv=True,
             pyFiles=(
@@ -84,34 +81,45 @@ class ResourceTest(ToilTest):
             ),
         )
 
-    def testStandAloneInPackage(self):
-        self.assertRaises(
-            ResourceException,
-            self._testExternal,
-            moduleName="userScript",
-            pyFiles=("__init__.py", "userScript.py", "helper.py"),
-        )
+    def testStandAloneInPackage(self, tmp_path: Path) -> None:
+        with pytest.raises(ResourceException):
+            self._testExternal(
+                tmp_path,
+                moduleName="userScript",
+                pyFiles=("__init__.py", "userScript.py", "helper.py"),
+            )
 
-    def _testExternal(self, moduleName, pyFiles, virtualenv=False):
-        dirPath = self._createTempDir()
+    def _testExternal(
+        self,
+        dirPath: Path,
+        moduleName: str,
+        pyFiles: Iterable[str],
+        virtualenv: bool = False,
+    ) -> None:
         if virtualenv:
-            self.assertTrue(inVirtualEnv())
+            assert inVirtualEnv()
             # --never-download prevents silent upgrades to pip, wheel and setuptools
             subprocess.check_call(
-                ["virtualenv", "--never-download", "--python", exactPython, dirPath]
+                [
+                    "virtualenv",
+                    "--never-download",
+                    "--python",
+                    exactPython,
+                    str(dirPath),
+                ]
             )
-            sitePackages = os.path.join(dirPath, "lib", exactPython, "site-packages")
+            sitePackages = dirPath / "lib" / exactPython / "site-packages"
             # tuple assignment is necessary to make this line immediately precede the try:
-            oldPrefix, sys.prefix, dirPath = sys.prefix, dirPath, sitePackages
+            oldPrefix, sys.prefix, dirPath = sys.prefix, str(dirPath), sitePackages
         else:
             oldPrefix = None
         try:
             for relPath in pyFiles:
-                path = os.path.join(dirPath, relPath)
-                os.makedirs(os.path.dirname(path), exist_ok=True)
-                with open(path, "w") as f:
+                path = dirPath / relPath
+                path.parent.mkdir(parents=True, exist_ok=True)
+                with path.open("w") as f:
                     f.write("pass\n")
-            sys.path.append(dirPath)
+            sys.path.append(str(dirPath))
             try:
                 userScript = importlib.import_module(moduleName)
                 try:
@@ -124,39 +132,39 @@ class ResourceTest(ToilTest):
                     del userScript
                     while moduleName:
                         del sys.modules[moduleName]
-                        self.assertFalse(moduleName in sys.modules)
+                        assert moduleName not in sys.modules
                         moduleName = ".".join(moduleName.split(".")[:-1])
 
             finally:
-                sys.path.remove(dirPath)
+                sys.path.remove(str(dirPath))
         finally:
             if oldPrefix:
                 sys.prefix = oldPrefix
 
-    def testBuiltIn(self):
+    def testBuiltIn(self) -> None:
         # Create a ModuleDescriptor for the module containing ModuleDescriptor, i.e. toil.resource
         module_name = ModuleDescriptor.__module__
-        self.assertEqual(module_name, "toil.resource")
+        assert module_name == "toil.resource"
         self._test(module_name, shouldBelongToToil=True)
 
     def _test(
         self,
-        module_name,
-        shouldBelongToToil=False,
-        expectedContents=None,
-        allowExtraContents=True,
-    ):
+        module_name: str,
+        shouldBelongToToil: bool = False,
+        expectedContents: Optional[Iterable[str]] = None,
+        allowExtraContents: bool = True,
+    ) -> None:
         module = ModuleDescriptor.forModule(module_name)
         # Assert basic attributes and properties
-        self.assertEqual(module.belongsToToil, shouldBelongToToil)
-        self.assertEqual(module.name, module_name)
+        assert module.belongsToToil == shouldBelongToToil
+        assert module.name == module_name
         if shouldBelongToToil:
-            self.assertTrue(module.dirPath.endswith("/src"))
+            assert module.dirPath.endswith("/src")
 
         # Before the module is saved as a resource, localize() and globalize() are identity
         # methods. This should log.warnings.
-        self.assertIs(module.localize(), module)
-        self.assertIs(module.globalize(), module)
+        assert module.localize() is module
+        assert module.globalize() is module
         # Create a mock job store ...
         jobStore = MagicMock()
         # ... to generate a fake URL for the resource ...
@@ -185,18 +193,18 @@ class ResourceTest(ToilTest):
         # contents. This is a bit brittle since it assumes that all the data is written in a
         # single call to write(). If more calls are made we can easily concatenate them.
         zipFile = file_handle.write.call_args_list[0][0][0]
-        self.assertTrue(zipFile.startswith(b"PK"))  # the magic header for ZIP files
+        assert zipFile.startswith(b"PK")  # the magic header for ZIP files
 
         # Check contents if requested
         if expectedContents is not None:
             with ZipFile(BytesIO(zipFile)) as _zipFile:
                 actualContents = set(_zipFile.namelist())
                 if allowExtraContents:
-                    self.assertTrue(actualContents.issuperset(expectedContents))
+                    assert actualContents.issuperset(expectedContents)
                 else:
-                    self.assertEqual(actualContents, expectedContents)
+                    assert actualContents == expectedContents
 
-        self.assertEqual(resource.url, url)
+        assert resource.url == url
         # Now we're on the worker. Prepare the storage for localized resources
         Resource.prepareSystem()
         try:
@@ -206,8 +214,8 @@ class ResourceTest(ToilTest):
             # original resource. Lookup will also be used when we localize the module that was
             # originally used to create the resource.
             localResource = Resource.lookup(module._resourcePath)
-            self.assertEqual(resource, localResource)
-            self.assertIsNot(resource, localResource)
+            assert resource == localResource
+            assert resource is not localResource
             # Now show that we can localize the module using the registered resource. Set up a mock
             # urlopen() that yields the zipped tree ...
             mock_urlopen = MagicMock()
@@ -216,16 +224,16 @@ class ResourceTest(ToilTest):
                 # ... and use it to download and unpack the resource
                 localModule = module.localize()
             # The name should be equal between original and localized resource ...
-            self.assertEqual(module.name, localModule.name)
+            assert module.name == localModule.name
             # ... but the directory should be different.
-            self.assertNotEqual(module.dirPath, localModule.dirPath)
+            assert module.dirPath != localModule.dirPath
             # Show that we can 'undo' localization. This is necessary when the user script's jobs
             #  are invoked on the worker where they generate more child jobs.
-            self.assertEqual(localModule.globalize(), module)
+            assert localModule.globalize() == module
         finally:
             Resource.cleanSystem()
 
-    def testNonPyStandAlone(self):
+    def testNonPyStandAlone(self, tmp_path: Path) -> None:
         """
         Asserts that Toil enforces the user script to have a .py or .pyc extension because that's
         the only way auto-deployment can re-import the module on a worker. See
@@ -234,13 +242,13 @@ class ResourceTest(ToilTest):
         https://github.com/BD2KGenomics/toil/issues/858
         """
 
-        def script():
+        def script() -> None:
             from configargparse import ArgumentParser
 
             from toil.common import Toil
             from toil.job import Job
 
-            def fn():
+            def fn() -> None:
                 pass
 
             if __name__ == "__main__":
@@ -253,17 +261,15 @@ class ResourceTest(ToilTest):
 
         scriptBody = dedent("\n".join(getsource(script).split("\n")[1:]))
         shebang = "#! %s\n" % sys.executable
-        with tempFileContaining(shebang + scriptBody) as scriptPath:
-            self.assertFalse(scriptPath.endswith((".py", ".pyc")))
-            os.chmod(scriptPath, 0o755)
-            jobStorePath = scriptPath + ".jobStore"
-            process = subprocess.Popen(
-                [scriptPath, jobStorePath], stderr=subprocess.PIPE
-            )
-            stdout, stderr = process.communicate()
-            self.assertTrue(
-                "The name of a user script/module must end in .py or .pyc."
-                in stderr.decode("utf-8")
-            )
-            self.assertNotEqual(0, process.returncode)
-            self.assertFalse(os.path.exists(jobStorePath))
+        scriptPath = tempFileContaining(tmp_path, shebang + scriptBody)
+        assert not scriptPath.endswith((".py", ".pyc"))
+        os.chmod(scriptPath, 0o755)
+        jobStorePath = scriptPath + ".jobStore"
+        process = subprocess.Popen([scriptPath, jobStorePath], stderr=subprocess.PIPE)
+        stdout, stderr = process.communicate()
+        assert (
+            "The name of a user script/module must end in .py or .pyc."
+            in stderr.decode("utf-8")
+        )
+        assert 0 != process.returncode
+        assert not os.path.exists(jobStorePath)

toil/test/src/restartDAGTest.py

@@ -15,41 +15,43 @@
 
 import logging
 import os
-import shutil
+from pathlib import Path
 import signal
+from typing import Optional
 
 from toil.common import Toil
 from toil.exceptions import FailedJobsException
 from toil.job import Job
-from toil.test import ToilTest, slow
+from toil.test import pslow as slow
+
+import pytest
 
 logger = logging.getLogger(__name__)
 
 
-class RestartDAGTest(ToilTest):
+class TestRestartDAG:
     """
     Tests that restarted job DAGs don't run children of jobs that failed in the first run till the
     parent completes successfully in the restart.
     """
 
-    def setUp(self):
-        super().setUp()
-        self.tempDir = self._createTempDir(purpose="tempDir")
-        self.testJobStore = self._getTestJobStorePath()
-
-    def tearDown(self):
-        super().tearDown()
-        shutil.rmtree(self.testJobStore)
-
     @slow
-    def testRestartedWorkflowSchedulesCorrectJobsOnFailedParent(self):
-        self._testRestartedWorkflowSchedulesCorrectJobs("raise")
+    @pytest.mark.slow
+    def testRestartedWorkflowSchedulesCorrectJobsOnFailedParent(
+        self, tmp_path: Path
+    ) -> None:
+        self._testRestartedWorkflowSchedulesCorrectJobs(tmp_path, "raise")
 
     @slow
-    def testRestartedWorkflowSchedulesCorrectJobsOnKilledParent(self):
-        self._testRestartedWorkflowSchedulesCorrectJobs("kill")
-
-    def _testRestartedWorkflowSchedulesCorrectJobs(self, failType):
+    @pytest.mark.slow
+    def testRestartedWorkflowSchedulesCorrectJobsOnKilledParent(
+        self, tmp_path: Path
+    ) -> None:
+        self._testRestartedWorkflowSchedulesCorrectJobs(tmp_path, "kill")
+
+    def _testRestartedWorkflowSchedulesCorrectJobs(
+        self, tmp_path: Path, failType: str
+    ) -> None:
         """
         Creates a diamond DAG
             /->passingParent-\
@@ -63,19 +65,19 @@ class RestartDAGTest(ToilTest):
         :param str failType: Does failingParent fail on an assertionError, or is it killed.
         """
         # Specify options
-        options = Job.Runner.getDefaultOptions(self.testJobStore)
+        options = Job.Runner.getDefaultOptions(tmp_path / "jobstore")
         options.logLevel = "DEBUG"
         options.retryCount = 0
         options.clean = "never"
 
-        parentFile = os.path.join(self.tempDir, "parent")
-        childFile = os.path.join(self.tempDir, "child")
+        parentFile = tmp_path / "parent"
+        childFile = tmp_path / "child"
 
         # Make the first job
         root = Job.wrapJobFn(passingFn)
         passingParent = Job.wrapJobFn(passingFn)
-        failingParent = Job.wrapJobFn(failingFn, failType=failType, fileName=parentFile)
-        child = Job.wrapJobFn(passingFn, fileName=childFile)
+        failingParent = Job.wrapJobFn(failingFn, failType=failType, file=parentFile)
+        child = Job.wrapJobFn(passingFn, file=childFile)
 
         # define the DAG
         root.addChild(passingParent)
@@ -85,11 +87,12 @@ class RestartDAGTest(ToilTest):
 
         failReasons = []
 
-        assert not os.path.exists(childFile)
+        assert not childFile.exists()
 
+        errorRaised: Optional[BaseException] = None
         # Run the test
         for runMode in "start", "restart":
-            self.errorRaised = None
+            errorRaised = None
             try:
                 with Toil(options) as toil:
                     if runMode == "start":
@@ -98,70 +101,71 @@ class RestartDAGTest(ToilTest):
                         toil.restart()
             except Exception as e:
                 logger.exception(e)
-                self.errorRaised = e
+                errorRaised = e
             finally:
                 # The processing of an AssertionError and FailedJobsException is the same so we do
                 # it together in this finally clause.
-                if self.errorRaised is not None:
-                    if not os.path.exists(parentFile):
+                if errorRaised is not None:
+                    if not parentFile.exists():
                         failReasons.append(
                             'The failing parent file did not exist on toil "%s".'
                             % runMode
                         )
-                    if os.path.exists(childFile):
+                    if childFile.exists():
                         failReasons.append(
                             "The child file existed.  i.e. the child was run on "
                             'toil "%s".' % runMode
                         )
-                    if isinstance(self.errorRaised, FailedJobsException):
-                        if self.errorRaised.numberOfFailedJobs != 3:
+                    if isinstance(errorRaised, FailedJobsException):
+                        if errorRaised.numberOfFailedJobs != 3:
                             failReasons.append(
                                 'FailedJobsException was raised on toil "%s" but '
                                 "the number of failed jobs (%s) was not 3."
-                                % (runMode, self.errorRaised.numberOfFailedJobs)
+                                % (runMode, errorRaised.numberOfFailedJobs)
                             )
-                    elif isinstance(self.errorRaised, AssertionError):
+                    elif isinstance(errorRaised, AssertionError):
                         failReasons.append(
                             "Toil raised an AssertionError instead of a "
                             'FailedJobsException on toil "%s".' % runMode
                         )
                     else:
-                        failReasons.append("Toil raised error: %s" % self.errorRaised)
-                    self.errorRaised = None
+                        failReasons.append("Toil raised error: %s" % errorRaised)
+                    errorRaised = None
                     options.restart = True
                 else:
-                    self.fail('No errors were raised on toil "%s".' % runMode)
+                    pytest.fail('No errors were raised on toil "%s".' % runMode)
         if failReasons:
-            self.fail(
+            pytest.fail(
                 "Test failed for ({}) reasons:\n\t{}".format(
                     len(failReasons), "\n\t".join(failReasons)
                 )
             )
 
 
-def passingFn(job, fileName=None):
+def passingFn(job: Job, file: Optional[Path] = None) -> None:
     """
-    This function is guaranteed to pass as it does nothing out of the ordinary.  If fileName is
-    provided, it will be created.
+    This function is guaranteed to pass as it does nothing out of the ordinary.
+
+    If "file" is provided, it will be created.
 
-    :param str fileName: The name of a file that must be created if provided.
+    :param file: The path of a file that must be created if provided.
     """
-    if fileName is not None:
+    if file is not None:
         # Emulates system touch.
-        open(fileName, "w").close()
+        file.open("w").close()
 
 
-def failingFn(job, failType, fileName):
+def failingFn(job: Job, failType: str, file: Path) -> None:
     """
     This function is guaranteed to fail via a raised assertion, or an os.kill
 
     :param job: Job
-    :param str failType: 'raise' or 'kill
-    :param str fileName: The name of a file that must be created.
+    :param failType: 'raise' or 'kill
+    :param file: The path of a file that must be created.
     """
     assert failType in ("raise", "kill")
     # Use that function to avoid code redundancy
-    passingFn(job, fileName)
+    passingFn(job, file)
 
     if failType == "raise":
         assert False