toil 8.0.0__py3-none-any.whl → 8.2.0__py3-none-any.whl

toil/test/src/importExportFileTest.py

@@ -13,54 +13,58 @@
 # limitations under the License.
 
 
+from argparse import Namespace
 import os
+from pathlib import Path
 import stat
 import uuid
 
 from toil.common import Toil
 from toil.exceptions import FailedJobsException
 from toil.fileStores import FileID
+from toil.fileStores.abstractFileStore import AbstractFileStore
 from toil.job import Job
-from toil.test import ToilTest, slow
+from toil.test import pslow as slow
 
+import pytest
+from pytest_subtests import SubTests
 
-class ImportExportFileTest(ToilTest):
-    def setUp(self):
-        super().setUp()
-        self.tmp_dir = self._createTempDir()
-        self.output_file_path = f"{self.tmp_dir}/out"
-        self.message_portion_1 = (
-            "What do you get when you cross a seal and a polar bear?"
-        )
-        self.message_portion_2 = "  A polar bear."
 
-    def create_file(self, content, executable=False):
-        file_path = f"{self.tmp_dir}/{uuid.uuid4()}"
+def create_file(tmp_path: Path, content: str, executable: bool = False) -> Path:
+    file_path = tmp_path / str(uuid.uuid4())
 
-        with open(file_path, "w") as f:
-            f.write(content)
+    with file_path.open("w") as f:
+        f.write(content)
 
-        if executable:
-            # Add file owner execute permissions
-            os.chmod(file_path, os.stat(file_path).st_mode | stat.S_IXUSR)
+    if executable:
+        # Add file owner execute permissions
+        file_path.chmod(file_path.stat().st_mode | stat.S_IXUSR)
 
-        return file_path
+    return file_path
 
-    def _import_export_workflow(self, options, fail):
+
+class TestImportExportFile:
+    message_portion_1 = "What do you get when you cross a seal and a polar bear?"
+    message_portion_2 = "  A polar bear."
+
+    def _import_export_workflow(
+        self, tmp_path: Path, options: Namespace, fail: bool
+    ) -> None:
         with Toil(options) as toil:
             if not options.restart:
-                msg_portion_file_path = self.create_file(content=self.message_portion_1)
-                msg_portion_file_id = toil.importFile(f"file://{msg_portion_file_path}")
-                self.assertIsInstance(msg_portion_file_id, FileID)
-                self.assertEqual(
-                    os.stat(msg_portion_file_path).st_size, msg_portion_file_id.size
+                msg_portion_file_path = create_file(
+                    tmp_path, content=self.message_portion_1
                 )
+                msg_portion_file_id = toil.importFile(msg_portion_file_path.as_uri())
+                assert isinstance(msg_portion_file_id, FileID)
+                assert msg_portion_file_path.stat().st_size == msg_portion_file_id.size
 
-                file_that_can_trigger_failure_when_job_starts = self.create_file(
-                    content="Time to freak out!" if fail else "Keep calm and carry on."
+                file_that_can_trigger_failure_when_job_starts = create_file(
+                    tmp_path,
+                    content="Time to freak out!" if fail else "Keep calm and carry on.",
                 )
                 self.trigger_file_id = toil.importFile(
-                    f"file://{file_that_can_trigger_failure_when_job_starts}"
+                    file_that_can_trigger_failure_when_job_starts.as_uri()
                 )
                 workflow_final_output_file_id = toil.start(
                     RestartingJob(
@@ -81,89 +85,93 @@ class ImportExportFileTest(ToilTest):
 
                 workflow_final_output_file_id = toil.restart()
 
-            toil.exportFile(
-                workflow_final_output_file_id, f"file://{self.output_file_path}"
-            )
-            with open(self.output_file_path) as f:
-                self.assertEqual(
-                    f.read(), f"{self.message_portion_1}{self.message_portion_2}"
-                )
+            toil.exportFile(workflow_final_output_file_id, str(tmp_path / "out"))
+            with (tmp_path / "out").open() as f:
+                assert f.read() == f"{self.message_portion_1}{self.message_portion_2}"
 
-    def _run_import_export_workflow(self, restart):
-        options = Job.Runner.getDefaultOptions(self._getTestJobStorePath())
+    def _run_import_export_workflow(self, tmp_path: Path, restart: bool) -> None:
+        options = Job.Runner.getDefaultOptions(tmp_path / "jobstore")
         options.logLevel = "INFO"
 
         if restart:
             try:
-                self._import_export_workflow(options, fail=True)
+                self._import_export_workflow(tmp_path, options, fail=True)
             except FailedJobsException:
                 options.restart = True
 
-        self._import_export_workflow(options, fail=False)
+        self._import_export_workflow(tmp_path, options, fail=False)
 
     @slow
-    def test_import_export_restart_true(self):
-        self._run_import_export_workflow(restart=True)
+    @pytest.mark.slow
+    def test_import_export_restart_true(self, tmp_path: Path) -> None:
+        self._run_import_export_workflow(tmp_path, restart=True)
 
-    def test_import_export_restart_false(self):
-        self._run_import_export_workflow(restart=False)
+    def test_import_export_restart_false(self, tmp_path: Path) -> None:
+        self._run_import_export_workflow(tmp_path, restart=False)
 
-    def test_basic_import_export(self):
+    def test_basic_import_export(self, tmp_path: Path, subtests: SubTests) -> None:
         """
         Ensures that uploaded files preserve their file permissions when they
         are downloaded again. This function checks that an imported executable file
         maintains its executability after being exported.
         """
-        options = Job.Runner.getDefaultOptions(self._getTestJobStorePath())
+        options = Job.Runner.getDefaultOptions(tmp_path / "jobstore")
         options.logLevel = "INFO"
 
         with Toil(options) as toil:
             # TODO: test this with non-local (AWS, Google)
             #  Note: this is somewhat done in src/toil/test/src/fileStoreTest.py
-            with self.subTest(
-                "Testing permissions are preserved for local importFile/exportFile"
+
+            with subtests.test(
+                msg="Testing permissions are preserved for local importFile/exportFile"
             ):
                 for executable in True, False:
-                    file_path = self.create_file(content="Hello", executable=executable)
-                    initial_permissions = os.stat(file_path).st_mode & stat.S_IXUSR
-                    file_id = toil.importFile(f"file://{file_path}")
-                    toil.exportFile(file_id, f"file://{self.output_file_path}")
-                    current_permissions = (
-                        os.stat(self.output_file_path).st_mode & stat.S_IXUSR
+                    file_path = create_file(
+                        tmp_path, content="Hello", executable=executable
                     )
+                    initial_permissions = file_path.stat().st_mode & stat.S_IXUSR
+                    file_id = toil.importFile(file_path.as_uri())
+                    output_file_path = tmp_path / f"out_{executable}"
+                    toil.exportFile(file_id, output_file_path.as_uri())
+                    current_permissions = output_file_path.stat().st_mode & stat.S_IXUSR
                     assert initial_permissions == current_permissions
 
-            with self.subTest("Testing relative paths without the file:// schema."):
+            with subtests.test(
+                msg="Testing relative paths without the file:// schema."
+            ):
                 relative_path_data = "Everything is relative."
-                file_path = self.create_file(content=relative_path_data)
+                file_path = create_file(tmp_path, content=relative_path_data)
 
                 file_id = toil.importFile(os.path.relpath(file_path))
-                toil.exportFile(file_id, os.path.relpath(self.output_file_path))
-                with open(self.output_file_path) as f:
-                    self.assertEqual(f.read(), relative_path_data)
+                output_file_path = tmp_path / "out2"
+                toil.exportFile(file_id, os.path.relpath(output_file_path))
+                with output_file_path.open() as f:
+                    assert f.read() == relative_path_data
 
-            with self.subTest("Test local importFile accepts a shared_file_name."):
+            with subtests.test(msg="Test local importFile accepts a shared_file_name."):
                 # TODO: whyyyy do we allow this?  shared file names are not unique and can overwrite each other
                 #  ...not only that... we can't use exportFile on them afterwards!?
-                file_path = self.create_file(content="why")
+                file_path = create_file(tmp_path, content="why")
                 shared_file_name = (
                     "users_should_probably_not_be_allowed_to_make_shared_files.bad"
                 )
-                toil.importFile(f"file://{file_path}", sharedFileName=shared_file_name)
+                toil.importFile(file_path.as_uri(), sharedFileName=shared_file_name)
                 with toil._jobStore.read_shared_file_stream(
                     shared_file_name, encoding="utf-8"
                 ) as f:
-                    self.assertEqual(f.read(), "why")
+                    assert f.read() == "why"
 
 
 class RestartingJob(Job):
-    def __init__(self, msg_portion_file_id, trigger_file_id, message_portion_2):
+    def __init__(
+        self, msg_portion_file_id: str, trigger_file_id: str, message_portion_2: str
+    ) -> None:
         Job.__init__(self, memory=100000, cores=1, disk="1M")
         self.msg_portion_file_id = msg_portion_file_id
         self.trigger_file_id = trigger_file_id
         self.message_portion_2 = message_portion_2
 
-    def run(self, file_store):
+    def run(self, file_store: AbstractFileStore) -> str:
         with file_store.readGlobalFileStream(self.trigger_file_id) as readable:
             if readable.read() == b"Time to freak out!":
                 raise RuntimeError("D:")

toil/test/src/jobEncapsulationTest.py

@@ -11,43 +11,42 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-import os
+from pathlib import Path
 
-from toil.job import Job
-from toil.test import ToilTest, get_temp_file
+from toil.job import Job, Promise
 from toil.test.src.jobTest import fn1Test
 
 
-class JobEncapsulationTest(ToilTest):
+class TestJobEncapsulation:
     """Tests testing the EncapsulationJob class."""
 
-    def testEncapsulation(self):
+    def testEncapsulation(self, tmp_path: Path) -> None:
         """
         Tests the Job.encapsulation method, which uses the EncapsulationJob
         class.
         """
         # Temporary file
-        outFile = get_temp_file(rootDir=self._createTempDir())
-        try:
-            # Encapsulate a job graph
-            a = Job.wrapJobFn(encapsulatedJobFn, "A", outFile, name="a")
-            a = a.encapsulate(name="a-encap")
-            # Now add children/follow to the encapsulated graph
-            d = Job.wrapFn(fn1Test, a.rv(), outFile, name="d")
-            e = Job.wrapFn(fn1Test, d.rv(), outFile, name="e")
-            a.addChild(d)
-            a.addFollowOn(e)
-            # Create the runner for the workflow.
-            options = Job.Runner.getDefaultOptions(self._getTestJobStorePath())
-            options.logLevel = "INFO"
-            # Run the workflow, the return value being the number of failed jobs
-            Job.Runner.startToil(a, options)
-            # Check output
-            self.assertEqual(open(outFile).readline(), "ABCDE")
-        finally:
-            os.remove(outFile)
+        rootDir = tmp_path / "out"
+        rootDir.mkdir()
+        outFile = rootDir / "out"
+        # Encapsulate a job graph
+        a = Job.wrapJobFn(encapsulatedJobFn, "A", outFile, name="a").encapsulate(
+            name="a-encap"
+        )
+        # Now add children/follow to the encapsulated graph
+        d = Job.wrapFn(fn1Test, a.rv(), outFile, name="d")
+        e = Job.wrapFn(fn1Test, d.rv(), outFile, name="e")
+        a.addChild(d)
+        a.addFollowOn(e)
+        # Create the runner for the workflow.
+        options = Job.Runner.getDefaultOptions(tmp_path / "jobstore")
+        options.logLevel = "INFO"
+        # Run the workflow, the return value being the number of failed jobs
+        Job.Runner.startToil(a, options)
+        # Check output
+        assert outFile.read_text() == "ABCDE"
 
-    def testAddChildEncapsulate(self):
+    def testAddChildEncapsulate(self) -> None:
         """
         Make sure that the encapsulate child does not have two parents
         with unique roots.
@@ -56,14 +55,14 @@ class JobEncapsulationTest(ToilTest):
         a = Job.wrapFn(noOp)
         b = Job.wrapFn(noOp)
         a.addChild(b).encapsulate()
-        self.assertEqual(len(a.getRootJobs()), 1)
+        assert len(a.getRootJobs()) == 1
 
 
-def noOp():
+def noOp() -> None:
     pass
 
 
-def encapsulatedJobFn(job, string, outFile):
+def encapsulatedJobFn(job: Job, string: str, outFile: Path) -> Promise:
     a = job.addChildFn(fn1Test, string, outFile, name="inner-a")
     b = a.addFollowOnFn(fn1Test, a.rv(), outFile, name="inner-b")
     return b.rv()