toil 8.1.0b1__py3-none-any.whl → 8.2.0__py3-none-any.whl

toil/test/lib/test_history.py

@@ -12,44 +12,42 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import os
+from collections.abc import Generator
+from pathlib import Path
 import logging
-import pytest
 import time
-from toil.test import ToilTest
 
 from toil.lib.history import HistoryManager
 
+import pytest
+
 logger = logging.getLogger(__name__)
 logging.basicConfig(level=logging.DEBUG)
 
-class HistoryTest(ToilTest):
+
+class TestHistory:
     """
     Tests for Toil history tracking.
 
     Each test gets its own history database.
     """
 
-    def setUp(self) -> None:
-        super().setUp()
-        
-        # Apply a temp dir override to history tracking
-        temp_dir = self._createTempDir()
-        HistoryManager.database_path_override = os.path.join(temp_dir, "test-db.sqlite")
-
-        # Flag on job history tracking
-        self.original_flag = HistoryManager.JOB_HISTORY_ENABLED
-        HistoryManager.JOB_HISTORY_ENABLED = True
-        
-
-    def tearDown(self) -> None:
-        # Remove the temp dir override from history tracking
-        HistoryManager.database_path_override = None
-
-        # Restore job history tracking flag
-        HistoryManager.JOB_HISTORY_ENABLED = self.original_flag
-
-        super().tearDown()
+    @pytest.fixture(autouse=True, scope="function")
+    def private_history_manager(
+        self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+    ) -> Generator[None]:
+        try:
+            with monkeypatch.context() as m:
+                m.setattr(
+                    HistoryManager,
+                    "database_path_override",
+                    str(tmp_path / "test-db.sqlite"),
+                )
+                m.setattr(HistoryManager, "enabled", lambda: True)
+                m.setattr(HistoryManager, "enabled_job", lambda: True)
+                yield
+        finally:
+            pass  # no cleanup needed
 
     def make_fake_workflow(self, workflow_id: str) -> None:
         # Make a fake workflow
@@ -57,15 +55,17 @@ class HistoryTest(ToilTest):
         HistoryManager.record_workflow_creation(workflow_id, workflow_jobstore_spec)
         workflow_name = "SuperCoolWF"
         workflow_trs_spec = "#wf:v1"
-        HistoryManager.record_workflow_metadata(workflow_id, workflow_name, workflow_trs_spec)
-        
+        HistoryManager.record_workflow_metadata(
+            workflow_id, workflow_name, workflow_trs_spec
+        )
+
         # Give it a job
         workflow_attempt_number = 1
         job_name = "DoThing"
         succeeded = True
         start_time = time.time()
         runtime = 0.1
-        HistoryManager.record_job_attempt( 
+        HistoryManager.record_job_attempt(
             workflow_id,
             workflow_attempt_number,
             job_name,
@@ -93,92 +93,126 @@ class HistoryTest(ToilTest):
         workflow_attempt_number = 1
 
         # Make sure we have data
-        self.assertEqual(HistoryManager.count_workflows(), 1)
-        self.assertEqual(HistoryManager.count_workflow_attempts(), 1)
-        self.assertEqual(HistoryManager.count_job_attempts(), 1)
+        assert HistoryManager.count_workflows() == 1
+        assert HistoryManager.count_workflow_attempts() == 1
+        assert HistoryManager.count_job_attempts() == 1
 
         # Make sure we see it as submittable
-        submittable_workflow_attempts = HistoryManager.get_submittable_workflow_attempts()
-        self.assertEqual(len(submittable_workflow_attempts), 1)
+        submittable_workflow_attempts = (
+            HistoryManager.get_submittable_workflow_attempts()
+        )
+        assert len(submittable_workflow_attempts) == 1
 
         # Make sure we see its jobs as submittable
-        with_submittable_job_attempts = HistoryManager.get_workflow_attempts_with_submittable_job_attempts()
-        self.assertEqual(len(with_submittable_job_attempts), 1)
+        with_submittable_job_attempts = (
+            HistoryManager.get_workflow_attempts_with_submittable_job_attempts()
+        )
+        assert len(with_submittable_job_attempts) == 1
 
         # Make sure we actually see the job
-        submittable_job_attempts = HistoryManager.get_unsubmitted_job_attempts(workflow_id, workflow_attempt_number)
-        self.assertEqual(len(submittable_job_attempts), 1)
+        submittable_job_attempts = HistoryManager.get_unsubmitted_job_attempts(
+            workflow_id, workflow_attempt_number
+        )
+        assert len(submittable_job_attempts) == 1
 
         # Pretend we submitted them.
-        HistoryManager.mark_job_attempts_submitted([j.id for j in submittable_job_attempts])
-        HistoryManager.mark_workflow_attempt_submitted(workflow_id, workflow_attempt_number)
+        HistoryManager.mark_job_attempts_submitted(
+            [j.id for j in submittable_job_attempts]
+        )
+        HistoryManager.mark_workflow_attempt_submitted(
+            workflow_id, workflow_attempt_number
+        )
 
         # Make sure they are no longer matching
-        self.assertEqual(len(HistoryManager.get_submittable_workflow_attempts()), 0)
-        self.assertEqual(len(HistoryManager.get_workflow_attempts_with_submittable_job_attempts()), 0)
-        self.assertEqual(len(HistoryManager.get_unsubmitted_job_attempts(workflow_id, workflow_attempt_number)), 0)
+        assert len(HistoryManager.get_submittable_workflow_attempts()) == 0
+        assert (
+            len(HistoryManager.get_workflow_attempts_with_submittable_job_attempts())
+            == 0
+        )
+        assert (
+            len(
+                HistoryManager.get_unsubmitted_job_attempts(
+                    workflow_id, workflow_attempt_number
+                )
+            )
+            == 0
+        )
 
         # Make sure we still have data
-        self.assertEqual(HistoryManager.count_workflows(), 1)
-        self.assertEqual(HistoryManager.count_workflow_attempts(), 1)
-        self.assertEqual(HistoryManager.count_job_attempts(), 1)
+        assert HistoryManager.count_workflows() == 1
+        assert HistoryManager.count_workflow_attempts() == 1
+        assert HistoryManager.count_job_attempts() == 1
 
     def test_history_deletion(self) -> None:
         workflow_id = "123"
         self.make_fake_workflow(workflow_id)
         workflow_attempt_number = 1
-        
+
         # Make sure we can see the workflow for deletion by age but not by done-ness
-        self.assertEqual(len(HistoryManager.get_oldest_workflow_ids()), 1)
-        self.assertEqual(len(HistoryManager.get_fully_submitted_workflow_ids()), 0)
+        assert len(HistoryManager.get_oldest_workflow_ids()) == 1
+        assert len(HistoryManager.get_fully_submitted_workflow_ids()) == 0
 
         # Pretend we submitted the workflow.
-        HistoryManager.mark_job_attempts_submitted([j.id for j in HistoryManager.get_unsubmitted_job_attempts(workflow_id, workflow_attempt_number)])
-        HistoryManager.mark_workflow_attempt_submitted(workflow_id, workflow_attempt_number)
+        HistoryManager.mark_job_attempts_submitted(
+            [
+                j.id
+                for j in HistoryManager.get_unsubmitted_job_attempts(
+                    workflow_id, workflow_attempt_number
+                )
+            ]
+        )
+        HistoryManager.mark_workflow_attempt_submitted(
+            workflow_id, workflow_attempt_number
+        )
 
         # Make sure we can see the workflow for deletion by done-ness
-        self.assertEqual(len(HistoryManager.get_fully_submitted_workflow_ids()), 1)
+        assert len(HistoryManager.get_fully_submitted_workflow_ids()) == 1
 
         # Add a new workflow
         other_workflow_id = "456"
         self.make_fake_workflow(other_workflow_id)
 
         # Make sure we can see the both for deletion by age but only one by done-ness
-        self.assertEqual(len(HistoryManager.get_oldest_workflow_ids()), 2)
-        self.assertEqual(len(HistoryManager.get_fully_submitted_workflow_ids()), 1)
+        assert len(HistoryManager.get_oldest_workflow_ids()) == 2
+        assert len(HistoryManager.get_fully_submitted_workflow_ids()) == 1
 
         # Make sure the older workflow is first.
-        self.assertEqual(HistoryManager.get_oldest_workflow_ids(), [workflow_id, other_workflow_id])
+        assert HistoryManager.get_oldest_workflow_ids() == [
+            workflow_id,
+            other_workflow_id,
+        ]
 
         # Delete the new workflow
         HistoryManager.delete_workflow(other_workflow_id)
 
         # Make sure we can see the old one
-        self.assertEqual(HistoryManager.get_oldest_workflow_ids(), [workflow_id])
-        self.assertEqual(HistoryManager.get_fully_submitted_workflow_ids(), [workflow_id])
+        assert HistoryManager.get_oldest_workflow_ids() == [workflow_id]
+        assert HistoryManager.get_fully_submitted_workflow_ids() == [workflow_id]
 
         # Delete the old workflow
         HistoryManager.delete_workflow(workflow_id)
 
         # Make sure we have no data
-        self.assertEqual(HistoryManager.count_workflows(), 0)
-        self.assertEqual(HistoryManager.count_workflow_attempts(), 0)
-        self.assertEqual(HistoryManager.count_job_attempts(), 0)
-
+        assert HistoryManager.count_workflows() == 0
+        assert HistoryManager.count_workflow_attempts() == 0
+        assert HistoryManager.count_job_attempts() == 0
 
     def test_history_size_limit(self) -> None:
         """
         Make sure the database size can be controlled.
         """
 
-        for workflow_id in ("WorkflowThatTakesUpSomeSpace,ActuallyMoreThanTheLaterOnesTake" + str(i) for i in range(10)):
+        for workflow_id in (
+            "WorkflowThatTakesUpSomeSpace,ActuallyMoreThanTheLaterOnesTake" + str(i)
+            for i in range(10)
+        ):
             self.make_fake_workflow(workflow_id)
 
         # We should see the workflows.
-        self.assertEqual(HistoryManager.count_workflows(), 10)
+        assert HistoryManager.count_workflows() == 10
         # And they take up space.
         small_size = HistoryManager.get_database_byte_size()
-        self.assertGreater(small_size, 0)
+        assert small_size > 0
 
         # Add a bunch more
         for workflow_id in ("WorkflowThatTakesUpSpace" + str(i) for i in range(50)):
@@ -187,26 +221,16 @@ class HistoryTest(ToilTest):
         # We should see that this is now a much larger database
         large_size = HistoryManager.get_database_byte_size()
         logger.info("Increased database size from %s to %s", small_size, large_size)
-        self.assertGreater(large_size, small_size)
-        
+        large_size > small_size
+
         # We should be able to shrink it back down
         HistoryManager.enforce_byte_size_limit(small_size)
-        
+
         reduced_size = HistoryManager.get_database_byte_size()
         logger.info("Decreased database size from %s to %s", large_size, reduced_size)
         # The database should be small enough
-        self.assertLessEqual(reduced_size, small_size)
+        reduced_size <= small_size
         # There should still be some workflow attempts left in the smaller database (though probably not the first ones)
         remaining_workflows = HistoryManager.count_workflows()
         logger.info("Still have %s workflows", remaining_workflows)
-        self.assertGreater(remaining_workflows, 0)
-        
-
-
-
-
-       
-
-
-
-        
+        assert remaining_workflows > 0

toil/test/provisioners/aws/awsProvisionerTest.py

@@ -28,6 +28,7 @@ from toil.provisioners import cluster_factory
 from toil.provisioners.aws.awsProvisioner import AWSProvisioner
 from toil.test import (
     ToilTest,
+    get_data,
     integrative,
     needs_aws_ec2,
     needs_fetchable_appliance,
@@ -135,7 +136,7 @@ class AbstractAWSAutoscaleTest(AbstractClusterTest):
         """
         return os.path.join(self.dataDir, filename)
 
-    def rsyncUtil(self, src, dest):
+    def rsyncUtil(self, src: str, dest: str) -> None:
         subprocess.check_call(
             [
                 "toil",
@@ -285,10 +286,11 @@ class AWSAutoscaleTest(AbstractAWSAutoscaleTest):
         with open(fileToSort, "w") as f:
             # Fixme: making this file larger causes the test to hang
             f.write("01234567890123456789012345678901")
-        self.rsyncUtil(
-            os.path.join(self._projectRootPath(), "src/toil/test/sort/sort.py"),
-            ":" + self.script(),
-        )
+        with get_data("test/sort/sort.py") as sort_py:
+            self.rsyncUtil(
+                sort_py,
+                ":" + self.script(),
+            )
         self.rsyncUtil(fileToSort, ":" + self.data("sortFile"))
         os.unlink(fileToSort)
 
@@ -501,10 +503,11 @@ class AWSAutoscaleTestMultipleNodeTypes(AbstractAWSAutoscaleTest):
         sseKeyFile = os.path.join(os.getcwd(), "keyFile")
         with open(sseKeyFile, "w") as f:
             f.write("01234567890123456789012345678901")
-        self.rsyncUtil(
-            os.path.join(self._projectRootPath(), "src/toil/test/sort/sort.py"),
-            ":" + self.script(),
-        )
+        with get_data("test/sort/sort.py") as sort_py:
+            self.rsyncUtil(
+                sort_py,
+                ":" + self.script(),
+            )
         self.rsyncUtil(sseKeyFile, ":" + self.data("keyFile"))
         os.unlink(sseKeyFile)
 

toil/test/provisioners/clusterTest.py

@@ -39,7 +39,7 @@ log = logging.getLogger(__name__)
 class AbstractClusterTest(ToilTest):
     def __init__(self, methodName: str) -> None:
         super().__init__(methodName=methodName)
-        self.keyName = os.getenv("TOIL_AWS_KEYNAME").strip() or "id_rsa"
+        self.keyName = os.getenv("TOIL_AWS_KEYNAME", "id_rsa").strip()
         self.clusterName = f"aws-provisioner-test-{uuid4()}"
         self.leaderNodeType = "t2.medium"
         self.clusterType = "mesos"
@@ -276,12 +276,12 @@ class CWLOnARMTest(AbstractClusterTest):
             ]
         )
 
-        # Runs the CWLv12Test on an ARM instance
+        # Runs the TestCWLv12 on an ARM instance
         self.sshUtil(
             [
                 "bash",
                 "-c",
-                f". .{self.venvDir}/bin/activate && cd {self.cwl_test_dir}/toil && pytest --log-cli-level DEBUG -r s src/toil/test/cwl/cwlTest.py::CWLv12Test::test_run_conformance",
+                f". .{self.venvDir}/bin/activate && cd {self.cwl_test_dir}/toil && pytest --log-cli-level DEBUG -r s src/toil/test/cwl/cwlTest.py::TestCWLv12::test_run_conformance",
             ]
         )
 
@@ -289,5 +289,5 @@ class CWLOnARMTest(AbstractClusterTest):
         # Bring it back to be an artifact.
         self.rsync_util(
             f":{self.cwl_test_dir}/toil/conformance-1.2.junit.xml",
-            os.path.join(self._projectRootPath(), "arm-conformance-1.2.junit.xml"),
+            str(self._rootpath / "arm-conformance-1.2.junit.xml"),
         )

toil/test/provisioners/gceProvisionerTest.py

@@ -21,6 +21,7 @@ import pytest
 
 from toil.test import (
     ToilTest,
+    get_data,
     integrative,
     needs_fetchable_appliance,
     needs_google_project,
@@ -214,10 +215,11 @@ class GCEAutoscaleTest(AbstractGCEAutoscaleTest):
         with open(fileToSort, "w") as f:
             # Fixme: making this file larger causes the test to hang
             f.write("01234567890123456789012345678901")
-        self.rsyncUtil(
-            os.path.join(self._projectRootPath(), "src/toil/test/sort/sort.py"),
-            ":/home/sort.py",
-        )
+        with get_data("test/sort/sort.py") as sort_py:
+            self.rsyncUtil(
+                sort_py,
+                ":/home/sort.py",
+            )
         self.rsyncUtil(fileToSort, ":/home/sortFile")
         os.unlink(fileToSort)
 
@@ -324,10 +326,11 @@ class GCEAutoscaleTestMultipleNodeTypes(AbstractGCEAutoscaleTest):
         sseKeyFile = os.path.join(os.getcwd(), "keyFile")
         with open(sseKeyFile, "w") as f:
             f.write("01234567890123456789012345678901")
-        self.rsyncUtil(
-            os.path.join(self._projectRootPath(), "src/toil/test/sort/sort.py"),
-            ":/home/sort.py",
-        )
+        with get_data("test/sort/sort.py") as sort_py:
+            self.rsyncUtil(
+                sort_py,
+                ":/home/sort.py",
+            )
         self.rsyncUtil(sseKeyFile, ":/home/keyFile")
         os.unlink(sseKeyFile)
 
@@ -376,12 +379,11 @@ class GCERestartTest(AbstractGCEAutoscaleTest):
         self.jobStore = f"google:{self.projectID}:restart-{uuid4()}"
 
     def _getScript(self):
-        self.rsyncUtil(
-            os.path.join(
-                self._projectRootPath(), "src/toil/test/provisioners/restartScript.py"
-            ),
-            ":" + self.scriptName,
-        )
+        with get_data("test/provisioners/restartScript.py") as restartScript:
+            self.rsyncUtil(
+                restartScript,
+                ":" + self.scriptName,
+            )
 
     def _runScript(self, toilOptions):
         # clean = onSuccess

toil/test/sort/sort.py

@@ -23,6 +23,7 @@ from configargparse import ArgumentParser
 
 from toil.common import Toil
 from toil.job import Job
+from toil.lib.misc import StrPath
 from toil.realtimeLogger import RealtimeLogger
 
 defaultLines = 1000
@@ -207,7 +208,9 @@ def getMidPoint(file, fileStart, fileEnd):
     return len(line) + fileStart - 1
 
 
-def makeFileToSort(fileName, lines=defaultLines, lineLen=defaultLineLen):
+def makeFileToSort(
+    fileName: StrPath, lines: int = defaultLines, lineLen: int = defaultLineLen
+) -> None:
     with open(fileName, "w") as f:
         for _ in range(lines):
             line = (

toil/test/src/busTest.py

@@ -14,7 +14,9 @@
 
 import logging
 import os
+from pathlib import Path
 from threading import Thread, current_thread
+from typing import NoReturn
 
 from toil.batchSystems.abstractBatchSystem import BatchJobExitReason
 from toil.bus import (
@@ -26,18 +28,17 @@ from toil.bus import (
 from toil.common import Toil
 from toil.exceptions import FailedJobsException
 from toil.job import Job
-from toil.test import ToilTest, get_temp_file
 
 logger = logging.getLogger(__name__)
 
 
-class MessageBusTest(ToilTest):
+class TestMessageBus:
 
-    def test_enum_ints_in_file(self) -> None:
+    def test_enum_ints_in_file(self, tmp_path: Path) -> None:
         """
         Make sure writing bus messages to files works with enums.
         """
-        bus_file = get_temp_file()
+        bus_file = tmp_path / "bus"
 
         bus = MessageBus()
         # Connect the handler and hold the result to protect it from GC
@@ -73,7 +74,7 @@ class MessageBusTest(ToilTest):
             # Message should always arrive in the main thread.
             nonlocal message_count
             logger.debug("Got message: %s", received)
-            self.assertEqual(current_thread(), main_thread)
+            assert current_thread() == main_thread
             message_count += 1
 
         bus.subscribe(JobIssuedMessage, handler)
@@ -101,28 +102,27 @@ class MessageBusTest(ToilTest):
             t.join()
 
         # We should ge tone message per thread, plus our own
-        self.assertEqual(box.count(JobIssuedMessage), 11)
+        assert box.count(JobIssuedMessage) == 11
         # And having polled for those, our handler should have run
-        self.assertEqual(message_count, 11)
+        assert message_count == 11
 
-    def test_restart_without_bus_path(self) -> None:
+    def test_restart_without_bus_path(self, tmp_path: Path) -> None:
         """
         Test the ability to restart a workflow when the message bus path used
         by the previous attempt is gone.
         """
-        temp_dir = self._createTempDir(purpose="tempDir")
-        job_store = self._getTestJobStorePath()
+        temp_dir = tmp_path / "tempDir"
+        temp_dir.mkdir()
+        job_store = tmp_path / "jobstore"
 
-        bus_holder_dir = os.path.join(temp_dir, "bus_holder")
-        os.mkdir(bus_holder_dir)
+        bus_holder_dir = temp_dir / "bus_holder"
+        bus_holder_dir.mkdir()
 
         start_options = Job.Runner.getDefaultOptions(job_store)
         start_options.logLevel = "DEBUG"
         start_options.retryCount = 0
         start_options.clean = "never"
-        start_options.write_messages = os.path.abspath(
-            os.path.join(bus_holder_dir, "messagebus.txt")
-        )
+        start_options.write_messages = str(bus_holder_dir / "messagebus.txt")
 
         root = Job.wrapJobFn(failing_job_fn)
 
@@ -137,7 +137,7 @@ class MessageBusTest(ToilTest):
 
         # Get rid of the bus
         os.unlink(start_options.write_messages)
-        os.rmdir(bus_holder_dir)
+        bus_holder_dir.rmdir()
 
         logger.info("Making second attempt")
 
@@ -158,7 +158,7 @@ class MessageBusTest(ToilTest):
         logger.info("Second attempt successfully failed")
 
 
-def failing_job_fn(job: Job) -> None:
+def failing_job_fn(job: Job) -> NoReturn:
     """
     This function is guaranteed to fail.
     """