toil 6.1.0a1__py3-none-any.whl → 8.0.0__py3-none-any.whl

toil/test/utils/utilsTest.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 import logging
 import os
+import re
 import shutil
 import subprocess
 import sys
@@ -22,7 +23,7 @@ from unittest.mock import patch
 
 import pytest
 
-pkg_root = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))  # noqa
+pkg_root = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))  # noqa
 sys.path.insert(0, pkg_root)  # noqa
 
 import toil
@@ -30,16 +31,18 @@ from toil import resolveEntryPoint
 from toil.common import Config, Toil
 from toil.job import Job
 from toil.lib.bioio import system
-from toil.test import (ToilTest,
-                       get_temp_file,
-                       integrative,
-                       needs_aws_ec2,
-                       needs_cwl,
-                       needs_docker,
-                       needs_rsync3,
-                       slow)
+from toil.test import (
+    ToilTest,
+    get_temp_file,
+    integrative,
+    needs_aws_ec2,
+    needs_cwl,
+    needs_docker,
+    needs_rsync3,
+    slow,
+)
 from toil.test.sort.sortTest import makeFileToSort
-from toil.utils.toilStats import getStats, processData
+from toil.utils.toilStats import get_stats, process_data
 from toil.utils.toilStatus import ToilStatus
 from toil.version import python
 
@@ -56,7 +59,8 @@ class UtilsTest(ToilTest):
         super().setUp()
         self.tempDir = self._createTempDir()
         self.tempFile = get_temp_file(rootDir=self.tempDir)
-        self.outputFile = 'someSortedStuff.txt'
+        self.outputFile = get_temp_file(rootDir=self.tempDir)
+        self.outputFile = "someSortedStuff.txt"
         self.toilDir = os.path.join(self.tempDir, "jobstore")
         self.assertFalse(os.path.exists(self.toilDir))
         self.lines = 1000
@@ -70,19 +74,19 @@ class UtilsTest(ToilTest):
 
         self.sort_workflow_cmd = [
             python,
-            '-m',
-            'toil.test.sort.sort',
-            f'file:{self.toilDir}',
-            '--clean=never',
-            '--numLines=1',
-            '--lineLength=1'
+            "-m",
+            "toil.test.sort.sort",
+            f"file:{self.toilDir}",
+            f"--fileToSort={self.tempFile}",
+            f"--outputFile={self.outputFile}",
+            "--clean=never",
         ]
 
         self.restart_sort_workflow_cmd = [
             python,
-            '-m',
-            'toil.test.sort.restart_sort',
-            f'file:{self.toilDir}'
+            "-m",
+            "toil.test.sort.restart_sort",
+            f"file:{self.toilDir}",
         ]
 
     def tearDown(self):
@@ -91,7 +95,11 @@ class UtilsTest(ToilTest):
         if os.path.exists(self.toilDir):
             shutil.rmtree(self.toilDir)
 
-        for f in ['fileToSort.txt', 'sortedFile.txt', 'output.txt']:
+        for f in [
+            self.tempFile,
+            self.outputFile,
+            os.path.join(self.tempDir, "output.txt"),
+        ]:
             if os.path.exists(f):
                 os.remove(f)
 
@@ -99,26 +107,26 @@ class UtilsTest(ToilTest):
 
     @property
     def toilMain(self):
-        return resolveEntryPoint('toil')
+        return resolveEntryPoint("toil")
 
     @property
     def cleanCommand(self):
-        return [self.toilMain, 'clean', self.toilDir]
+        return [self.toilMain, "clean", self.toilDir]
 
     @property
     def statsCommand(self):
-        return [self.toilMain, 'stats', self.toilDir, '--pretty']
+        return [self.toilMain, "stats", self.toilDir, "--pretty"]
 
     def statusCommand(self, failIfNotComplete=False):
-        commandTokens = [self.toilMain, 'status', self.toilDir]
+        commandTokens = [self.toilMain, "status", self.toilDir]
         if failIfNotComplete:
-            commandTokens.append('--failIfNotComplete')
+            commandTokens.append("--failIfNotComplete")
         return commandTokens
 
     def test_config_functionality(self):
         """Ensure that creating and reading back the config file works"""
         config_file = os.path.abspath("config.yaml")
-        config_command = [self.toilMain, 'config', config_file]
+        config_command = [self.toilMain, "config", config_file]
         # make sure the command `toil config file_path` works
         try:
             subprocess.check_call(config_command)
@@ -129,12 +137,18 @@ class UtilsTest(ToilTest):
         # make sure that toil can read from the generated config file
         try:
             parser.parse_args(["random_jobstore", "--config", config_file])
+            with open(config_file) as cm:
+                payload = cm.read()
+                expected = "workDir batchSystem symlinkImports defaultMemory retryCount"
+                assert all(
+                    re.search(rf"^#*{ param }:", payload, re.MULTILINE)
+                    for param in expected.split(" ")
+                ), f"Generated config contains { expected }"
         except SystemExit:
             self.fail("Failed to parse the default generated config file!")
         finally:
             os.remove(config_file)
 
-
     @needs_rsync3
     @pytest.mark.timeout(1200)
     @needs_aws_ec2
@@ -155,30 +169,63 @@ class UtilsTest(ToilTest):
         :return:
         """
         # TODO: Run these for the other clouds.
-        clusterName = f'cluster-utils-test{uuid.uuid4()}'
-        keyName = os.getenv('TOIL_AWS_KEYNAME').strip() or 'id_rsa'
-        expected_owner = os.getenv('TOIL_OWNER_TAG') or keyName
+        clusterName = f"cluster-utils-test{uuid.uuid4()}"
+        keyName = os.getenv("TOIL_AWS_KEYNAME").strip() or "id_rsa"
+        expected_owner = os.getenv("TOIL_OWNER_TAG") or keyName
 
         try:
             from toil.provisioners.aws.awsProvisioner import AWSProvisioner
+
             aws_provisioner = AWSProvisioner.__module__
             logger.debug(f"Found AWSProvisioner: {aws_provisioner}.")
 
             # launch master with an assortment of custom tags
-            system([self.toilMain, 'launch-cluster', '--clusterType', 'mesos',
-                    '-t', 'key1=value1', '-t', 'key2=value2', '--tag', 'key3=value3',
-                    '--leaderNodeType=t2.medium', '--keyPairName=' + keyName, clusterName,
-                    '--provisioner=aws', '--zone=us-west-2a', '--logLevel=DEBUG'])
-
-            cluster = toil.provisioners.cluster_factory(provisioner='aws', zone='us-west-2a', clusterName=clusterName)
+            system(
+                [
+                    self.toilMain,
+                    "launch-cluster",
+                    "--clusterType",
+                    "mesos",
+                    "-t",
+                    "key1=value1",
+                    "-t",
+                    "key2=value2",
+                    "--tag",
+                    "key3=value3",
+                    "--leaderNodeType=t2.medium",
+                    "--keyPairName=" + keyName,
+                    clusterName,
+                    "--provisioner=aws",
+                    "--zone=us-west-2a",
+                    "--logLevel=DEBUG",
+                ]
+            )
+
+            cluster = toil.provisioners.cluster_factory(
+                provisioner="aws", zone="us-west-2a", clusterName=clusterName
+            )
             leader = cluster.getLeader()
 
             # check that the leader carries the appropriate tags
-            tags = {'key1': 'value1', 'key2': 'value2', 'key3': 'value3', 'Name': clusterName, 'Owner': expected_owner}
+            tags = {
+                "key1": "value1",
+                "key2": "value2",
+                "key3": "value3",
+                "Name": clusterName,
+                "Owner": expected_owner,
+            }
             for key in tags:
                 self.assertEqual(leader.tags.get(key), tags[key])
         finally:
-            system([self.toilMain, 'destroy-cluster', '--zone=us-west-2a', '--provisioner=aws', clusterName])
+            system(
+                [
+                    self.toilMain,
+                    "destroy-cluster",
+                    "--zone=us-west-2a",
+                    "--provisioner=aws",
+                    clusterName,
+                ]
+            )
 
     @slow
     def testUtilsSort(self):
@@ -187,19 +234,27 @@ class UtilsTest(ToilTest):
         sort example with the --restart flag.
         """
         # Get the sort command to run
-        toilCommand = [sys.executable,
-                       '-m', toil.test.sort.sort.__name__,
-                       self.toilDir,
-                       '--logLevel=DEBUG',
-                       '--fileToSort', self.tempFile,
-                       '--outputFile', self.outputFile,
-                       '--N', str(self.N),
-                       '--stats',
-                       '--retryCount=2',
-                       '--badWorker=0.5',
-                       '--badWorkerFailInterval=0.05']
+        toilCommand = [
+            sys.executable,
+            "-m",
+            toil.test.sort.sort.__name__,
+            self.toilDir,
+            "--logLevel=DEBUG",
+            "--fileToSort",
+            self.tempFile,
+            "--outputFile",
+            self.outputFile,
+            "--N",
+            str(self.N),
+            "--stats",
+            "--retryCount=2",
+            "--badWorker=0.5",
+            "--badWorkerFailInterval=0.05",
+        ]
         # Try restarting it to check that a JobStoreException is thrown
-        self.assertRaises(subprocess.CalledProcessError, system, toilCommand + ['--restart'])
+        self.assertRaises(
+            subprocess.CalledProcessError, system, toilCommand + ["--restart"]
+        )
         # Check that trying to run it in restart mode does not create the jobStore
         self.assertFalse(os.path.exists(self.toilDir))
 
@@ -208,9 +263,15 @@ class UtilsTest(ToilTest):
         try:
             system(toilCommand)
             finished = True
-        except subprocess.CalledProcessError:  # This happens when the script fails due to having unfinished jobs
+        except (
+            subprocess.CalledProcessError
+        ):  # This happens when the script fails due to having unfinished jobs
             system(self.statusCommand())
-            self.assertRaises(subprocess.CalledProcessError, system, self.statusCommand(failIfNotComplete=True))
+            self.assertRaises(
+                subprocess.CalledProcessError,
+                system,
+                self.statusCommand(failIfNotComplete=True),
+            )
             finished = False
         self.assertTrue(os.path.exists(self.toilDir))
 
@@ -221,11 +282,17 @@ class UtilsTest(ToilTest):
         totalTrys = 1
         while not finished:
             try:
-                system(toilCommand + ['--restart'])
+                system(toilCommand + ["--restart"])
                 finished = True
-            except subprocess.CalledProcessError:  # This happens when the script fails due to having unfinished jobs
+            except (
+                subprocess.CalledProcessError
+            ):  # This happens when the script fails due to having unfinished jobs
                 system(self.statusCommand())
-                self.assertRaises(subprocess.CalledProcessError, system, self.statusCommand(failIfNotComplete=True))
+                self.assertRaises(
+                    subprocess.CalledProcessError,
+                    system,
+                    self.statusCommand(failIfNotComplete=True),
+                )
                 if totalTrys > 16:
                     self.fail()  # Exceeded a reasonable number of restarts
                 totalTrys += 1
@@ -253,17 +320,23 @@ class UtilsTest(ToilTest):
         Tests the stats commands on a complete run of the stats test.
         """
         # Get the sort command to run
-        toilCommand = [sys.executable,
-                       '-m', toil.test.sort.sort.__name__,
-                       self.toilDir,
-                       '--logLevel=DEBUG',
-                       '--fileToSort', self.tempFile,
-                       '--outputFile', self.outputFile,
-                       '--N', str(self.N),
-                       '--stats',
-                       '--retryCount=99',
-                       '--badWorker=0.5',
-                       '--badWorkerFailInterval=0.01']
+        toilCommand = [
+            sys.executable,
+            "-m",
+            toil.test.sort.sort.__name__,
+            self.toilDir,
+            "--logLevel=DEBUG",
+            "--fileToSort",
+            self.tempFile,
+            "--outputFile",
+            self.outputFile,
+            "--N",
+            str(self.N),
+            "--stats",
+            "--retryCount=99",
+            "--badWorker=0.5",
+            "--badWorkerFailInterval=0.01",
+        ]
 
         # Run the script for the first time
         system(toilCommand)
@@ -282,8 +355,8 @@ class UtilsTest(ToilTest):
 
     def testUnicodeSupport(self):
         options = Job.Runner.getDefaultOptions(self._getTestJobStorePath())
-        options.clean = 'always'
-        options.logLevel = 'debug'
+        options.clean = "always"
+        options.logLevel = "debug"
         Job.Runner.startToil(Job.wrapFn(printUnicodeCharacter), options)
 
     @slow
@@ -292,36 +365,60 @@ class UtilsTest(ToilTest):
         Tests case where multiple jobs are run on 1 worker to ensure that all jobs report back their data
         """
         options = Job.Runner.getDefaultOptions(self._getTestJobStorePath())
-        options.clean = 'never'
+        options.clean = "never"
         options.stats = True
         Job.Runner.startToil(RunTwoJobsPerWorker(), options)
         config = Config()
         config.setOptions(options)
         jobStore = Toil.resumeJobStore(config.jobStore)
-        stats = getStats(jobStore)
-        collatedStats = processData(jobStore.config, stats)
-        self.assertTrue(len(collatedStats.job_types) == 2, "Some jobs are not represented in the stats.")
-
-    def check_status(self, status, status_fn, seconds=20):
-        i = 0.0
-        while status_fn(self.toilDir) != status:
+        stats = get_stats(jobStore)
+        collatedStats = process_data(jobStore.config, stats)
+        self.assertTrue(
+            len(collatedStats.job_types) == 2,
+            "Some jobs are not represented in the stats.",
+        )
+
+    def check_status(self, status, status_fn, process=None, seconds=20):
+        time_elapsed = 0.0
+        has_stopped = process.poll() is not None if process else False
+        current_status = status_fn(self.toilDir)
+        while current_status != status:
+            if has_stopped:
+                # If the process has stopped and the stratus is wrong, it will never be right.
+                self.assertEqual(
+                    current_status,
+                    status,
+                    f"Process returned {process.returncode} without status reaching {status}; stuck at {current_status}",
+                )
+            logger.debug(
+                "Workflow is %s; waiting for %s (%s/%s elapsed)",
+                current_status,
+                status,
+                time_elapsed,
+                seconds
+            )
             time.sleep(0.5)
-            i += 0.5
-            if i > seconds:
-                s = status_fn(self.toilDir)
-                self.assertEqual(s, status, f'Waited {seconds} seconds without status reaching {status}; stuck at {s}')
+            time_elapsed += 0.5
+            has_stopped = process.poll() is not None if process else False
+            current_status = status_fn(self.toilDir)
+            if time_elapsed > seconds:
+                self.assertEqual(
+                    current_status,
+                    status,
+                    f"Waited {seconds} seconds without status reaching {status}; stuck at {current_status}",
+                )
 
     def testGetPIDStatus(self):
         """Test that ToilStatus.getPIDStatus() behaves as expected."""
         wf = subprocess.Popen(self.sort_workflow_cmd)
-        self.check_status('RUNNING', status_fn=ToilStatus.getPIDStatus, seconds=20)
+        self.check_status("RUNNING", status_fn=ToilStatus.getPIDStatus, process=wf, seconds=60)
         wf.wait()
-        self.check_status('COMPLETED', status_fn=ToilStatus.getPIDStatus)
+        self.check_status("COMPLETED", status_fn=ToilStatus.getPIDStatus, process=wf, seconds=60)
 
         # TODO: we need to reach into the FileJobStore's files and delete this
         #  shared file. We assume we know its internal layout.
-        os.remove(os.path.join(self.toilDir, 'files/shared/pid.log'))
-        self.check_status('QUEUED', status_fn=ToilStatus.getPIDStatus)
+        os.remove(os.path.join(self.toilDir, "files/shared/pid.log"))
+        self.check_status("QUEUED", status_fn=ToilStatus.getPIDStatus, process=wf, seconds=60)
 
     def testGetStatusFailedToilWF(self):
         """
@@ -330,41 +427,71 @@ class UtilsTest(ToilTest):
         opportunity to test the 'RUNNING' functionality of getStatus().
         """
         # --badWorker is set to force failure.
-        wf = subprocess.Popen(self.sort_workflow_cmd + ['--badWorker=1'])
-        self.check_status('RUNNING', status_fn=ToilStatus.getStatus)
+        wf = subprocess.Popen(self.sort_workflow_cmd + ["--badWorker=1"])
+        self.check_status("RUNNING", status_fn=ToilStatus.getStatus, process=wf, seconds=60)
         wf.wait()
-        self.check_status('ERROR', status_fn=ToilStatus.getStatus)
+        self.check_status("ERROR", status_fn=ToilStatus.getStatus, process=wf, seconds=60)
 
     @needs_cwl
     @needs_docker
     def testGetStatusFailedCWLWF(self):
         """Test that ToilStatus.getStatus() behaves as expected with a failing CWL workflow."""
         # --badWorker is set to force failure.
-        cmd = ['toil-cwl-runner', '--jobStore', self.toilDir, '--clean=never', '--badWorker=1',
-               'src/toil/test/cwl/sorttool.cwl', '--reverse', '--input', 'src/toil/test/cwl/whale.txt']
+        cmd = [
+            "toil-cwl-runner",
+            "--logDebug",
+            "--jobStore",
+            self.toilDir,
+            "--clean=never",
+            "--badWorker=1",
+            "src/toil/test/cwl/sorttool.cwl",
+            "--reverse",
+            "--input",
+            "src/toil/test/cwl/whale.txt",
+            f"--outdir={self.tempDir}",
+        ]
+        logger.info("Run command: %s", " ".join(cmd))
         wf = subprocess.Popen(cmd)
-        self.check_status('RUNNING', status_fn=ToilStatus.getStatus)
+        self.check_status("RUNNING", status_fn=ToilStatus.getStatus, process=wf, seconds=60)
         wf.wait()
-        self.check_status('ERROR', status_fn=ToilStatus.getStatus)
+        self.check_status("ERROR", status_fn=ToilStatus.getStatus, process=wf, seconds=60)
 
     @needs_cwl
     @needs_docker
     def testGetStatusSuccessfulCWLWF(self):
         """Test that ToilStatus.getStatus() behaves as expected with a successful CWL workflow."""
-        cmd = ['toil-cwl-runner', '--jobStore', self.toilDir, '--clean=never',
-               'src/toil/test/cwl/sorttool.cwl', '--reverse', '--input', 'src/toil/test/cwl/whale.txt']
+        cmd = [
+            "toil-cwl-runner",
+            "--jobStore",
+            self.toilDir,
+            "--clean=never",
+            "src/toil/test/cwl/sorttool.cwl",
+            "--reverse",
+            "--input",
+            "src/toil/test/cwl/whale.txt",
+            f"--outdir={self.tempDir}",
+        ]
         wf = subprocess.Popen(cmd)
-        self.check_status('RUNNING', status_fn=ToilStatus.getStatus, seconds=20)
+        self.check_status("RUNNING", status_fn=ToilStatus.getStatus, process=wf, seconds=60)
         wf.wait()
-        self.check_status('COMPLETED', status_fn=ToilStatus.getStatus)
+        self.check_status("COMPLETED", status_fn=ToilStatus.getStatus, process=wf, seconds=60)
 
     @needs_cwl
-    @patch('builtins.print')
+    @patch("builtins.print")
     def testPrintJobLog(self, mock_print):
         """Test that ToilStatus.printJobLog() reads the log from a failed command without error."""
         # Run a workflow that will always fail
-        cmd = ['toil-cwl-runner', '--jobStore', self.toilDir, '--clean=never',
-               'src/toil/test/cwl/alwaysfails.cwl', '--message', 'Testing']
+        cmd = [
+            "toil-cwl-runner",
+            "--logDebug",
+            "--jobStore",
+            self.toilDir,
+            "--clean=never",
+            "src/toil/test/cwl/alwaysfails.cwl",
+            "--message",
+            "Testing",
+        ]
+        logger.info("Run command: %s", " ".join(cmd))
         wf = subprocess.Popen(cmd)
         wf.wait()
         # print log and check output
@@ -373,26 +500,37 @@ class UtilsTest(ToilTest):
 
         # Make sure it printed some kind of complaint about the missing command.
         args, kwargs = mock_print.call_args
-        self.assertIn('invalidcommand', args[0])
+        self.assertIn("invalidcommand", args[0])
 
+    @pytest.mark.timeout(1200)
     def testRestartAttribute(self):
         """
-        Test that the job store is only destroyed when we observe a succcessful workflow run.
+        Test that the job store is only destroyed when we observe a successful workflow run.
         The following simulates a failing workflow that attempts to resume without restart().
         In this case, the job store should not be destroyed until restart() is called.
         """
         # Run a workflow that will always fail
-        cmd = self.restart_sort_workflow_cmd + ['--badWorker=1']
+        cmd = self.restart_sort_workflow_cmd + ["--badWorker=1", "--logDebug"]
         subprocess.run(cmd)
 
-        restart_cmd = self.restart_sort_workflow_cmd + ['--badWorker=0', '--restart']
+        restart_cmd = self.restart_sort_workflow_cmd + [
+            "--badWorker=0",
+            "--logDebug",
+            "--restart",
+        ]
         subprocess.run(restart_cmd)
 
         # Check the job store exists after restart attempt
         self.assertTrue(os.path.exists(self.toilDir))
 
-        successful_cmd = [python, '-m', 'toil.test.sort.sort', 'file:' + self.toilDir,
-                                  '--restart']
+        successful_cmd = [
+            python,
+            "-m",
+            "toil.test.sort.sort",
+            "--logDebug",
+            "file:" + self.toilDir,
+            "--restart",
+        ]
         subprocess.run(successful_cmd)
 
         # Check the job store is destroyed after calling restart()
@@ -403,13 +541,14 @@ def printUnicodeCharacter():
     # We want to get a unicode character to stdout but we can't print it directly because of
     # Python encoding issues. To work around this we print in a separate Python process. See
     # http://stackoverflow.com/questions/492483/setting-the-correct-encoding-when-piping-stdout-in-python
-    subprocess.check_call([sys.executable, '-c', "print('\\xc3\\xbc')"])
+    subprocess.check_call([sys.executable, "-c", "print('\\xc3\\xbc')"])
 
 
 class RunTwoJobsPerWorker(Job):
     """
     Runs child job with same resources as self in an attempt to chain the jobs on the same worker
     """
+
     def __init__(self):
         Job.__init__(self)