toil 7.0.0__py3-none-any.whl → 8.0.0__py3-none-any.whl

toil/test/batchSystems/batchSystemTest.py

@@ -24,36 +24,43 @@ from abc import ABCMeta, abstractmethod
 from fractions import Fraction
 from unittest import skipIf
 
-from toil.batchSystems.abstractBatchSystem import (AbstractBatchSystem,
-                                                   BatchSystemSupport,
-                                                   InsufficientSystemResources)
+from toil.batchSystems.abstractBatchSystem import (
+    AbstractBatchSystem,
+    BatchSystemSupport,
+    InsufficientSystemResources,
+)
+
 # Don't import any batch systems here that depend on extras
 # in order to import properly. Import them later, in tests
 # protected by annotations.
 from toil.batchSystems.mesos.test import MesosTestSupport
-from toil.batchSystems.registry import (add_batch_system_factory,
-                                        get_batch_system,
-                                        get_batch_systems,
-                                        restore_batch_system_plugin_state,
-                                        save_batch_system_plugin_state)
+from toil.batchSystems.registry import (
+    add_batch_system_factory,
+    get_batch_system,
+    get_batch_systems,
+    restore_batch_system_plugin_state,
+    save_batch_system_plugin_state,
+)
 from toil.batchSystems.singleMachine import SingleMachineBatchSystem
 from toil.common import Config, Toil
 from toil.job import Job, JobDescription, Requirer
 from toil.lib.retry import retry_flaky_test
 from toil.lib.threading import cpu_count
-from toil.test import (ToilTest,
-                       needs_aws_batch,
-                       needs_aws_s3,
-                       needs_fetchable_appliance,
-                       needs_gridengine,
-                       needs_htcondor,
-                       needs_kubernetes,
-                       needs_kubernetes_installed,
-                       needs_lsf,
-                       needs_mesos,
-                       needs_slurm,
-                       needs_torque,
-                       slow)
+from toil.test import (
+    ToilTest,
+    needs_aws_batch,
+    needs_aws_s3,
+    needs_fetchable_appliance,
+    needs_gridengine,
+    needs_htcondor,
+    needs_kubernetes,
+    needs_kubernetes_installed,
+    needs_lsf,
+    needs_mesos,
+    needs_slurm,
+    needs_torque,
+    slow,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -66,7 +73,10 @@ preemptible = False
 
 # Since we aren't always attaching the config to the jobs for these tests, we
 # need to use fully specified requirements.
-defaultRequirements = dict(memory=int(100e6), cores=1, disk=1000, preemptible=preemptible, accelerators=[])
+defaultRequirements = dict(
+    memory=int(100e6), cores=1, disk=1000, preemptible=preemptible, accelerators=[]
+)
+
 
 class BatchSystemPluginTest(ToilTest):
     """
@@ -91,9 +101,10 @@ class BatchSystemPluginTest(ToilTest):
             # add its arguments.
             return SingleMachineBatchSystem
 
-        add_batch_system_factory('testBatchSystem', test_batch_system_factory)
-        assert 'testBatchSystem' in get_batch_systems()
-        assert get_batch_system('testBatchSystem') == SingleMachineBatchSystem
+        add_batch_system_factory("testBatchSystem", test_batch_system_factory)
+        assert "testBatchSystem" in get_batch_systems()
+        assert get_batch_system("testBatchSystem") == SingleMachineBatchSystem
+
 
 class hidden:
     """
@@ -127,8 +138,9 @@ class hidden:
             """
             config = Config()
             from uuid import uuid4
+
             config.workflowID = str(uuid4())
-            config.cleanWorkDir = 'always'
+            config.cleanWorkDir = "always"
             return config
 
         def _createConfig(self):
@@ -164,7 +176,7 @@ class hidden:
             super().setUp()
             self.config = self._createConfig()
             self.batchSystem = self.createBatchSystem()
-            self.tempDir = self._createTempDir('testFiles')
+            self.tempDir = self._createTempDir("testFiles")
 
         def tearDown(self):
             self.batchSystem.shutdown()
@@ -182,12 +194,20 @@ class hidden:
 
         @retry_flaky_test(prepare=[tearDown, setUp])
         def test_run_jobs(self):
-            jobDesc1 = self._mockJobDescription(jobName='test1', unitName=None,
-                                                jobStoreID='1', requirements=defaultRequirements)
-            jobDesc2 = self._mockJobDescription(jobName='test2', unitName=None,
-                                                jobStoreID='2', requirements=defaultRequirements)
-            job1 = self.batchSystem.issueBatchJob('sleep 1000', jobDesc1)
-            job2 = self.batchSystem.issueBatchJob('sleep 1000', jobDesc2)
+            jobDesc1 = self._mockJobDescription(
+                jobName="test1",
+                unitName=None,
+                jobStoreID="1",
+                requirements=defaultRequirements,
+            )
+            jobDesc2 = self._mockJobDescription(
+                jobName="test2",
+                unitName=None,
+                jobStoreID="2",
+                requirements=defaultRequirements,
+            )
+            job1 = self.batchSystem.issueBatchJob("sleep 1000", jobDesc1)
+            job2 = self.batchSystem.issueBatchJob("sleep 1000", jobDesc2)
 
             issuedIDs = self._waitForJobsToIssue(2)
             self.assertEqual(set(issuedIDs), {job1, job2})
@@ -202,7 +222,9 @@ class hidden:
             # getUpdatedBatchJob, and the sleep time is longer than the time we
             # should spend waiting for both to start, so if our cluster can
             # only run one job at a time, we will fail the test.
-            runningJobIDs = self._waitForJobsToStart(2, tries=self.get_max_startup_seconds())
+            runningJobIDs = self._waitForJobsToStart(
+                2, tries=self.get_max_startup_seconds()
+            )
             self.assertEqual(set(runningJobIDs), {job1, job2})
 
             # Killing the jobs instead of allowing them to complete means this test can run very
@@ -216,13 +238,21 @@ class hidden:
             # then check for it having happened, but we can't guarantee that
             # the batch system will run against the same filesystem we are
             # looking at.
-            jobDesc3 = self._mockJobDescription(jobName='test3', unitName=None,
-                                                jobStoreID='3', requirements=defaultRequirements)
+            jobDesc3 = self._mockJobDescription(
+                jobName="test3",
+                unitName=None,
+                jobStoreID="3",
+                requirements=defaultRequirements,
+            )
             job3 = self.batchSystem.issueBatchJob("mktemp -d", jobDesc3)
 
             jobUpdateInfo = self.batchSystem.getUpdatedBatchJob(maxWait=1000)
-            jobID, exitStatus, wallTime = jobUpdateInfo.jobID, jobUpdateInfo.exitStatus, jobUpdateInfo.wallTime
-            logger.info(f'Third job completed: {jobID} {exitStatus} {wallTime}')
+            jobID, exitStatus, wallTime = (
+                jobUpdateInfo.jobID,
+                jobUpdateInfo.exitStatus,
+                jobUpdateInfo.wallTime,
+            )
+            logger.info(f"Third job completed: {jobID} {exitStatus} {wallTime}")
 
             # Since the first two jobs were killed, the only job in the updated jobs queue should
             # be job 3. If the first two jobs were (incorrectly) added to the queue, this will
@@ -242,46 +272,68 @@ class hidden:
 
         def test_set_env(self):
             # Start with a relatively safe script
-            script_shell = 'if [ "x${FOO}" == "xbar" ] ; then exit 23 ; else exit 42 ; fi'
+            script_shell = (
+                'if [ "x${FOO}" == "xbar" ] ; then exit 23 ; else exit 42 ; fi'
+            )
 
             # Escape the semicolons
-            script_protected = script_shell.replace(';', r'\;')
+            script_protected = script_shell.replace(";", r"\;")
 
             # Turn into a string which convinces bash to take all args and paste them back together and run them
-            command = "bash -c \"\\${@}\" bash eval " + script_protected
-            jobDesc4 = self._mockJobDescription(jobName='test4', unitName=None,
-                                                jobStoreID='4', requirements=defaultRequirements)
+            command = 'bash -c "\\${@}" bash eval ' + script_protected
+            jobDesc4 = self._mockJobDescription(
+                jobName="test4",
+                unitName=None,
+                jobStoreID="4",
+                requirements=defaultRequirements,
+            )
             job4 = self.batchSystem.issueBatchJob(command, jobDesc4)
             jobUpdateInfo = self.batchSystem.getUpdatedBatchJob(maxWait=1000)
-            jobID, exitStatus, wallTime = jobUpdateInfo.jobID, jobUpdateInfo.exitStatus, jobUpdateInfo.wallTime
+            jobID, exitStatus, wallTime = (
+                jobUpdateInfo.jobID,
+                jobUpdateInfo.exitStatus,
+                jobUpdateInfo.wallTime,
+            )
             self.assertEqual(exitStatus, 42)
             self.assertEqual(jobID, job4)
             # Now set the variable and ensure that it is present
-            self.batchSystem.setEnv('FOO', 'bar')
-            jobDesc5 = self._mockJobDescription(jobName='test5', unitName=None,
-                                                jobStoreID='5', requirements=defaultRequirements)
+            self.batchSystem.setEnv("FOO", "bar")
+            jobDesc5 = self._mockJobDescription(
+                jobName="test5",
+                unitName=None,
+                jobStoreID="5",
+                requirements=defaultRequirements,
+            )
             job5 = self.batchSystem.issueBatchJob(command, jobDesc5)
             jobUpdateInfo = self.batchSystem.getUpdatedBatchJob(maxWait=1000)
             self.assertEqual(jobUpdateInfo.exitStatus, 23)
             self.assertEqual(jobUpdateInfo.jobID, job5)
 
         def test_set_job_env(self):
-            """ Test the mechanism for setting per-job environment variables to batch system jobs."""
+            """Test the mechanism for setting per-job environment variables to batch system jobs."""
             script = 'if [ "x${FOO}" == "xbar" ] ; then exit 23 ; else exit 42 ; fi'
-            command = "bash -c \"\\${@}\" bash eval " + script.replace(';', r'\;')
+            command = 'bash -c "\\${@}" bash eval ' + script.replace(";", r"\;")
 
             # Issue a job with a job environment variable
-            job_desc_6 = self._mockJobDescription(jobName='test6', unitName=None,
-                                                  jobStoreID='6', requirements=defaultRequirements)
-            job6 = self.batchSystem.issueBatchJob(command, job_desc_6, job_environment={
-                'FOO': 'bar'
-            })
+            job_desc_6 = self._mockJobDescription(
+                jobName="test6",
+                unitName=None,
+                jobStoreID="6",
+                requirements=defaultRequirements,
+            )
+            job6 = self.batchSystem.issueBatchJob(
+                command, job_desc_6, job_environment={"FOO": "bar"}
+            )
             job_update_info = self.batchSystem.getUpdatedBatchJob(maxWait=1000)
             self.assertEqual(job_update_info.exitStatus, 23)  # this should succeed
             self.assertEqual(job_update_info.jobID, job6)
             # Now check that the environment variable doesn't exist for other jobs
-            job_desc_7 = self._mockJobDescription(jobName='test7', unitName=None,
-                                                  jobStoreID='7', requirements=defaultRequirements)
+            job_desc_7 = self._mockJobDescription(
+                jobName="test7",
+                unitName=None,
+                jobStoreID="7",
+                requirements=defaultRequirements,
+            )
             job7 = self.batchSystem.issueBatchJob(command, job_desc_7)
             job_update_info = self.batchSystem.getUpdatedBatchJob(maxWait=1000)
             self.assertEqual(job_update_info.exitStatus, 42)
@@ -291,34 +343,67 @@ class hidden:
             if isinstance(self.batchSystem, BatchSystemSupport):
                 check_resource_request = self.batchSystem.check_resource_request
                 # Assuming we have <2000 cores, this should be too many cores
-                self.assertRaises(InsufficientSystemResources, check_resource_request,
-                                  Requirer(dict(memory=1000, cores=2000, disk='1G', accelerators=[])))
-                self.assertRaises(InsufficientSystemResources, check_resource_request,
-                                  Requirer(dict(memory=5, cores=2000, disk='1G', accelerators=[])))
+                self.assertRaises(
+                    InsufficientSystemResources,
+                    check_resource_request,
+                    Requirer(dict(memory=1000, cores=2000, disk="1G", accelerators=[])),
+                )
+                self.assertRaises(
+                    InsufficientSystemResources,
+                    check_resource_request,
+                    Requirer(dict(memory=5, cores=2000, disk="1G", accelerators=[])),
+                )
 
                 # This should be too much memory
-                self.assertRaises(InsufficientSystemResources, check_resource_request,
-                                  Requirer(dict(memory='5000G', cores=1, disk='1G', accelerators=[])))
+                self.assertRaises(
+                    InsufficientSystemResources,
+                    check_resource_request,
+                    Requirer(dict(memory="5000G", cores=1, disk="1G", accelerators=[])),
+                )
 
                 # This should be too much disk
-                self.assertRaises(InsufficientSystemResources, check_resource_request,
-                                  Requirer(dict(memory=5, cores=1, disk='2G', accelerators=[])))
+                self.assertRaises(
+                    InsufficientSystemResources,
+                    check_resource_request,
+                    Requirer(dict(memory=5, cores=1, disk="2G", accelerators=[])),
+                )
 
                 # This should be an accelerator we don't have.
                 # All the batch systems need code to know they don't have these accelerators.
-                self.assertRaises(InsufficientSystemResources, check_resource_request,
-                                  Requirer(dict(memory=5, cores=1, disk=100, accelerators=[{'kind': 'turbo-encabulator', 'count': 1}])))
+                self.assertRaises(
+                    InsufficientSystemResources,
+                    check_resource_request,
+                    Requirer(
+                        dict(
+                            memory=5,
+                            cores=1,
+                            disk=100,
+                            accelerators=[{"kind": "turbo-encabulator", "count": 1}],
+                        )
+                    ),
+                )
 
                 # These should be missing attributes
-                self.assertRaises(AttributeError, check_resource_request,
-                                  Requirer(dict(memory=5, cores=1, disk=1000)))
-                self.assertRaises(AttributeError, check_resource_request,
-                                  Requirer(dict(cores=1, disk=1000, accelerators=[])))
-                self.assertRaises(AttributeError, check_resource_request,
-                                  Requirer(dict(memory=10, disk=1000, accelerators=[])))
+                self.assertRaises(
+                    AttributeError,
+                    check_resource_request,
+                    Requirer(dict(memory=5, cores=1, disk=1000)),
+                )
+                self.assertRaises(
+                    AttributeError,
+                    check_resource_request,
+                    Requirer(dict(cores=1, disk=1000, accelerators=[])),
+                )
+                self.assertRaises(
+                    AttributeError,
+                    check_resource_request,
+                    Requirer(dict(memory=10, disk=1000, accelerators=[])),
+                )
 
                 # This should actually work
-                check_resource_request(Requirer(dict(memory=10, cores=1, disk=100, accelerators=[])))
+                check_resource_request(
+                    Requirer(dict(memory=10, cores=1, disk=100, accelerators=[]))
+                )
 
         def testScalableBatchSystem(self):
             # If instance of scalable batch system
@@ -345,7 +430,7 @@ class hidden:
             # prevent an endless loop, give it a few tries
             for it in range(tries):
                 running = self.batchSystem.getRunningBatchJobIDs()
-                logger.info(f'Running jobs now: {running}')
+                logger.info(f"Running jobs now: {running}")
                 runningIDs = list(running.keys())
                 if len(runningIDs) == numJobs:
                     break
@@ -360,7 +445,7 @@ class hidden:
 
         cpuCount = cpu_count()
         allocatedCores = sorted({1, 2, cpuCount})
-        sleepTime = 5
+        sleepTime = 30
 
         @abstractmethod
         def getBatchSystemName(self):
@@ -395,18 +480,26 @@ class hidden:
             Tests that the batch system is allocating core resources properly for concurrent tasks.
             """
             for coresPerJob in self.allocatedCores:
-                tempDir = self._createTempDir('testFiles')
+                tempDir = self._createTempDir("testFiles")
                 options = self.getOptions(tempDir)
 
-                counterPath = os.path.join(tempDir, 'counter')
+                counterPath = os.path.join(tempDir, "counter")
                 resetCounters(counterPath)
                 value, maxValue = getCounters(counterPath)
                 assert (value, maxValue) == (0, 0)
 
                 root = Job()
                 for _ in range(self.cpuCount):
-                    root.addFollowOn(Job.wrapFn(measureConcurrency, counterPath, self.sleepTime,
-                                                cores=coresPerJob, memory='1M', disk='1Mi'))
+                    root.addFollowOn(
+                        Job.wrapFn(
+                            measureConcurrency,
+                            counterPath,
+                            self.sleepTime,
+                            cores=coresPerJob,
+                            memory="1M",
+                            disk="1Mi",
+                        )
+                    )
                 with Toil(options) as toil:
                     toil.start(root)
                 _, maxValue = getCounters(counterPath)
@@ -420,18 +513,24 @@ class hidden:
                 # mapping of the number of cores to the OMP_NUM_THREADS value
                 0.1: "1",
                 1: "1",
-                2: "2"
+                2: "2",
             }
 
             temp_dir = self._createTempDir()
             options = self.getOptions(temp_dir)
 
             for cores, expected_omp_threads in test_cases.items():
-                if os.environ.get('OMP_NUM_THREADS'):
-                    expected_omp_threads = os.environ.get('OMP_NUM_THREADS')
-                    logger.info(f"OMP_NUM_THREADS is set.  Using OMP_NUM_THREADS={expected_omp_threads} instead.")
+                if os.environ.get("OMP_NUM_THREADS"):
+                    expected_omp_threads = os.environ.get("OMP_NUM_THREADS")
+                    logger.info(
+                        f"OMP_NUM_THREADS is set.  Using OMP_NUM_THREADS={expected_omp_threads} instead."
+                    )
                 with Toil(options) as toil:
-                    output = toil.start(Job.wrapFn(get_omp_threads, memory='1Mi', cores=cores, disk='1Mi'))
+                    output = toil.start(
+                        Job.wrapFn(
+                            get_omp_threads, memory="1Mi", cores=cores, disk="1Mi"
+                        )
+                    )
                 self.assertEqual(output, expected_omp_threads)
 
     class AbstractGridEngineBatchSystemTest(AbstractBatchSystemTest):
@@ -444,9 +543,10 @@ class hidden:
             config = super()._createConfig()
             config.statePollingWait = 0.5  # Reduce polling wait so tests run faster
             # can't use _getTestJobStorePath since that method removes the directory
-            config.jobStore = 'file:' + self._createTempDir('jobStore')
+            config.jobStore = "file:" + self._createTempDir("jobStore")
             return config
 
+
 @needs_kubernetes
 @needs_aws_s3
 @needs_fetchable_appliance
@@ -461,8 +561,11 @@ class KubernetesBatchSystemTest(hidden.AbstractBatchSystemTest):
     def createBatchSystem(self):
         # We know we have Kubernetes so we can import the batch system
         from toil.batchSystems.kubernetes import KubernetesBatchSystem
-        return KubernetesBatchSystem(config=self.config,
-                                     maxCores=numCores, maxMemory=1e9, maxDisk=2001)
+
+        return KubernetesBatchSystem(
+            config=self.config, maxCores=numCores, maxMemory=1e9, maxDisk=2001
+        )
+
 
 @needs_kubernetes_installed
 class KubernetesBatchSystemBenchTest(ToilTest):
@@ -486,7 +589,9 @@ class KubernetesBatchSystemBenchTest(ToilTest):
         constraints = KubernetesBatchSystem.Placement()
         constraints.set_preemptible(False)
         constraints.apply(normal_spec)
-        self.assertEqual(textwrap.dedent("""
+        self.assertEqual(
+            textwrap.dedent(
+                """
         {'node_affinity': {'preferred_during_scheduling_ignored_during_execution': None,
                            'required_during_scheduling_ignored_during_execution': {'node_selector_terms': [{'match_expressions': [{'key': 'eks.amazonaws.com/capacityType',
                                                                                                                                    'operator': 'NotIn',
@@ -497,14 +602,19 @@ class KubernetesBatchSystemBenchTest(ToilTest):
                                                                                                             'match_fields': None}]}},
          'pod_affinity': None,
          'pod_anti_affinity': None}
-        """).strip(), str(normal_spec.affinity))
+        """
+            ).strip(),
+            str(normal_spec.affinity),
+        )
         self.assertEqual(str(normal_spec.tolerations), "None")
 
         spot_spec = V1PodSpec(containers=[])
         constraints = KubernetesBatchSystem.Placement()
         constraints.set_preemptible(True)
         constraints.apply(spot_spec)
-        self.assertEqual(textwrap.dedent("""
+        self.assertEqual(
+            textwrap.dedent(
+                """
         {'node_affinity': {'preferred_during_scheduling_ignored_during_execution': [{'preference': {'match_expressions': [{'key': 'eks.amazonaws.com/capacityType',
                                                                                                                            'operator': 'In',
                                                                                                                            'values': ['SPOT']}],
@@ -518,14 +628,22 @@ class KubernetesBatchSystemBenchTest(ToilTest):
                            'required_during_scheduling_ignored_during_execution': None},
          'pod_affinity': None,
          'pod_anti_affinity': None}
-        """).strip(), str(spot_spec.affinity), )
-        self.assertEqual(textwrap.dedent("""
+        """
+            ).strip(),
+            str(spot_spec.affinity),
+        )
+        self.assertEqual(
+            textwrap.dedent(
+                """
         [{'effect': None,
          'key': 'cloud.google.com/gke-preemptible',
          'operator': None,
          'toleration_seconds': None,
          'value': 'true'}]
-        """).strip(), str(spot_spec.tolerations))
+        """
+            ).strip(),
+            str(spot_spec.tolerations),
+        )
 
     def test_label_constraints(self):
         """
@@ -541,11 +659,13 @@ class KubernetesBatchSystemBenchTest(ToilTest):
 
         spec = V1PodSpec(containers=[])
         constraints = KubernetesBatchSystem.Placement()
-        constraints.required_labels = [('GottaBeSetTo', ['This'])]
-        constraints.desired_labels = [('OutghtToBeSetTo', ['That'])]
-        constraints.prohibited_labels = [('CannotBe', ['ABadThing'])]
+        constraints.required_labels = [("GottaBeSetTo", ["This"])]
+        constraints.desired_labels = [("OutghtToBeSetTo", ["That"])]
+        constraints.prohibited_labels = [("CannotBe", ["ABadThing"])]
         constraints.apply(spec)
-        self.assertEqual(textwrap.dedent("""
+        self.assertEqual(
+            textwrap.dedent(
+                """
         {'node_affinity': {'preferred_during_scheduling_ignored_during_execution': [{'preference': {'match_expressions': [{'key': 'OutghtToBeSetTo',
                                                                                                                            'operator': 'In',
                                                                                                                            'values': ['That']}],
@@ -560,7 +680,10 @@ class KubernetesBatchSystemBenchTest(ToilTest):
                                                                                                             'match_fields': None}]}},
          'pod_affinity': None,
          'pod_anti_affinity': None}
-        """).strip(), str(spec.affinity),)
+        """
+            ).strip(),
+            str(spec.affinity),
+        )
         self.assertEqual(str(spec.tolerations), "None")
 
 
@@ -576,13 +699,16 @@ class AWSBatchBatchSystemTest(hidden.AbstractBatchSystemTest):
 
     def createBatchSystem(self):
         from toil.batchSystems.awsBatch import AWSBatchBatchSystem
-        return AWSBatchBatchSystem(config=self.config,
-                                   maxCores=numCores, maxMemory=1e9, maxDisk=2001)
+
+        return AWSBatchBatchSystem(
+            config=self.config, maxCores=numCores, maxMemory=1e9, maxDisk=2001
+        )
 
     def get_max_startup_seconds(self) -> int:
         # AWS Batch may need to scale out the compute environment.
         return 300
 
+
 @slow
 @needs_mesos
 class MesosBatchSystemTest(hidden.AbstractBatchSystemTest, MesosTestSupport):
@@ -597,7 +723,7 @@ class MesosBatchSystemTest(hidden.AbstractBatchSystemTest, MesosTestSupport):
         private IP address
         """
         config = super().createConfig()
-        config.mesos_endpoint = 'localhost:5050'
+        config.mesos_endpoint = "localhost:5050"
         return config
 
     def supportsWallTime(self):
@@ -606,19 +732,25 @@ class MesosBatchSystemTest(hidden.AbstractBatchSystemTest, MesosTestSupport):
     def createBatchSystem(self):
         # We know we have Mesos so we can import the batch system
         from toil.batchSystems.mesos.batchSystem import MesosBatchSystem
+
         self._startMesos(numCores)
-        return MesosBatchSystem(config=self.config,
-                                maxCores=numCores, maxMemory=1e9, maxDisk=1001)
+        return MesosBatchSystem(
+            config=self.config, maxCores=numCores, maxMemory=1e9, maxDisk=1001
+        )
 
     def tearDown(self):
         self._stopMesos()
         super().tearDown()
 
     def testIgnoreNode(self):
-        self.batchSystem.ignoreNode('localhost')
-        jobDesc = self._mockJobDescription(jobName='test2', unitName=None,
-                                           jobStoreID='1', requirements=defaultRequirements)
-        job = self.batchSystem.issueBatchJob('sleep 1000', jobDesc)
+        self.batchSystem.ignoreNode("localhost")
+        jobDesc = self._mockJobDescription(
+            jobName="test2",
+            unitName=None,
+            jobStoreID="1",
+            requirements=defaultRequirements,
+        )
+        job = self.batchSystem.issueBatchJob("sleep 1000", jobDesc)
 
         issuedID = self._waitForJobsToIssue(1)
         self.assertEqual(set(issuedID), {job})
@@ -635,7 +767,7 @@ def write_temp_file(s: str, temp_dir: str) -> str:
     """
     fd, path = tempfile.mkstemp(dir=temp_dir)
     try:
-        encoded = s.encode('utf-8')
+        encoded = s.encode("utf-8")
         assert os.write(fd, encoded) == len(encoded)
     except:
         os.unlink(path)
@@ -655,8 +787,9 @@ class SingleMachineBatchSystemTest(hidden.AbstractBatchSystemTest):
         return True
 
     def createBatchSystem(self) -> AbstractBatchSystem:
-        return SingleMachineBatchSystem(config=self.config,
-                                        maxCores=numCores, maxMemory=1e9, maxDisk=2001)
+        return SingleMachineBatchSystem(
+            config=self.config, maxCores=numCores, maxMemory=1e9, maxDisk=2001
+        )
 
     def testProcessEscape(self, hide: bool = False) -> None:
         """
@@ -677,14 +810,18 @@ class SingleMachineBatchSystemTest(hidden.AbstractBatchSystemTest):
             from typing import Any
 
             def handle_signal(sig: Any, frame: Any) -> None:
-                sys.stderr.write(f'{os.getpid()} ignoring signal {sig}\n')
+                sys.stderr.write(f"{os.getpid()} ignoring signal {sig}\n")
 
-            if hasattr(signal, 'valid_signals'):
+            if hasattr(signal, "valid_signals"):
                 # We can just ask about the signals
                 all_signals = signal.valid_signals()
             else:
                 # Fish them out by name
-                all_signals = [getattr(signal, n) for n in dir(signal) if n.startswith('SIG') and not n.startswith('SIG_')]
+                all_signals = [
+                    getattr(signal, n)
+                    for n in dir(signal)
+                    if n.startswith("SIG") and not n.startswith("SIG_")
+                ]
 
             for sig in all_signals:
                 # Set up to ignore all signals we can and generally be obstinate
@@ -706,7 +843,7 @@ class SingleMachineBatchSystemTest(hidden.AbstractBatchSystemTest):
                 fd = os.open(sys.argv[1], os.O_RDONLY)
                 fcntl.lockf(fd, fcntl.LOCK_SH)
 
-            sys.stderr.write(f'{os.getpid()} waiting...\n')
+            sys.stderr.write(f"{os.getpid()} waiting...\n")
 
             while True:
                 # Wait around forever
@@ -718,27 +855,26 @@ class SingleMachineBatchSystemTest(hidden.AbstractBatchSystemTest):
         script_path = write_temp_file(self._getScriptSource(script), temp_dir)
 
         # We will have all the job processes try and lock this file shared while they are alive.
-        lockable_path = write_temp_file('', temp_dir)
+        lockable_path = write_temp_file("", temp_dir)
 
         try:
-            command = f'{sys.executable} {script_path} {lockable_path}'
+            command = f"{sys.executable} {script_path} {lockable_path}"
             if hide:
                 # Tell the children to stop the first child and hide out in the
                 # process group it made.
-                command += ' hide'
+                command += " hide"
 
             # Start the job
             self.batchSystem.issueBatchJob(
-                command, 
+                command,
                 self._mockJobDescription(
-                    jobName='fork',
-                    jobStoreID='1',
-                    requirements=defaultRequirements)
+                    jobName="fork", jobStoreID="1", requirements=defaultRequirements
+                ),
             )
             # Wait
             time.sleep(10)
 
-            lockfile = open(lockable_path, 'w')
+            lockfile = open(lockable_path, "w")
 
             if not hide:
                 # In hiding mode the job will finish, and the batch system will
@@ -793,13 +929,14 @@ class MaxCoresSingleMachineBatchSystemTest(ToilTest):
 
         # Write initial value of counter file containing a tuple of two integers (i, n) where i
         # is the number of currently executing tasks and n the maximum observed value of i
-        self.counterPath = write_temp_file('0,0', temp_dir)
+        self.counterPath = write_temp_file("0,0", temp_dir)
 
         def script() -> None:
             import fcntl
             import os
             import sys
             import time
+
             def count(delta: int) -> None:
                 """
                 Adjust the first integer value in a file by the given amount. If the result
@@ -809,13 +946,14 @@ class MaxCoresSingleMachineBatchSystemTest(ToilTest):
                 try:
                     fcntl.flock(fd, fcntl.LOCK_EX)
                     try:
-                        s = os.read(fd, 10).decode('utf-8')
-                        value, maxValue = list(map(int, s.split(',')))
+                        s = os.read(fd, 10).decode("utf-8")
+                        value, maxValue = list(map(int, s.split(",")))
                         value += delta
-                        if value > maxValue: maxValue = value
+                        if value > maxValue:
+                            maxValue = value
                         os.lseek(fd, 0, 0)
                         os.ftruncate(fd, 0)
-                        os.write(fd, f'{value},{maxValue}'.encode())
+                        os.write(fd, f"{value},{maxValue}".encode())
                     finally:
                         fcntl.flock(fd, fcntl.LOCK_UN)
                 finally:
@@ -839,7 +977,7 @@ class MaxCoresSingleMachineBatchSystemTest(ToilTest):
         os.unlink(self.counterPath)
 
     def scriptCommand(self) -> str:
-        return ' '.join([sys.executable, self.scriptPath, self.counterPath])
+        return " ".join([sys.executable, self.scriptPath, self.counterPath])
 
     @retry_flaky_test(prepare=[tearDown, setUp])
     def test(self):
@@ -851,7 +989,13 @@ class MaxCoresSingleMachineBatchSystemTest(ToilTest):
         minCores = F(1, 10)
         self.assertEqual(float(minCores), SingleMachineBatchSystem.minCores)
         for maxCores in {F(minCores), minCores * 10, F(1), F(numCores, 2), F(numCores)}:
-            for coresPerJob in {F(minCores), F(minCores * 10), F(1), F(maxCores, 2), F(maxCores)}:
+            for coresPerJob in {
+                F(minCores),
+                F(minCores * 10),
+                F(1),
+                F(maxCores, 2),
+                F(maxCores),
+            }:
                 for load in (F(1, 10), F(1), F(10)):
                     jobs = int(maxCores / coresPerJob * load)
                     if jobs >= 1 and minCores <= coresPerJob < maxCores:
@@ -861,7 +1005,8 @@ class MaxCoresSingleMachineBatchSystemTest(ToilTest):
                             maxCores=float(maxCores),
                             # Ensure that memory or disk requirements don't get in the way.
                             maxMemory=jobs * 10,
-                            maxDisk=jobs * 10)
+                            maxDisk=jobs * 10,
+                        )
                         try:
                             jobIds = set()
                             for i in range(0, int(jobs)):
@@ -871,48 +1016,62 @@ class MaxCoresSingleMachineBatchSystemTest(ToilTest):
                                         memory=1,
                                         disk=1,
                                         accelerators=[],
-                                        preemptible=preemptible
+                                        preemptible=preemptible,
                                     ),
                                     jobName=str(i),
-                                    unitName=''
+                                    unitName="",
                                 )
                                 jobIds.add(bs.issueBatchJob(self.scriptCommand(), desc))
                             self.assertEqual(len(jobIds), jobs)
                             while jobIds:
                                 job = bs.getUpdatedBatchJob(maxWait=10)
                                 self.assertIsNotNone(job)
-                                jobId, status, wallTime = job.jobID, job.exitStatus, job.wallTime
+                                jobId, status, wallTime = (
+                                    job.jobID,
+                                    job.exitStatus,
+                                    job.wallTime,
+                                )
                                 self.assertEqual(status, 0)
                                 # would raise KeyError on absence
                                 jobIds.remove(jobId)
                         finally:
                             bs.shutdown()
-                        concurrentTasks, maxConcurrentTasks = getCounters(self.counterPath)
+                        concurrentTasks, maxConcurrentTasks = getCounters(
+                            self.counterPath
+                        )
                         self.assertEqual(concurrentTasks, 0)
-                        logger.info(f'maxCores: {maxCores}, '
-                                 f'coresPerJob: {coresPerJob}, '
-                                 f'load: {load}')
+                        logger.info(
+                            f"maxCores: {maxCores}, "
+                            f"coresPerJob: {coresPerJob}, "
+                            f"load: {load}"
+                        )
                         # This is the key assertion: we shouldn't run too many jobs.
                         # Because of nondeterminism we can't guarantee hitting the limit.
                         expectedMaxConcurrentTasks = min(maxCores // coresPerJob, jobs)
-                        self.assertLessEqual(maxConcurrentTasks, expectedMaxConcurrentTasks)
+                        self.assertLessEqual(
+                            maxConcurrentTasks, expectedMaxConcurrentTasks
+                        )
                         resetCounters(self.counterPath)
 
-    @skipIf(SingleMachineBatchSystem.numCores < 3, 'Need at least three cores to run this test')
+    @skipIf(
+        SingleMachineBatchSystem.numCores < 3,
+        "Need at least three cores to run this test",
+    )
     def testServices(self):
         options = Job.Runner.getDefaultOptions(self._getTestJobStorePath())
         options.logLevel = "DEBUG"
         options.maxCores = 3
         self.assertTrue(options.maxCores <= SingleMachineBatchSystem.numCores)
         Job.Runner.startToil(Job.wrapJobFn(parentJob, self.scriptCommand()), options)
-        with open(self.counterPath, 'r+') as f:
+        with open(self.counterPath, "r+") as f:
             s = f.read()
-        logger.info('Counter is %s', s)
+        logger.info("Counter is %s", s)
         self.assertEqual(getCounters(self.counterPath), (0, 3))
 
 
 # Toil can use only top-level functions so we have to add them here:
 
+
 def parentJob(job, cmd):
     job.addChildJobFn(childJob, cmd)
 
@@ -939,13 +1098,13 @@ class Service(Job.Service):
         self.cmd = cmd
 
     def start(self, fileStore):
-        subprocess.check_call(self.cmd + ' 1', shell=True)
+        subprocess.check_call(self.cmd + " 1", shell=True)
 
     def check(self):
         return True
 
     def stop(self, fileStore):
-        subprocess.check_call(self.cmd + ' -1', shell=True)
+        subprocess.check_call(self.cmd + " -1", shell=True)
 
 
 @slow
@@ -957,14 +1116,17 @@ class GridEngineBatchSystemTest(hidden.AbstractGridEngineBatchSystemTest):
 
     def createBatchSystem(self) -> AbstractBatchSystem:
         from toil.batchSystems.gridengine import GridEngineBatchSystem
-        return GridEngineBatchSystem(config=self.config, maxCores=numCores, maxMemory=1000e9,
-                                     maxDisk=1e9)
+
+        return GridEngineBatchSystem(
+            config=self.config, maxCores=numCores, maxMemory=1000e9, maxDisk=1e9
+        )
 
     def tearDown(self):
         super().tearDown()
         # Cleanup GridEngine output log file from qsub
         from glob import glob
-        for f in glob('toil_job*.o*'):
+
+        for f in glob("toil_job*.o*"):
             os.unlink(f)
 
 
@@ -977,14 +1139,17 @@ class SlurmBatchSystemTest(hidden.AbstractGridEngineBatchSystemTest):
 
     def createBatchSystem(self) -> AbstractBatchSystem:
         from toil.batchSystems.slurm import SlurmBatchSystem
-        return SlurmBatchSystem(config=self.config, maxCores=numCores, maxMemory=1000e9,
-                                maxDisk=1e9)
+
+        return SlurmBatchSystem(
+            config=self.config, maxCores=numCores, maxMemory=1000e9, maxDisk=1e9
+        )
 
     def tearDown(self):
         super().tearDown()
         # Cleanup 'slurm-%j.out' produced by sbatch
         from glob import glob
-        for f in glob('slurm-*.out'):
+
+        for f in glob("slurm-*.out"):
             os.unlink(f)
 
 
@@ -994,10 +1159,13 @@ class LSFBatchSystemTest(hidden.AbstractGridEngineBatchSystemTest):
     """
     Tests against the LSF batch system
     """
+
     def createBatchSystem(self) -> AbstractBatchSystem:
         from toil.batchSystems.lsf import LSFBatchSystem
-        return LSFBatchSystem(config=self.config, maxCores=numCores,
-                              maxMemory=1000e9, maxDisk=1e9)
+
+        return LSFBatchSystem(
+            config=self.config, maxCores=numCores, maxMemory=1000e9, maxDisk=1e9
+        )
 
 
 @slow
@@ -1010,19 +1178,22 @@ class TorqueBatchSystemTest(hidden.AbstractGridEngineBatchSystemTest):
     def _createDummyConfig(self):
         config = super()._createDummyConfig()
         # can't use _getTestJobStorePath since that method removes the directory
-        config.jobStore = self._createTempDir('jobStore')
+        config.jobStore = self._createTempDir("jobStore")
         return config
 
     def createBatchSystem(self) -> AbstractBatchSystem:
         from toil.batchSystems.torque import TorqueBatchSystem
-        return TorqueBatchSystem(config=self.config, maxCores=numCores, maxMemory=1000e9,
-                                     maxDisk=1e9)
+
+        return TorqueBatchSystem(
+            config=self.config, maxCores=numCores, maxMemory=1000e9, maxDisk=1e9
+        )
 
     def tearDown(self):
         super().tearDown()
         # Cleanup 'toil_job-%j.out' produced by sbatch
         from glob import glob
-        for f in glob('toil_job_*.[oe]*'):
+
+        for f in glob("toil_job_*.[oe]*"):
             os.unlink(f)
 
 
@@ -1035,8 +1206,10 @@ class HTCondorBatchSystemTest(hidden.AbstractGridEngineBatchSystemTest):
 
     def createBatchSystem(self) -> AbstractBatchSystem:
         from toil.batchSystems.htcondor import HTCondorBatchSystem
-        return HTCondorBatchSystem(config=self.config, maxCores=numCores, maxMemory=1000e9,
-                                   maxDisk=1e9)
+
+        return HTCondorBatchSystem(
+            config=self.config, maxCores=numCores, maxMemory=1000e9, maxDisk=1e9
+        )
 
     def tearDown(self):
         super().tearDown()
@@ -1051,46 +1224,71 @@ class SingleMachineBatchSystemJobTest(hidden.AbstractBatchSystemJobTest):
         return "single_machine"
 
     @slow
-    @retry_flaky_test(prepare=[hidden.AbstractBatchSystemJobTest.tearDown, hidden.AbstractBatchSystemJobTest.setUp])
+    @retry_flaky_test(
+        prepare=[
+            hidden.AbstractBatchSystemJobTest.tearDown,
+            hidden.AbstractBatchSystemJobTest.setUp,
+        ]
+    )
     def testConcurrencyWithDisk(self):
         """
         Tests that the batch system is allocating disk resources properly
         """
-        tempDir = self._createTempDir('testFiles')
+        tempDir = self._createTempDir("testFiles")
 
         options = Job.Runner.getDefaultOptions(self._getTestJobStorePath())
         options.workDir = tempDir
         from toil import physicalDisk
+
         availableDisk = physicalDisk(options.workDir)
-        logger.info('Testing disk concurrency limits with %s disk space', availableDisk)
+        logger.info("Testing disk concurrency limits with %s disk space", availableDisk)
         # More disk might become available by the time Toil starts, so we limit it here
         options.maxDisk = availableDisk
         options.batchSystem = self.batchSystemName
 
-        counterPath = os.path.join(tempDir, 'counter')
+        counterPath = os.path.join(tempDir, "counter")
         resetCounters(counterPath)
         value, maxValue = getCounters(counterPath)
         assert (value, maxValue) == (0, 0)
 
         half_disk = availableDisk // 2
         more_than_half_disk = half_disk + 500
-        logger.info('Dividing into parts of %s and %s', half_disk, more_than_half_disk)
+        logger.info("Dividing into parts of %s and %s", half_disk, more_than_half_disk)
 
         root = Job()
         # Physically, we're asking for 50% of disk and 50% of disk + 500bytes in the two jobs. The
         # batchsystem should not allow the 2 child jobs to run concurrently.
-        root.addChild(Job.wrapFn(measureConcurrency, counterPath, self.sleepTime, cores=1,
-                                 memory='1M', disk=half_disk))
-        root.addChild(Job.wrapFn(measureConcurrency, counterPath, self.sleepTime, cores=1,
-                                 memory='1M', disk=more_than_half_disk))
+        root.addChild(
+            Job.wrapFn(
+                measureConcurrency,
+                counterPath,
+                self.sleepTime,
+                cores=1,
+                memory="1M",
+                disk=half_disk,
+            )
+        )
+        root.addChild(
+            Job.wrapFn(
+                measureConcurrency,
+                counterPath,
+                self.sleepTime,
+                cores=1,
+                memory="1M",
+                disk=more_than_half_disk,
+            )
+        )
         Job.Runner.startToil(root, options)
         _, maxValue = getCounters(counterPath)
 
-        logger.info('After run: %s disk space', physicalDisk(options.workDir))
+        logger.info("After run: %s disk space", physicalDisk(options.workDir))
 
         self.assertEqual(maxValue, 1)
 
-    @skipIf(SingleMachineBatchSystem.numCores < 4, 'Need at least four cores to run this test')
+    @skipIf(
+        SingleMachineBatchSystem.numCores < 4,
+        "Need at least four cores to run this test",
+    )
     @slow
     def testNestedResourcesDoNotBlock(self):
         """
@@ -1098,39 +1296,80 @@ class SingleMachineBatchSystemJobTest(hidden.AbstractBatchSystemJobTest):
         Test that unavailability of cpus for one job that is scheduled does not block another job
         that can run.
         """
-        tempDir = self._createTempDir('testFiles')
+        tempDir = self._createTempDir("testFiles")
 
         options = Job.Runner.getDefaultOptions(self._getTestJobStorePath())
         options.workDir = tempDir
         options.maxCores = 4
         from toil import physicalMemory
+
         availableMemory = physicalMemory()
         options.batchSystem = self.batchSystemName
 
-        outFile = os.path.join(tempDir, 'counter')
-        open(outFile, 'w').close()
+        outFile = os.path.join(tempDir, "counter")
+        open(outFile, "w").close()
 
         root = Job()
 
-        blocker = Job.wrapFn(_resourceBlockTestAuxFn, outFile=outFile, sleepTime=30, writeVal='b',
-                             cores=2, memory='1M', disk='1M')
-        firstJob = Job.wrapFn(_resourceBlockTestAuxFn, outFile=outFile, sleepTime=5, writeVal='fJ',
-                              cores=1, memory='1M', disk='1M')
-        secondJob = Job.wrapFn(_resourceBlockTestAuxFn, outFile=outFile, sleepTime=10,
-                               writeVal='sJ', cores=1, memory='1M', disk='1M')
+        blocker = Job.wrapFn(
+            _resourceBlockTestAuxFn,
+            outFile=outFile,
+            sleepTime=30,
+            writeVal="b",
+            cores=2,
+            memory="1M",
+            disk="1M",
+        )
+        firstJob = Job.wrapFn(
+            _resourceBlockTestAuxFn,
+            outFile=outFile,
+            sleepTime=5,
+            writeVal="fJ",
+            cores=1,
+            memory="1M",
+            disk="1M",
+        )
+        secondJob = Job.wrapFn(
+            _resourceBlockTestAuxFn,
+            outFile=outFile,
+            sleepTime=10,
+            writeVal="sJ",
+            cores=1,
+            memory="1M",
+            disk="1M",
+        )
 
         # Should block off 50% of memory while waiting for it's 3 cores
-        firstJobChild = Job.wrapFn(_resourceBlockTestAuxFn, outFile=outFile, sleepTime=0,
-                                   writeVal='fJC', cores=3, memory=int(availableMemory // 2), disk='1M')
+        firstJobChild = Job.wrapFn(
+            _resourceBlockTestAuxFn,
+            outFile=outFile,
+            sleepTime=0,
+            writeVal="fJC",
+            cores=3,
+            memory=int(availableMemory // 2),
+            disk="1M",
+        )
 
         # These two shouldn't be able to run before B because there should be only
         # (50% of memory - 1M) available (firstJobChild should be blocking 50%)
-        secondJobChild = Job.wrapFn(_resourceBlockTestAuxFn, outFile=outFile, sleepTime=5,
-                                    writeVal='sJC', cores=2, memory=int(availableMemory // 1.5),
-                                    disk='1M')
-        secondJobGrandChild = Job.wrapFn(_resourceBlockTestAuxFn, outFile=outFile, sleepTime=5,
-                                         writeVal='sJGC', cores=2, memory=int(availableMemory // 1.5),
-                                         disk='1M')
+        secondJobChild = Job.wrapFn(
+            _resourceBlockTestAuxFn,
+            outFile=outFile,
+            sleepTime=5,
+            writeVal="sJC",
+            cores=2,
+            memory=int(availableMemory // 1.5),
+            disk="1M",
+        )
+        secondJobGrandChild = Job.wrapFn(
+            _resourceBlockTestAuxFn,
+            outFile=outFile,
+            sleepTime=5,
+            writeVal="sJGC",
+            cores=2,
+            memory=int(availableMemory // 1.5),
+            disk="1M",
+        )
 
         root.addChild(blocker)
         root.addChild(firstJob)
@@ -1160,9 +1399,11 @@ class SingleMachineBatchSystemJobTest(hidden.AbstractBatchSystemJobTest):
             outString = oFH.read()
         # The ordering of b, fJ and sJ is non-deterministic since they are scheduled at the same
         # time. We look for all possible permutations.
-        possibleStarts = tuple(''.join(x) for x in itertools.permutations(['b', 'fJ', 'sJ']))
+        possibleStarts = tuple(
+            "".join(x) for x in itertools.permutations(["b", "fJ", "sJ"])
+        )
         assert outString.startswith(possibleStarts)
-        assert outString.endswith('sJCsJGCfJC')
+        assert outString.endswith("sJCsJGCfJC")
 
 
 def _resourceBlockTestAuxFn(outFile, sleepTime, writeVal):
@@ -1172,7 +1413,7 @@ def _resourceBlockTestAuxFn(outFile, sleepTime, writeVal):
     :param int sleepTime: Time to sleep for
     :param str writeVal: Character to write
     """
-    with open(outFile, 'a') as oFH:
+    with open(outFile, "a") as oFH:
         fcntl.flock(oFH, fcntl.LOCK_EX)
         oFH.write(writeVal)
     time.sleep(sleepTime)
@@ -1184,9 +1425,10 @@ class MesosBatchSystemJobTest(hidden.AbstractBatchSystemJobTest, MesosTestSuppor
     """
     Tests Toil workflow against the Mesos batch system
     """
+
     def getOptions(self, tempDir):
         options = super().getOptions(tempDir)
-        options.mesos_endpoint = 'localhost:5050'
+        options.mesos_endpoint = "localhost:5050"
         return options
 
     def getBatchSystemName(self):
@@ -1227,12 +1469,13 @@ def count(delta, file_path):
         fcntl.flock(fd, fcntl.LOCK_EX)
         try:
             s = os.read(fd, 10)
-            value, maxValue = (int(i) for i in s.decode('utf-8').split(','))
+            value, maxValue = (int(i) for i in s.decode("utf-8").split(","))
             value += delta
-            if value > maxValue: maxValue = value
+            if value > maxValue:
+                maxValue = value
             os.lseek(fd, 0, 0)
             os.ftruncate(fd, 0)
-            os.write(fd, f'{value},{maxValue}'.encode())
+            os.write(fd, f"{value},{maxValue}".encode())
         finally:
             fcntl.flock(fd, fcntl.LOCK_UN)
     finally:
@@ -1241,8 +1484,8 @@ def count(delta, file_path):
 
 
 def getCounters(path):
-    with open(path, 'r+') as f:
-        concurrentTasks, maxConcurrentTasks = (int(i) for i in f.read().split(','))
+    with open(path, "r+") as f:
+        concurrentTasks, maxConcurrentTasks = (int(i) for i in f.read().split(","))
     return concurrentTasks, maxConcurrentTasks
 
 
@@ -1253,4 +1496,4 @@ def resetCounters(path):
 
 
 def get_omp_threads() -> str:
-    return os.environ['OMP_NUM_THREADS']
+    return os.environ["OMP_NUM_THREADS"]