toil 5.12.0__py3-none-any.whl → 6.1.0a1__py3-none-any.whl

toil/server/wes/abstract_backend.py

@@ -3,7 +3,6 @@ import functools
 import json
 import logging
 import os
-import tempfile
 from abc import abstractmethod
 from typing import Any, Callable, Dict, List, Optional, Tuple, Union
 from urllib.parse import urldefrag
@@ -11,6 +10,8 @@ from urllib.parse import urldefrag
 import connexion  # type: ignore
 from werkzeug.utils import secure_filename
 
+from toil.lib.io import mkdtemp
+
 logger = logging.getLogger(__name__)
 
 # Define a type for WES task log entries in responses
@@ -210,7 +211,7 @@ class WESBackend:
                          If None, a temporary directory is created.
         """
         if not temp_dir:
-            temp_dir = tempfile.mkdtemp()
+            temp_dir = mkdtemp()
         body: Dict[str, Any] = {}
         has_attachments = False
         for key, ls in connexion.request.files.lists():

toil/server/wes/amazon_wes_utils.py

@@ -20,11 +20,10 @@
 
 import json
 import logging
-import os
 import sys
 import zipfile
 from os import path
-from typing import IO, Any, Dict, List, Optional, Union, cast
+from typing import IO, List, Optional, cast
 
 if sys.version_info >= (3, 8):
     from typing import TypedDict
@@ -164,15 +163,17 @@ def parse_workflow_manifest_file(manifest_file: str) -> WorkflowPlan:
     :rtype: dict of `data` and `files`
 
     MANIFEST.json is expected to be formatted like:
+    
     .. code-block:: json
+        
        {
            "mainWorkflowURL": "relpath/to/workflow",
            "inputFileURLs": [
                "relpath/to/input-file-1",
                "relpath/to/input-file-2",
-               ...
+               "relpath/to/input-file-3"
            ],
-           "optionsFileURL" "relpath/to/option-file
+           "optionsFileURL": "relpath/to/option-file"
        }
 
     The `mainWorkflowURL` property that provides a relative file path in the zip to a workflow file, which will be set as `workflowSource`

toil/server/wes/tasks.py

@@ -11,7 +11,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-import fcntl
 import json
 import logging
 import multiprocessing
@@ -134,8 +133,8 @@ class ToilWorkflowRunner:
         the workflow execution engine.
 
         :param workflow_engine_parameters: User-specified parameters for this
-        particular workflow. Keys are command-line options, and values are
-        option arguments, or None for options that are flags.
+            particular workflow. Keys are command-line options, and values are
+            option arguments, or None for options that are flags.
         """
         options = []
 

toil/server/wes/toil_backend.py

@@ -16,9 +16,8 @@ import logging
 import os
 import shutil
 import uuid
-from collections import Counter, defaultdict
+from collections import Counter
 from contextlib import contextmanager
-from tempfile import NamedTemporaryFile
 from typing import (Any,
                     Callable,
                     Dict,
@@ -32,18 +31,11 @@ from typing import (Any,
                     overload)
 
 from flask import send_from_directory
-from flask.globals import request as flask_request
 from werkzeug.utils import redirect
 from werkzeug.wrappers.response import Response
 
 import toil.server.wes.amazon_wes_utils as amazon_wes_utils
-from toil.bus import (JobAnnotationMessage,
-                      JobCompletedMessage,
-                      JobFailedMessage,
-                      JobIssuedMessage,
-                      JobUpdatedMessage,
-                      MessageBus,
-                      replay_message_bus, JobStatus)
+from toil.bus import JobStatus, replay_message_bus
 from toil.lib.io import AtomicFileCreate
 from toil.lib.threading import global_mutex
 from toil.server.utils import (WorkflowStateMachine,

toil/server/wsgi_app.py

@@ -21,6 +21,8 @@ class GunicornApplication(BaseApplication):  # type: ignore
     An entry point to integrate a Gunicorn WSGI server in Python. To start a
     WSGI application with callable `app`, run the following code:
 
+    .. code-block:: python
+
         WSGIApplication(app, options={
             ...
         }).run()

toil/serviceManager.py

@@ -122,7 +122,8 @@ class ServiceManager:
         try:
             client_id = self.__clients_out.get(timeout=maxWait)
             self.__waiting_clients.remove(client_id)
-            assert self.__service_manager_jobs >= 0
+            if self.__service_manager_jobs < 0:
+                raise RuntimeError("The number of jobs scheduled by the service manager cannot be negative.")
             self.__service_manager_jobs -= 1
             return client_id
         except Empty:
@@ -139,7 +140,8 @@ class ServiceManager:
         try:
             client_id = self.__failed_clients_out.get(timeout=maxWait)
             self.__waiting_clients.remove(client_id)
-            assert self.__service_manager_jobs >= 0
+            if self.__service_manager_jobs < 0:
+                raise RuntimeError("The number of jobs scheduled by the service manager cannot be negative.")
             self.__service_manager_jobs -= 1
             return client_id
         except Empty:
@@ -154,7 +156,8 @@ class ServiceManager:
         """
         try:
             service_id = self.__services_out.get(timeout=maxWait)
-            assert self.__service_manager_jobs >= 0
+            if self.__service_manager_jobs < 0:
+                raise RuntimeError("The number of jobs scheduled by the service manager cannot be negative.")
             self.__service_manager_jobs -= 1
             return service_id
         except Empty:
@@ -304,7 +307,8 @@ class ServiceManager:
                         starting_services.remove(service_id)
                         client_id = service_to_client[service_id]
                         remaining_services_by_client[client_id] -= 1
-                        assert remaining_services_by_client[client_id] >= 0
+                        if remaining_services_by_client[client_id] < 0:
+                            raise RuntimeError("The number of remaining services cannot be negative.")
                         del service_to_client[service_id]
                         if not self.__job_store.file_exists(service_job_desc.errorJobStoreID):
                             logger.error(
@@ -356,12 +360,10 @@ class ServiceManager:
                     service_job_desc,
                     service_job_desc.startJobStoreID,
                 )
-                assert self.__job_store.file_exists(
-                    service_job_desc.startJobStoreID
-                ), f"Service manager attempted to start service {service_job_desc} that has already started"
-                assert self.__toil_state.job_exists(
-                    str(service_job_desc.jobStoreID)
-                ), f"Service manager attempted to start service {service_job_desc} that is not in the job store"
+                if not self.__job_store.file_exists(service_job_desc.startJobStoreID):
+                    raise RuntimeError(f"Service manager attempted to start service {service_job_desc} that has already started")
+                if not self.__toil_state.job_exists(str(service_job_desc.jobStoreID)):
+                    raise RuntimeError(f"Service manager attempted to start service {service_job_desc} that is not in the job store")
                 # At this point the terminateJobStoreID and errorJobStoreID
                 # could have been deleted, since the service can be killed at
                 # any time! So we can't assert their presence here.

toil/statsAndLogging.py

@@ -148,10 +148,12 @@ class StatsAndLogging:
             if not isinstance(statsStr, str):
                 statsStr = statsStr.decode()
             stats = json.loads(statsStr, object_hook=Expando)
+            if not stats:
+                return
             try:
                 logs = stats.workers.logsToMaster
             except AttributeError:
-                # To be expected if there were no calls to logToMaster()
+                # To be expected if there were no calls to log_to_leader()
                 pass
             else:
                 for message in logs:
@@ -225,6 +227,8 @@ def add_logging_options(parser: ArgumentParser) -> None:
     levels += [l.lower() for l in levels] + [l.upper() for l in levels]
     group.add_argument("--logOff", dest="logLevel", default=default_loglevel,
                        action="store_const", const="CRITICAL", help="Same as --logCRITICAL.")
+    # Maybe deprecate the above in favor of --logLevel?
+
     group.add_argument("--logLevel", dest="logLevel", default=default_loglevel, choices=levels,
                        help=f"Set the log level. Default: {default_loglevel}.  Options: {levels}.")
     group.add_argument("--logFile", dest="logFile", help="File to log in.")

toil/test/__init__.py

@@ -21,7 +21,6 @@ import shutil
 import signal
 import subprocess
 import sys
-import tempfile
 import threading
 import time
 import unittest
@@ -30,6 +29,7 @@ from abc import ABCMeta, abstractmethod
 from contextlib import contextmanager
 from inspect import getsource
 from shutil import which
+from tempfile import mkstemp
 from textwrap import dedent
 from typing import (Any,
                     Callable,
@@ -57,6 +57,7 @@ from toil import ApplianceImageNotFound, applianceSelf, toilPackageDirPath
 from toil.lib.accelerators import (have_working_nvidia_docker_runtime,
                                    have_working_nvidia_smi)
 from toil.lib.aws import running_on_ec2
+from toil.lib.io import mkdtemp
 from toil.lib.iterables import concat
 from toil.lib.memoize import memoize
 from toil.lib.threading import ExceptionalThread, cpu_count
@@ -188,7 +189,7 @@ class ToilTest(unittest.TestCase):
         prefix.extend([_f for _f in names if _f])
         prefix.append('')
         temp_dir_path = os.path.realpath(
-            tempfile.mkdtemp(dir=cls._tempBaseDir, prefix="-".join(prefix))
+            mkdtemp(dir=cls._tempBaseDir, prefix="-".join(prefix))
         )
         cls._tempDirs.append(temp_dir_path)
         return temp_dir_path
@@ -314,7 +315,7 @@ else:
 def get_temp_file(suffix: str = "", rootDir: Optional[str] = None) -> str:
     """Return a string representing a temporary file, that must be manually deleted."""
     if rootDir is None:
-        handle, tmp_file = tempfile.mkstemp(suffix)
+        handle, tmp_file = mkstemp(suffix)
         os.close(handle)
         return tmp_file
     else:
@@ -359,10 +360,17 @@ def needs_rsync3(test_item: MT) -> MT:
     return test_item
 
 
+def needs_online(test_item: MT) -> MT:
+    """Use as a decorator before test classes or methods to run only if we are meant to talk to the Internet."""
+    test_item = _mark_test('online', test_item)
+    if os.getenv('TOIL_SKIP_ONLINE', '').lower() == 'true':
+        return unittest.skip('Skipping online test.')(test_item)
+    return test_item
+
 def needs_aws_s3(test_item: MT) -> MT:
     """Use as a decorator before test classes or methods to run only if AWS S3 is usable."""
     # TODO: we just check for generic access to the AWS account
-    test_item = _mark_test('aws-s3', test_item)
+    test_item = _mark_test('aws-s3', needs_online(test_item))
     try:
         from boto import config
         boto_credentials = config.get('Credentials', 'aws_access_key_id')
@@ -415,7 +423,7 @@ def needs_google_storage(test_item: MT) -> MT:
     Cloud is installed and we ought to be able to access public Google Storage
     URIs.
     """
-    test_item = _mark_test('google-storage', test_item)
+    test_item = _mark_test('google-storage', needs_online(test_item))
     try:
         from google.cloud import storage  # noqa
     except ImportError:
@@ -427,7 +435,7 @@ def needs_google_project(test_item: MT) -> MT:
     """
     Use as a decorator before test classes or methods to run only if we have a Google Cloud project set.
     """
-    test_item = _mark_test('google-project', test_item)
+    test_item = _mark_test('google-project', needs_online(test_item))
     test_item = needs_env_var('TOIL_GOOGLE_PROJECTID', "a Google project ID")(test_item)
     return test_item
 
@@ -447,44 +455,19 @@ def needs_torque(test_item: MT) -> MT:
         return test_item
     return unittest.skip("Install PBS/Torque to include this test.")(test_item)
 
-
-def needs_tes(test_item: MT) -> MT:
-    """Use as a decorator before test classes or methods to run only if TES is available."""
-    test_item = _mark_test('tes', test_item)
-
-    try:
-        from toil.batchSystems.tes import TESBatchSystem
-    except ImportError:
-        return unittest.skip("Install py-tes to include this test")(test_item)
-
-    tes_url = os.environ.get('TOIL_TES_ENDPOINT', TESBatchSystem.get_default_tes_endpoint())
-    try:
-        urlopen(tes_url)
-    except HTTPError:
-        # Funnel happens to 404 if TES is working. But any HTTPError means we
-        # dialed somebody who picked up.
-        pass
-    except URLError:
-        # Will give connection refused if we can't connect because the server's
-        # not there. We can also get a "cannot assign requested address" if
-        # we're on Kubernetes dialing localhost and !!creative things!! have
-        # been done to the network stack.
-        return unittest.skip(f"Run a TES server on {tes_url} to include this test")(test_item)
-    return test_item
-
-
 def needs_kubernetes_installed(test_item: MT) -> MT:
     """Use as a decorator before test classes or methods to run only if Kubernetes is installed."""
     test_item = _mark_test('kubernetes', test_item)
     try:
         import kubernetes
+        str(kubernetes)  # to prevent removal of this import
     except ImportError:
         return unittest.skip("Install Toil with the 'kubernetes' extra to include this test.")(test_item)
     return test_item
 
 def needs_kubernetes(test_item: MT) -> MT:
     """Use as a decorator before test classes or methods to run only if Kubernetes is installed and configured."""
-    test_item = needs_kubernetes_installed(test_item)
+    test_item = needs_kubernetes_installed(needs_online(test_item))
     try:
         import kubernetes
         try:
@@ -514,14 +497,6 @@ def needs_mesos(test_item: MT) -> MT:
     return test_item
 
 
-def needs_parasol(test_item: MT) -> MT:
-    """Use as decorator so tests are only run if Parasol is installed."""
-    test_item = _mark_test('parasol', test_item)
-    if which('parasol'):
-        return test_item
-    return unittest.skip("Install Parasol to include this test.")(test_item)
-
-
 def needs_slurm(test_item: MT) -> MT:
     """Use as a decorator before test classes or methods to run only if Slurm is installed."""
     test_item = _mark_test('slurm', test_item)
@@ -571,20 +546,20 @@ def needs_docker(test_item: MT) -> MT:
     Use as a decorator before test classes or methods to only run them if
     docker is installed and docker-based tests are enabled.
     """
-    test_item = _mark_test('docker', test_item)
+    test_item = _mark_test('docker', needs_online(test_item))
     if os.getenv('TOIL_SKIP_DOCKER', '').lower() == 'true':
         return unittest.skip('Skipping docker test.')(test_item)
     if which('docker'):
         return test_item
     else:
         return unittest.skip("Install docker to include this test.")(test_item)
-        
+
 def needs_singularity(test_item: MT) -> MT:
     """
     Use as a decorator before test classes or methods to only run them if
     singularity is installed.
     """
-    test_item = _mark_test('singularity', test_item)
+    test_item = _mark_test('singularity', needs_online(test_item))
     if which('singularity'):
         return test_item
     else:
@@ -621,7 +596,7 @@ def needs_docker_cuda(test_item: MT) -> MT:
     Use as a decorator before test classes or methods to only run them if
     a CUDA setup is available through Docker.
     """
-    test_item = _mark_test('docker_cuda', test_item)
+    test_item = _mark_test('docker_cuda', needs_online(test_item))
     if have_working_nvidia_docker_runtime():
         return test_item
     else:
@@ -677,7 +652,7 @@ def needs_celery_broker(test_item: MT) -> MT:
     """
     Use as a decorator before test classes or methods to run only if RabbitMQ is set up to take Celery jobs.
     """
-    test_item = _mark_test('celery', test_item)
+    test_item = _mark_test('celery', needs_online(test_item))
     test_item = needs_env_var('TOIL_WES_BROKER_URL', "a URL to a RabbitMQ broker for Celery")(test_item)
     return test_item
 
@@ -686,7 +661,7 @@ def needs_wes_server(test_item: MT) -> MT:
     Use as a decorator before test classes or methods to run only if a WES
     server is available to run against.
     """
-    test_item = _mark_test('wes_server', test_item)
+    test_item = _mark_test('wes_server', needs_online(test_item))
 
     wes_url = os.environ.get('TOIL_WES_ENDPOINT')
     if not wes_url:
@@ -744,7 +719,7 @@ def needs_fetchable_appliance(test_item: MT) -> MT:
     the Toil appliance Docker image is able to be downloaded from the Internet.
     """
 
-    test_item = _mark_test('fetchable_appliance', test_item)
+    test_item = _mark_test('fetchable_appliance', needs_online(test_item))
     if os.getenv('TOIL_SKIP_DOCKER', '').lower() == 'true':
         return unittest.skip('Skipping docker test.')(test_item)
     try:
@@ -765,9 +740,7 @@ def integrative(test_item: MT) -> MT:
     Use this to decorate integration tests so as to skip them during regular builds.
 
     We define integration tests as A) involving other, non-Toil software components
-    that we develop and/or B) having a higher cost (time or money). Note that brittleness
-    does not qualify a test for being integrative. Neither does involvement of external
-    services such as AWS, since that would cover most of Toil's test.
+    that we develop and/or B) having a higher cost (time or money).
     """
     test_item = _mark_test('integrative', test_item)
     if os.getenv('TOIL_TEST_INTEGRATIVE', '').lower() == 'true':
@@ -797,11 +770,13 @@ methodNamePartRegex = re.compile('^[a-zA-Z_0-9]+$')
 @contextmanager
 def timeLimit(seconds: int) -> Generator[None, None, None]:
     """
-    http://stackoverflow.com/a/601168
-    Use to limit the execution time of a function. Raises an exception if the execution of the
-    function takes more than the specified amount of time.
+    Use to limit the execution time of a function.
+
+    Raises an exception if the execution of the function takes more than the
+    specified amount of time. See <http://stackoverflow.com/a/601168>.
 
     :param seconds: maximum allowable time, in seconds
+    
     >>> import time
     >>> with timeLimit(2):
     ...    time.sleep(1)

toil/test/batchSystems/batchSystemTest.py

@@ -31,10 +31,9 @@ from toil.batchSystems.abstractBatchSystem import (AbstractBatchSystem,
 # in order to import properly. Import them later, in tests
 # protected by annotations.
 from toil.batchSystems.mesos.test import MesosTestSupport
-from toil.batchSystems.parasol import ParasolBatchSystem
-from toil.batchSystems.registry import (BATCH_SYSTEM_FACTORY_REGISTRY,
-                                        BATCH_SYSTEMS,
-                                        addBatchSystemFactory,
+from toil.batchSystems.registry import (add_batch_system_factory,
+                                        get_batch_system,
+                                        get_batch_systems,
                                         restore_batch_system_plugin_state,
                                         save_batch_system_plugin_state)
 from toil.batchSystems.singleMachine import SingleMachineBatchSystem
@@ -52,12 +51,9 @@ from toil.test import (ToilTest,
                        needs_kubernetes_installed,
                        needs_lsf,
                        needs_mesos,
-                       needs_parasol,
                        needs_slurm,
-                       needs_tes,
                        needs_torque,
                        slow)
-from toil.test.batchSystems.parasolTestSupport import ParasolTestSupport
 
 logger = logging.getLogger(__name__)
 
@@ -88,16 +84,16 @@ class BatchSystemPluginTest(ToilTest):
         restore_batch_system_plugin_state(self.__state)
         super().tearDown()
 
-    def testAddBatchSystemFactory(self):
+    def test_add_batch_system_factory(self):
         def test_batch_system_factory():
             # TODO: Adding the same batch system under multiple names means we
             # can't actually create Toil options, because each version tries to
             # add its arguments.
             return SingleMachineBatchSystem
 
-        addBatchSystemFactory('testBatchSystem', test_batch_system_factory)
-        assert ('testBatchSystem', test_batch_system_factory) in BATCH_SYSTEM_FACTORY_REGISTRY.items()
-        assert 'testBatchSystem' in BATCH_SYSTEMS
+        add_batch_system_factory('testBatchSystem', test_batch_system_factory)
+        assert 'testBatchSystem' in get_batch_systems()
+        assert get_batch_system('testBatchSystem') == SingleMachineBatchSystem
 
 class hidden:
     """
@@ -248,10 +244,6 @@ class hidden:
             self.batchSystem.killBatchJobs([10])
 
         def test_set_env(self):
-            # Parasol disobeys shell rules and splits the command at the space
-            # character into arguments before exec'ing it, whether the space is
-            # quoted, escaped or not.
-
             # Start with a relatively safe script
             script_shell = 'if [ "x${FOO}" == "xbar" ] ; then exit 23 ; else exit 42 ; fi'
 
@@ -575,23 +567,6 @@ class KubernetesBatchSystemBenchTest(ToilTest):
         self.assertEqual(str(spec.tolerations), "None")
 
 
-@needs_tes
-@needs_fetchable_appliance
-class TESBatchSystemTest(hidden.AbstractBatchSystemTest):
-    """
-    Tests against the TES batch system
-    """
-
-    def supportsWallTime(self):
-        return True
-
-    def createBatchSystem(self):
-        # Import the batch system when we know we have it.
-        # Doesn't really matter for TES right now, but someday it might.
-        from toil.batchSystems.tes import TESBatchSystem
-        return TESBatchSystem(config=self.config,
-                              maxCores=numCores, maxMemory=1e9, maxDisk=2001)
-
 @needs_aws_batch
 @needs_fetchable_appliance
 class AWSBatchBatchSystemTest(hidden.AbstractBatchSystemTest):
@@ -849,7 +824,7 @@ class MaxCoresSingleMachineBatchSystemTest(ToilTest):
             if len(sys.argv) < 3:
                 count(1)
                 try:
-                    time.sleep(1)
+                    time.sleep(0.5)
                 finally:
                     count(-1)
             else:
@@ -910,9 +885,10 @@ class MaxCoresSingleMachineBatchSystemTest(ToilTest):
                         logger.info(f'maxCores: {maxCores}, '
                                  f'coresPerJob: {coresPerJob}, '
                                  f'load: {load}')
-                        # This is the key assertion:
+                        # This is the key assertion: we shouldn't run too many jobs.
+                        # Because of nondeterminism we can't guarantee hitting the limit.
                         expectedMaxConcurrentTasks = min(maxCores // coresPerJob, jobs)
-                        self.assertEqual(maxConcurrentTasks, expectedMaxConcurrentTasks)
+                        self.assertLessEqual(maxConcurrentTasks, expectedMaxConcurrentTasks)
                         resetCounters(self.counterPath)
 
     @skipIf(SingleMachineBatchSystem.numCores < 3, 'Need at least three cores to run this test')
@@ -965,82 +941,6 @@ class Service(Job.Service):
         subprocess.check_call(self.cmd + ' -1', shell=True)
 
 
-@slow
-@needs_parasol
-class ParasolBatchSystemTest(hidden.AbstractBatchSystemTest, ParasolTestSupport):
-    """
-    Tests the Parasol batch system
-    """
-
-    def supportsWallTime(self):
-        return True
-
-    def _createConfig(self):
-        config = super()._createConfig()
-        # can't use _getTestJobStorePath since that method removes the directory
-        config.jobStore = self._createTempDir('jobStore')
-        return config
-
-    def createBatchSystem(self) -> AbstractBatchSystem:
-        memory = int(3e9)
-        self._startParasol(numCores=numCores, memory=memory)
-
-        return ParasolBatchSystem(config=self.config,
-                                  maxCores=numCores,
-                                  maxMemory=memory,
-                                  maxDisk=1001)
-
-    def tearDown(self):
-        super().tearDown()
-        self._stopParasol()
-
-    def testBatchResourceLimits(self):
-        jobDesc1 = JobDescription(command="sleep 1000",
-                                  requirements=dict(memory=1 << 30, cores=1,
-                                                    disk=1000, accelerators=[],
-                                                    preemptible=preemptible),
-                                  jobName='testResourceLimits')
-        job1 = self.batchSystem.issueBatchJob(jobDesc1)
-        self.assertIsNotNone(job1)
-        jobDesc2 = JobDescription(command="sleep 1000",
-                                  requirements=dict(memory=2 << 30, cores=1,
-                                                    disk=1000, accelerators=[],
-                                                    preemptible=preemptible),
-                                  jobName='testResourceLimits')
-        job2 = self.batchSystem.issueBatchJob(jobDesc2)
-        self.assertIsNotNone(job2)
-        batches = self._getBatchList()
-        self.assertEqual(len(batches), 2)
-        # It would be better to directly check that the batches have the correct memory and cpu
-        # values, but Parasol seems to slightly change the values sometimes.
-        self.assertNotEqual(batches[0]['ram'], batches[1]['ram'])
-        # Need to kill one of the jobs because there are only two cores available
-        self.batchSystem.killBatchJobs([job2])
-        job3 = self.batchSystem.issueBatchJob(jobDesc1)
-        self.assertIsNotNone(job3)
-        batches = self._getBatchList()
-        self.assertEqual(len(batches), 1)
-
-    def _parseBatchString(self, batchString):
-        import re
-        batchInfo = dict()
-        memPattern = re.compile(r"(\d+\.\d+)([kgmbt])")
-        items = batchString.split()
-        batchInfo["cores"] = int(items[7])
-        memMatch = memPattern.match(items[8])
-        ramValue = float(memMatch.group(1))
-        ramUnits = memMatch.group(2)
-        ramConversion = {'b': 1e0, 'k': 1e3, 'm': 1e6, 'g': 1e9, 't': 1e12}
-        batchInfo["ram"] = ramValue * ramConversion[ramUnits]
-        return batchInfo
-
-    def _getBatchList(self):
-        # noinspection PyUnresolvedReferences
-        exitStatus, batchLines = self.batchSystem._runParasol(['list', 'batches'])
-        self.assertEqual(exitStatus, 0)
-        return [self._parseBatchString(line) for line in batchLines[1:] if line]
-
-
 @slow
 @needs_gridengine
 class GridEngineBatchSystemTest(hidden.AbstractGridEngineBatchSystemTest):

toil/test/batchSystems/test_slurm.py

@@ -16,7 +16,8 @@ def call_sacct(args, **_) -> str:
     The arguments passed to `call_command` when executing `sacct` are:
     ['sacct', '-n', '-j', '<comma-separated list of job-ids>', '--format',
     'JobIDRaw,State,ExitCode', '-P', '-S', '1970-01-01']
-    The multi-line output is something like:
+    The multi-line output is something like::
+
         1234|COMPLETED|0:0
         1234.batch|COMPLETED|0:0
         1235|PENDING|0:0
@@ -46,7 +47,7 @@ def call_sacct(args, **_) -> str:
 def call_scontrol(args, **_) -> str:
     """
     The arguments passed to `call_command` when executing `scontrol` are:
-    ['scontrol', 'show', 'job'] or ['scontrol', 'show', 'job', '<job-id>']
+    ``['scontrol', 'show', 'job']`` or ``['scontrol', 'show', 'job', '<job-id>']``
     """
     job_id = int(args[3]) if len(args) > 3 else None
     # Fake output per fake job-id.