toil 6.1.0__py3-none-any.whl → 7.0.0__py3-none-any.whl

toil/__init__.py

@@ -22,7 +22,6 @@ from datetime import datetime
 from typing import TYPE_CHECKING, Optional, Tuple
 
 import requests
-from pytz import timezone
 
 from docker.errors import ImageNotFound
 from toil.lib.memoize import memoize
@@ -210,7 +209,7 @@ def customDockerInitCmd() -> str:
     private docker registries). Any single quotes are escaped and the command cannot contain a
     set of blacklisted chars (newline or tab).
 
-    :returns: The custom commmand, or an empty string is returned if the environment variable is not set.
+    :returns: The custom command, or an empty string is returned if the environment variable is not set.
     """
     command = lookupEnvVar(name='user-defined custom docker init command',
                            envName='TOIL_CUSTOM_DOCKER_INIT_COMMAND',
@@ -440,7 +439,6 @@ def logProcessContext(config: "Config") -> None:
 
 
 try:
-    from boto import provider
     from botocore.credentials import (JSONFileCache,
                                       RefreshableCredentials,
                                       create_credential_resolver)
@@ -477,234 +475,5 @@ try:
         """
         return datetime.strptime(s, datetime_format)
 
-
-    class BotoCredentialAdapter(provider.Provider):
-        """
-        Boto 2 Adapter to use AWS credentials obtained via Boto 3's credential finding logic.
-
-        This allows for automatic role assumption
-        respecting the Boto 3 config files, even when parts of the app still use
-        Boto 2.
-
-        This class also handles caching credentials in multi-process environments
-        to avoid loads of processes swamping the EC2 metadata service.
-        """
-
-        # TODO: We take kwargs because new boto2 versions have an 'anon'
-        # argument and we want to be future proof
-
-        def __init__(self, name, access_key=None, secret_key=None,
-                     security_token=None, profile_name=None, **kwargs):
-            """Create a new BotoCredentialAdapter."""
-            # TODO: We take kwargs because new boto2 versions have an 'anon'
-            # argument and we want to be future proof
-
-            if (name == 'aws' or name is None) and access_key is None and not kwargs.get('anon', False):
-                # We are on AWS and we don't have credentials passed along and we aren't anonymous.
-                # We will backend into a boto3 resolver for getting credentials.
-                # Make sure to enable boto3's own caching, so we can share that
-                # cache with pure boto3 code elsewhere in Toil.
-                # Keep synced with toil.lib.aws.session.establish_boto3_session
-                self._boto3_resolver = create_credential_resolver(Session(profile=profile_name), cache=JSONFileCache())
-            else:
-                # We will use the normal flow
-                self._boto3_resolver = None
-
-            # Pass along all the arguments
-            super().__init__(name, access_key=access_key,
-                                                        secret_key=secret_key, security_token=security_token,
-                                                        profile_name=profile_name, **kwargs)
-
-        def get_credentials(self, access_key=None, secret_key=None, security_token=None, profile_name=None):
-            """
-            Make sure our credential fields are populated.
-
-            Called by the base class constructor.
-            """
-            if self._boto3_resolver is not None:
-                # Go get the credentials from the cache, or from boto3 if not cached.
-                # We need to be eager here; having the default None
-                # _credential_expiry_time makes the accessors never try to refresh.
-                self._obtain_credentials_from_cache_or_boto3()
-            else:
-                # We're not on AWS, or they passed a key, or we're anonymous.
-                # Use the normal route; our credentials shouldn't expire.
-                super().get_credentials(access_key=access_key,
-                                                                   secret_key=secret_key, security_token=security_token,
-                                                                   profile_name=profile_name)
-
-        def _populate_keys_from_metadata_server(self):
-            """
-            Hack to catch _credential_expiry_time being too soon and refresh the credentials.
-
-            This override is misnamed; it's actually the only hook we have to catch
-            _credential_expiry_time being too soon and refresh the credentials. We
-            actually just go back and poke the cache to see if it feels like
-            getting us new credentials.
-
-            Boto 2 hardcodes a refresh within 5 minutes of expiry:
-            https://github.com/boto/boto/blob/591911db1029f2fbb8ba1842bfcc514159b37b32/boto/provider.py#L247
-
-            Boto 3 wants to refresh 15 or 10 minutes before expiry:
-            https://github.com/boto/botocore/blob/8d3ea0e61473fba43774eb3c74e1b22995ee7370/botocore/credentials.py#L279
-
-            So if we ever want to refresh, Boto 3 wants to refresh too.
-            """
-            # This should only happen if we have expiring credentials, which we should only get from boto3
-            if self._boto3_resolver is None:
-                raise RuntimeError("The Boto3 resolver should not be None.")
-
-            self._obtain_credentials_from_cache_or_boto3()
-
-        @retry()
-        def _obtain_credentials_from_boto3(self):
-            """
-            Fill our credential fields from Boto 3.
-
-            We know the current cached credentials are not good, and that we
-            need to get them from Boto 3. Fill in our credential fields
-            (_access_key, _secret_key, _security_token,
-            _credential_expiry_time) from Boto 3.
-            """
-            # We get a Credentials object
-            # <https://github.com/boto/botocore/blob/8d3ea0e61473fba43774eb3c74e1b22995ee7370/botocore/credentials.py#L227>
-            # or a RefreshableCredentials, or None on failure.
-            creds = self._boto3_resolver.load_credentials()
-
-            if creds is None:
-                try:
-                    resolvers = str(self._boto3_resolver.providers)
-                except:
-                    resolvers = "(Resolvers unavailable)"
-                raise RuntimeError("Could not obtain AWS credentials from Boto3. Resolvers tried: " + resolvers)
-
-            # Make sure the credentials actually has some credentials if it is lazy
-            creds.get_frozen_credentials()
-
-            # Get when the credentials will expire, if ever
-            if isinstance(creds, RefreshableCredentials):
-                # Credentials may expire.
-                # Get a naive UTC datetime like boto 2 uses from the boto 3 time.
-                self._credential_expiry_time = creds._expiry_time.astimezone(timezone('UTC')).replace(tzinfo=None)
-            else:
-                # Credentials never expire
-                self._credential_expiry_time = None
-
-            # Then, atomically get all the credentials bits. They may be newer than we think they are, but never older.
-            frozen = creds.get_frozen_credentials()
-
-            # Copy them into us
-            self._access_key = frozen.access_key
-            self._secret_key = frozen.secret_key
-            self._security_token = frozen.token
-
-        def _obtain_credentials_from_cache_or_boto3(self):
-            """
-            Get the cached credentials.
-
-            Or retrieve them from Boto 3 and cache them
-            (or wait for another cooperating process to do so) if they are missing
-            or not fresh enough.
-            """
-            cache_path = '~/.cache/aws/cached_temporary_credentials'
-            path = os.path.expanduser(cache_path)
-            tmp_path = path + '.tmp'
-            while True:
-                log.debug('Attempting to read cached credentials from %s.', path)
-                try:
-                    with open(path) as f:
-                        content = f.read()
-                        if content:
-                            record = content.split('\n')
-                            if len(record) != 4:
-                                raise RuntimeError("Number of cached credentials is not 4.")
-                            self._access_key = record[0]
-                            self._secret_key = record[1]
-                            self._security_token = record[2]
-                            self._credential_expiry_time = str_to_datetime(record[3])
-                        else:
-                            log.debug('%s is empty. Credentials are not temporary.', path)
-                            self._obtain_credentials_from_boto3()
-                            return
-                except OSError as e:
-                    if e.errno == errno.ENOENT:
-                        log.debug('Cached credentials are missing.')
-                        dir_path = os.path.dirname(path)
-                        if not os.path.exists(dir_path):
-                            log.debug('Creating parent directory %s', dir_path)
-                            try:
-                                # A race would be ok at this point
-                                os.makedirs(dir_path, exist_ok=True)
-                            except OSError as e2:
-                                if e2.errno == errno.EROFS:
-                                    # Sometimes we don't actually have write access to ~.
-                                    # We may be running in a non-writable Toil container.
-                                    # We should just go get our own credentials
-                                    log.debug('Cannot use the credentials cache because we are working on a read-only filesystem.')
-                                    self._obtain_credentials_from_boto3()
-                                else:
-                                    raise
-                    else:
-                        raise
-                else:
-                    if self._credentials_need_refresh():
-                        log.debug('Cached credentials are expired.')
-                    else:
-                        log.debug('Cached credentials exist and are still fresh.')
-                        return
-                # We get here if credentials are missing or expired
-                log.debug('Racing to create %s.', tmp_path)
-                # Only one process, the winner, will succeed
-                try:
-                    fd = os.open(tmp_path, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o600)
-                except OSError as e:
-                    if e.errno == errno.EEXIST:
-                        log.debug('Lost the race to create %s. Waiting on winner to remove it.', tmp_path)
-                        while os.path.exists(tmp_path):
-                            time.sleep(0.1)
-                        log.debug('Winner removed %s. Trying from the top.', tmp_path)
-                    else:
-                        raise
-                else:
-                    try:
-                        log.debug('Won the race to create %s.  Requesting credentials from backend.', tmp_path)
-                        self._obtain_credentials_from_boto3()
-                    except:
-                        os.close(fd)
-                        fd = None
-                        log.debug('Failed to obtain credentials, removing %s.', tmp_path)
-                        # This unblocks the losers.
-                        os.unlink(tmp_path)
-                        # Bail out. It's too likely to happen repeatedly
-                        raise
-                    else:
-                        if self._credential_expiry_time is None:
-                            os.close(fd)
-                            fd = None
-                            log.debug('Credentials are not temporary.  Leaving %s empty and renaming it to %s.',
-                                      tmp_path, path)
-                            # No need to actually cache permanent credentials,
-                            # because we know we aren't getting them from the
-                            # metadata server or by assuming a role. Those both
-                            # give temporary credentials.
-                        else:
-                            log.debug('Writing credentials to %s.', tmp_path)
-                            with os.fdopen(fd, 'w') as fh:
-                                fd = None
-                                fh.write('\n'.join([
-                                    self._access_key,
-                                    self._secret_key,
-                                    self._security_token,
-                                    datetime_to_str(self._credential_expiry_time)]))
-                            log.debug('Wrote credentials to %s. Renaming to %s.', tmp_path, path)
-                        os.rename(tmp_path, path)
-                        return
-                    finally:
-                        if fd is not None:
-                            os.close(fd)
-
-
-    provider.Provider = BotoCredentialAdapter
-
 except ImportError:
     pass

toil/batchSystems/abstractBatchSystem.py

@@ -58,6 +58,13 @@ class BatchJobExitReason(enum.IntEnum):
     """Internal error."""
     MEMLIMIT: int = 6
     """Job hit batch system imposed memory limit."""
+    MISSING: int = 7
+    """Job disappeared from the scheduler without actually stopping, so Toil killed it."""
+    MAXJOBDURATION: int = 8
+    """Job ran longer than --maxJobDuration, so Toil killed it."""
+    PARTITION: int = 9
+    """Job was not able to talk to the leader via the job store, so Toil declared it failed."""
+
 
     @classmethod
     def to_string(cls, value: int) -> str:
@@ -156,17 +163,19 @@ class AbstractBatchSystem(ABC):
         """
 
     @abstractmethod
-    def issueBatchJob(self, jobDesc: JobDescription, job_environment: Optional[Dict[str, str]] = None) -> int:
+    def issueBatchJob(self, command: str, job_desc: JobDescription, job_environment: Optional[Dict[str, str]] = None) -> int:
         """
         Issues a job with the specified command to the batch system and returns
-        a unique jobID.
+        a unique job ID number.
 
-        :param jobDesc: a toil.job.JobDescription
+        :param command: the command to execute somewhere to run the Toil
+            worker process
+        :param job_desc: the JobDescription for the job being run
         :param job_environment: a collection of job-specific environment
-                                variables to be set on the worker.
+            variables to be set on the worker.
 
-        :return: a unique jobID that can be used to reference the newly issued
-                 job
+        :return: a unique job ID number that can be used to reference the newly
+            issued job
         """
         raise NotImplementedError()
 
@@ -188,20 +197,20 @@ class AbstractBatchSystem(ABC):
         """
         Gets all currently issued jobs
 
-        :return: A list of jobs (as jobIDs) currently issued (may be running, or may be
-                 waiting to be run). Despite the result being a list, the ordering should not
-                 be depended upon.
+        :return: A list of jobs (as job ID numbers) currently issued (may be
+            running, or may be waiting to be run). Despite the result being a
+            list, the ordering should not be depended upon.
         """
         raise NotImplementedError()
 
     @abstractmethod
     def getRunningBatchJobIDs(self) -> Dict[int, float]:
         """
-        Gets a map of jobs as jobIDs that are currently running (not just waiting)
-        and how long they have been running, in seconds.
+        Gets a map of jobs as job ID numbers that are currently running (not
+        just waiting) and how long they have been running, in seconds.
 
-        :return: dictionary with currently running jobID keys and how many seconds they have
-                 been running as the value
+        :return: dictionary with currently running job ID number keys and how
+            many seconds they have been running as the value
         """
         raise NotImplementedError()
 

toil/batchSystems/abstractGridEngineBatchSystem.py

@@ -25,6 +25,7 @@ from toil.batchSystems.cleanup_support import BatchSystemCleanupSupport
 from toil.bus import ExternalBatchIdMessage, get_job_kind
 from toil.job import AcceleratorRequirement
 from toil.lib.misc import CalledProcessErrorStderr
+from toil.lib.retry import old_retry, DEFAULT_DELAYS
 
 logger = logging.getLogger(__name__)
 
@@ -44,26 +45,29 @@ class AbstractGridEngineBatchSystem(BatchSystemCleanupSupport):
     A partial implementation of BatchSystemSupport for batch systems run on a
     standard HPC cluster. By default auto-deployment is not implemented.
     """
+    class GridEngineThreadException(Exception):
+        pass
 
-    class Worker(Thread, metaclass=ABCMeta):
-
+    class GridEngineThread(Thread, metaclass=ABCMeta):
         def __init__(self, newJobsQueue: Queue, updatedJobsQueue: Queue, killQueue: Queue, killedJobsQueue: Queue, boss: 'AbstractGridEngineBatchSystem') -> None:
             """
-            Abstract worker interface class. All instances are created with five
+            Abstract thread interface class. All instances are created with five
             initial arguments (below). Note the Queue instances passed are empty.
 
             :param newJobsQueue: a Queue of new (unsubmitted) jobs
             :param updatedJobsQueue: a Queue of jobs that have been updated
             :param killQueue: a Queue of active jobs that need to be killed
-            :param killedJobsQueue: Queue of killed jobs for this worker
+            :param killedJobsQueue: Queue of killed jobs for this thread
             :param boss: the AbstractGridEngineBatchSystem instance that
-                         controls this AbstractGridEngineWorker
+                         controls this GridEngineThread
 
             """
             Thread.__init__(self)
             self.boss = boss
             self.boss.config.statePollingWait = \
                 self.boss.config.statePollingWait or self.boss.getWaitDuration()
+            self.boss.config.state_polling_timeout = \
+                self.boss.config.state_polling_timeout or self.boss.config.statePollingWait * 10
             self.newJobsQueue = newJobsQueue
             self.updatedJobsQueue = updatedJobsQueue
             self.killQueue = killQueue
@@ -74,6 +78,7 @@ class AbstractGridEngineBatchSystem(BatchSystemCleanupSupport):
             self.batchJobIDs: Dict[int, str] = dict()
             self._checkOnJobsCache = None
             self._checkOnJobsTimestamp = None
+            self.exception = None
 
         def getBatchSystemID(self, jobID: int) -> str:
             """
@@ -107,7 +112,7 @@ class AbstractGridEngineBatchSystem(BatchSystemCleanupSupport):
             """
             Create a new job with the given attributes.
 
-            Implementation-specific; called by AbstractGridEngineWorker.run()
+            Implementation-specific; called by GridEngineThread.run()
             """
             activity = False
             # Load new job id if present:
@@ -143,7 +148,7 @@ class AbstractGridEngineBatchSystem(BatchSystemCleanupSupport):
 
         def killJobs(self):
             """
-            Kill any running jobs within worker
+            Kill any running jobs within thread
             """
             killList = list()
             while True:
@@ -175,7 +180,8 @@ class AbstractGridEngineBatchSystem(BatchSystemCleanupSupport):
             while killList:
                 for jobID in list(killList):
                     batchJobID = self.getBatchSystemID(jobID)
-                    if self.boss.with_retries(self.getJobExitCode, batchJobID) is not None:
+                    exit_code = self.boss.with_retries(self.getJobExitCode, batchJobID)
+                    if exit_code is not None:
                         logger.debug('Adding jobID %s to killedJobsQueue', jobID)
                         self.killedJobsQueue.put(jobID)
                         killList.remove(jobID)
@@ -273,14 +279,17 @@ class AbstractGridEngineBatchSystem(BatchSystemCleanupSupport):
                 while self._runStep():
                     pass
             except Exception as ex:
-                logger.error("GridEngine like batch system failure", exc_info=ex)
-                raise
+                self.exception = ex
+                logger.error("GridEngine like batch system failure: %s", ex)
+                # don't raise exception as is_alive will still be set to false,
+                # signalling exception in the thread as we expect the thread to
+                # always be running for the duration of the workflow
 
         def coalesce_job_exit_codes(self, batch_job_id_list: list) -> List[Union[int, Tuple[int, Optional[BatchJobExitReason]], None]]:
             """
             Returns exit codes and possibly exit reasons for a list of jobs, or None if they are running.
 
-            Called by AbstractGridEngineWorker.checkOnJobs().
+            Called by GridEngineThread.checkOnJobs().
 
             This is an optional part of the interface. It should raise
             NotImplementedError if not actually implemented for a particular
@@ -341,7 +350,7 @@ class AbstractGridEngineBatchSystem(BatchSystemCleanupSupport):
         def killJob(self, jobID):
             """
             Kill specific job with the Toil job ID. Implementation-specific; called
-            by AbstractGridEngineWorker.killJobs()
+            by GridEngineThread.killJobs()
 
             :param string jobID: Toil job ID
             """
@@ -356,7 +365,7 @@ class AbstractGridEngineBatchSystem(BatchSystemCleanupSupport):
 
             If the job is not running but the exit code is not available, it
             will be EXIT_STATUS_UNAVAILABLE_VALUE. Implementation-specific;
-            called by AbstractGridEngineWorker.checkOnJobs().
+            called by GridEngineThread.checkOnJobs().
 
             The exit code will only be 0 if the job affirmatively succeeded.
 
@@ -375,24 +384,20 @@ class AbstractGridEngineBatchSystem(BatchSystemCleanupSupport):
         self.updatedJobsQueue = Queue()
         self.killQueue = Queue()
         self.killedJobsQueue = Queue()
-        # get the associated worker class here
-        self.worker = self.Worker(self.newJobsQueue, self.updatedJobsQueue,
-                                  self.killQueue, self.killedJobsQueue, self)
-        self.worker.start()
+        # get the associated thread class here
+        self.background_thread = self.GridEngineThread(self.newJobsQueue, self.updatedJobsQueue,
+                                                       self.killQueue, self.killedJobsQueue, self)
+        self.background_thread.start()
         self._getRunningBatchJobIDsTimestamp = None
         self._getRunningBatchJobIDsCache = {}
 
-    @classmethod
-    def supportsWorkerCleanup(cls):
-        return False
-
     @classmethod
     def supportsAutoDeployment(cls):
         return False
 
-    def issueBatchJob(self, jobDesc, job_environment: Optional[Dict[str, str]] = None):
+    def issueBatchJob(self, command: str, jobDesc, job_environment: Optional[Dict[str, str]] = None):
         # Avoid submitting internal jobs to the batch queue, handle locally
-        localID = self.handleLocalJob(jobDesc)
+        localID = self.handleLocalJob(command, jobDesc)
         if localID is not None:
             return localID
         else:
@@ -406,10 +411,10 @@ class AbstractGridEngineBatchSystem(BatchSystemCleanupSupport):
                         gpus = accelerator['count']
             else:
                 gpus = jobDesc.accelerators
-
-            self.newJobsQueue.put((jobID, jobDesc.cores, jobDesc.memory, jobDesc.command, get_job_kind(jobDesc.get_names()),
+            
+            self.newJobsQueue.put((jobID, jobDesc.cores, jobDesc.memory, command, get_job_kind(jobDesc.get_names()),
                                    job_environment, gpus))
-            logger.debug("Issued the job command: %s with job id: %s and job name %s", jobDesc.command, str(jobID),
+            logger.debug("Issued the job command: %s with job id: %s and job name %s", command, str(jobID),
                          get_job_kind(jobDesc.get_names()))
         return jobID
 
@@ -424,7 +429,12 @@ class AbstractGridEngineBatchSystem(BatchSystemCleanupSupport):
         for jobID in jobIDs:
             self.killQueue.put(jobID)
         while jobIDs:
-            killedJobId = self.killedJobsQueue.get()
+            try:
+                killedJobId = self.killedJobsQueue.get(timeout=10)
+            except Empty:
+                if not self.background_thread.is_alive():
+                    raise self.GridEngineThreadException("Grid engine thread failed unexpectedly") from self.background_thread.exception
+                continue
             if killedJobId is None:
                 break
             jobIDs.remove(killedJobId)
@@ -456,7 +466,7 @@ class AbstractGridEngineBatchSystem(BatchSystemCleanupSupport):
                 self.config.statePollingWait):
             batchIds = self._getRunningBatchJobIDsCache
         else:
-            batchIds = self.with_retries(self.worker.getRunningJobIDs)
+            batchIds = self.with_retries(self.background_thread.getRunningJobIDs)
             self._getRunningBatchJobIDsCache = batchIds
             self._getRunningBatchJobIDsTimestamp = datetime.now()
         batchIds.update(self.getRunningLocalJobIDs())
@@ -464,6 +474,11 @@ class AbstractGridEngineBatchSystem(BatchSystemCleanupSupport):
 
     def getUpdatedBatchJob(self, maxWait):
         local_tuple = self.getUpdatedLocalJob(0)
+
+        if not self.background_thread.is_alive():
+            # kill remaining jobs on the thread
+            self.background_thread.killJobs()
+            raise self.GridEngineThreadException("Unexpected GridEngineThread failure") from self.background_thread.exception
         if local_tuple:
             return local_tuple
         else:
@@ -477,14 +492,14 @@ class AbstractGridEngineBatchSystem(BatchSystemCleanupSupport):
 
     def shutdown(self) -> None:
         """
-        Signals worker to shutdown (via sentinel) then cleanly joins the thread
+        Signals thread to shutdown (via sentinel) then cleanly joins the thread
         """
         self.shutdownLocal()
         newJobsQueue = self.newJobsQueue
         self.newJobsQueue = None
 
         newJobsQueue.put(None)
-        self.worker.join()
+        self.background_thread.join()
 
     def setEnv(self, name, value=None):
         if value and ',' in value:
@@ -503,21 +518,20 @@ class AbstractGridEngineBatchSystem(BatchSystemCleanupSupport):
 
     def with_retries(self, operation, *args, **kwargs):
         """
-        Call operation with args and kwargs. If one of the calls to an SGE
-        command fails, sleep and try again for a set number of times.
+        Call operation with args and kwargs. If one of the calls to a
+        command fails, sleep and try again.
         """
-        maxTries = 3
-        tries = 0
-        while True:
-            tries += 1
-            try:
-                return operation(*args, **kwargs)
-            except CalledProcessErrorStderr as err:
-                if tries < maxTries:
-                    logger.error("Will retry errored operation %s, code %d: %s",
-                                 operation.__name__, err.returncode, err.stderr)
-                    time.sleep(self.config.statePollingWait)
-                else:
-                    logger.error("Failed operation %s, code %d: %s",
+        for attempt in old_retry(
+            # Don't retry more often than the state polling wait.
+            delays=[max(delay, self.config.statePollingWait) for delay in DEFAULT_DELAYS],
+            timeout=self.config.state_polling_timeout,
+            predicate=lambda e: isinstance(e, CalledProcessErrorStderr)
+        ):
+            with attempt:
+                try:
+                    return operation(*args, **kwargs)
+                except CalledProcessErrorStderr as err:
+                    logger.error("Errored operation %s, code %d: %s",
                                  operation.__name__, err.returncode, err.stderr)
+                    # Raise up to the retry logic, which will retry until timeout
                     raise err

toil/batchSystems/awsBatch.py

@@ -36,7 +36,7 @@ import uuid
 from argparse import ArgumentParser, _ArgumentGroup
 from typing import Any, Dict, Iterator, List, Optional, Set, Union
 
-from boto.exception import BotoServerError
+from botocore.exceptions import ClientError
 
 from toil import applianceSelf
 from toil.batchSystems.abstractBatchSystem import (EXIT_STATUS_UNAVAILABLE_VALUE,
@@ -156,9 +156,9 @@ class AWSBatchBatchSystem(BatchSystemCleanupSupport):
                     'AWS Batch can only provide nvidia gpu accelerators.'
                 ])
 
-    def issueBatchJob(self, job_desc: JobDescription, job_environment: Optional[Dict[str, str]] = None) -> int:
+    def issueBatchJob(self, command: str, job_desc: JobDescription, job_environment: Optional[Dict[str, str]] = None) -> int:
         # Try the job as local
-        local_id = self.handleLocalJob(job_desc)
+        local_id = self.handleLocalJob(command, job_desc)
         if local_id is not None:
             # It is a local job
             return local_id
@@ -184,7 +184,7 @@ class AWSBatchBatchSystem(BatchSystemCleanupSupport):
                 environment.update(job_environment)
 
             # Make a command to run it in the executor
-            command_list = pack_job(job_desc, self.user_script)
+            command_list = pack_job(command, self.user_script)
 
             # Compose a job spec to submit
             job_spec = {
@@ -376,7 +376,7 @@ class AWSBatchBatchSystem(BatchSystemCleanupSupport):
         # Get rid of the job definition we are using if we can.
         self._destroy_job_definition()
 
-    @retry(errors=[BotoServerError])
+    @retry(errors=[ClientError])
     def _try_terminate(self, aws_id: str) -> None:
         """
         Internal function. Should not be called outside this class.
@@ -392,7 +392,7 @@ class AWSBatchBatchSystem(BatchSystemCleanupSupport):
         # Kill the AWS Batch job
         self.client.terminate_job(jobId=aws_id, reason='Killed by Toil')
 
-    @retry(errors=[BotoServerError])
+    @retry(errors=[ClientError])
     def _wait_until_stopped(self, aws_id: str) -> None:
         """
         Internal function. Should not be called outside this class.
@@ -418,7 +418,7 @@ class AWSBatchBatchSystem(BatchSystemCleanupSupport):
             logger.info('Waiting for killed job %s to stop', self.aws_id_to_bs_id.get(aws_id, aws_id))
             time.sleep(2)
 
-    @retry(errors=[BotoServerError])
+    @retry(errors=[ClientError])
     def _get_or_create_job_definition(self) -> str:
         """
         Internal function. Should not be called outside this class.
@@ -482,7 +482,7 @@ class AWSBatchBatchSystem(BatchSystemCleanupSupport):
 
         return self.job_definition
 
-    @retry(errors=[BotoServerError])
+    @retry(errors=[ClientError])
     def _destroy_job_definition(self) -> None:
         """
         Internal function. Should not be called outside this class.

toil/batchSystems/contained_executor.py

@@ -25,18 +25,17 @@ import sys
 from typing import Any, Dict, List, Optional
 
 from toil.batchSystems.abstractBatchSystem import EXIT_STATUS_UNAVAILABLE_VALUE
-from toil.job import JobDescription
 from toil.resource import Resource
 from toil.statsAndLogging import configure_root_logger, set_log_level
 
 logger = logging.getLogger(__name__)
 
 
-def pack_job(job_desc: JobDescription, user_script: Optional[Resource] = None, environment: Optional[Dict[str, str]] = None) -> List[str]:
+def pack_job(command: str, user_script: Optional[Resource] = None, environment: Optional[Dict[str, str]] = None) -> List[str]:
     """
-    Create a command that, when run, will execute the given job.
+    Create a command that runs the given command in an environment. 
 
-    :param job_desc: Job description for the job to run.
+    :param command: Worker command to run to run the job.
     :param user_script: User script that will be loaded before the job is run.
     :param environment: Environment variable dict that will be applied before
         the job is run.
@@ -46,7 +45,7 @@ def pack_job(job_desc: JobDescription, user_script: Optional[Resource] = None, e
     """
     # Make a job dict to send to the executor.
     # TODO: Factor out executor setup from here and Kubernetes and TES
-    job: Dict[str, Any] = {"command": job_desc.command}
+    job: Dict[str, Any] = {"command": command}
     if user_script is not None:
         # If there's a user script resource be sure to send it along
         job['userScript'] = user_script