toil 5.12.0__py3-none-any.whl → 6.1.0a1__py3-none-any.whl

toil/__init__.py

@@ -20,6 +20,7 @@ import sys
 import time
 from datetime import datetime
 from typing import TYPE_CHECKING, Optional, Tuple
+
 import requests
 from pytz import timezone
 
@@ -106,7 +107,8 @@ def toilPackageDirPath() -> str:
     The return value is guaranteed to end in '/toil'.
     """
     result = os.path.dirname(os.path.realpath(__file__))
-    assert result.endswith('/toil')
+    if not result.endswith('/toil'):
+        raise RuntimeError("The top-level toil package is not named Toil.")
     return result
 
 
@@ -132,7 +134,8 @@ def resolveEntryPoint(entryPoint: str) -> str:
             # opposed to being included via --system-site-packages). For clusters this means that
             # if Toil is installed in a virtualenv on the leader, it must be installed in
             # a virtualenv located at the same path on each worker as well.
-            assert os.access(path, os.X_OK)
+            if not os.access(path, os.X_OK):
+                raise RuntimeError("Cannot access the Toil virtualenv. If installed in a virtualenv on a cluster, make sure that the virtualenv path is the same for the leader and workers.")
             return path
     # Otherwise, we aren't in a virtualenv, or we're in a virtualenv but Toil
     # came in via --system-site-packages, or we think the virtualenv might not
@@ -238,7 +241,8 @@ def customInitCmd() -> str:
 
 def _check_custom_bash_cmd(cmd_str):
     """Ensure that the Bash command doesn't contain invalid characters."""
-    assert not re.search(r'[\n\r\t]', cmd_str), f'"{cmd_str}" contains invalid characters (newline and/or tab).'
+    if re.search(r'[\n\r\t]', cmd_str):
+        raise RuntimeError(f'"{cmd_str}" contains invalid characters (newline and/or tab).')
 
 
 def lookupEnvVar(name: str, envName: str, defaultValue: str) -> str:
@@ -370,11 +374,10 @@ def requestCheckRegularDocker(origAppliance: str, registryName: str, imageName:
     separate check is done for docker.io images.
 
     :param origAppliance: The full url of the docker image originally
-                          specified by the user (or the default).
-                           e.g. ``quay.io/ucsc_cgl/toil:latest``
-    :param registryName: The url of a docker image's registry.  e.g. ``quay.io``
-    :param imageName: The image, including path and excluding the tag. e.g. ``ucsc_cgl/toil``
-    :param tag: The tag used at that docker image's registry.  e.g. ``latest``
+        specified by the user (or the default). For example, ``quay.io/ucsc_cgl/toil:latest``.
+    :param registryName: The url of a docker image's registry. For example, ``quay.io``.
+    :param imageName: The image, including path and excluding the tag. For example, ``ucsc_cgl/toil``.
+    :param tag: The tag used at that docker image's registry. For example, ``latest``.
     :raises: ApplianceImageNotFound if no match is found.
     :return: Return True if match found.
     """
@@ -399,9 +402,9 @@ def requestCheckDockerIo(origAppliance: str, imageName: str, tag: str) -> bool:
     URL is based on the docker v2 schema.  Requires that an access token be fetched first.
 
     :param origAppliance: The full url of the docker image originally
-                          specified by the user (or the default).  e.g. "ubuntu:latest"
-    :param imageName: The image, including path and excluding the tag. e.g. "ubuntu"
-    :param tag: The tag used at that docker image's registry.  e.g. "latest"
+        specified by the user (or the default). For example, ``ubuntu:latest``.
+    :param imageName: The image, including path and excluding the tag. For example, ``ubuntu``.
+    :param tag: The tag used at that docker image's registry. For example, ``latest``.
     :raises: ApplianceImageNotFound if no match is found.
     :return: Return True if match found.
     """
@@ -548,7 +551,8 @@ try:
             So if we ever want to refresh, Boto 3 wants to refresh too.
             """
             # This should only happen if we have expiring credentials, which we should only get from boto3
-            assert (self._boto3_resolver is not None)
+            if self._boto3_resolver is None:
+                raise RuntimeError("The Boto3 resolver should not be None.")
 
             self._obtain_credentials_from_cache_or_boto3()
 
@@ -612,7 +616,8 @@ try:
                         content = f.read()
                         if content:
                             record = content.split('\n')
-                            assert len(record) == 4
+                            if len(record) != 4:
+                                raise RuntimeError("Number of cached credentials is not 4.")
                             self._access_key = record[0]
                             self._secret_key = record[1]
                             self._security_token = record[2]

toil/batchSystems/abstractBatchSystem.py

@@ -15,19 +15,19 @@ import enum
 import logging
 import os
 import shutil
+import time
 from abc import ABC, abstractmethod
 from argparse import ArgumentParser, _ArgumentGroup
 from contextlib import contextmanager
 from threading import Condition
-import time
 from typing import (Any,
                     ContextManager,
                     Dict,
-                    List,
-                    Set,
                     Iterator,
+                    List,
                     NamedTuple,
                     Optional,
+                    Set,
                     Union,
                     cast)
 
@@ -37,6 +37,7 @@ from toil.common import Config, Toil, cacheDirName
 from toil.deferred import DeferredFunctionManager
 from toil.fileStores.abstractFileStore import AbstractFileStore
 from toil.job import JobDescription, ParsedRequirement, Requirer
+from toil.lib.memoize import memoize
 from toil.resource import Resource
 
 logger = logging.getLogger(__name__)
@@ -106,6 +107,8 @@ class AbstractBatchSystem(ABC):
     @abstractmethod
     def supportsWorkerCleanup(cls) -> bool:
         """
+        Whether this batch system supports worker cleanup.
+
         Indicates whether this batch system invokes
         :meth:`BatchSystemSupport.workerCleanup` after the last job for a
         particular workflow invocation finishes. Note that the term *worker*
@@ -119,7 +122,9 @@ class AbstractBatchSystem(ABC):
 
     def setUserScript(self, userScript: Resource) -> None:
         """
-        Set the user script for this workflow. This method must be called before the first job is
+        Set the user script for this workflow.
+
+        This method must be called before the first job is
         issued to this batch system, and only if :meth:`.supportsAutoDeployment` returns True,
         otherwise it will raise an exception.
 
@@ -134,7 +139,6 @@ class AbstractBatchSystem(ABC):
         bus, so that it can send informational messages about the jobs it is
         running to other Toil components.
         """
-        pass
 
     @abstractmethod
     def issueBatchJob(self, jobDesc: JobDescription, job_environment: Optional[Dict[str, str]] = None) -> int:
@@ -263,7 +267,6 @@ class AbstractBatchSystem(ABC):
             setOption(option_name, parsing_function=None, check_function=None, default=None, env=None)
             returning nothing, used to update run configuration as a side effect.
         """
-        pass
 
     def getWorkerContexts(self) -> List[ContextManager[Any]]:
         """
@@ -372,7 +375,7 @@ class BatchSystemSupport(AbstractBatchSystem):
         :param name: the environment variable to be set on the worker.
 
         :param value: if given, the environment variable given by name will be set to this value.
-        if None, the variable's current value will be used as the value on the worker
+            If None, the variable's current value will be used as the value on the worker
 
         :raise RuntimeError: if value is None and the name cannot be found in the environment
         """
@@ -392,6 +395,7 @@ class BatchSystemSupport(AbstractBatchSystem):
         # We do in fact send messages to the message bus.
         self._outbox = message_bus.outbox()
 
+    @memoize
     def get_batch_logs_dir(self) -> str:
         """
         Get the directory where the backing batch system should save its logs.
@@ -404,6 +408,9 @@ class BatchSystemSupport(AbstractBatchSystem):
         """
         if self.config.batch_logs_dir:
             # Use what is specified
+            if not os.path.isdir(self.config.batch_logs_dir):
+                # But if it doesn't exist, make it exist
+                os.makedirs(self.config.batch_logs_dir, exist_ok=True)
             return self.config.batch_logs_dir
         # And if nothing is specified use the workDir.
         return Toil.getToilWorkDir(self.config.workDir)
@@ -442,7 +449,9 @@ class BatchSystemSupport(AbstractBatchSystem):
     @staticmethod
     def workerCleanup(info: WorkerCleanupInfo) -> None:
         """
-        Cleans up the worker node on batch system shutdown. Also see :meth:`supportsWorkerCleanup`.
+        Cleans up the worker node on batch system shutdown.
+
+        Also see :meth:`supportsWorkerCleanup`.
 
         :param WorkerCleanupInfo info: A named tuple consisting of all the relevant information
                for cleaning up the worker.
@@ -498,8 +507,10 @@ class NodeInfo:
 
 class AbstractScalableBatchSystem(AbstractBatchSystem):
     """
-    A batch system that supports a variable number of worker nodes. Used by :class:`toil.
-    provisioners.clusterScaler.ClusterScaler` to scale the number of worker nodes in the cluster
+    A batch system that supports a variable number of worker nodes.
+
+    Used by :class:`toil.provisioners.clusterScaler.ClusterScaler` 
+    to scale the number of worker nodes in the cluster
     up or down depending on overall load.
     """
 

toil/batchSystems/abstractGridEngineBatchSystem.py

@@ -17,13 +17,13 @@ from abc import ABCMeta, abstractmethod
 from datetime import datetime
 from queue import Empty, Queue
 from threading import Lock, Thread
-from typing import Any, Dict, List, Optional, Tuple, Union
+from typing import Dict, List, Optional, Tuple, Union
 
 from toil.batchSystems.abstractBatchSystem import (BatchJobExitReason,
                                                    UpdatedBatchJobInfo)
 from toil.batchSystems.cleanup_support import BatchSystemCleanupSupport
 from toil.bus import ExternalBatchIdMessage
-from toil.job import JobDescription, AcceleratorRequirement
+from toil.job import AcceleratorRequirement
 from toil.lib.misc import CalledProcessErrorStderr
 
 logger = logging.getLogger(__name__)

toil/batchSystems/awsBatch.py

@@ -34,25 +34,25 @@ import tempfile
 import time
 import uuid
 from argparse import ArgumentParser, _ArgumentGroup
-from typing import Any, Callable, Dict, Iterator, List, Optional, Set, Union
+from typing import Any, Dict, Iterator, List, Optional, Set, Union
 
 from boto.exception import BotoServerError
 
 from toil import applianceSelf
 from toil.batchSystems.abstractBatchSystem import (EXIT_STATUS_UNAVAILABLE_VALUE,
                                                    BatchJobExitReason,
-                                                   UpdatedBatchJobInfo,
-                                                   InsufficientSystemResources)
-from toil.batchSystems.options import OptionSetter
+                                                   InsufficientSystemResources,
+                                                   UpdatedBatchJobInfo)
 from toil.batchSystems.cleanup_support import BatchSystemCleanupSupport
 from toil.batchSystems.contained_executor import pack_job
-from toil.bus import ExternalBatchIdMessage, MessageBus, MessageOutbox
+from toil.batchSystems.options import OptionSetter
+from toil.bus import ExternalBatchIdMessage
 from toil.common import Config, Toil
 from toil.job import JobDescription, Requirer
-from toil.lib.aws import get_current_aws_region, zone_to_region
+from toil.lib.aws import get_current_aws_region
 from toil.lib.aws.session import establish_boto3_session
-from toil.lib.conversions import b_to_mib, mib_to_b
-from toil.lib.misc import slow_down, unix_now_ms, utc_now
+from toil.lib.conversions import b_to_mib
+from toil.lib.misc import slow_down, unix_now_ms
 from toil.lib.retry import retry
 from toil.resource import Resource
 
@@ -559,17 +559,17 @@ class AWSBatchBatchSystem(BatchSystemCleanupSupport):
 
     @classmethod
     def add_options(cls, parser: Union[ArgumentParser, _ArgumentGroup]) -> None:
-        parser.add_argument("--awsBatchRegion", dest="aws_batch_region", default=None,
+        parser.add_argument("--awsBatchRegion", dest="aws_batch_region", default=None, env_var="TOIL_AWS_REGION",
                             help="The AWS region containing the AWS Batch queue to submit to.")
-        parser.add_argument("--awsBatchQueue", dest="aws_batch_queue", default=None,
+        parser.add_argument("--awsBatchQueue", dest="aws_batch_queue", default=None, env_var="TOIL_AWS_BATCH_QUEUE",
                             help="The name or ARN of the AWS Batch queue to submit to.")
-        parser.add_argument("--awsBatchJobRoleArn", dest="aws_batch_job_role_arn", default=None,
+        parser.add_argument("--awsBatchJobRoleArn", dest="aws_batch_job_role_arn", default=None, env_var="TOIL_AWS_BATCH_JOB_ROLE_ARN",
                             help=("The ARN of an IAM role to run AWS Batch jobs as, so they "
                                   "can e.g. access a job store. Must be assumable by "
                                   "ecs-tasks.amazonaws.com."))
 
     @classmethod
     def setOptions(cls, setOption: OptionSetter) -> None:
-        setOption("aws_batch_region", default=None)
-        setOption("aws_batch_queue", default=None, env=["TOIL_AWS_BATCH_QUEUE"])
-        setOption("aws_batch_job_role_arn", default=None, env=["TOIL_AWS_BATCH_JOB_ROLE_ARN"])
+        setOption("aws_batch_region")
+        setOption("aws_batch_queue")
+        setOption("aws_batch_job_role_arn")

toil/batchSystems/contained_executor.py

@@ -14,7 +14,7 @@
 """
 Executor for running inside a container.
 
-Useful for Kubernetes and TES batch systems.
+Useful for Kubernetes batch system and TES batch system plugin.
 """
 import base64
 import logging
@@ -39,10 +39,10 @@ def pack_job(job_desc: JobDescription, user_script: Optional[Resource] = None, e
     :param job_desc: Job description for the job to run.
     :param user_script: User script that will be loaded before the job is run.
     :param environment: Environment variable dict that will be applied before
-    the job is run.
+        the job is run.
 
     :returns: Command to run the job, as an argument list that can be run
-    inside the Toil appliance container.
+        inside the Toil appliance container.
     """
     # Make a job dict to send to the executor.
     # TODO: Factor out executor setup from here and Kubernetes and TES

toil/batchSystems/htcondor.py

@@ -24,7 +24,6 @@ import htcondor
 
 from toil.batchSystems.abstractGridEngineBatchSystem import \
     AbstractGridEngineBatchSystem
-
 from toil.job import AcceleratorRequirement
 from toil.lib.retry import retry
 

toil/batchSystems/kubernetes.py

@@ -24,22 +24,22 @@ import datetime
 import logging
 import math
 import os
-from queue import Empty, Queue
 import string
 import sys
 import tempfile
-from threading import Event, Thread, Condition, RLock
 import time
 import uuid
 from argparse import ArgumentParser, _ArgumentGroup
+from queue import Empty, Queue
+from threading import Condition, Event, RLock, Thread
 from typing import (Any,
                     Callable,
                     Dict,
                     Iterator,
                     List,
-                    Set,
                     Literal,
                     Optional,
+                    Set,
                     Tuple,
                     Type,
                     TypeVar,
@@ -104,7 +104,8 @@ from toil.batchSystems.abstractBatchSystem import (EXIT_STATUS_UNAVAILABLE_VALUE
 from toil.batchSystems.cleanup_support import BatchSystemCleanupSupport
 from toil.batchSystems.contained_executor import pack_job
 from toil.batchSystems.options import OptionSetter
-from toil.common import Config, Toil, SYS_MAX_SIZE
+from toil.common import Config, Toil
+from toil.options.common import SYS_MAX_SIZE
 from toil.job import JobDescription, Requirer
 from toil.lib.conversions import human2bytes
 from toil.lib.misc import get_user_name, slow_down, utc_now
@@ -152,6 +153,7 @@ class KubernetesBatchSystem(BatchSystemCleanupSupport):
         super().__init__(config, maxCores, maxMemory, maxDisk)
 
         # Re-type the config to make sure it has all the fields we need.
+        # This convinces MyPy we really do have this type.
         assert isinstance(config, KubernetesBatchSystem.KubernetesConfig)
 
         # Turn down log level for Kubernetes modules and dependencies.
@@ -167,26 +169,26 @@ class KubernetesBatchSystem(BatchSystemCleanupSupport):
         self._apis: KubernetesBatchSystem._ApiStorageDict = {}
 
         # Get our namespace (and our Kubernetes credentials to make sure they exist)
-        self.namespace = self._api('namespace')
+        self.namespace: str = self._api('namespace')
 
         # Decide if we are going to mount a Kubernetes host path as the Toil
         # work dir in the workers, for shared caching.
-        self.host_path = config.kubernetes_host_path
+        self.host_path: Optional[str] = config.kubernetes_host_path
 
         # Get the service account name to use, if any.
-        self.service_account = config.kubernetes_service_account
+        self.service_account: Optional[str] = config.kubernetes_service_account
 
         # Get how long we should wait for a pod that lands on a node to
         # actually start.
-        self.pod_timeout = config.kubernetes_pod_timeout
+        self.pod_timeout: float = config.kubernetes_pod_timeout
 
         # Get the username to mark jobs with
-        username = config.kubernetes_owner
+        username = config.kubernetes_owner or self.get_default_kubernetes_owner()
         # And a unique ID for the run
         self.unique_id = uuid.uuid4()
 
         # Create a prefix for jobs, starting with our username
-        self.job_prefix = f'{username}-toil-{self.unique_id}-'
+        self.job_prefix: str = f'{username}-toil-{self.unique_id}-'
         # Instead of letting Kubernetes assign unique job names, we assign our
         # own based on a numerical job ID. This functionality is managed by the
         # BatchSystemLocalSupport.
@@ -199,17 +201,17 @@ class KubernetesBatchSystem(BatchSystemCleanupSupport):
         # conformance tests. To work around this, we tag all our jobs with an
         # explicit TTL that is long enough that we're sure we can deal with all
         # the finished jobs before they expire.
-        self.finished_job_ttl = 3600  # seconds
+        self.finished_job_ttl: int = 3600  # seconds
 
         # Here is where we will store the user script resource object if we get one.
         self.user_script: Optional[Resource] = None
 
         # Ge the image to deploy from Toil's configuration
-        self.docker_image = applianceSelf()
+        self.docker_image: str = applianceSelf()
 
         # Try and guess what Toil work dir the workers will use.
         # We need to be able to provision (possibly shared) space there.
-        self.worker_work_dir = Toil.getToilWorkDir(config.workDir)
+        self.worker_work_dir: str = Toil.getToilWorkDir(config.workDir)
         if (config.workDir is None and
             os.getenv('TOIL_WORKDIR') is None and
             self.worker_work_dir == tempfile.gettempdir()):
@@ -226,17 +228,17 @@ class KubernetesBatchSystem(BatchSystemCleanupSupport):
         self.environment['TMPDIR'] = '/var/tmp'
 
         # Get the name of the AWS secret, if any, to mount in containers.
-        self.aws_secret_name = os.environ.get("TOIL_AWS_SECRET_NAME", None)
+        self.aws_secret_name: Optional[str] = os.environ.get("TOIL_AWS_SECRET_NAME", None)
 
         # Set this to True to enable the experimental wait-for-job-update code
-        self.enable_watching = os.environ.get("KUBE_WATCH_ENABLED", False)
+        self.enable_watching: bool = os.environ.get("KUBE_WATCH_ENABLED", False)
 
         # This will be a label to select all our jobs.
-        self.run_id = f'toil-{self.unique_id}'
+        self.run_id: str = f'toil-{self.unique_id}'
 
         # Keep track of available resources.
         maxMillicores = int(SYS_MAX_SIZE if self.maxCores == SYS_MAX_SIZE else self.maxCores * 1000)
-        self.resource_sources = [
+        self.resource_sources: List[ResourcePool] = [
             # A pool representing available job slots
             ResourcePool(self.config.max_jobs, 'job slots'),
             # A pool representing available CPU in units of millicores (1 CPU
@@ -261,16 +263,16 @@ class KubernetesBatchSystem(BatchSystemCleanupSupport):
         self._killed_queue_jobs: Set[int] = set()
 
         # We use this event to signal shutdown
-        self._shutting_down = Event()
+        self._shutting_down: Event = Event()
 
         # A lock to protect critical regions when working with queued jobs.
-        self._mutex = RLock()
+        self._mutex: RLock = RLock()
 
         # A condition set to true when there is more work to do. e.g.: new job
         # in the queue or any resource becomes available.
-        self._work_available = Condition(lock=self._mutex)
+        self._work_available: Condition = Condition(lock=self._mutex)
 
-        self.schedulingThread = Thread(target=self._scheduler, daemon=True)
+        self.schedulingThread: Thread = Thread(target=self._scheduler, daemon=True)
         self.schedulingThread.start()
 
     def _pretty_print(self, kubernetes_object: Any) -> str:
@@ -1864,24 +1866,25 @@ class KubernetesBatchSystem(BatchSystemCleanupSupport):
 
     @classmethod
     def add_options(cls, parser: Union[ArgumentParser, _ArgumentGroup]) -> None:
-        parser.add_argument("--kubernetesHostPath", dest="kubernetes_host_path", default=None,
+        parser.add_argument("--kubernetesHostPath", dest="kubernetes_host_path", default=None, env_var="TOIL_KUBERNETES_HOST_PATH",
                             help="Path on Kubernetes hosts to use as shared inter-pod temp directory.  "
                                  "(default: %(default)s)")
-        parser.add_argument("--kubernetesOwner", dest="kubernetes_owner", default=cls.get_default_kubernetes_owner(),
-                            help="Username to mark Kubernetes jobs with.  "
-                                 "(default: %(default)s)")
-        parser.add_argument("--kubernetesServiceAccount", dest="kubernetes_service_account", default=None,
+        parser.add_argument("--kubernetesOwner", dest="kubernetes_owner", default=None, env_var="TOIL_KUBERNETES_OWNER",
+                            help=f"Username to mark Kubernetes jobs with. If the provided value is None, the value will "
+                                 f"be generated at runtime. "
+                                 f"(Generated default: {cls.get_default_kubernetes_owner()})")
+        parser.add_argument("--kubernetesServiceAccount", dest="kubernetes_service_account", default=None, env_var="TOIL_KUBERNETES_SERVICE_ACCOUNT",
                             help="Service account to run jobs as.  "
                                  "(default: %(default)s)")
-        parser.add_argument("--kubernetesPodTimeout", dest="kubernetes_pod_timeout", default=120,
+        parser.add_argument("--kubernetesPodTimeout", dest="kubernetes_pod_timeout", default=120, env_var="TOIL_KUBERNETES_POD_TIMEOUT", type=float,
                             help="Seconds to wait for a scheduled Kubernetes pod to start running.  "
                                  "(default: %(default)s)")
 
     OptionType = TypeVar('OptionType')
     @classmethod
     def setOptions(cls, setOption: OptionSetter) -> None:
-        setOption("kubernetes_host_path", default=None, env=['TOIL_KUBERNETES_HOST_PATH'])
-        setOption("kubernetes_owner", default=cls.get_default_kubernetes_owner(), env=['TOIL_KUBERNETES_OWNER'])
-        setOption("kubernetes_service_account", default=None, env=['TOIL_KUBERNETES_SERVICE_ACCOUNT'])
-        setOption("kubernetes_pod_timeout", default=120, env=['TOIL_KUBERNETES_POD_TIMEOUT'])
+        setOption("kubernetes_host_path")
+        setOption("kubernetes_owner")
+        setOption("kubernetes_service_account",)
+        setOption("kubernetes_pod_timeout")
 

toil/batchSystems/local_support.py

@@ -19,6 +19,7 @@ from toil.batchSystems.abstractBatchSystem import (BatchSystemSupport,
 from toil.batchSystems.singleMachine import SingleMachineBatchSystem
 from toil.common import Config
 from toil.job import JobDescription
+from toil.lib.threading import cpu_count
 
 logger = logging.getLogger(__name__)
 
@@ -28,8 +29,9 @@ class BatchSystemLocalSupport(BatchSystemSupport):
 
     def __init__(self, config: Config, maxCores: float, maxMemory: int, maxDisk: int) -> None:
         super().__init__(config, maxCores, maxMemory, maxDisk)
+        max_local_jobs = config.max_local_jobs if config.max_local_jobs is not None else cpu_count()
         self.localBatch: SingleMachineBatchSystem = SingleMachineBatchSystem(
-            config, maxCores, maxMemory, maxDisk, max_jobs=config.max_local_jobs
+            config, maxCores, maxMemory, maxDisk, max_jobs=max_local_jobs
         )
 
     def handleLocalJob(self, jobDesc: JobDescription) -> Optional[int]:

toil/batchSystems/mesos/batchSystem.py

@@ -18,11 +18,9 @@ import os
 import pickle
 import pwd
 import socket
-import sys
 import time
 import traceback
 from argparse import ArgumentParser, _ArgumentGroup
-from contextlib import contextmanager
 from queue import Empty, Queue
 from typing import Dict, Optional, Union
 from urllib.parse import quote_plus
@@ -93,7 +91,7 @@ class MesosBatchSystem(BatchSystemLocalSupport,
         self.jobQueues = JobQueue()
 
         # Address of the Mesos master in the form host:port where host can be an IP or a hostname
-        self.mesos_endpoint = config.mesos_endpoint
+        self.mesos_endpoint = config.mesos_endpoint or self.get_default_mesos_endpoint()
         if config.mesos_role is not None:
             self.mesos_role = config.mesos_role
         self.mesos_name = config.mesos_name
@@ -846,8 +844,10 @@ class MesosBatchSystem(BatchSystemLocalSupport,
 
     @classmethod
     def add_options(cls, parser: Union[ArgumentParser, _ArgumentGroup]) -> None:
-        parser.add_argument("--mesosEndpoint", "--mesosMaster", dest="mesos_endpoint", default=cls.get_default_mesos_endpoint(),
-                            help="The host and port of the Mesos master separated by colon.  (default: %(default)s)")
+        parser.add_argument("--mesosEndpoint", "--mesosMaster", dest="mesos_endpoint", default=None,
+                            help=f"The host and port of the Mesos master separated by colon. If the provided value "
+                                 f"is None, the value will be generated at runtime.  "
+                                 f"(Generated default: {cls.get_default_mesos_endpoint})")
         parser.add_argument("--mesosFrameworkId", dest="mesos_framework_id",
                             help="Use a specific Mesos framework ID.")
         parser.add_argument("--mesosRole", dest="mesos_role",
@@ -857,8 +857,8 @@ class MesosBatchSystem(BatchSystemLocalSupport,
 
     @classmethod
     def setOptions(cls, setOption: OptionSetter):
-        setOption("mesos_endpoint", None, None, cls.get_default_mesos_endpoint(), old_names=["mesosMasterAddress"])
-        setOption("mesos_name", None, None, "toil")
+        setOption("mesos_endpoint", old_names=["mesosMasterAddress"])
+        setOption("mesos_name")
         setOption("mesos_role")
         setOption("mesos_framework_id")