toil 6.1.0a1__py3-none-any.whl → 7.0.0__py3-none-any.whl

toil/lib/io.py

@@ -16,7 +16,7 @@ def mkdtemp(suffix: Optional[str] = None, prefix: Optional[str] = None, dir: Opt
 
     The permissions on the directory will be 711 instead of 700, allowing the
     group and all other users to traverse the directory. This is necessary if
-    the direcotry is on NFS and the Docker daemon would like to mount it or a
+    the directory is on NFS and the Docker daemon would like to mount it or a
     file inside it into a container, because on NFS even the Docker daemon
     appears bound by the file permissions.
 
@@ -159,14 +159,26 @@ def atomic_copyobj(src_fh: BytesIO, dest_path: str, length: int = 16384, executa
             os.chmod(dest_path_tmp, os.stat(dest_path_tmp).st_mode | stat.S_IXUSR)
 
 
-def make_public_dir(in_directory: Optional[str] = None) -> str:
+def make_public_dir(in_directory: str, suggested_name: Optional[str] = None) -> str:
     """
+    Make a publicly-accessible directory in the given directory.
+
+    :param suggested_name: Use this directory name first if possible.
+
     Try to make a random directory name with length 4 that doesn't exist, with the given prefix.
     Otherwise, try length 5, length 6, etc, up to a max of 32 (len of uuid4 with dashes replaced).
     This function's purpose is mostly to avoid having long file names when generating directories.
     If somehow this fails, which should be incredibly unlikely, default to a normal uuid4, which was
     our old default.
     """
+    if suggested_name is not None:
+        generated_dir_path: str = os.path.join(in_directory, suggested_name)
+        try:
+            os.mkdir(generated_dir_path)
+            os.chmod(generated_dir_path, 0o777)
+            return generated_dir_path
+        except FileExistsError:
+            pass
     for i in range(4, 32 + 1):  # make random uuids and truncate to lengths starting at 4 and working up to max 32
         for _ in range(10):  # make 10 attempts for each length
             truncated_uuid: str = str(uuid.uuid4()).replace('-', '')[:i]
@@ -182,17 +194,43 @@ def make_public_dir(in_directory: Optional[str] = None) -> str:
     os.chmod(this_should_never_happen, 0o777)
     return this_should_never_happen
 
-def try_path(path: str) -> Optional[str]:
+def try_path(path: str, min_size: int = 100 * 1024 * 1024) -> Optional[str]:
     """
     Try to use the given path. Return it if it exists or can be made,
     and we can make things within it, or None otherwise.
+
+    :param min_size: Reject paths on filesystems smaller than this many bytes.
     """
+
     try:
         os.makedirs(path, exist_ok=True)
     except OSError:
         # Maybe we lack permissions
         return None
-    return path if os.path.exists(path) and os.access(path, os.W_OK) else None
+
+    if not os.path.exists(path):
+        # We didn't manage to make it
+        return None
+
+    if not os.access(path, os.W_OK):
+        # It doesn't look writable
+        return None
+
+    try:
+        stats = os.statvfs(path)
+    except OSError:
+        # Maybe we lack permissions
+        return None
+
+    # Is the filesystem big enough?
+    # We need to look at the FS size and not the free space so we don't change
+    # over to a different filesystem when this one fills up.
+    fs_size = stats.f_frsize * stats.f_blocks
+    if fs_size < min_size:
+        # Too small
+        return None
+
+    return path
 
 
 class WriteWatchingStream:

toil/lib/misc.py

@@ -11,8 +11,6 @@ import typing
 from contextlib import closing
 from typing import Iterator, List, Optional
 
-import pytz
-
 logger = logging.getLogger(__name__)
 
 
@@ -56,7 +54,7 @@ def get_user_name() -> str:
 
 def utc_now() -> datetime.datetime:
     """Return a datetime in the UTC timezone corresponding to right now."""
-    return datetime.datetime.utcnow().replace(tzinfo=pytz.UTC)
+    return datetime.datetime.utcnow().replace(tzinfo=datetime.timezone.utc)
 
 def unix_now_ms() -> float:
     """Return the current time in milliseconds since the Unix epoch."""

toil/lib/resources.py

@@ -13,27 +13,68 @@
 # limitations under the License.
 import fnmatch
 import os
+import math
+import sys
 import resource
 from typing import List, Tuple
 
-
-def get_total_cpu_time_and_memory_usage() -> Tuple[float, int]:
+class ResourceMonitor:
     """
-    Gives the total cpu time of itself and all its children, and the maximum RSS memory usage of
-    itself and its single largest child.
+    Global resource monitoring widget.
+
+    Presents class methods to get the resource usage of this process and child
+    processes, and other class methods to adjust the statistics so they can
+    account for e.g. resources used inside containers, or other resource usage
+    that *should* be billable to the current process.
     """
-    me = resource.getrusage(resource.RUSAGE_SELF)
-    children = resource.getrusage(resource.RUSAGE_CHILDREN)
-    total_cpu_time = me.ru_utime + me.ru_stime + children.ru_utime + children.ru_stime
-    total_memory_usage = me.ru_maxrss + children.ru_maxrss
-    return total_cpu_time, total_memory_usage
-
-
-def get_total_cpu_time() -> float:
-    """Gives the total cpu time, including the children."""
-    me = resource.getrusage(resource.RUSAGE_SELF)
-    childs = resource.getrusage(resource.RUSAGE_CHILDREN)
-    return me.ru_utime + me.ru_stime + childs.ru_utime + childs.ru_stime
+
+    # Store some extra usage to tack onto the stats as module-level globals
+    _extra_cpu_seconds: float = 0
+    _extra_memory_ki: int = 0
+
+    @classmethod
+    def record_extra_memory(cls, peak_ki: int) -> None:
+        """
+        Become responsible for the given peak memory usage, in kibibytes.
+
+        The memory will be treated as if it was used by a child process at the time
+        our real child processes were also using their peak memory.
+        """
+        cls._extra_memory_ki = max(cls._extra_memory_ki, peak_ki)
+
+    @classmethod
+    def record_extra_cpu(cls, seconds: float) -> None:
+        """
+        Become responsible for the given CPU time.
+
+        The CPU time will be treated as if it had been used by a child process.
+        """
+        cls._extra_cpu_seconds += seconds
+
+    @classmethod
+    def get_total_cpu_time_and_memory_usage(cls) -> Tuple[float, int]:
+        """
+        Gives the total cpu time of itself and all its children, and the maximum RSS memory usage of
+        itself and its single largest child (in kibibytes).
+        """
+        me = resource.getrusage(resource.RUSAGE_SELF)
+        children = resource.getrusage(resource.RUSAGE_CHILDREN)
+        total_cpu_time = me.ru_utime + me.ru_stime + children.ru_utime + children.ru_stime + cls._extra_cpu_seconds
+        total_memory_usage = me.ru_maxrss + children.ru_maxrss
+        if sys.platform == "darwin":
+            # On Linux, getrusage works in "kilobytes" (really kibibytes), but on
+            # Mac it works in bytes. See
+            # <https://github.com/python/cpython/issues/74698>
+            total_memory_usage = int(math.ceil(total_memory_usage / 1024))
+        total_memory_usage += cls._extra_memory_ki
+        return total_cpu_time, total_memory_usage
+
+    @classmethod
+    def get_total_cpu_time(cls) -> float:
+        """Gives the total cpu time, including the children."""
+        me = resource.getrusage(resource.RUSAGE_SELF)
+        childs = resource.getrusage(resource.RUSAGE_CHILDREN)
+        return me.ru_utime + me.ru_stime + childs.ru_utime + childs.ru_stime + cls._extra_cpu_seconds
 
 
 def glob(glob_pattern: str, directoryname: str) -> List[str]:

toil/lib/retry.py

@@ -142,7 +142,7 @@ from typing import (Any,
                     Sequence,
                     Tuple,
                     Type,
-                    Union)
+                    Union, TypeVar)
 
 import requests.exceptions
 import urllib3.exceptions
@@ -224,13 +224,16 @@ class ErrorCondition:
                 )
 
 
+# There is a better way to type hint this with python 3.10
+# https://stackoverflow.com/a/68290080
+RT = TypeVar("RT")
 def retry(
     intervals: Optional[List] = None,
     infinite_retries: bool = False,
     errors: Optional[Sequence[Union[ErrorCondition, Type[Exception]]]] = None,
     log_message: Optional[Tuple[Callable, str]] = None,
     prepare: Optional[List[Callable]] = None,
-) -> Callable[[Any], Any]:
+) -> Callable[[Callable[..., RT]], Callable[..., RT]]:
     """
     Retry a function if it fails with any Exception defined in "errors".
 
@@ -281,9 +284,9 @@ def retry(
         if error_condition.retry_on_this_condition:
             retriable_errors.add(error_condition.error)
 
-    def decorate(func):
+    def decorate(func: Callable[..., RT]) -> Callable[..., RT]:
         @functools.wraps(func)
-        def call(*args, **kwargs):
+        def call(*args, **kwargs) -> RT:
             intervals_remaining = copy.deepcopy(intervals)
             while True:
                 try:
@@ -488,13 +491,15 @@ def error_meets_conditions(e, error_conditions):
 DEFAULT_DELAYS = (0, 1, 1, 4, 16, 64)
 DEFAULT_TIMEOUT = 300
 
+E = TypeVar("E", bound=Exception)  # so mypy understands passed through types
+
 # TODO: Replace the use of this with retry()
 #  The aws provisioner and jobstore need a large refactoring to be boto3 compliant, so this is
 #  still used there to avoid the duplication of future work
 def old_retry(
     delays: Iterable[float] = DEFAULT_DELAYS,
     timeout: float = DEFAULT_TIMEOUT,
-    predicate: Callable[[Exception], bool] = lambda e: False,
+    predicate: Callable[[E], bool] = lambda e: False,
 ) -> Generator[ContextManager, None, None]:
     """
     Deprecated.
@@ -567,6 +572,8 @@ def old_retry(
     >>> i
     1
     """
+    if timeout is None:
+        timeout = DEFAULT_TIMEOUT
     if timeout > 0:
         go = [ None ]
 

toil/lib/threading.py

@@ -28,7 +28,7 @@ import traceback
 from contextlib import contextmanager
 from typing import Dict, Iterator, Optional, Union, cast
 
-import psutil  # type: ignore
+import psutil
 
 from toil.lib.exceptions import raise_
 from toil.lib.io import robust_rmtree
@@ -41,7 +41,7 @@ class ExceptionalThread(threading.Thread):
     A thread whose join() method re-raises exceptions raised during run(). While join() is
     idempotent, the exception is only during the first invocation of join() that successfully
     joined the thread. If join() times out, no exception will be re reraised even though an
-    exception might already have occured in run().
+    exception might already have occurred in run().
 
     When subclassing this thread, override tryRun() instead of run().
 
@@ -109,9 +109,12 @@ def cpu_count() -> int:
         return cast(int, cached)
 
     # Get the fallback answer of all the CPUs on the machine
-    total_machine_size = cast(int, psutil.cpu_count(logical=True))
+    psutil_cpu_count = cast(Optional[int], psutil.cpu_count(logical=True))
+    if psutil_cpu_count is None:
+        logger.debug('Could not retrieve the logical CPU count.')
 
-    logger.debug('Total machine size: %d cores', total_machine_size)
+    total_machine_size: Union[float, int] = psutil_cpu_count if psutil_cpu_count is not None else float('inf')
+    logger.debug('Total machine size: %s core(s)', total_machine_size)
 
     # cgroups may limit the size
     cgroup_size: Union[float, int] = float('inf')
@@ -151,13 +154,13 @@ def cpu_count() -> int:
             if quota == -1:
                 # But the quota can be -1 for unset.
                 # Assume we can use the whole machine.
-                return total_machine_size
-
-            # The thread count is how many multiples of a wall clock period we
-            # can burn in that period.
-            cgroup_size = int(math.ceil(float(quota)/float(period)))
+                cgroup_size = float('inf')
+            else:
+                # The thread count is how many multiples of a wall clock period we
+                # can burn in that period.
+                cgroup_size = int(math.ceil(float(quota)/float(period)))
 
-            logger.debug('Control group size in cores: %d', cgroup_size)
+            logger.debug('Control group size in cores: %s', cgroup_size)
     except:
         # We can't actually read these cgroup fields. Maybe we are a mac or something.
         logger.debug('Could not inspect cgroup: %s', traceback.format_exc())
@@ -175,9 +178,16 @@ def cpu_count() -> int:
     else:
         logger.debug('CPU affinity not available')
 
-    # Return the smaller of the actual thread count and the cgroup's limit, minimum 1.
-    result = cast(int, max(1, min(min(affinity_size, cgroup_size), total_machine_size)))
-    logger.debug('cpu_count: %s', str(result))
+    limit: Union[float, int] = float('inf')
+    # Apply all the limits to take the smallest
+    limit = min(limit, total_machine_size)
+    limit = min(limit, cgroup_size)
+    limit = min(limit, affinity_size)
+    if limit < 1 or limit == float('inf'):
+        # Fall back to 1 if we can't get a size
+        limit = 1
+    result = int(limit)
+    logger.debug('cpu_count: %s', result)
     # Make sure to remember it for the next call
     setattr(cpu_count, 'result', result)
     return result
@@ -529,9 +539,14 @@ class LastProcessStandingArena:
                 os.mkdir(self.lockfileDir)
             except FileExistsError:
                 pass
+            except Exception as e:
+                raise RuntimeError("Could not make lock file directory " + self.lockfileDir) from e
 
             # Make ourselves a file in it and lock it to prove we are alive.
-            self.lockfileFD, self.lockfileName = tempfile.mkstemp(dir=self.lockfileDir) # type: ignore
+            try:
+                self.lockfileFD, self.lockfileName = tempfile.mkstemp(dir=self.lockfileDir) # type: ignore
+            except Exception as e:
+                raise RuntimeError("Could not make lock file in " + self.lockfileDir) from e
             # Nobody can see it yet, so lock it right away
             fcntl.lockf(self.lockfileFD, fcntl.LOCK_EX) # type: ignore
 

toil/lib/throttle.py

@@ -43,7 +43,7 @@ class LocalThrottle:
         thread as necessary to ensure that no less than the configured minimum interval has
         passed since the last invocation of this method in the current thread returned True.
 
-        If the wait parameter is False, this method immediatly returns True (if at least the
+        If the wait parameter is False, this method immediatley returns True (if at least the
         configured minimum interval has passed since the last time this method returned True in
         the current thread) or False otherwise.
         """

toil/options/common.py

@@ -2,13 +2,12 @@ import os
 from argparse import ArgumentParser, Action, _AppendAction
 from typing import Any, Optional, Union, Type, Callable, List, Dict, TYPE_CHECKING
 
-from distutils.util import strtobool
 from configargparse import SUPPRESS
 import logging
 
 from ruamel.yaml import YAML
 
-from toil.lib.conversions import bytes2human, human2bytes
+from toil.lib.conversions import bytes2human, human2bytes, strtobool, opt_strtobool
 
 from toil.batchSystems.options import add_all_batchsystem_options
 from toil.provisioners import parse_node_types
@@ -138,8 +137,14 @@ def make_open_interval_action(min: Union[int, float], max: Optional[Union[int, f
                 func = fC(min, max)
             try:
                 if not func(values):
-                    raise parser.error(
-                        f"{option_string} ({values}) must be within the range: [{min}, {'infinity' if max is None else max})")
+                    if max is None:
+                        raise parser.error(
+                            f"{option_string} ({values}) must be at least {min}"
+                        )
+                    else:
+                        raise parser.error(
+                            f"{option_string} ({values}) must be at least {min} and strictly less than {max})"
+                        )
             except AssertionError:
                 raise RuntimeError(f"The {option_string} option has an invalid value: {values}")
             setattr(namespace, self.dest, values)
@@ -174,7 +179,7 @@ JOBSTORE_HELP = ("The location of the job store for the workflow.  "
                  "store must be accessible by all worker nodes. Depending on the desired "
                  "job store implementation, the location should be formatted according to "
                  "one of the following schemes:\n\n"
-                 "file:<path> where <path> points to a directory on the file systen\n\n"
+                 "file:<path> where <path> points to a directory on the file system\n\n"
                  "aws:<region>:<prefix> where <region> is the name of an AWS region like "
                  "us-west-2 and <prefix> will be prepended to the names of any top-level "
                  "AWS resources in use by job store, e.g. S3 buckets.\n\n "
@@ -201,14 +206,6 @@ def add_base_toil_options(parser: ArgumentParser, jobstore_as_flag: bool = False
     config.add_argument('--config', dest='config', is_config_file_arg=True, default=None, metavar="PATH",
                         help="Get options from a config file.")
 
-    def convert_bool(b: str) -> bool:
-        """Convert a string representation of bool to bool"""
-        return bool(strtobool(b))
-
-    def opt_strtobool(b: Optional[str]) -> Optional[bool]:
-        """Convert an optional string representation of bool to None or bool"""
-        return b if b is None else convert_bool(b)
-
     add_logging_options(parser)
     parser.register("type", "bool", parseBool)  # Custom type for arg=True/False.
 
@@ -354,7 +351,7 @@ def add_base_toil_options(parser: ArgumentParser, jobstore_as_flag: bool = False
                          "them from being modified externally.  When set to False, as long as caching is enabled, "
                          "Toil will protect the file automatically by changing the permissions to read-only."
                          "default=%(default)s")
-    link_imports.add_argument("--symlinkImports", dest="symlinkImports", type=convert_bool, default=True,
+    link_imports.add_argument("--symlinkImports", dest="symlinkImports", type=strtobool, default=True,
                               metavar="BOOL", help=link_imports_help)
     move_exports = file_store_options.add_mutually_exclusive_group()
     move_exports_help = ('When using a filesystem based job store, output files are by default moved to the '
@@ -362,7 +359,7 @@ def add_base_toil_options(parser: ArgumentParser, jobstore_as_flag: bool = False
                          'location.  Setting this option to True instead copies the files into the output directory.  '
                          'Applies to filesystem-based job stores only.'
                          'default=%(default)s')
-    move_exports.add_argument("--moveOutputs", dest="moveOutputs", type=convert_bool, default=False, metavar="BOOL",
+    move_exports.add_argument("--moveOutputs", dest="moveOutputs", type=strtobool, default=False, metavar="BOOL",
                               help=move_exports_help)
 
     caching = file_store_options.add_mutually_exclusive_group()
@@ -472,11 +469,11 @@ def add_base_toil_options(parser: ArgumentParser, jobstore_as_flag: bool = False
                                           "This is useful for heterogeneous jobs where some tasks require much more "
                                           "disk than others.")
 
-    autoscaling_options.add_argument("--metrics", dest="metrics", default=False, type=convert_bool, metavar="BOOL",
+    autoscaling_options.add_argument("--metrics", dest="metrics", default=False, type=strtobool, metavar="BOOL",
                                      help="Enable the prometheus/grafana dashboard for monitoring CPU/RAM usage, "
                                           "queue size, and issued jobs.")
     autoscaling_options.add_argument("--assumeZeroOverhead", dest="assume_zero_overhead", default=False,
-                                     type=convert_bool, metavar="BOOL",
+                                     type=strtobool, metavar="BOOL",
                                      help="Ignore scheduler and OS overhead and assume jobs can use every last byte "
                                           "of memory and disk on a node when autoscaling.")
 
@@ -547,7 +544,7 @@ def add_base_toil_options(parser: ArgumentParser, jobstore_as_flag: bool = False
                                   help=resource_help_msg.format('default', 'accelerators', accelerators_note, []))
     resource_options.add_argument('--defaultPreemptible', '--defaultPreemptable', dest='defaultPreemptible',
                                   metavar='BOOL',
-                                  type=convert_bool, nargs='?', const=True, default=False,
+                                  type=strtobool, nargs='?', const=True, default=False,
                                   help='Make all jobs able to run on preemptible (spot) nodes by default.')
     resource_options.add_argument('--maxCores', dest='maxCores', default=SYS_MAX_SIZE, metavar='INT', type=int,
                                   action=make_open_interval_action(1),
@@ -572,10 +569,10 @@ def add_base_toil_options(parser: ArgumentParser, jobstore_as_flag: bool = False
                                   f"labeling job failed. default={1}")
     job_options.add_argument("--enableUnlimitedPreemptibleRetries", "--enableUnlimitedPreemptableRetries",
                              dest="enableUnlimitedPreemptibleRetries",
-                             type=convert_bool, default=False, metavar="BOOL",
+                             type=strtobool, default=False, metavar="BOOL",
                              help="If set, preemptible failures (or any failure due to an instance getting "
                                   "unexpectedly terminated) will not count towards job failures and --retryCount.")
-    job_options.add_argument("--doubleMem", dest="doubleMem", type=convert_bool, default=False, metavar="BOOL",
+    job_options.add_argument("--doubleMem", dest="doubleMem", type=strtobool, default=False, metavar="BOOL",
                              help="If set, batch jobs which die to reaching memory limit on batch schedulers "
                                   "will have their memory doubled and they will be retried. The remaining "
                                   "retry count will be reduced by 1. Currently supported by LSF.")
@@ -589,18 +586,23 @@ def add_base_toil_options(parser: ArgumentParser, jobstore_as_flag: bool = False
                              help=f"Period of time to wait (in seconds) between checking for missing/overlong jobs, "
                                   f"that is jobs which get lost by the batch system. Expert parameter.  "
                                   f"default=%(default)s")
+    job_options.add_argument("--jobStoreTimeout", dest="job_store_timeout", default=30, type=float,
+                             action=make_open_interval_action(0), metavar="FLOAT",
+                             help=f"Maximum time (in seconds) to wait for a job's update to the job store "
+                                  f"before declaring it failed. default=%(default)s")
+
 
     # Log management options
     log_options = parser.add_argument_group(
         title="Toil log management options.",
         description="Options for how Toil should manage its logs."
     )
-    log_options.add_argument("--maxLogFileSize", dest="maxLogFileSize", default=64000, type=h2b,
+    log_options.add_argument("--maxLogFileSize", dest="maxLogFileSize", default=100 * 1024 * 1024, type=h2b,
                              action=make_open_interval_action(1),
                              help=f"The maximum size of a job log file to keep (in bytes), log files larger than "
                                   f"this will be truncated to the last X bytes. Setting this option to zero will "
                                   f"prevent any truncation. Setting this option to a negative value will truncate "
-                                  f"from the beginning.  Default={bytes2human(64000)}")
+                                  f"from the beginning.  Default={bytes2human(100 * 1024 * 1024)}")
     log_options.add_argument("--writeLogs", dest="writeLogs", nargs='?', action='store', default=None,
                              const=os.getcwd(), metavar="OPT_PATH",
                              help="Write worker logs received by the leader into their own files at the specified "
@@ -613,7 +615,7 @@ def add_base_toil_options(parser: ArgumentParser, jobstore_as_flag: bool = False
     log_options.add_argument("--writeLogsGzip", dest="writeLogsGzip", nargs='?', action='store', default=None,
                              const=os.getcwd(), metavar="OPT_PATH",
                              help="Identical to --writeLogs except the logs files are gzipped on the leader.")
-    log_options.add_argument("--writeLogsFromAllJobs", dest="writeLogsFromAllJobs", type=convert_bool,
+    log_options.add_argument("--writeLogsFromAllJobs", dest="writeLogsFromAllJobs", type=strtobool,
                              default=False, metavar="BOOL",
                              help="Whether to write logs from all jobs (including the successful ones) without "
                                   "necessarily setting the log level to 'debug'. Ensure that either --writeLogs "
@@ -621,7 +623,7 @@ def add_base_toil_options(parser: ArgumentParser, jobstore_as_flag: bool = False
     log_options.add_argument("--writeMessages", dest="write_messages", default=None,
                              type=lambda x: None if x is None else os.path.abspath(x), metavar="PATH",
                              help="File to send messages from the leader's message bus to.")
-    log_options.add_argument("--realTimeLogging", dest="realTimeLogging", type=convert_bool, default=False,
+    log_options.add_argument("--realTimeLogging", dest="realTimeLogging", type=strtobool, default=False,
                              help="Enable real-time logging from workers to leader")
 
     # Misc options
@@ -629,12 +631,12 @@ def add_base_toil_options(parser: ArgumentParser, jobstore_as_flag: bool = False
         title="Toil miscellaneous options.",
         description="Everything else."
     )
-    misc_options.add_argument('--disableChaining', dest='disableChaining', type=convert_bool, default=False,
+    misc_options.add_argument('--disableChaining', dest='disableChaining', type=strtobool, default=False,
                               metavar="BOOL",
                               help="Disables chaining of jobs (chaining uses one job's resource allocation "
                                    "for its successor job if possible).")
     misc_options.add_argument("--disableJobStoreChecksumVerification", dest="disableJobStoreChecksumVerification",
-                              default=False, type=convert_bool, metavar="BOOL",
+                              default=False, type=strtobool, metavar="BOOL",
                               help="Disables checksum verification for files transferred to/from the job store.  "
                                    "Checksum verification is a safety check to ensure the data is not corrupted "
                                    "during transfer. Currently only supported for non-streaming AWS files.")
@@ -686,14 +688,13 @@ def add_base_toil_options(parser: ArgumentParser, jobstore_as_flag: bool = False
                               action=make_open_interval_action(0.0), metavar="FLOAT",
                               help=f"Interval of time service jobs wait between polling for the existence of the "
                                    f"keep-alive flag.  Default: {60.0}")
-    misc_options.add_argument('--forceDockerAppliance', dest='forceDockerAppliance', type=convert_bool, default=False,
+    misc_options.add_argument('--forceDockerAppliance', dest='forceDockerAppliance', type=strtobool, default=False,
                               metavar="BOOL",
                               help='Disables sanity checking the existence of the docker image specified by '
                                    'TOIL_APPLIANCE_SELF, which Toil uses to provision mesos for autoscaling.')
     misc_options.add_argument('--statusWait', dest='statusWait', type=int, default=3600, metavar="INT",
                               help="Seconds to wait between reports of running jobs.")
-    misc_options.add_argument('--disableProgress', dest='disableProgress', type=convert_bool, default=False,
-                              metavar="BOOL",
+    misc_options.add_argument('--disableProgress', dest='disableProgress', action="store_true", default=False,
                               help="Disables the progress bar shown when standard error is a terminal.")
 
     # Debug options
@@ -735,4 +736,4 @@ def add_base_toil_options(parser: ArgumentParser, jobstore_as_flag: bool = False
 
     # dest is set to enableCaching to not conflict with the current --caching destination
     caching.add_argument('--disableCaching', dest='enableCaching', action='store_false', help=SUPPRESS)
-    caching.set_defaults(disableCaching=None)
+    caching.set_defaults(enableCaching=None)

toil/options/wdl.py

@@ -13,6 +13,8 @@ def add_wdl_options(parser: ArgumentParser, suppress: bool = True) -> None:
     suppress_help = SUPPRESS if suppress else None
     # include arg names without a wdl specifier if suppress is False
     # this is to avoid possible duplicate options in custom toil scripts, ex outputFile can be a common argument name
+    # TODO: Why do we even need them at all in other Toil scripts? Do we have to worry about dest= collisions?
+    # TODO: Can the better option name be first?
     output_dialect_arguments = ["--wdlOutputDialect"] + (["--outputDialect"] if not suppress else [])
     parser.add_argument(*output_dialect_arguments, dest="output_dialect", type=str, default='cromwell',
                         choices=['cromwell', 'miniwdl'],
@@ -30,3 +32,6 @@ def add_wdl_options(parser: ArgumentParser, suppress: bool = True) -> None:
     reference_inputs_arguments = ["--wdlReferenceInputs"] + (["--referenceInputs"] if not suppress else [])
     parser.add_argument(*reference_inputs_arguments, dest="reference_inputs", type=bool, default=False,
                         help=suppress_help or "Pass input files by URL")
+    container_arguments = ["--wdlContainer"] + (["--container"] if not suppress else [])
+    parser.add_argument(*container_arguments, dest="container", type=str, choices=["singularity", "docker", "auto"], default="auto",
+                        help=suppress_help or "Container engine to use to run WDL tasks")

toil/provisioners/__init__.py

@@ -32,6 +32,7 @@ def cluster_factory(
     nodeStorage: int = 50,
     nodeStorageOverrides: Optional[List[str]] = None,
     sseKey: Optional[str] = None,
+    enable_fuse: bool = False
 ) -> Union["AWSProvisioner", "GCEProvisioner"]:
     """
     Find and instantiate the appropriate provisioner instance to make clusters in the given cloud.
@@ -51,14 +52,14 @@ def cluster_factory(
         except ImportError:
             logger.error('The aws extra must be installed to use this provisioner')
             raise
-        return AWSProvisioner(clusterName, clusterType, zone, nodeStorage, nodeStorageOverrides, sseKey)
+        return AWSProvisioner(clusterName, clusterType, zone, nodeStorage, nodeStorageOverrides, sseKey, enable_fuse)
     elif provisioner == 'gce':
         try:
             from toil.provisioners.gceProvisioner import GCEProvisioner
         except ImportError:
             logger.error('The google extra must be installed to use this provisioner')
             raise
-        return GCEProvisioner(clusterName, clusterType, zone, nodeStorage, nodeStorageOverrides, sseKey)
+        return GCEProvisioner(clusterName, clusterType, zone, nodeStorage, nodeStorageOverrides, sseKey, enable_fuse)
     else:
         raise RuntimeError("Invalid provisioner '%s'" % provisioner)
 
@@ -174,9 +175,14 @@ def check_valid_node_types(provisioner, node_types: List[Tuple[Set[str], Optiona
 
 class NoSuchClusterException(Exception):
     """Indicates that the specified cluster does not exist."""
-    def __init__(self, cluster_name):
+    def __init__(self, cluster_name: str) -> None:
         super().__init__(f"The cluster '{cluster_name}' could not be found")
 
+class NoSuchZoneException(Exception):
+    """Indicates that a valid zone could not be found."""
+    def __init__(self) -> None:
+        super().__init__(f"No valid zone could be found!")
+
 
 class ClusterTypeNotSupportedException(Exception):
     """Indicates that a provisioner does not support a given cluster type."""

toil/provisioners/abstractProvisioner.py

@@ -137,6 +137,7 @@ class AbstractProvisioner(ABC):
         zone: Optional[str] = None,
         nodeStorage: int = 50,
         nodeStorageOverrides: Optional[List[str]] = None,
+        enable_fuse: bool = False
     ) -> None:
         """
         Initialize provisioner.
@@ -162,11 +163,14 @@ class AbstractProvisioner(ABC):
         for override in nodeStorageOverrides or []:
             nodeShape, storageOverride = override.split(':')
             self._nodeStorageOverrides[nodeShape] = int(storageOverride)
-        self._leaderPrivateIP = None
+        self._leaderPrivateIP: Optional[str] = None
         # This will hold an SSH public key for Mesos clusters, or the
         # Kubernetes joining information as a dict for Kubernetes clusters.
         self._leaderWorkerAuthentication = None
 
+        # Whether or not to use FUSE on the cluster. If true, the cluster's Toil containers will be launched in privileged mode
+        self.enable_fuse = enable_fuse
+
         if clusterName:
             # Making a new cluster
             self.createClusterSettings()
@@ -812,6 +816,12 @@ class AbstractProvisioner(ABC):
                 -v /opt:/opt \\
                 -v /etc/kubernetes:/etc/kubernetes \\
                 -v /etc/kubernetes/admin.conf:/root/.kube/config \\
+                {"-e TOIL_KUBERNETES_PRIVILEGED=True --privileged" if self.enable_fuse else 
+                "--security-opt seccomp=unconfined --security-opt systempaths=unconfined"} \\
+                -e TOIL_KUBERNETES_HOST_PATH=/var/lib/toil \\
+                # Pass in a path to use for singularity image caching into the container
+                -e SINGULARITY_CACHEDIR=/var/lib/toil/singularity \\
+                -e MINIWDL__SINGULARITY__IMAGE_CACHE=/var/lib/toil/miniwdl \\
                 --name=toil_{role} \\
                 {applianceSelf()} \\
                 {entryPointArgs}
@@ -1228,7 +1238,7 @@ class AbstractProvisioner(ABC):
             WantedBy=multi-user.target
             ''').format(**values))
 
-    def _getIgnitionUserData(self, role, keyPath=None, preemptible=False, architecture='amd64'):
+    def _getIgnitionUserData(self, role: str, keyPath: Optional[str] = None, preemptible: bool = False, architecture: str = 'amd64') -> str:
         """
         Return the text (not bytes) user data to pass to a provisioned node.