toil 6.1.0__py3-none-any.whl → 7.0.0__py3-none-any.whl

toil/lib/throttle.py

@@ -43,7 +43,7 @@ class LocalThrottle:
         thread as necessary to ensure that no less than the configured minimum interval has
         passed since the last invocation of this method in the current thread returned True.
 
-        If the wait parameter is False, this method immediatly returns True (if at least the
+        If the wait parameter is False, this method immediatley returns True (if at least the
         configured minimum interval has passed since the last time this method returned True in
         the current thread) or False otherwise.
         """

toil/options/common.py

@@ -7,7 +7,7 @@ import logging
 
 from ruamel.yaml import YAML
 
-from toil.lib.conversions import bytes2human, human2bytes, strtobool
+from toil.lib.conversions import bytes2human, human2bytes, strtobool, opt_strtobool
 
 from toil.batchSystems.options import add_all_batchsystem_options
 from toil.provisioners import parse_node_types
@@ -137,8 +137,14 @@ def make_open_interval_action(min: Union[int, float], max: Optional[Union[int, f
                 func = fC(min, max)
             try:
                 if not func(values):
-                    raise parser.error(
-                        f"{option_string} ({values}) must be within the range: [{min}, {'infinity' if max is None else max})")
+                    if max is None:
+                        raise parser.error(
+                            f"{option_string} ({values}) must be at least {min}"
+                        )
+                    else:
+                        raise parser.error(
+                            f"{option_string} ({values}) must be at least {min} and strictly less than {max})"
+                        )
             except AssertionError:
                 raise RuntimeError(f"The {option_string} option has an invalid value: {values}")
             setattr(namespace, self.dest, values)
@@ -173,7 +179,7 @@ JOBSTORE_HELP = ("The location of the job store for the workflow.  "
                  "store must be accessible by all worker nodes. Depending on the desired "
                  "job store implementation, the location should be formatted according to "
                  "one of the following schemes:\n\n"
-                 "file:<path> where <path> points to a directory on the file systen\n\n"
+                 "file:<path> where <path> points to a directory on the file system\n\n"
                  "aws:<region>:<prefix> where <region> is the name of an AWS region like "
                  "us-west-2 and <prefix> will be prepended to the names of any top-level "
                  "AWS resources in use by job store, e.g. S3 buckets.\n\n "
@@ -200,14 +206,6 @@ def add_base_toil_options(parser: ArgumentParser, jobstore_as_flag: bool = False
     config.add_argument('--config', dest='config', is_config_file_arg=True, default=None, metavar="PATH",
                         help="Get options from a config file.")
 
-    def convert_bool(b: str) -> bool:
-        """Convert a string representation of bool to bool"""
-        return bool(strtobool(b))
-
-    def opt_strtobool(b: Optional[str]) -> Optional[bool]:
-        """Convert an optional string representation of bool to None or bool"""
-        return b if b is None else convert_bool(b)
-
     add_logging_options(parser)
     parser.register("type", "bool", parseBool)  # Custom type for arg=True/False.
 
@@ -353,7 +351,7 @@ def add_base_toil_options(parser: ArgumentParser, jobstore_as_flag: bool = False
                          "them from being modified externally.  When set to False, as long as caching is enabled, "
                          "Toil will protect the file automatically by changing the permissions to read-only."
                          "default=%(default)s")
-    link_imports.add_argument("--symlinkImports", dest="symlinkImports", type=convert_bool, default=True,
+    link_imports.add_argument("--symlinkImports", dest="symlinkImports", type=strtobool, default=True,
                               metavar="BOOL", help=link_imports_help)
     move_exports = file_store_options.add_mutually_exclusive_group()
     move_exports_help = ('When using a filesystem based job store, output files are by default moved to the '
@@ -361,7 +359,7 @@ def add_base_toil_options(parser: ArgumentParser, jobstore_as_flag: bool = False
                          'location.  Setting this option to True instead copies the files into the output directory.  '
                          'Applies to filesystem-based job stores only.'
                          'default=%(default)s')
-    move_exports.add_argument("--moveOutputs", dest="moveOutputs", type=convert_bool, default=False, metavar="BOOL",
+    move_exports.add_argument("--moveOutputs", dest="moveOutputs", type=strtobool, default=False, metavar="BOOL",
                               help=move_exports_help)
 
     caching = file_store_options.add_mutually_exclusive_group()
@@ -471,11 +469,11 @@ def add_base_toil_options(parser: ArgumentParser, jobstore_as_flag: bool = False
                                           "This is useful for heterogeneous jobs where some tasks require much more "
                                           "disk than others.")
 
-    autoscaling_options.add_argument("--metrics", dest="metrics", default=False, type=convert_bool, metavar="BOOL",
+    autoscaling_options.add_argument("--metrics", dest="metrics", default=False, type=strtobool, metavar="BOOL",
                                      help="Enable the prometheus/grafana dashboard for monitoring CPU/RAM usage, "
                                           "queue size, and issued jobs.")
     autoscaling_options.add_argument("--assumeZeroOverhead", dest="assume_zero_overhead", default=False,
-                                     type=convert_bool, metavar="BOOL",
+                                     type=strtobool, metavar="BOOL",
                                      help="Ignore scheduler and OS overhead and assume jobs can use every last byte "
                                           "of memory and disk on a node when autoscaling.")
 
@@ -546,7 +544,7 @@ def add_base_toil_options(parser: ArgumentParser, jobstore_as_flag: bool = False
                                   help=resource_help_msg.format('default', 'accelerators', accelerators_note, []))
     resource_options.add_argument('--defaultPreemptible', '--defaultPreemptable', dest='defaultPreemptible',
                                   metavar='BOOL',
-                                  type=convert_bool, nargs='?', const=True, default=False,
+                                  type=strtobool, nargs='?', const=True, default=False,
                                   help='Make all jobs able to run on preemptible (spot) nodes by default.')
     resource_options.add_argument('--maxCores', dest='maxCores', default=SYS_MAX_SIZE, metavar='INT', type=int,
                                   action=make_open_interval_action(1),
@@ -571,10 +569,10 @@ def add_base_toil_options(parser: ArgumentParser, jobstore_as_flag: bool = False
                                   f"labeling job failed. default={1}")
     job_options.add_argument("--enableUnlimitedPreemptibleRetries", "--enableUnlimitedPreemptableRetries",
                              dest="enableUnlimitedPreemptibleRetries",
-                             type=convert_bool, default=False, metavar="BOOL",
+                             type=strtobool, default=False, metavar="BOOL",
                              help="If set, preemptible failures (or any failure due to an instance getting "
                                   "unexpectedly terminated) will not count towards job failures and --retryCount.")
-    job_options.add_argument("--doubleMem", dest="doubleMem", type=convert_bool, default=False, metavar="BOOL",
+    job_options.add_argument("--doubleMem", dest="doubleMem", type=strtobool, default=False, metavar="BOOL",
                              help="If set, batch jobs which die to reaching memory limit on batch schedulers "
                                   "will have their memory doubled and they will be retried. The remaining "
                                   "retry count will be reduced by 1. Currently supported by LSF.")
@@ -588,6 +586,11 @@ def add_base_toil_options(parser: ArgumentParser, jobstore_as_flag: bool = False
                              help=f"Period of time to wait (in seconds) between checking for missing/overlong jobs, "
                                   f"that is jobs which get lost by the batch system. Expert parameter.  "
                                   f"default=%(default)s")
+    job_options.add_argument("--jobStoreTimeout", dest="job_store_timeout", default=30, type=float,
+                             action=make_open_interval_action(0), metavar="FLOAT",
+                             help=f"Maximum time (in seconds) to wait for a job's update to the job store "
+                                  f"before declaring it failed. default=%(default)s")
+
 
     # Log management options
     log_options = parser.add_argument_group(
@@ -612,7 +615,7 @@ def add_base_toil_options(parser: ArgumentParser, jobstore_as_flag: bool = False
     log_options.add_argument("--writeLogsGzip", dest="writeLogsGzip", nargs='?', action='store', default=None,
                              const=os.getcwd(), metavar="OPT_PATH",
                              help="Identical to --writeLogs except the logs files are gzipped on the leader.")
-    log_options.add_argument("--writeLogsFromAllJobs", dest="writeLogsFromAllJobs", type=convert_bool,
+    log_options.add_argument("--writeLogsFromAllJobs", dest="writeLogsFromAllJobs", type=strtobool,
                              default=False, metavar="BOOL",
                              help="Whether to write logs from all jobs (including the successful ones) without "
                                   "necessarily setting the log level to 'debug'. Ensure that either --writeLogs "
@@ -620,7 +623,7 @@ def add_base_toil_options(parser: ArgumentParser, jobstore_as_flag: bool = False
     log_options.add_argument("--writeMessages", dest="write_messages", default=None,
                              type=lambda x: None if x is None else os.path.abspath(x), metavar="PATH",
                              help="File to send messages from the leader's message bus to.")
-    log_options.add_argument("--realTimeLogging", dest="realTimeLogging", type=convert_bool, default=False,
+    log_options.add_argument("--realTimeLogging", dest="realTimeLogging", type=strtobool, default=False,
                              help="Enable real-time logging from workers to leader")
 
     # Misc options
@@ -628,12 +631,12 @@ def add_base_toil_options(parser: ArgumentParser, jobstore_as_flag: bool = False
         title="Toil miscellaneous options.",
         description="Everything else."
     )
-    misc_options.add_argument('--disableChaining', dest='disableChaining', type=convert_bool, default=False,
+    misc_options.add_argument('--disableChaining', dest='disableChaining', type=strtobool, default=False,
                               metavar="BOOL",
                               help="Disables chaining of jobs (chaining uses one job's resource allocation "
                                    "for its successor job if possible).")
     misc_options.add_argument("--disableJobStoreChecksumVerification", dest="disableJobStoreChecksumVerification",
-                              default=False, type=convert_bool, metavar="BOOL",
+                              default=False, type=strtobool, metavar="BOOL",
                               help="Disables checksum verification for files transferred to/from the job store.  "
                                    "Checksum verification is a safety check to ensure the data is not corrupted "
                                    "during transfer. Currently only supported for non-streaming AWS files.")
@@ -685,7 +688,7 @@ def add_base_toil_options(parser: ArgumentParser, jobstore_as_flag: bool = False
                               action=make_open_interval_action(0.0), metavar="FLOAT",
                               help=f"Interval of time service jobs wait between polling for the existence of the "
                                    f"keep-alive flag.  Default: {60.0}")
-    misc_options.add_argument('--forceDockerAppliance', dest='forceDockerAppliance', type=convert_bool, default=False,
+    misc_options.add_argument('--forceDockerAppliance', dest='forceDockerAppliance', type=strtobool, default=False,
                               metavar="BOOL",
                               help='Disables sanity checking the existence of the docker image specified by '
                                    'TOIL_APPLIANCE_SELF, which Toil uses to provision mesos for autoscaling.')

toil/provisioners/__init__.py

@@ -32,6 +32,7 @@ def cluster_factory(
     nodeStorage: int = 50,
     nodeStorageOverrides: Optional[List[str]] = None,
     sseKey: Optional[str] = None,
+    enable_fuse: bool = False
 ) -> Union["AWSProvisioner", "GCEProvisioner"]:
     """
     Find and instantiate the appropriate provisioner instance to make clusters in the given cloud.
@@ -51,14 +52,14 @@ def cluster_factory(
         except ImportError:
             logger.error('The aws extra must be installed to use this provisioner')
             raise
-        return AWSProvisioner(clusterName, clusterType, zone, nodeStorage, nodeStorageOverrides, sseKey)
+        return AWSProvisioner(clusterName, clusterType, zone, nodeStorage, nodeStorageOverrides, sseKey, enable_fuse)
     elif provisioner == 'gce':
         try:
             from toil.provisioners.gceProvisioner import GCEProvisioner
         except ImportError:
             logger.error('The google extra must be installed to use this provisioner')
             raise
-        return GCEProvisioner(clusterName, clusterType, zone, nodeStorage, nodeStorageOverrides, sseKey)
+        return GCEProvisioner(clusterName, clusterType, zone, nodeStorage, nodeStorageOverrides, sseKey, enable_fuse)
     else:
         raise RuntimeError("Invalid provisioner '%s'" % provisioner)
 
@@ -174,9 +175,14 @@ def check_valid_node_types(provisioner, node_types: List[Tuple[Set[str], Optiona
 
 class NoSuchClusterException(Exception):
     """Indicates that the specified cluster does not exist."""
-    def __init__(self, cluster_name):
+    def __init__(self, cluster_name: str) -> None:
         super().__init__(f"The cluster '{cluster_name}' could not be found")
 
+class NoSuchZoneException(Exception):
+    """Indicates that a valid zone could not be found."""
+    def __init__(self) -> None:
+        super().__init__(f"No valid zone could be found!")
+
 
 class ClusterTypeNotSupportedException(Exception):
     """Indicates that a provisioner does not support a given cluster type."""

toil/provisioners/abstractProvisioner.py

@@ -137,6 +137,7 @@ class AbstractProvisioner(ABC):
         zone: Optional[str] = None,
         nodeStorage: int = 50,
         nodeStorageOverrides: Optional[List[str]] = None,
+        enable_fuse: bool = False
     ) -> None:
         """
         Initialize provisioner.
@@ -162,11 +163,14 @@ class AbstractProvisioner(ABC):
         for override in nodeStorageOverrides or []:
             nodeShape, storageOverride = override.split(':')
             self._nodeStorageOverrides[nodeShape] = int(storageOverride)
-        self._leaderPrivateIP = None
+        self._leaderPrivateIP: Optional[str] = None
         # This will hold an SSH public key for Mesos clusters, or the
         # Kubernetes joining information as a dict for Kubernetes clusters.
         self._leaderWorkerAuthentication = None
 
+        # Whether or not to use FUSE on the cluster. If true, the cluster's Toil containers will be launched in privileged mode
+        self.enable_fuse = enable_fuse
+
         if clusterName:
             # Making a new cluster
             self.createClusterSettings()
@@ -812,14 +816,12 @@ class AbstractProvisioner(ABC):
                 -v /opt:/opt \\
                 -v /etc/kubernetes:/etc/kubernetes \\
                 -v /etc/kubernetes/admin.conf:/root/.kube/config \\
-                # Pass in a path to use for singularity image caching into the container
+                {"-e TOIL_KUBERNETES_PRIVILEGED=True --privileged" if self.enable_fuse else 
+                "--security-opt seccomp=unconfined --security-opt systempaths=unconfined"} \\
                 -e TOIL_KUBERNETES_HOST_PATH=/var/lib/toil \\
+                # Pass in a path to use for singularity image caching into the container
                 -e SINGULARITY_CACHEDIR=/var/lib/toil/singularity \\
                 -e MINIWDL__SINGULARITY__IMAGE_CACHE=/var/lib/toil/miniwdl \\
-                # These rules are necessary in order to get user namespaces working
-                # https://github.com/apptainer/singularity/issues/5806
-                --security-opt seccomp=unconfined \\
-                --security-opt systempaths=unconfined \\
                 --name=toil_{role} \\
                 {applianceSelf()} \\
                 {entryPointArgs}
@@ -1236,7 +1238,7 @@ class AbstractProvisioner(ABC):
             WantedBy=multi-user.target
             ''').format(**values))
 
-    def _getIgnitionUserData(self, role, keyPath=None, preemptible=False, architecture='amd64'):
+    def _getIgnitionUserData(self, role: str, keyPath: Optional[str] = None, preemptible: bool = False, architecture: str = 'amd64') -> str:
         """
         Return the text (not bytes) user data to pass to a provisioned node.
 

toil/provisioners/aws/__init__.py

@@ -18,6 +18,8 @@ from operator import attrgetter
 from statistics import mean, stdev
 from typing import List, Optional
 
+from botocore.client import BaseClient
+
 from toil.lib.aws import (get_aws_zone_from_boto,
                           get_aws_zone_from_environment,
                           get_aws_zone_from_environment_region,
@@ -27,8 +29,10 @@ logger = logging.getLogger(__name__)
 
 ZoneTuple = namedtuple('ZoneTuple', ['name', 'price_deviation'])
 
+
 def get_aws_zone_from_spot_market(spotBid: Optional[float], nodeType: Optional[str],
-                                  boto2_ec2: Optional["boto.connection.AWSAuthConnection"], zone_options: Optional[List[str]]) -> Optional[str]:
+                                  boto3_ec2: Optional[BaseClient], zone_options: Optional[List[str]]) -> \
+Optional[str]:
     """
     If a spot bid, node type, and Boto2 EC2 connection are specified, picks a
     zone where instances are easy to buy from the zones in the region of the
@@ -40,21 +44,22 @@ def get_aws_zone_from_spot_market(spotBid: Optional[float], nodeType: Optional[s
     """
     if spotBid:
         # if spot bid is present, all the other parameters must be as well
-        assert bool(spotBid) == bool(nodeType) == bool(boto2_ec2)
+        assert bool(spotBid) == bool(nodeType) == bool(boto3_ec2)
         # if the zone is unset and we are using the spot market, optimize our
         # choice based on the spot history
 
         if zone_options is None:
             # We can use all the zones in the region
-            zone_options = [z.name for z in boto2_ec2.get_all_zones()]
+            zone_options = [z.name for z in boto3_ec2.describe_availability_zones()]
 
-        return optimize_spot_bid(boto2_ec2, instance_type=nodeType, spot_bid=float(spotBid), zone_options=zone_options)
+        return optimize_spot_bid(boto3_ec2, instance_type=nodeType, spot_bid=float(spotBid), zone_options=zone_options)
     else:
         return None
 
 
 def get_best_aws_zone(spotBid: Optional[float] = None, nodeType: Optional[str] = None,
-                      boto2_ec2: Optional["boto.connection.AWSAuthConnection"] = None, zone_options: Optional[List[str]] = None) -> Optional[str]:
+                      boto3_ec2: Optional[BaseClient] = None,
+                      zone_options: Optional[List[str]] = None) -> Optional[str]:
     """
     Get the right AWS zone to use.
 
@@ -81,12 +86,13 @@ def get_best_aws_zone(spotBid: Optional[float] = None, nodeType: Optional[str] =
     """
     return get_aws_zone_from_environment() or \
         get_aws_zone_from_metadata() or \
-        get_aws_zone_from_spot_market(spotBid, nodeType, boto2_ec2, zone_options) or \
+        get_aws_zone_from_spot_market(spotBid, nodeType, boto3_ec2, zone_options) or \
         get_aws_zone_from_environment_region() or \
         get_aws_zone_from_boto()
 
 
-def choose_spot_zone(zones: List[str], bid: float, spot_history: List['boto.ec2.spotpricehistory.SpotPriceHistory']) -> str:
+def choose_spot_zone(zones: List[str], bid: float,
+                     spot_history: List['boto.ec2.spotpricehistory.SpotPriceHistory']) -> str:
     """
     Returns the zone to put the spot request based on, in order of priority:
 
@@ -137,7 +143,7 @@ def choose_spot_zone(zones: List[str], bid: float, spot_history: List['boto.ec2.
     return min(markets_under_bid or markets_over_bid, key=attrgetter('price_deviation')).name
 
 
-def optimize_spot_bid(boto2_ec2, instance_type, spot_bid, zone_options: List[str]):
+def optimize_spot_bid(boto3_ec2: BaseClient, instance_type: str, spot_bid: float, zone_options: List[str]):
     """
     Check whether the bid is in line with history and makes an effort to place
     the instance in a sensible zone.
@@ -145,7 +151,7 @@ def optimize_spot_bid(boto2_ec2, instance_type, spot_bid, zone_options: List[str
     :param zone_options: The collection of allowed zones to consider, within
         the region associated with the Boto2 connection.
     """
-    spot_history = _get_spot_history(boto2_ec2, instance_type)
+    spot_history = _get_spot_history(boto3_ec2, instance_type)
     if spot_history:
         _check_spot_bid(spot_bid, spot_history)
     most_stable_zone = choose_spot_zone(zone_options, spot_bid, spot_history)
@@ -183,20 +189,19 @@ def _check_spot_bid(spot_bid, spot_history):
     average = mean([datum.price for datum in spot_history])
     if spot_bid > average * 2:
         logger.warning("Your bid $ %f is more than double this instance type's average "
-                 "spot price ($ %f) over the last week", spot_bid, average)
+                       "spot price ($ %f) over the last week", spot_bid, average)
 
 
-def _get_spot_history(boto2_ec2, instance_type):
+def _get_spot_history(boto3_ec2: BaseClient, instance_type: str):
     """
     Returns list of 1,000 most recent spot market data points represented as SpotPriceHistory
     objects. Note: The most recent object/data point will be first in the list.
 
     :rtype: list[SpotPriceHistory]
     """
-
     one_week_ago = datetime.datetime.now() - datetime.timedelta(days=7)
-    spot_data = boto2_ec2.get_spot_price_history(start_time=one_week_ago.isoformat(),
-                                                 instance_type=instance_type,
-                                                 product_description="Linux/UNIX")
+    spot_data = boto3_ec2.describe_spot_price_history(StartTime=one_week_ago.isoformat(),
+                                                      InstanceTypes=[instance_type],
+                                                      ProductDescriptions=["Linux/UNIX"])
     spot_data.sort(key=attrgetter("timestamp"), reverse=True)
     return spot_data