toil 8.1.0b1__py3-none-any.whl → 9.0.0__py3-none-any.whl

toil/lib/threading.py

@@ -36,12 +36,13 @@ import psutil
 
 from toil.lib.exceptions import raise_
 from toil.lib.io import robust_rmtree
+from toil.lib.misc import StrPath
 
 logger = logging.getLogger(__name__)
 
 
 def ensure_filesystem_lockable(
-    path: str, timeout: float = 30, hint: Optional[str] = None
+    path: StrPath, timeout: float = 30, hint: Optional[str] = None
 ) -> None:
     """
     Make sure that the filesystem used at the given path is one where locks are safe to use.
@@ -71,7 +72,7 @@ def ensure_filesystem_lockable(
             # Start a child process to stat the path. See <https://unix.stackexchange.com/a/402236>.
             # We really should call statfs but no bindings for it are in PyPI.
             completed = subprocess.run(
-                ["stat", "-f", "-c", "%T", path],
+                ["stat", "-f", "-c", "%T", str(path)],
                 check=True,
                 capture_output=True,
                 timeout=timeout,
@@ -85,7 +86,7 @@ def ensure_filesystem_lockable(
             # Stat didn't work. Maybe we don't have the right version of stat installed?
             logger.warning(
                 "Could not determine filesystem type at %s because of: %s",
-                path,
+                str(path),
                 e.stderr.decode("utf-8", errors="replace").strip(),
             )
             # If we don't know the filesystem type, keep going anyway.
@@ -107,7 +108,7 @@ def ensure_filesystem_lockable(
             # flaky with regard to locks actually locking anything).
             logger.debug(
                 "Detected that %s has lockable filesystem type: %s",
-                path,
+                str(path),
                 filesystem_type,
             )
 
@@ -518,7 +519,7 @@ def process_name_exists(base_dir: str, name: str) -> bool:
 # Similar to the process naming system above, we define a global mutex system
 # for critical sections, based just around file locks.
 @contextmanager
-def global_mutex(base_dir: str, mutex: str) -> Iterator[None]:
+def global_mutex(base_dir: StrPath, mutex: str) -> Iterator[None]:
     """
     Context manager that locks a mutex. The mutex is identified by the given
     name, and scoped to the given directory. Works across all containers that
@@ -527,7 +528,7 @@ def global_mutex(base_dir: str, mutex: str) -> Iterator[None]:
 
     Only works between processes, NOT between threads.
 
-    :param str base_dir: Base directory to work in. Defines the shared namespace.
+    :param base_dir: Base directory to work in. Defines the shared namespace.
     :param str mutex: Mutex to lock. Must be a permissible path component.
     """
 
@@ -674,7 +675,7 @@ class LastProcessStandingArena:
     Consider using a try/finally; this class is not a context manager.
     """
 
-    def __init__(self, base_dir: str, name: str) -> None:
+    def __init__(self, base_dir: StrPath, name: str) -> None:
         """
         Connect to the arena specified by the given base_dir and name.
 
@@ -683,12 +684,12 @@ class LastProcessStandingArena:
 
         Doesn't enter or leave the arena.
 
-        :param str base_dir: Base directory to work in. Defines the shared namespace.
-        :param str name: Name of the arena. Must be a permissible path component.
+        :param base_dir: Base directory to work in. Defines the shared namespace.
+        :param name: Name of the arena. Must be a permissible path component.
         """
 
         # Save the base_dir which namespaces everything
-        self.base_dir = base_dir
+        self.base_dir = str(base_dir)
 
         # We need a mutex name to allow only one process to be entering or
         # leaving at a time.

toil/lib/url.py

@@ -0,0 +1,320 @@
+# Copyright (C) 2015-2025 Regents of the University of California
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import logging
+import os
+from abc import ABC, ABCMeta, abstractmethod
+from typing import (
+    IO,
+    TYPE_CHECKING,
+    Any,
+    Callable,
+    ContextManager,
+    Literal,
+    Optional,
+    Union,
+    cast,
+    overload,
+    Type,
+)
+from urllib.parse import ParseResult, urlparse
+
+from toil.lib.exceptions import UnimplementedURLException
+from toil.lib.memoize import memoize
+from toil.lib.plugins import register_plugin, get_plugin
+
+try:
+    from botocore.exceptions import ProxyConnectionError
+except ImportError:
+
+    class ProxyConnectionError(BaseException):  # type: ignore
+        """Dummy class."""
+
+logger = logging.getLogger(__name__)
+
+class URLAccess:
+    """
+    Widget for accessing external storage (URLs).
+    """
+
+    @classmethod
+    def url_exists(cls, src_uri: str) -> bool:
+        """
+        Return True if the file at the given URI exists, and False otherwise.
+
+        May raise an error if file existence cannot be determined.
+
+        :param src_uri: URL that points to a file or object in the storage
+               mechanism of a supported URL scheme e.g. a blob in an AWS s3 bucket.
+        """
+        parseResult = urlparse(src_uri)
+        otherCls = cls._find_url_implementation(parseResult)
+        return otherCls._url_exists(parseResult)
+
+    @classmethod
+    def get_size(cls, src_uri: str) -> Optional[int]:
+        """
+        Get the size in bytes of the file at the given URL, or None if it cannot be obtained.
+
+        :param src_uri: URL that points to a file or object in the storage
+               mechanism of a supported URL scheme e.g. a blob in an AWS s3 bucket.
+        """
+        parseResult = urlparse(src_uri)
+        otherCls = cls._find_url_implementation(parseResult)
+        return otherCls._get_size(parseResult)
+
+    @classmethod
+    def get_is_directory(cls, src_uri: str) -> bool:
+        """
+        Return True if the thing at the given URL is a directory, and False if
+        it is a file. The URL may or may not end in '/'.
+        """
+        parseResult = urlparse(src_uri)
+        otherCls = cls._find_url_implementation(parseResult)
+        return otherCls._get_is_directory(parseResult)
+
+    @classmethod
+    def list_url(cls, src_uri: str) -> list[str]:
+        """
+        List the directory at the given URL. Returned path components can be
+        joined with '/' onto the passed URL to form new URLs. Those that end in
+        '/' correspond to directories. The provided URL may or may not end with
+        '/'.
+
+        Currently supported schemes are:
+
+            - 's3' for objects in Amazon S3
+                e.g. s3://bucket/prefix/
+
+            - 'file' for local files
+                e.g. file:///local/dir/path/
+
+        :param str src_uri: URL that points to a directory or prefix in the storage mechanism of a
+                supported URL scheme e.g. a prefix in an AWS s3 bucket.
+
+        :return: A list of URL components in the given directory, already URL-encoded.
+        """
+        parseResult = urlparse(src_uri)
+        otherCls = cls._find_url_implementation(parseResult)
+        return otherCls._list_url(parseResult)
+
+    @classmethod
+    def read_from_url(cls, src_uri: str, writable: IO[bytes]) -> tuple[int, bool]:
+        """
+        Read the given URL and write its content into the given writable stream.
+
+        Raises FileNotFoundError if the URL doesn't exist.
+
+        :return: The size of the file in bytes and whether the executable permission bit is set
+        """
+        parseResult = urlparse(src_uri)
+        otherCls = cls._find_url_implementation(parseResult)
+        return otherCls._read_from_url(parseResult, writable)
+
+    @classmethod
+    def open_url(cls, src_uri: str) -> IO[bytes]:
+        """
+        Read from the given URI.
+
+        Raises FileNotFoundError if the URL doesn't exist.
+
+        Has a readable stream interface, unlike :meth:`read_from_url` which
+        takes a writable stream.
+        """
+        parseResult = urlparse(src_uri)
+        otherCls = cls._find_url_implementation(parseResult)
+        return otherCls._open_url(parseResult)
+
+    @classmethod
+    @abstractmethod
+    def _url_exists(cls, url: ParseResult) -> bool:
+        """
+        Return True if the item at the given URL exists, and Flase otherwise.
+
+        May raise an error if file existence cannot be determined.
+        """
+        raise NotImplementedError(f"No implementation for {url}")
+
+    @classmethod
+    @abstractmethod
+    def _get_size(cls, url: ParseResult) -> Optional[int]:
+        """
+        Get the size of the object at the given URL, or None if it cannot be obtained.
+        """
+        raise NotImplementedError(f"No implementation for {url}")
+
+    @classmethod
+    @abstractmethod
+    def _get_is_directory(cls, url: ParseResult) -> bool:
+        """
+        Return True if the thing at the given URL is a directory, and False if
+        it is a file or it is known not to exist. The URL may or may not end in
+        '/'.
+
+        :param url: URL that points to a file or object, or directory or prefix,
+               in the storage mechanism of a supported URL scheme e.g. a blob
+               in an AWS s3 bucket.
+        """
+        raise NotImplementedError(f"No implementation for {url}")
+
+    @classmethod
+    @abstractmethod
+    def _read_from_url(cls, url: ParseResult, writable: IO[bytes]) -> tuple[int, bool]:
+        """
+        Reads the contents of the object at the specified location and writes it to the given
+        writable stream.
+
+        Raises FileNotFoundError if the thing at the URL is not found.
+
+        :param ParseResult url: URL that points to a file or object in the storage
+               mechanism of a supported URL scheme e.g. a blob in an AWS s3 bucket.
+
+        :param IO[bytes] writable: a writable stream
+
+        :return: The size of the file in bytes and whether the executable permission bit is set
+        """
+        raise NotImplementedError(f"No implementation for {url}")
+
+    @classmethod
+    @abstractmethod
+    def _list_url(cls, url: ParseResult) -> list[str]:
+        """
+        List the contents of the given URL, which may or may not end in '/'
+
+        Returns a list of URL components. Those that end in '/' are meant to be
+        directories, while those that do not are meant to be files.
+
+        :param ParseResult url: URL that points to a directory or prefix in the
+        storage mechanism of a supported URL scheme e.g. a prefix in an AWS s3
+        bucket.
+
+        :return: The children of the given URL, already URL-encoded if
+        appropriate. (If the URL is a bare path, no encoding is done.)
+        """
+        raise NotImplementedError(f"No implementation for {url}")
+
+    @classmethod
+    @abstractmethod
+    def _open_url(cls, url: ParseResult) -> IO[bytes]:
+        """
+        Get a stream of the object at the specified location.
+
+        Raises FileNotFoundError if the thing at the URL is not found.
+        """
+        raise NotImplementedError(f"No implementation for {url}")
+
+    @classmethod
+    @abstractmethod
+    def _write_to_url(
+        cls,
+        readable: Union[IO[bytes], IO[str]],
+        url: ParseResult,
+        executable: bool = False,
+    ) -> None:
+        """
+        Reads the contents of the given readable stream and writes it to the object at the
+        specified location. Raises FileNotFoundError if the URL doesn't exist.
+
+        :param Union[IO[bytes], IO[str]] readable: a readable stream
+
+        :param ParseResult url: URL that points to a file or object in the storage
+               mechanism of a supported URL scheme e.g. a blob in an AWS s3 bucket.
+
+        :param bool executable: determines if the file has executable permissions
+        """
+        raise NotImplementedError(f"No implementation for {url}")
+
+    @classmethod
+    @abstractmethod
+    def _supports_url(cls, url: ParseResult, export: bool = False) -> bool:
+        """
+        Returns True if the url access implementation supports the URL's scheme.
+
+        :param ParseResult url: a parsed URL that may be supported
+
+        :param bool export: Determines if the url is supported for exported
+
+        :return bool: returns true if the cls supports the URL
+        """
+        raise NotImplementedError(f"No implementation for {url}")
+
+    @classmethod
+    def _find_url_implementation(
+        cls, url: ParseResult, export: bool = False
+    ) -> type["URLAccess"]:
+        """
+        Returns the URLAccess subclass that supports the given URL.
+
+        :param ParseResult url: The given URL
+
+        :param bool export: Determines if the url is supported for exporting
+
+        """
+        try:
+            implementation_factory = get_plugin("url_access", url.scheme.lower())
+        except KeyError:
+            raise UnimplementedURLException(url, "export" if export else "import")
+        
+        try:
+            implementation = cast(Type[URLAccess], implementation_factory())
+        except (ImportError, ProxyConnectionError):
+            logger.debug(
+                "Unable to import implementation for scheme '%s', as is expected if the corresponding extra was "
+                "omitted at installation time.",
+                url.scheme.lower(),
+            )
+            raise UnimplementedURLException(url, "export" if export else "import")
+
+        if implementation._supports_url(url, export):
+            return implementation
+        raise UnimplementedURLException(url, "export" if export else "import")
+
+#####
+# Built-in url access
+#####
+
+def file_job_store_factory() -> type[URLAccess]:
+    from toil.jobStores.fileJobStore import FileJobStore
+
+    return FileJobStore
+
+
+def google_job_store_factory() -> type[URLAccess]:
+    from toil.jobStores.googleJobStore import GoogleJobStore
+
+    return GoogleJobStore
+
+
+def aws_job_store_factory() -> type[URLAccess]:
+    from toil.jobStores.aws.jobStore import AWSJobStore
+
+    return AWSJobStore
+
+
+def job_store_support_factory() -> type[URLAccess]:
+    from toil.jobStores.abstractJobStore import JobStoreSupport
+
+    return JobStoreSupport
+
+#make sure my py still works and the tests work
+# can then get rid of _url_access_classes method
+
+#####
+# Registers all built-in urls
+#####
+register_plugin("url_access", "file", file_job_store_factory)
+register_plugin("url_access", "gs", google_job_store_factory)
+register_plugin("url_access", "s3", aws_job_store_factory)
+register_plugin("url_access", "http", job_store_support_factory)
+register_plugin("url_access", "https", job_store_support_factory)
+register_plugin("url_access", "ftp", job_store_support_factory)

toil/options/common.py

@@ -860,6 +860,14 @@ def add_base_toil_options(
         help=f"Number of times to retry a failing job before giving up and "
         f"labeling job failed. default={1}",
     )
+    job_options.add_argument(
+        "--stopOnFirstFailure",
+        dest="stop_on_first_failure",
+        type=strtobool,
+        default=False,
+        metavar="BOOL",
+        help="Stop the workflow at the first complete job failure.",
+    )
     job_options.add_argument(
         "--enableUnlimitedPreemptibleRetries",
         "--enableUnlimitedPreemptableRetries",

toil/options/cwl.py

@@ -3,6 +3,8 @@ from argparse import ArgumentParser
 
 from configargparse import SUPPRESS
 
+from toil.lib.conversions import human2bytes
+from toil.options.common import make_open_interval_action
 from toil.version import baseVersion
 
 
@@ -411,9 +413,19 @@ def add_cwl_options(parser: ArgumentParser, suppress: bool = True) -> None:
         "--no-cwl-default-ram",
         action="store_false",
         help=suppress_help
-        or "Do not apply CWL specification default ramMin, so that Toil --defaultMemory applies.",
+        or "Do not apply CWL specification default ramMin, so that Toil --defaultMemory applies. This can help jobs get to Slurm with no memory limit assigned.",
         dest="cwl_default_ram",
     )
+    parser.add_argument(
+        "--cwl-min-ram",
+        type=human2bytes,
+        action=make_open_interval_action(1),
+        help=suppress_help
+        or "Specify a minimum memory allocation for all tasks ."
+        "If --no-cwl-default-ram is passed, this does not apply to tools that do not "
+        "specify a memory requirement; --defaultMemory is used for those tools"
+        "in that case."
+    )
     parser.add_argument(
         "--destBucket",
         type=str,

toil/options/runner.py

@@ -1,4 +1,4 @@
-from argparse import ArgumentParser
+from argparse import ArgumentParser, SUPPRESS
 
 from toil.lib.conversions import human2bytes
 
@@ -25,17 +25,23 @@ def add_runner_options(
         help="Run the file imports on a worker instead of the leader. This is useful if the leader is not optimized for high network performance. "
         "If set to true, the argument --importWorkersDisk must also be set."
     )
-    import_workers_threshold_argument = ["--importWorkersThreshold"]
+    import_workers_batchsize_argument = ["--importWorkersBatchSize"]
     if cwl:
-        import_workers_threshold_argument.append("--import-workers-threshold")
+        import_workers_batchsize_argument.append("--import-workers-batch-size")
     parser.add_argument(
-        *import_workers_threshold_argument,
-        dest="import_workers_threshold",
+        *import_workers_batchsize_argument,
+        dest="import_workers_batchsize",
         type=lambda x: human2bytes(str(x)),
         default="1 GiB",
-        help="Specify the file size threshold that determines how many files go into a batched import. As many files will go into a batch import job until this threshold "
-             "is reached. This should be set in conjunction with the argument --runImportsOnWorkers."
+        help="Specify the target total file size for file import batches. "
+        "As many files as can fit will go into each batch import job. This should be set in conjunction with the argument --runImportsOnWorkers."
     )
+
+    # Deprecated
+    parser.add_argument(
+        "--importWorkersThreshold", "--import-workers-threshold", dest="import_workers_batchsize",type=lambda x: human2bytes(str(x)), help=SUPPRESS
+    )
+
     import_workers_disk_argument = ["--importWorkersDisk"]
     if cwl:
         import_workers_disk_argument.append("--import-workers-disk")
@@ -44,7 +50,8 @@ def add_runner_options(
         dest="import_workers_disk",
         type=lambda x: human2bytes(str(x)),
         default="1 MiB",
-        help="Specify the disk size each import worker will get. This may be necessary when file streaming is not possible. For example, downloading from AWS to a GCE "
-             "job store. If specified, this should be set to the largest file size of all files to import. This should be set in conjunction with the arguments "
-             "--runImportsOnWorkers and --importWorkersThreshold."
+        help="Specify the disk size each import worker will get. This usually will not need to be set as Toil will attempt to use file streaming when downloading files. "
+             "If not possible, for example, when downloading from AWS to a GCE job store, "
+             "this should be set to the largest file size of all files to import. This should be set in conjunction with the arguments "
+             "--runImportsOnWorkers and --importWorkersBatchSize."
     )

toil/options/wdl.py

@@ -86,3 +86,25 @@ def add_wdl_options(parser: ArgumentParser, suppress: bool = True) -> None:
         default=None,
         help=suppress_help or "Keep and return all call outputs as workflow outputs"
     )
+
+    strict_arguments = ["--wdlStrict"] + (
+        ["--strict"] if not suppress else []
+    )
+    parser.add_argument(
+        *strict_arguments,
+        dest="strict",
+        type=strtobool,
+        default=False,
+        help=suppress_help or "Exit runner if workflow has any lint warnings"
+    )
+
+    quant_check_arguments = ["--wdlQuantCheck"] + (
+        ["--quantCheck"] if not suppress else []
+    )
+    parser.add_argument(
+        *quant_check_arguments,
+        dest="quant_check",
+        type=strtobool,
+        default=True,
+        help=suppress_help or "Whether to relax quantifier validation rules"
+    )

toil/provisioners/aws/awsProvisioner.py

@@ -109,6 +109,11 @@ _INSTANCE_PROFILE_ROLE_NAME = "toil"
 _TAG_KEY_TOIL_NODE_TYPE = "ToilNodeType"
 # The tag that specifies the cluster name on all nodes
 _TAG_KEY_TOIL_CLUSTER_NAME = "clusterName"
+# The tag we use to store the SSH key name.
+# TODO: Get rid of this once
+# <https://github.com/adamchainz/ec2-metadata/pull/562> is merged and we can
+# get the SSH key name from the instance metadata.
+_TAG_KEY_TOIL_SSH_KEY = "sshKeyName"
 # How much storage on the root volume is expected to go to overhead and be
 # unavailable to jobs when the node comes up?
 # TODO: measure
@@ -309,11 +314,28 @@ class AWSProvisioner(AbstractProvisioner):
         for tag in instance["Tags"]:
             if tag.get("Key") == "Name":
                 self.clusterName = tag["Value"]
+            elif tag.get("Key") == _TAG_KEY_TOIL_SSH_KEY:
+                # If we can't get an SSH key from the instance metadata, we
+                # might be able to use this one from the tags.
+                self._keyName = tag["Value"]
         # Determine what subnet we, the leader, are in
         self._leader_subnet = instance["SubnetId"]
         # Determine where to deploy workers.
         self._worker_subnets_by_zone = self._get_good_subnets_like(self._leader_subnet)
 
+        # Find the SSH key name to use to start instances
+        if hasattr(ec2_metadata, 'public_keys') and isinstance(ec2_metadata.public_keys, dict):
+            key_names = list(ec2_metadata.public_keys.keys())
+            if len(key_names) > 0 and isinstance(key_names[0], str):
+                # We have a key name from the EC2 metadata. This should always
+                # be the case once
+                # <https://github.com/adamchainz/ec2-metadata/pull/562> is
+                # merged. Override anything from the tags.
+                self._keyName = key_names[0]
+
+        if not hasattr(self, '_keyName'):
+            raise RuntimeError("Unable to determine the SSH key name the cluster is using")
+
         self._leaderPrivateIP = ec2_metadata.private_ipv4  # this is PRIVATE IP
         self._tags = {
             k: v
@@ -495,6 +517,7 @@ class AWSProvisioner(AbstractProvisioner):
         # Make tags for the leader specifically
         leader_tags = dict(self._tags)
         leader_tags[_TAG_KEY_TOIL_NODE_TYPE] = "leader"
+        leader_tags[_TAG_KEY_TOIL_SSH_KEY] = self._keyName
         logger.debug("Launching leader with tags: %s", leader_tags)
 
         instances: list[Instance] = create_instances(
@@ -1144,7 +1167,7 @@ class AWSProvisioner(AbstractProvisioner):
             workerInstances = [
                 i
                 for i in workerInstances
-                if preemptible == (i["SpotInstanceRequestId"] is not None)
+                if preemptible == (i.get("SpotInstanceRequestId") is not None)
             ]
             logger.debug(
                 "%spreemptible workers found in cluster: %s",
@@ -1161,7 +1184,7 @@ class AWSProvisioner(AbstractProvisioner):
                 name=i["InstanceId"],
                 launchTime=i["LaunchTime"],
                 nodeType=i["InstanceType"],
-                preemptible=i["SpotInstanceRequestId"] is not None,
+                preemptible=i.get("SpotInstanceRequestId") is not None,
                 tags=collapse_tags(i["Tags"]),
             )
             for i in workerInstances