toil 8.2.0__py3-none-any.whl → 9.1.0__py3-none-any.whl

toil/lib/url.py

@@ -0,0 +1,320 @@
+# Copyright (C) 2015-2025 Regents of the University of California
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import logging
+import os
+from abc import ABC, ABCMeta, abstractmethod
+from typing import (
+    IO,
+    TYPE_CHECKING,
+    Any,
+    Callable,
+    ContextManager,
+    Literal,
+    Optional,
+    Union,
+    cast,
+    overload,
+    Type,
+)
+from urllib.parse import ParseResult, urlparse
+
+from toil.lib.exceptions import UnimplementedURLException
+from toil.lib.memoize import memoize
+from toil.lib.plugins import register_plugin, get_plugin
+
+try:
+    from botocore.exceptions import ProxyConnectionError
+except ImportError:
+
+    class ProxyConnectionError(BaseException):  # type: ignore
+        """Dummy class."""
+
+logger = logging.getLogger(__name__)
+
+class URLAccess:
+    """
+    Widget for accessing external storage (URLs).
+    """
+
+    @classmethod
+    def url_exists(cls, src_uri: str) -> bool:
+        """
+        Return True if the file at the given URI exists, and False otherwise.
+
+        May raise an error if file existence cannot be determined.
+
+        :param src_uri: URL that points to a file or object in the storage
+               mechanism of a supported URL scheme e.g. a blob in an AWS s3 bucket.
+        """
+        parseResult = urlparse(src_uri)
+        otherCls = cls._find_url_implementation(parseResult)
+        return otherCls._url_exists(parseResult)
+
+    @classmethod
+    def get_size(cls, src_uri: str) -> Optional[int]:
+        """
+        Get the size in bytes of the file at the given URL, or None if it cannot be obtained.
+
+        :param src_uri: URL that points to a file or object in the storage
+               mechanism of a supported URL scheme e.g. a blob in an AWS s3 bucket.
+        """
+        parseResult = urlparse(src_uri)
+        otherCls = cls._find_url_implementation(parseResult)
+        return otherCls._get_size(parseResult)
+
+    @classmethod
+    def get_is_directory(cls, src_uri: str) -> bool:
+        """
+        Return True if the thing at the given URL is a directory, and False if
+        it is a file. The URL may or may not end in '/'.
+        """
+        parseResult = urlparse(src_uri)
+        otherCls = cls._find_url_implementation(parseResult)
+        return otherCls._get_is_directory(parseResult)
+
+    @classmethod
+    def list_url(cls, src_uri: str) -> list[str]:
+        """
+        List the directory at the given URL. Returned path components can be
+        joined with '/' onto the passed URL to form new URLs. Those that end in
+        '/' correspond to directories. The provided URL may or may not end with
+        '/'.
+
+        Currently supported schemes are:
+
+            - 's3' for objects in Amazon S3
+                e.g. s3://bucket/prefix/
+
+            - 'file' for local files
+                e.g. file:///local/dir/path/
+
+        :param str src_uri: URL that points to a directory or prefix in the storage mechanism of a
+                supported URL scheme e.g. a prefix in an AWS s3 bucket.
+
+        :return: A list of URL components in the given directory, already URL-encoded.
+        """
+        parseResult = urlparse(src_uri)
+        otherCls = cls._find_url_implementation(parseResult)
+        return otherCls._list_url(parseResult)
+
+    @classmethod
+    def read_from_url(cls, src_uri: str, writable: IO[bytes]) -> tuple[int, bool]:
+        """
+        Read the given URL and write its content into the given writable stream.
+
+        Raises FileNotFoundError if the URL doesn't exist.
+
+        :return: The size of the file in bytes and whether the executable permission bit is set
+        """
+        parseResult = urlparse(src_uri)
+        otherCls = cls._find_url_implementation(parseResult)
+        return otherCls._read_from_url(parseResult, writable)
+
+    @classmethod
+    def open_url(cls, src_uri: str) -> IO[bytes]:
+        """
+        Read from the given URI.
+
+        Raises FileNotFoundError if the URL doesn't exist.
+
+        Has a readable stream interface, unlike :meth:`read_from_url` which
+        takes a writable stream.
+        """
+        parseResult = urlparse(src_uri)
+        otherCls = cls._find_url_implementation(parseResult)
+        return otherCls._open_url(parseResult)
+
+    @classmethod
+    @abstractmethod
+    def _url_exists(cls, url: ParseResult) -> bool:
+        """
+        Return True if the item at the given URL exists, and Flase otherwise.
+
+        May raise an error if file existence cannot be determined.
+        """
+        raise NotImplementedError(f"No implementation for {url}")
+
+    @classmethod
+    @abstractmethod
+    def _get_size(cls, url: ParseResult) -> Optional[int]:
+        """
+        Get the size of the object at the given URL, or None if it cannot be obtained.
+        """
+        raise NotImplementedError(f"No implementation for {url}")
+
+    @classmethod
+    @abstractmethod
+    def _get_is_directory(cls, url: ParseResult) -> bool:
+        """
+        Return True if the thing at the given URL is a directory, and False if
+        it is a file or it is known not to exist. The URL may or may not end in
+        '/'.
+
+        :param url: URL that points to a file or object, or directory or prefix,
+               in the storage mechanism of a supported URL scheme e.g. a blob
+               in an AWS s3 bucket.
+        """
+        raise NotImplementedError(f"No implementation for {url}")
+
+    @classmethod
+    @abstractmethod
+    def _read_from_url(cls, url: ParseResult, writable: IO[bytes]) -> tuple[int, bool]:
+        """
+        Reads the contents of the object at the specified location and writes it to the given
+        writable stream.
+
+        Raises FileNotFoundError if the thing at the URL is not found.
+
+        :param ParseResult url: URL that points to a file or object in the storage
+               mechanism of a supported URL scheme e.g. a blob in an AWS s3 bucket.
+
+        :param IO[bytes] writable: a writable stream
+
+        :return: The size of the file in bytes and whether the executable permission bit is set
+        """
+        raise NotImplementedError(f"No implementation for {url}")
+
+    @classmethod
+    @abstractmethod
+    def _list_url(cls, url: ParseResult) -> list[str]:
+        """
+        List the contents of the given URL, which may or may not end in '/'
+
+        Returns a list of URL components. Those that end in '/' are meant to be
+        directories, while those that do not are meant to be files.
+
+        :param ParseResult url: URL that points to a directory or prefix in the
+        storage mechanism of a supported URL scheme e.g. a prefix in an AWS s3
+        bucket.
+
+        :return: The children of the given URL, already URL-encoded if
+        appropriate. (If the URL is a bare path, no encoding is done.)
+        """
+        raise NotImplementedError(f"No implementation for {url}")
+
+    @classmethod
+    @abstractmethod
+    def _open_url(cls, url: ParseResult) -> IO[bytes]:
+        """
+        Get a stream of the object at the specified location.
+
+        Raises FileNotFoundError if the thing at the URL is not found.
+        """
+        raise NotImplementedError(f"No implementation for {url}")
+
+    @classmethod
+    @abstractmethod
+    def _write_to_url(
+        cls,
+        readable: Union[IO[bytes], IO[str]],
+        url: ParseResult,
+        executable: bool = False,
+    ) -> None:
+        """
+        Reads the contents of the given readable stream and writes it to the object at the
+        specified location. Raises FileNotFoundError if the URL doesn't exist.
+
+        :param Union[IO[bytes], IO[str]] readable: a readable stream
+
+        :param ParseResult url: URL that points to a file or object in the storage
+               mechanism of a supported URL scheme e.g. a blob in an AWS s3 bucket.
+
+        :param bool executable: determines if the file has executable permissions
+        """
+        raise NotImplementedError(f"No implementation for {url}")
+
+    @classmethod
+    @abstractmethod
+    def _supports_url(cls, url: ParseResult, export: bool = False) -> bool:
+        """
+        Returns True if the url access implementation supports the URL's scheme.
+
+        :param ParseResult url: a parsed URL that may be supported
+
+        :param bool export: Determines if the url is supported for exported
+
+        :return bool: returns true if the cls supports the URL
+        """
+        raise NotImplementedError(f"No implementation for {url}")
+
+    @classmethod
+    def _find_url_implementation(
+        cls, url: ParseResult, export: bool = False
+    ) -> type["URLAccess"]:
+        """
+        Returns the URLAccess subclass that supports the given URL.
+
+        :param ParseResult url: The given URL
+
+        :param bool export: Determines if the url is supported for exporting
+
+        """
+        try:
+            implementation_factory = get_plugin("url_access", url.scheme.lower())
+        except KeyError:
+            raise UnimplementedURLException(url, "export" if export else "import")
+        
+        try:
+            implementation = cast(Type[URLAccess], implementation_factory())
+        except (ImportError, ProxyConnectionError):
+            logger.debug(
+                "Unable to import implementation for scheme '%s', as is expected if the corresponding extra was "
+                "omitted at installation time.",
+                url.scheme.lower(),
+            )
+            raise UnimplementedURLException(url, "export" if export else "import")
+
+        if implementation._supports_url(url, export):
+            return implementation
+        raise UnimplementedURLException(url, "export" if export else "import")
+
+#####
+# Built-in url access
+#####
+
+def file_job_store_factory() -> type[URLAccess]:
+    from toil.jobStores.fileJobStore import FileJobStore
+
+    return FileJobStore
+
+
+def google_job_store_factory() -> type[URLAccess]:
+    from toil.jobStores.googleJobStore import GoogleJobStore
+
+    return GoogleJobStore
+
+
+def aws_job_store_factory() -> type[URLAccess]:
+    from toil.jobStores.aws.jobStore import AWSJobStore
+
+    return AWSJobStore
+
+
+def job_store_support_factory() -> type[URLAccess]:
+    from toil.jobStores.abstractJobStore import JobStoreSupport
+
+    return JobStoreSupport
+
+#make sure my py still works and the tests work
+# can then get rid of _url_access_classes method
+
+#####
+# Registers all built-in urls
+#####
+register_plugin("url_access", "file", file_job_store_factory)
+register_plugin("url_access", "gs", google_job_store_factory)
+register_plugin("url_access", "s3", aws_job_store_factory)
+register_plugin("url_access", "http", job_store_support_factory)
+register_plugin("url_access", "https", job_store_support_factory)
+register_plugin("url_access", "ftp", job_store_support_factory)

toil/lib/web.py

@@ -17,13 +17,12 @@ Contains functions for making web requests with Toil.
 
 All web requests should go through this module, to make sure they use the right
 user agent.
-
+>>> httpserver = getfixture("httpserver")
+>>> handler = httpserver.expect_request("/path").respond_with_json({})
 >>> from toil.lib.web import web_session
->>> web_session.get("https://example.com")
-
+>>> web_session.get(httpserver.url_for("/path"))
+<Response [200]>
 """
-
-import logging
 import requests
 import sys
 

toil/options/cwl.py

@@ -3,6 +3,8 @@ from argparse import ArgumentParser
 
 from configargparse import SUPPRESS
 
+from toil.lib.conversions import human2bytes
+from toil.options.common import make_open_interval_action
 from toil.version import baseVersion
 
 
@@ -411,9 +413,19 @@ def add_cwl_options(parser: ArgumentParser, suppress: bool = True) -> None:
         "--no-cwl-default-ram",
         action="store_false",
         help=suppress_help
-        or "Do not apply CWL specification default ramMin, so that Toil --defaultMemory applies.",
+        or "Do not apply CWL specification default ramMin, so that Toil --defaultMemory applies. This can help jobs get to Slurm with no memory limit assigned.",
         dest="cwl_default_ram",
     )
+    parser.add_argument(
+        "--cwl-min-ram",
+        type=human2bytes,
+        action=make_open_interval_action(1),
+        help=suppress_help
+        or "Specify a minimum memory allocation for all tasks ."
+        "If --no-cwl-default-ram is passed, this does not apply to tools that do not "
+        "specify a memory requirement; --defaultMemory is used for those tools"
+        "in that case."
+    )
     parser.add_argument(
         "--destBucket",
         type=str,

toil/options/runner.py

@@ -1,4 +1,4 @@
-from argparse import ArgumentParser
+from argparse import ArgumentParser, SUPPRESS
 
 from toil.lib.conversions import human2bytes
 
@@ -25,17 +25,23 @@ def add_runner_options(
         help="Run the file imports on a worker instead of the leader. This is useful if the leader is not optimized for high network performance. "
         "If set to true, the argument --importWorkersDisk must also be set."
     )
-    import_workers_threshold_argument = ["--importWorkersThreshold"]
+    import_workers_batchsize_argument = ["--importWorkersBatchSize"]
     if cwl:
-        import_workers_threshold_argument.append("--import-workers-threshold")
+        import_workers_batchsize_argument.append("--import-workers-batch-size")
     parser.add_argument(
-        *import_workers_threshold_argument,
-        dest="import_workers_threshold",
+        *import_workers_batchsize_argument,
+        dest="import_workers_batchsize",
         type=lambda x: human2bytes(str(x)),
         default="1 GiB",
-        help="Specify the file size threshold that determines how many files go into a batched import. As many files will go into a batch import job until this threshold "
-             "is reached. This should be set in conjunction with the argument --runImportsOnWorkers."
+        help="Specify the target total file size for file import batches. "
+        "As many files as can fit will go into each batch import job. This should be set in conjunction with the argument --runImportsOnWorkers."
     )
+
+    # Deprecated
+    parser.add_argument(
+        "--importWorkersThreshold", "--import-workers-threshold", dest="import_workers_batchsize",type=lambda x: human2bytes(str(x)), help=SUPPRESS
+    )
+
     import_workers_disk_argument = ["--importWorkersDisk"]
     if cwl:
         import_workers_disk_argument.append("--import-workers-disk")
@@ -44,7 +50,8 @@ def add_runner_options(
         dest="import_workers_disk",
         type=lambda x: human2bytes(str(x)),
         default="1 MiB",
-        help="Specify the disk size each import worker will get. This may be necessary when file streaming is not possible. For example, downloading from AWS to a GCE "
-             "job store. If specified, this should be set to the largest file size of all files to import. This should be set in conjunction with the arguments "
-             "--runImportsOnWorkers and --importWorkersThreshold."
+        help="Specify the disk size each import worker will get. This usually will not need to be set as Toil will attempt to use file streaming when downloading files. "
+             "If not possible, for example, when downloading from AWS to a GCE job store, "
+             "this should be set to the largest file size of all files to import. This should be set in conjunction with the arguments "
+             "--runImportsOnWorkers and --importWorkersBatchSize."
     )

toil/options/wdl.py

@@ -96,4 +96,15 @@ def add_wdl_options(parser: ArgumentParser, suppress: bool = True) -> None:
         type=strtobool,
         default=False,
         help=suppress_help or "Exit runner if workflow has any lint warnings"
-    )
+    )
+
+    quant_check_arguments = ["--wdlQuantCheck"] + (
+        ["--quantCheck"] if not suppress else []
+    )
+    parser.add_argument(
+        *quant_check_arguments,
+        dest="quant_check",
+        type=strtobool,
+        default=True,
+        help=suppress_help or "Whether to relax quantifier validation rules"
+    )

toil/provisioners/__init__.py

@@ -127,8 +127,11 @@ def parse_node_types(
 
     Inputs should look something like this:
 
-    >>> parse_node_types('c5.4xlarge/c5a.4xlarge:0.42,t2.large')
-    [({'c5.4xlarge', 'c5a.4xlarge'}, 0.42), ({'t2.large'}, None)]
+    >>> types = parse_node_types('c5.4xlarge/c5a.4xlarge:0.42,t2.large')
+    >>> sorted(types[0][0]), types[0][1]
+    (['c5.4xlarge', 'c5a.4xlarge'], 0.42)
+    >>> sorted(types[1][0]), types[1][1]
+    (['t2.large'], None)
 
     :param node_type_specs: A string defining node types
 

toil/provisioners/aws/__init__.py

@@ -16,7 +16,7 @@ import logging
 from collections import namedtuple
 from operator import attrgetter
 from statistics import mean, stdev
-from typing import Optional
+from typing import Optional, TYPE_CHECKING
 
 from botocore.client import BaseClient
 
@@ -26,11 +26,15 @@ from toil.lib.aws import (
     get_aws_zone_from_environment_region,
     get_aws_zone_from_metadata,
 )
+from toil.lib.aws.utils import boto3_pager
 
 logger = logging.getLogger(__name__)
 
 ZoneTuple = namedtuple("ZoneTuple", ["name", "price_deviation"])
 
+if TYPE_CHECKING:
+    from mypy_boto3_ec2.type_defs import SpotPriceTypeDef
+
 
 def get_aws_zone_from_spot_market(
     spotBid: Optional[float],
@@ -109,7 +113,7 @@ def get_best_aws_zone(
 def choose_spot_zone(
     zones: list[str],
     bid: float,
-    spot_history: list["boto.ec2.spotpricehistory.SpotPriceHistory"],
+    spot_history: list["SpotPriceTypeDef"],
 ) -> str:
     """
     Returns the zone to put the spot request based on, in order of priority:
@@ -120,27 +124,26 @@ def choose_spot_zone(
 
     :return: the name of the selected zone
 
-    >>> from collections import namedtuple
-    >>> FauxHistory = namedtuple('FauxHistory', ['price', 'availability_zone'])
     >>> zones = ['us-west-2a', 'us-west-2b']
-    >>> spot_history = [FauxHistory(0.1, 'us-west-2a'), \
-                        FauxHistory(0.2, 'us-west-2a'), \
-                        FauxHistory(0.3, 'us-west-2b'), \
-                        FauxHistory(0.6, 'us-west-2b')]
+    >>> FauxHistory = lambda p, z: {"SpotPrice": p, "AvailabilityZone": z}
+    >>> spot_history = [FauxHistory("0.1", 'us-west-2a'), \
+                        FauxHistory("0.2", 'us-west-2a'), \
+                        FauxHistory("0.3", 'us-west-2b'), \
+                        FauxHistory("0.6", 'us-west-2b')]
     >>> choose_spot_zone(zones, 0.15, spot_history)
     'us-west-2a'
 
-    >>> spot_history=[FauxHistory(0.3, 'us-west-2a'), \
-                      FauxHistory(0.2, 'us-west-2a'), \
-                      FauxHistory(0.1, 'us-west-2b'), \
-                      FauxHistory(0.6, 'us-west-2b')]
+    >>> spot_history=[FauxHistory("0.3", 'us-west-2a'), \
+                      FauxHistory("0.2", 'us-west-2a'), \
+                      FauxHistory("0.1", 'us-west-2b'), \
+                      FauxHistory("0.6", 'us-west-2b')]
     >>> choose_spot_zone(zones, 0.15, spot_history)
     'us-west-2b'
 
-    >>> spot_history=[FauxHistory(0.1, 'us-west-2a'), \
-                      FauxHistory(0.7, 'us-west-2a'), \
-                      FauxHistory(0.1, 'us-west-2b'), \
-                      FauxHistory(0.6, 'us-west-2b')]
+    >>> spot_history=[FauxHistory("0.1", 'us-west-2a'), \
+                      FauxHistory("0.7", 'us-west-2a'), \
+                      FauxHistory("0.1", 'us-west-2b'), \
+                      FauxHistory("0.6", 'us-west-2b')]
     >>> choose_spot_zone(zones, 0.15, spot_history)
     'us-west-2b'
     """
@@ -152,11 +155,11 @@ def choose_spot_zone(
         zone_histories = [
             zone_history
             for zone_history in spot_history
-            if zone_history.availability_zone == zone
+            if zone_history["AvailabilityZone"] == zone
         ]
         if zone_histories:
-            price_deviation = stdev([history.price for history in zone_histories])
-            recent_price = zone_histories[0].price
+            price_deviation = stdev([float(history["SpotPrice"]) for history in zone_histories])
+            recent_price = float(zone_histories[0]["SpotPrice"])
         else:
             price_deviation, recent_price = 0.0, bid
         zone_tuple = ZoneTuple(name=zone, price_deviation=price_deviation)
@@ -169,7 +172,7 @@ def choose_spot_zone(
 
 def optimize_spot_bid(
     boto3_ec2: BaseClient, instance_type: str, spot_bid: float, zone_options: list[str]
-):
+) -> str:
     """
     Check whether the bid is in line with history and makes an effort to place
     the instance in a sensible zone.
@@ -179,30 +182,29 @@ def optimize_spot_bid(
     """
     spot_history = _get_spot_history(boto3_ec2, instance_type)
     if spot_history:
-        _check_spot_bid(spot_bid, spot_history)
+        _check_spot_bid(spot_bid, spot_history, name=instance_type)
     most_stable_zone = choose_spot_zone(zone_options, spot_bid, spot_history)
     logger.debug("Placing spot instances in zone %s.", most_stable_zone)
     return most_stable_zone
 
 
-def _check_spot_bid(spot_bid, spot_history):
+def _check_spot_bid(spot_bid: float, spot_history: list["SpotPriceTypeDef"], name: Optional[str] = None) -> None:
     """
     Prevents users from potentially over-paying for instances
 
     Note: this checks over the whole region, not a particular zone
 
-    :param spot_bid: float
+    :param spot_bid: The proposed bid in dollars per hour.
 
-    :type spot_history: list[SpotPriceHistory]
+    :type spot_history: The recent history of the spot price
 
     :raises UserError: if bid is > 2X the spot price's average
 
-    >>> from collections import namedtuple
-    >>> FauxHistory = namedtuple( "FauxHistory", [ "price", "availability_zone" ] )
-    >>> spot_data = [ FauxHistory( 0.1, "us-west-2a" ), \
-                      FauxHistory( 0.2, "us-west-2a" ), \
-                      FauxHistory( 0.3, "us-west-2b" ), \
-                      FauxHistory( 0.6, "us-west-2b" ) ]
+    >>> FauxHistory = lambda p, z: {"SpotPrice": p, "AvailabilityZone": z}
+    >>> spot_data = [ FauxHistory( "0.1", "us-west-2a" ), \
+                      FauxHistory( "0.2", "us-west-2a" ), \
+                      FauxHistory( "0.3", "us-west-2b" ), \
+                      FauxHistory( "0.6", "us-west-2b" ) ]
     >>> # noinspection PyProtectedMember
     >>> _check_spot_bid( 0.1, spot_data )
     >>> # noinspection PyProtectedMember
@@ -212,17 +214,21 @@ def _check_spot_bid(spot_bid, spot_history):
     ...
     UserError: Your bid $ 2.000000 is more than double this instance type's average spot price ($ 0.300000) over the last week
     """
-    average = mean([datum.price for datum in spot_history])
+    if name is None:
+        # Describe the instance as something
+        name = "this instance type"
+    average = mean([float(datum["SpotPrice"]) for datum in spot_history])
     if spot_bid > average * 2:
         logger.warning(
-            "Your bid $ %f is more than double this instance type's average "
+            "Your bid $ %f is more than double %s's average "
             "spot price ($ %f) over the last week",
             spot_bid,
+            name,
             average,
         )
 
 
-def _get_spot_history(boto3_ec2: BaseClient, instance_type: str):
+def _get_spot_history(boto3_ec2: BaseClient, instance_type: str) -> list["SpotPriceTypeDef"]:
     """
     Returns list of 1,000 most recent spot market data points represented as SpotPriceHistory
     objects. Note: The most recent object/data point will be first in the list.
@@ -230,10 +236,11 @@ def _get_spot_history(boto3_ec2: BaseClient, instance_type: str):
     :rtype: list[SpotPriceHistory]
     """
     one_week_ago = datetime.datetime.now() - datetime.timedelta(days=7)
-    spot_data = boto3_ec2.describe_spot_price_history(
+    spot_data = boto3_pager(
+        boto3_ec2.describe_spot_price_history,
+        "SpotPriceHistory",
         StartTime=one_week_ago.isoformat(),
         InstanceTypes=[instance_type],
         ProductDescriptions=["Linux/UNIX"],
     )
-    spot_data.sort(key=attrgetter("timestamp"), reverse=True)
-    return spot_data
+    return sorted(spot_data, key=lambda d: d["Timestamp"], reverse=True)