toil 9.0.0__py3-none-any.whl → 9.1.1__py3-none-any.whl

toil/provisioners/aws/__init__.py

@@ -16,7 +16,7 @@ import logging
 from collections import namedtuple
 from operator import attrgetter
 from statistics import mean, stdev
-from typing import Optional
+from typing import Optional, TYPE_CHECKING
 
 from botocore.client import BaseClient
 
@@ -26,11 +26,15 @@ from toil.lib.aws import (
     get_aws_zone_from_environment_region,
     get_aws_zone_from_metadata,
 )
+from toil.lib.aws.utils import boto3_pager
 
 logger = logging.getLogger(__name__)
 
 ZoneTuple = namedtuple("ZoneTuple", ["name", "price_deviation"])
 
+if TYPE_CHECKING:
+    from mypy_boto3_ec2.type_defs import SpotPriceTypeDef
+
 
 def get_aws_zone_from_spot_market(
     spotBid: Optional[float],
@@ -109,7 +113,7 @@ def get_best_aws_zone(
 def choose_spot_zone(
     zones: list[str],
     bid: float,
-    spot_history: list["boto.ec2.spotpricehistory.SpotPriceHistory"],
+    spot_history: list["SpotPriceTypeDef"],
 ) -> str:
     """
     Returns the zone to put the spot request based on, in order of priority:
@@ -120,27 +124,26 @@ def choose_spot_zone(
 
     :return: the name of the selected zone
 
-    >>> from collections import namedtuple
-    >>> FauxHistory = namedtuple('FauxHistory', ['price', 'availability_zone'])
     >>> zones = ['us-west-2a', 'us-west-2b']
-    >>> spot_history = [FauxHistory(0.1, 'us-west-2a'), \
-                        FauxHistory(0.2, 'us-west-2a'), \
-                        FauxHistory(0.3, 'us-west-2b'), \
-                        FauxHistory(0.6, 'us-west-2b')]
+    >>> FauxHistory = lambda p, z: {"SpotPrice": p, "AvailabilityZone": z}
+    >>> spot_history = [FauxHistory("0.1", 'us-west-2a'), \
+                        FauxHistory("0.2", 'us-west-2a'), \
+                        FauxHistory("0.3", 'us-west-2b'), \
+                        FauxHistory("0.6", 'us-west-2b')]
     >>> choose_spot_zone(zones, 0.15, spot_history)
     'us-west-2a'
 
-    >>> spot_history=[FauxHistory(0.3, 'us-west-2a'), \
-                      FauxHistory(0.2, 'us-west-2a'), \
-                      FauxHistory(0.1, 'us-west-2b'), \
-                      FauxHistory(0.6, 'us-west-2b')]
+    >>> spot_history=[FauxHistory("0.3", 'us-west-2a'), \
+                      FauxHistory("0.2", 'us-west-2a'), \
+                      FauxHistory("0.1", 'us-west-2b'), \
+                      FauxHistory("0.6", 'us-west-2b')]
     >>> choose_spot_zone(zones, 0.15, spot_history)
     'us-west-2b'
 
-    >>> spot_history=[FauxHistory(0.1, 'us-west-2a'), \
-                      FauxHistory(0.7, 'us-west-2a'), \
-                      FauxHistory(0.1, 'us-west-2b'), \
-                      FauxHistory(0.6, 'us-west-2b')]
+    >>> spot_history=[FauxHistory("0.1", 'us-west-2a'), \
+                      FauxHistory("0.7", 'us-west-2a'), \
+                      FauxHistory("0.1", 'us-west-2b'), \
+                      FauxHistory("0.6", 'us-west-2b')]
     >>> choose_spot_zone(zones, 0.15, spot_history)
     'us-west-2b'
     """
@@ -152,11 +155,11 @@ def choose_spot_zone(
         zone_histories = [
             zone_history
             for zone_history in spot_history
-            if zone_history.availability_zone == zone
+            if zone_history["AvailabilityZone"] == zone
         ]
         if zone_histories:
-            price_deviation = stdev([history.price for history in zone_histories])
-            recent_price = zone_histories[0].price
+            price_deviation = stdev([float(history["SpotPrice"]) for history in zone_histories])
+            recent_price = float(zone_histories[0]["SpotPrice"])
         else:
             price_deviation, recent_price = 0.0, bid
         zone_tuple = ZoneTuple(name=zone, price_deviation=price_deviation)
@@ -169,7 +172,7 @@ def choose_spot_zone(
 
 def optimize_spot_bid(
     boto3_ec2: BaseClient, instance_type: str, spot_bid: float, zone_options: list[str]
-):
+) -> str:
     """
     Check whether the bid is in line with history and makes an effort to place
     the instance in a sensible zone.
@@ -179,30 +182,29 @@ def optimize_spot_bid(
     """
     spot_history = _get_spot_history(boto3_ec2, instance_type)
     if spot_history:
-        _check_spot_bid(spot_bid, spot_history)
+        _check_spot_bid(spot_bid, spot_history, name=instance_type)
     most_stable_zone = choose_spot_zone(zone_options, spot_bid, spot_history)
     logger.debug("Placing spot instances in zone %s.", most_stable_zone)
     return most_stable_zone
 
 
-def _check_spot_bid(spot_bid, spot_history):
+def _check_spot_bid(spot_bid: float, spot_history: list["SpotPriceTypeDef"], name: Optional[str] = None) -> None:
     """
     Prevents users from potentially over-paying for instances
 
     Note: this checks over the whole region, not a particular zone
 
-    :param spot_bid: float
+    :param spot_bid: The proposed bid in dollars per hour.
 
-    :type spot_history: list[SpotPriceHistory]
+    :type spot_history: The recent history of the spot price
 
     :raises UserError: if bid is > 2X the spot price's average
 
-    >>> from collections import namedtuple
-    >>> FauxHistory = namedtuple( "FauxHistory", [ "price", "availability_zone" ] )
-    >>> spot_data = [ FauxHistory( 0.1, "us-west-2a" ), \
-                      FauxHistory( 0.2, "us-west-2a" ), \
-                      FauxHistory( 0.3, "us-west-2b" ), \
-                      FauxHistory( 0.6, "us-west-2b" ) ]
+    >>> FauxHistory = lambda p, z: {"SpotPrice": p, "AvailabilityZone": z}
+    >>> spot_data = [ FauxHistory( "0.1", "us-west-2a" ), \
+                      FauxHistory( "0.2", "us-west-2a" ), \
+                      FauxHistory( "0.3", "us-west-2b" ), \
+                      FauxHistory( "0.6", "us-west-2b" ) ]
     >>> # noinspection PyProtectedMember
     >>> _check_spot_bid( 0.1, spot_data )
     >>> # noinspection PyProtectedMember
@@ -212,17 +214,21 @@ def _check_spot_bid(spot_bid, spot_history):
     ...
     UserError: Your bid $ 2.000000 is more than double this instance type's average spot price ($ 0.300000) over the last week
     """
-    average = mean([datum.price for datum in spot_history])
+    if name is None:
+        # Describe the instance as something
+        name = "this instance type"
+    average = mean([float(datum["SpotPrice"]) for datum in spot_history])
     if spot_bid > average * 2:
         logger.warning(
-            "Your bid $ %f is more than double this instance type's average "
+            "Your bid $ %f is more than double %s's average "
             "spot price ($ %f) over the last week",
             spot_bid,
+            name,
             average,
         )
 
 
-def _get_spot_history(boto3_ec2: BaseClient, instance_type: str):
+def _get_spot_history(boto3_ec2: BaseClient, instance_type: str) -> list["SpotPriceTypeDef"]:
     """
     Returns list of 1,000 most recent spot market data points represented as SpotPriceHistory
     objects. Note: The most recent object/data point will be first in the list.
@@ -230,10 +236,11 @@ def _get_spot_history(boto3_ec2: BaseClient, instance_type: str):
     :rtype: list[SpotPriceHistory]
     """
     one_week_ago = datetime.datetime.now() - datetime.timedelta(days=7)
-    spot_data = boto3_ec2.describe_spot_price_history(
+    spot_data = boto3_pager(
+        boto3_ec2.describe_spot_price_history,
+        "SpotPriceHistory",
         StartTime=one_week_ago.isoformat(),
         InstanceTypes=[instance_type],
         ProductDescriptions=["Linux/UNIX"],
     )
-    spot_data.sort(key=attrgetter("timestamp"), reverse=True)
-    return spot_data
+    return sorted(spot_data, key=lambda d: d["Timestamp"], reverse=True)

toil/provisioners/aws/awsProvisioner.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 from __future__ import annotations
 
+import base64
 import json
 import logging
 import os
@@ -25,7 +26,7 @@ import uuid
 from collections.abc import Collection, Iterable
 from functools import wraps
 from shlex import quote
-from typing import TYPE_CHECKING, Any, Callable, TypeVar, cast
+from typing import TYPE_CHECKING, Any, Callable, Literal, TypeVar, cast
 
 # We need these to exist as attributes we can get off of the boto object
 from botocore.exceptions import ClientError
@@ -449,7 +450,7 @@ class AWSProvisioner(AbstractProvisioner):
         :return: None
         """
 
-        if "network" in kwargs:
+        if kwargs.get("network") is not None:
             logger.warning(
                 "AWS provisioner does not support a network parameter. Ignoring %s!",
                 kwargs["network"],
@@ -1018,28 +1019,29 @@ class AWSProvisioner(AbstractProvisioner):
         userData: str = self._getIgnitionUserData(
             "worker", keyPath, preemptible, self._architecture
         )
-        userDataBytes: bytes = b""
-        if isinstance(userData, str):
-            # Spot-market provisioning requires bytes for user data.
-            userDataBytes = userData.encode("utf-8")
-
-        spot_kwargs = {
-            "KeyName": self._keyName,
+        # Boto 3 demands we base64 the user data ourselves *only* for spot
+        # instances, and still wants a str.
+        spot_user_data = base64.b64encode(
+            userData.encode("utf-8")
+        ).decode("utf-8")
+        spot_kwargs: dict[Literal["LaunchSpecification"], dict[str, Any]] = {
             "LaunchSpecification": {
+                "KeyName": self._keyName,
                 "SecurityGroupIds": self._getSecurityGroupIDs(),
                 "InstanceType": type_info.name,
-                "UserData": userDataBytes,
+                "UserData": spot_user_data,
                 "BlockDeviceMappings": bdm,
                 "IamInstanceProfile": {"Arn": self._leaderProfileArn},
                 "Placement": {"AvailabilityZone": zone},
                 "SubnetId": subnet_id,
             },
         }
+
         on_demand_kwargs = {
             "KeyName": self._keyName,
             "SecurityGroupIds": self._getSecurityGroupIDs(),
             "InstanceType": type_info.name,
-            "UserData": userDataBytes,
+            "UserData": userData,
             "BlockDeviceMappings": bdm,
             "IamInstanceProfile": {"Arn": self._leaderProfileArn},
             "Placement": {"AvailabilityZone": zone},
@@ -1055,6 +1057,10 @@ class AWSProvisioner(AbstractProvisioner):
                 # every request in this method
                 if not preemptible:
                     logger.debug("Launching %s non-preemptible nodes", numNodes)
+                    # TODO: Use create_instances() instead, which requires
+                    # refactoring both ondemand and spot sides here to use
+                    # mypy_boto3_ec2.service_resource.Instance objects instead
+                    # of mypy_boto3_ec2.type_defs.InstanceTypeDef
                     instancesLaunched = create_ondemand_instances(
                         boto3_ec2=boto3_ec2,
                         image_id=self._discoverAMI(),
@@ -1082,7 +1088,6 @@ class AWSProvisioner(AbstractProvisioner):
                         reservation
                         for subdict in generatedInstancesLaunched
                         for reservation in subdict["Reservations"]
-                        for key, value in subdict.items()
                     ]
                     # get a flattened list of all requested instances, as before instancesLaunched is a dict of reservations which is a dict of instance requests
                     instancesLaunched = [
@@ -1532,11 +1537,15 @@ class AWSProvisioner(AbstractProvisioner):
         tags: list[TagDescriptionTypeDef] = ec2.describe_tags(Filters=[tag_filter])[
             "Tags"
         ]
+        # TODO: Does this reference instance or spot request? Or can it be either?
         idsToCancel = [tag["ResourceId"] for tag in tags]
         return [
             request["SpotInstanceRequestId"]
             for request in requests
-            if request["InstanceId"] in idsToCancel
+            if (
+                request.get("InstanceId") in idsToCancel
+                or request["SpotInstanceRequestId"] in idsToCancel
+            )
         ]
 
     def _createSecurityGroups(self) -> list[str]:

toil/provisioners/node.py

@@ -41,6 +41,21 @@ class Node:
         tags: Optional[dict[str, str]] = None,
         use_private_ip: Optional[bool] = None,
     ) -> None:
+        """
+        Create a new node.
+
+        :param launchTime: Time when the node was launched. If a naive
+            datetime, or a string without timezone information, is assumed to
+            be in UTC.
+
+        :raises ValueError: If launchTime is an improperly formatted string.
+
+        >>> node = Node("127.0.0.1", "127.0.0.1", "localhost",
+        ...             "Decembruary Eleventeenth", None, False)
+        Traceback (most recent call last):
+        ...
+        ValueError: Invalid isoformat string: 'Decembruary Eleventeenth'
+        """
         self.publicIP = publicIP
         self.privateIP = privateIP
         if use_private_ip:
@@ -48,13 +63,25 @@ class Node:
         else:
             self.effectiveIP = self.publicIP or self.privateIP
         self.name = name
+        # Typing should prevent an empty launch time, but just to make sure,
+        # check it at runtime.
+        assert launchTime is not None, (
+            f"Attempted to create a Node {name} without a launch time"
+        )
         if isinstance(launchTime, datetime.datetime):
             self.launchTime = launchTime
         else:
             try:
+                # Parse an RFC 3339 ISO 8601 UTC datetime
                 self.launchTime = parse_iso_utc(launchTime)
             except ValueError:
+                # Parse (almost) any ISO 8601 datetime
                 self.launchTime = datetime.datetime.fromisoformat(launchTime)
+        if self.launchTime.tzinfo is None:
+            # Read naive datatimes as in UTC
+            self.launchTime = self.launchTime.replace(
+                tzinfo=datetime.timezone.utc
+            )
         self.nodeType = nodeType
         self.preemptible = preemptible
         self.tags = tags
@@ -70,22 +97,43 @@ class Node:
 
     def remainingBillingInterval(self) -> float:
         """
-        If the node has a launch time, this function returns a floating point value
-        between 0 and 1.0 representing how far we are into the
-        current billing cycle for the given instance. If the return value is .25, we are one
-        quarter into the billing cycle, with three quarters remaining before we will be charged
-        again for that instance.
+        Returns a floating point value between 0 and 1.0 representing how much
+        time is left in the current billing cycle for the given instance. If
+        the return value is .25, we are three quarters into the billing cycle,
+        with one quarters remaining before we will be charged again for that
+        instance.
 
         Assumes a billing cycle of one hour.
 
-        :return: Float from 0 -> 1.0 representing percentage of pre-paid time left in cycle.
+        :return: Float from 1.0 -> 0.0 representing fraction of pre-paid time
+            remaining in cycle.
+
+        >>> node = Node("127.0.0.1", "127.0.0.1", "localhost",
+        ...             datetime.datetime.utcnow(), None, False)
+        >>> node.remainingBillingInterval() >= 0
+        True
+        >>> node.remainingBillingInterval() <= 1.0
+        True
+        >>> node.remainingBillingInterval() > 0.5
+        True
+        >>> interval1 = node.remainingBillingInterval()
+        >>> time.sleep(1)
+        >>> interval2 = node.remainingBillingInterval()
+        >>> interval2 < interval1
+        True
+
+        >>> node = Node("127.0.0.1", "127.0.0.1", "localhost",
+        ...             datetime.datetime.now(datetime.timezone.utc) - 
+        ...             datetime.timedelta(minutes=5), None, False)
+        >>> node.remainingBillingInterval() < 0.99
+        True
+        >>> node.remainingBillingInterval() > 0.9
+        True
+        
         """
-        if self.launchTime:
-            now = datetime.datetime.utcnow()
-            delta = now - self.launchTime
-            return 1 - delta.total_seconds() / 3600.0 % 1.0
-        else:
-            return 1
+        now = datetime.datetime.now(datetime.timezone.utc)
+        delta = now - self.launchTime
+        return 1 - delta.total_seconds() / 3600.0 % 1.0
 
     def waitForNode(self, role: str, keyName: str = "core") -> None:
         self._waitForSSHPort()

toil/resource.py

@@ -32,7 +32,6 @@ from zipfile import ZipFile
 
 from toil import inVirtualEnv
 from toil.lib.io import mkdtemp
-from toil.lib.iterables import concat
 from toil.lib.memoize import strict_bool
 from toil.lib.retry import ErrorCondition, retry
 from toil.version import exactPython
@@ -619,18 +618,9 @@ class ModuleDescriptor(
             initName = self._initModuleName(self.dirPath)
             if initName:
                 raise ResourceException(
-                    "Toil does not support loading a user script from a package directory. You "
-                    "may want to remove %s from %s or invoke the user script as a module via "
-                    "'PYTHONPATH=\"%s\" %s -m %s.%s'."
-                    % tuple(
-                        concat(
-                            initName,
-                            self.dirPath,
-                            exactPython,
-                            os.path.split(self.dirPath),
-                            self.name,
-                        )
-                    )
+                    f"Toil does not support loading a user script from a package directory. You "
+                    f"may want to remove {initName} from {self.dirPath} or invoke the user script as a module via: "
+                    f"PYTHONPATH='{self.dirPath}' {exactPython} -m {self.dirPath}.{self.name}"
                 )
             return self.dirPath
 

toil/test/__init__.py

@@ -49,7 +49,6 @@ from toil.lib.accelerators import (
     have_working_nvidia_smi,
 )
 from toil.lib.io import mkdtemp
-from toil.lib.iterables import concat
 from toil.lib.memoize import memoize
 from toil.lib.threading import ExceptionalThread, cpu_count
 from toil.version import distVersion
@@ -224,7 +223,7 @@ class ToilTest(unittest.TestCase):
 
         :return: The output of the process' stdout if capture=True was passed, None otherwise.
         """
-        argl = list(concat(command, args))
+        argl = [command] + list(args)
         logger.info("Running %r", argl)
         capture = kwargs.pop("capture", False)
         _input = kwargs.pop("input", None)
@@ -395,6 +394,8 @@ def _aws_s3_avail() -> bool:
         boto3_credentials = session.get_credentials()
     except ImportError:
         return False
+    except ProxyConnectionError:
+        return False
     from toil.lib.aws import running_on_ec2
 
     if not (
@@ -440,7 +441,7 @@ def needs_aws_batch(test_item: MT) -> MT:
         test_item
     )
     test_item = needs_env_var(
-        "TOIL_AWS_BATCH_JOB_ROLE_ARN", "an IAM role ARN that grants S3 and SDB access"
+        "TOIL_AWS_BATCH_JOB_ROLE_ARN", "an IAM role ARN that grants S3 access"
     )(test_item)
     try:
         from toil.lib.aws import get_current_aws_region
@@ -1249,19 +1250,16 @@ class ApplianceTestSupport(ToilTest):
             with self.lock:
                 image = applianceSelf()
                 # Omitting --rm, it's unreliable, see https://github.com/docker/docker/issues/16575
-                args = list(
-                    concat(
-                        "docker",
-                        "run",
-                        "--entrypoint=" + self._entryPoint(),
-                        "--net=host",
-                        "-i",
-                        "--name=" + self.containerName,
-                        ["--volume=%s:%s" % mount for mount in self.mounts.items()],
-                        image,
-                        self._containerCommand(),
-                    )
-                )
+                args = [
+                    "docker",
+                    "run",
+                    f"--entrypoint={self._entryPoint()}",
+                    "--net=host",
+                    "-i",
+                    f"--name={self.containerName}"] + \
+                    ["--volume=%s:%s" % mount for mount in self.mounts.items()] + \
+                    [image] + \
+                    self._containerCommand()
                 logger.info("Running %r", args)
                 self.popen = subprocess.Popen(args)
             self.start()