toil 6.1.0a1__py3-none-any.whl → 7.0.0__py3-none-any.whl

toil/provisioners/aws/awsProvisioner.py

@@ -11,6 +11,8 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+from __future__ import annotations
+
 import json
 import logging
 import os
@@ -29,34 +31,34 @@ from typing import (Any,
                     Iterable,
                     List,
                     Optional,
-                    Set)
+                    Set,
+                    Union,
+                    Literal,
+                    cast,
+                    TypeVar)
 from urllib.parse import unquote
 
 # We need these to exist as attributes we can get off of the boto object
-import boto.ec2
-import boto.iam
-import boto.vpc
-from boto.ec2.blockdevicemapping import \
-    BlockDeviceMapping as Boto2BlockDeviceMapping
-from boto.ec2.blockdevicemapping import BlockDeviceType as Boto2BlockDeviceType
-from boto.ec2.instance import Instance as Boto2Instance
-from boto.exception import BotoServerError, EC2ResponseError
-from boto.utils import get_instance_metadata
 from botocore.exceptions import ClientError
+from mypy_boto3_autoscaling.client import AutoScalingClient
+from mypy_boto3_ec2.service_resource import Instance
+from mypy_boto3_iam.type_defs import InstanceProfileTypeDef, RoleTypeDef, ListRolePoliciesResponseTypeDef
+from mypy_extensions import VarArg, KwArg
 
-from toil.lib.aws import zone_to_region
+from toil.lib.aws import zone_to_region, AWSRegionName, AWSServerErrors
 from toil.lib.aws.ami import get_flatcar_ami
 from toil.lib.aws.iam import (CLUSTER_LAUNCHING_PERMISSIONS,
                               get_policy_permissions,
                               policy_permissions_allow)
 from toil.lib.aws.session import AWSConnectionManager
-from toil.lib.aws.utils import create_s3_bucket
+from toil.lib.aws.utils import create_s3_bucket, flatten_tags, boto3_pager
 from toil.lib.conversions import human2bytes
 from toil.lib.ec2 import (a_short_time,
                           create_auto_scaling_group,
                           create_instances,
                           create_launch_template,
                           create_ondemand_instances,
+                          increase_instance_hop_limit,
                           create_spot_instances,
                           wait_instances_running,
                           wait_transition,
@@ -73,12 +75,20 @@ from toil.lib.retry import (ErrorCondition,
                             old_retry,
                             retry)
 from toil.provisioners import (ClusterCombinationNotSupportedException,
-                               NoSuchClusterException)
+                               NoSuchClusterException,
+                               NoSuchZoneException)
 from toil.provisioners.abstractProvisioner import (AbstractProvisioner,
                                                    ManagedNodesNotSupportedException,
                                                    Shape)
 from toil.provisioners.aws import get_best_aws_zone
 from toil.provisioners.node import Node
+from toil.lib.aws.session import client as get_client
+
+from mypy_boto3_ec2.client import EC2Client
+from mypy_boto3_iam.client import IAMClient
+from mypy_boto3_ec2.type_defs import DescribeInstancesResultTypeDef, InstanceTypeDef, TagTypeDef, BlockDeviceMappingTypeDef, EbsBlockDeviceTypeDef, FilterTypeDef, SpotInstanceRequestTypeDef, TagDescriptionTypeDef, SecurityGroupTypeDef, \
+    CreateSecurityGroupResultTypeDef, IpPermissionTypeDef, ReservationTypeDef
+from mypy_boto3_s3.literals import BucketLocationConstraintType
 
 logger = logging.getLogger(__name__)
 logging.getLogger("boto").setLevel(logging.CRITICAL)
@@ -98,14 +108,8 @@ _S3_BUCKET_MAX_NAME_LEN = 63
 # The suffix of the S3 bucket associated with the cluster
 _S3_BUCKET_INTERNAL_SUFFIX = '--internal'
 
-# prevent removal of these imports
-str(boto.ec2)
-str(boto.iam)
-str(boto.vpc)
-
 
-
-def awsRetryPredicate(e):
+def awsRetryPredicate(e: Exception) -> bool:
     if isinstance(e, socket.gaierror):
         # Could be a DNS outage:
         # socket.gaierror: [Errno -2] Name or service not known
@@ -135,33 +139,38 @@ def expectedShutdownErrors(e: Exception) -> bool:
     return get_error_status(e) == 400 and 'dependent object' in get_error_body(e)
 
 
-def awsRetry(f):
+F = TypeVar("F")  # so mypy understands passed through types
+
+
+def awsRetry(f: Callable[..., F]) -> Callable[..., F]:
     """
-    This decorator retries the wrapped function if aws throws unexpected errors
-    errors.
+    This decorator retries the wrapped function if aws throws unexpected errors.
+
     It should wrap any function that makes use of boto
     """
+
     @wraps(f)
-    def wrapper(*args, **kwargs):
+    def wrapper(*args: Any, **kwargs: Any) -> Any:
         for attempt in old_retry(delays=truncExpBackoff(),
                                  timeout=300,
                                  predicate=awsRetryPredicate):
             with attempt:
                 return f(*args, **kwargs)
+
     return wrapper
 
 
-def awsFilterImpairedNodes(nodes, ec2):
+def awsFilterImpairedNodes(nodes: List[InstanceTypeDef], boto3_ec2: EC2Client) -> List[InstanceTypeDef]:
     # if TOIL_AWS_NODE_DEBUG is set don't terminate nodes with
     # failing status checks so they can be debugged
     nodeDebug = os.environ.get('TOIL_AWS_NODE_DEBUG') in ('True', 'TRUE', 'true', True)
     if not nodeDebug:
         return nodes
-    nodeIDs = [node.id for node in nodes]
-    statuses = ec2.get_all_instance_status(instance_ids=nodeIDs)
-    statusMap = {status.id: status.instance_status for status in statuses}
-    healthyNodes = [node for node in nodes if statusMap.get(node.id, None) != 'impaired']
-    impairedNodes = [node.id for node in nodes if statusMap.get(node.id, None) == 'impaired']
+    nodeIDs = [node["InstanceId"] for node in nodes]
+    statuses = boto3_ec2.describe_instance_status(InstanceIds=nodeIDs)
+    statusMap = {status["InstanceId"]: status["InstanceStatus"]["Status"] for status in statuses["InstanceStatuses"]}
+    healthyNodes = [node for node in nodes if statusMap.get(node["InstanceId"], None) != 'impaired']
+    impairedNodes = [node["InstanceId"] for node in nodes if statusMap.get(node["InstanceId"], None) == 'impaired']
     logger.warning('TOIL_AWS_NODE_DEBUG is set and nodes %s have failed EC2 status checks so '
                    'will not be terminated.', ' '.join(impairedNodes))
     return healthyNodes
@@ -170,8 +179,24 @@ def awsFilterImpairedNodes(nodes, ec2):
 class InvalidClusterStateException(Exception):
     pass
 
+
+def collapse_tags(instance_tags: List[TagTypeDef]) -> Dict[str, str]:
+    """
+    Collapse tags from boto3 format to node format
+    :param instance_tags: tags as list of TagTypeDef
+    :return: Dict of tags
+    """
+    collapsed_tags: Dict[str, str] = dict()
+    for tag in instance_tags:
+        if tag.get("Key") is not None:
+            collapsed_tags[tag["Key"]] = tag["Value"]
+    return collapsed_tags
+
+
 class AWSProvisioner(AbstractProvisioner):
-    def __init__(self, clusterName, clusterType, zone, nodeStorage, nodeStorageOverrides, sseKey):
+    def __init__(self, clusterName: Optional[str], clusterType: Optional[str], zone: Optional[str],
+                 nodeStorage: int, nodeStorageOverrides: Optional[List[str]], sseKey: Optional[str],
+                 enable_fuse: bool):
         self.cloud = 'aws'
         self._sseKey = sseKey
         # self._zone will be filled in by base class constructor
@@ -186,7 +211,7 @@ class AWSProvisioner(AbstractProvisioner):
 
         # Determine our region to work in, before readClusterSettings() which
         # might need it. TODO: support multiple regions in one cluster
-        self._region = zone_to_region(zone)
+        self._region: AWSRegionName = zone_to_region(zone)
 
         # Set up our connections to AWS
         self.aws = AWSConnectionManager()
@@ -197,16 +222,22 @@ class AWSProvisioner(AbstractProvisioner):
 
         # Call base class constructor, which will call createClusterSettings()
         # or readClusterSettings()
-        super().__init__(clusterName, clusterType, zone, nodeStorage, nodeStorageOverrides)
+        super().__init__(clusterName, clusterType, zone, nodeStorage, nodeStorageOverrides, enable_fuse)
+
+        if self._zone is None:
+            logger.warning("Leader zone was never initialized before creating AWS provisioner. Defaulting to cluster zone.")
+
+        self._leader_subnet: str = self._get_default_subnet(self._zone or zone)
+        self._tags: Dict[str, Any] = {}
 
         # After self.clusterName is set, generate a valid name for the S3 bucket associated with this cluster
         suffix = _S3_BUCKET_INTERNAL_SUFFIX
         self.s3_bucket_name = self.clusterName[:_S3_BUCKET_MAX_NAME_LEN - len(suffix)] + suffix
 
-    def supportedClusterTypes(self):
+    def supportedClusterTypes(self) -> Set[str]:
         return {'mesos', 'kubernetes'}
 
-    def createClusterSettings(self):
+    def createClusterSettings(self) -> None:
         """
         Create a new set of cluster settings for a cluster to be deployed into
         AWS.
@@ -216,41 +247,51 @@ class AWSProvisioner(AbstractProvisioner):
         # constructor.
         assert self._zone is not None
 
-    def readClusterSettings(self):
+    def readClusterSettings(self) -> None:
         """
         Reads the cluster settings from the instance metadata, which assumes
         the instance is the leader.
         """
-        instanceMetaData = get_instance_metadata()
-        ec2 = self.aws.boto2(self._region, 'ec2')
-        instance = ec2.get_all_instances(instance_ids=[instanceMetaData["instance-id"]])[0].instances[0]
+        from ec2_metadata import ec2_metadata
+        boto3_ec2 = self.aws.client(self._region, 'ec2')
+        instance: InstanceTypeDef = boto3_ec2.describe_instances(InstanceIds=[ec2_metadata.instance_id])["Reservations"][0]["Instances"][0]
         # The cluster name is the same as the name of the leader.
-        self.clusterName = str(instance.tags["Name"])
+        self.clusterName: str = "default-toil-cluster-name"
+        for tag in instance["Tags"]:
+            if tag.get("Key") == "Name":
+                self.clusterName = tag["Value"]
         # Determine what subnet we, the leader, are in
-        self._leader_subnet = instance.subnet_id
+        self._leader_subnet = instance["SubnetId"]
         # Determine where to deploy workers.
         self._worker_subnets_by_zone = self._get_good_subnets_like(self._leader_subnet)
 
-        self._leaderPrivateIP = instanceMetaData['local-ipv4']  # this is PRIVATE IP
-        self._keyName = list(instanceMetaData['public-keys'].keys())[0]
-        self._tags = {k: v for k, v in self.getLeader().tags.items() if k != _TAG_KEY_TOIL_NODE_TYPE}
+        self._leaderPrivateIP = ec2_metadata.private_ipv4  # this is PRIVATE IP
+        self._tags = {k: v for k, v in (self.getLeader().tags or {}).items() if k != _TAG_KEY_TOIL_NODE_TYPE}
         # Grab the ARN name of the instance profile (a str) to apply to workers
-        self._leaderProfileArn = instanceMetaData['iam']['info']['InstanceProfileArn']
+        leader_info = None
+        for attempt in old_retry(timeout=300, predicate=lambda e: True):
+            with attempt:
+                leader_info = ec2_metadata.iam_info
+                if leader_info is None:
+                    raise RuntimeError("Could not get EC2 metadata IAM info")
+        if leader_info is None:
+            # This is more for mypy as it is unable to see that the retry will guarantee this is not None
+            # and that this is not reachable
+            raise RuntimeError(f"Leader IAM metadata is unreachable.")
+        self._leaderProfileArn = leader_info["InstanceProfileArn"]
+
         # The existing metadata API returns a single string if there is one security group, but
         # a list when there are multiple: change the format to always be a list.
-        rawSecurityGroups = instanceMetaData['security-groups']
-        self._leaderSecurityGroupNames = {rawSecurityGroups} if not isinstance(rawSecurityGroups, list) else set(rawSecurityGroups)
+        rawSecurityGroups = ec2_metadata.security_groups
+        self._leaderSecurityGroupNames: Set[str] = set(rawSecurityGroups)
         # Since we have access to the names, we don't also need to use any IDs
-        self._leaderSecurityGroupIDs = set()
+        self._leaderSecurityGroupIDs: Set[str] = set()
 
         # Let the base provisioner work out how to deploy duly authorized
         # workers for this leader.
         self._setLeaderWorkerAuthentication()
 
-    @retry(errors=[ErrorCondition(
-        error=ClientError,
-        error_codes=[404, 500, 502, 503, 504]
-    )])
+    @retry(errors=[AWSServerErrors])
     def _write_file_to_cloud(self, key: str, contents: bytes) -> str:
         bucket_name = self.s3_bucket_name
 
@@ -289,7 +330,7 @@ class AWSProvisioner(AbstractProvisioner):
         obj = self.aws.resource(self._region, 's3').Object(bucket_name, key)
 
         try:
-            return obj.get().get('Body').read()
+            return obj.get()['Body'].read()
         except ClientError as e:
             if get_error_status(e) == 404:
                 logger.warning(f'Trying to read non-existent file "{key}" from {bucket_name}.')
@@ -305,11 +346,11 @@ class AWSProvisioner(AbstractProvisioner):
                       owner: str,
                       keyName: str,
                       botoPath: str,
-                      userTags: Optional[dict],
+                      userTags: Optional[Dict[str, str]],
                       vpcSubnet: Optional[str],
                       awsEc2ProfileArn: Optional[str],
-                      awsEc2ExtraSecurityGroupIds: Optional[list],
-                      **kwargs):
+                      awsEc2ExtraSecurityGroupIds: Optional[List[str]],
+                      **kwargs: Dict[str, Any]) -> None:
         """
         Starts a single leader node and populates this class with the leader's metadata.
 
@@ -352,9 +393,6 @@ class AWSProvisioner(AbstractProvisioner):
         if vpcSubnet:
             # This is where we put the leader
             self._leader_subnet = vpcSubnet
-        else:
-            # Find the default subnet for the zone
-            self._leader_subnet = self._get_default_subnet(self._zone)
 
         profileArn = awsEc2ProfileArn or self._createProfileArn()
 
@@ -370,7 +408,7 @@ class AWSProvisioner(AbstractProvisioner):
         if userTags is not None:
             self._tags.update(userTags)
 
-        #All user specified tags have been set
+        # All user specified tags have been set
         userData = self._getIgnitionUserData('leader', architecture=self._architecture)
 
         if self.clusterType == 'kubernetes':
@@ -383,18 +421,18 @@ class AWSProvisioner(AbstractProvisioner):
         leader_tags[_TAG_KEY_TOIL_NODE_TYPE] = 'leader'
         logger.debug('Launching leader with tags: %s', leader_tags)
 
-        instances = create_instances(self.aws.resource(self._region, 'ec2'),
-                                     image_id=self._discoverAMI(),
-                                     num_instances=1,
-                                     key_name=self._keyName,
-                                     security_group_ids=createdSGs + (awsEc2ExtraSecurityGroupIds or []),
-                                     instance_type=leader_type.name,
-                                     user_data=userData,
-                                     block_device_map=bdms,
-                                     instance_profile_arn=profileArn,
-                                     placement_az=self._zone,
-                                     subnet_id=self._leader_subnet,
-                                     tags=leader_tags)
+        instances: List[Instance] = create_instances(self.aws.resource(self._region, 'ec2'),
+                                                     image_id=self._discoverAMI(),
+                                                     num_instances=1,
+                                                     key_name=self._keyName,
+                                                     security_group_ids=createdSGs + (awsEc2ExtraSecurityGroupIds or []),
+                                                     instance_type=leader_type.name,
+                                                     user_data=userData,
+                                                     block_device_map=bdms,
+                                                     instance_profile_arn=profileArn,
+                                                     placement_az=self._zone,
+                                                     subnet_id=self._leader_subnet,
+                                                     tags=leader_tags)
 
         # wait for the leader to exist at all
         leader = instances[0]
@@ -425,7 +463,7 @@ class AWSProvisioner(AbstractProvisioner):
         leaderNode = Node(publicIP=leader.public_ip_address, privateIP=leader.private_ip_address,
                           name=leader.id, launchTime=leader.launch_time,
                           nodeType=leader_type.name, preemptible=False,
-                          tags=leader.tags)
+                          tags=collapse_tags(leader.tags))
         leaderNode.waitForNode('toil_leader')
 
         # Download credentials
@@ -483,7 +521,7 @@ class AWSProvisioner(AbstractProvisioner):
         acls = set(self._get_subnet_acls(base_subnet_id))
 
         # Compose a filter that selects the subnets we might want
-        filters = [{
+        filters: List[FilterTypeDef] = [{
             'Name': 'vpc-id',
             'Values': [vpc_id]
         }, {
@@ -495,7 +533,7 @@ class AWSProvisioner(AbstractProvisioner):
         }]
 
         # Fill in this collection
-        by_az = {}
+        by_az: Dict[str, List[str]] = {}
 
         # Go get all the subnets. There's no way to page manually here so it
         # must page automatically.
@@ -534,7 +572,7 @@ class AWSProvisioner(AbstractProvisioner):
         }]
 
         # TODO: Can't we use the resource's network_acls.filter(Filters=)?
-        return [item['NetworkAclId'] for item in self._pager(ec2.describe_network_acls,
+        return [item['NetworkAclId'] for item in boto3_pager(ec2.describe_network_acls,
                                                              'NetworkAcls',
                                                              Filters=filters)]
 
@@ -546,7 +584,7 @@ class AWSProvisioner(AbstractProvisioner):
         """
 
         # Compose a filter that selects the default subnet in the AZ
-        filters = [{
+        filters: List[FilterTypeDef] = [{
             'Name': 'default-for-az',
             'Values': ['true']
         }, {
@@ -586,7 +624,7 @@ class AWSProvisioner(AbstractProvisioner):
 
         return 'aws'
 
-    def getNodeShape(self, instance_type: str, preemptible=False) -> Shape:
+    def getNodeShape(self, instance_type: str, preemptible: bool=False) -> Shape:
         """
         Get the Shape for the given instance type (e.g. 't2.medium').
         """
@@ -603,13 +641,13 @@ class AWSProvisioner(AbstractProvisioner):
         # mesos about whether a job can run on a particular node type
         memory = (type_info.memory - 0.1) * 2 ** 30
         return Shape(wallTime=60 * 60,
-                     memory=memory,
+                     memory=int(memory),
                      cores=type_info.cores,
-                     disk=disk,
+                     disk=int(disk),
                      preemptible=preemptible)
 
     @staticmethod
-    def retryPredicate(e):
+    def retryPredicate(e: Exception) -> bool:
         return awsRetryPredicate(e)
 
     def destroyCluster(self) -> None:
@@ -619,8 +657,8 @@ class AWSProvisioner(AbstractProvisioner):
         # The leader may create more instances while we're terminating the workers.
         vpcId = None
         try:
-            leader = self._getLeaderInstance()
-            vpcId = leader.vpc_id
+            leader = self._getLeaderInstanceBoto3()
+            vpcId = leader.get("VpcId")
             logger.info('Terminating the leader first ...')
             self._terminateInstances([leader])
         except (NoSuchClusterException, InvalidClusterStateException):
@@ -651,14 +689,16 @@ class AWSProvisioner(AbstractProvisioner):
         # Do the workers after the ASGs because some may belong to ASGs
         logger.info('Terminating any remaining workers ...')
         removed = False
-        instances = self._get_nodes_in_cluster(include_stopped_nodes=True)
+        instances = self._get_nodes_in_cluster_boto3(include_stopped_nodes=True)
         spotIDs = self._getSpotRequestIDs()
+        boto3_ec2: EC2Client = self.aws.client(region=self._region, service_name="ec2")
         if spotIDs:
-            self.aws.boto2(self._region, 'ec2').cancel_spot_instance_requests(request_ids=spotIDs)
+            boto3_ec2.cancel_spot_instance_requests(SpotInstanceRequestIds=spotIDs)
+            # self.aws.boto2(self._region, 'ec2').cancel_spot_instance_requests(request_ids=spotIDs)
             removed = True
-        instancesToTerminate = awsFilterImpairedNodes(instances, self.aws.boto2(self._region, 'ec2'))
+        instancesToTerminate = awsFilterImpairedNodes(instances, self.aws.client(self._region, 'ec2'))
         if instancesToTerminate:
-            vpcId = vpcId or instancesToTerminate[0].vpc_id
+            vpcId = vpcId or instancesToTerminate[0].get("VpcId")
             self._terminateInstances(instancesToTerminate)
             removed = True
         if removed:
@@ -672,7 +712,7 @@ class AWSProvisioner(AbstractProvisioner):
                 # for some LuanchTemplate.
                 mistake = False
                 for ltID in self._get_launch_template_ids():
-                    response = self.aws.client(self._region, 'ec2').delete_launch_template(LaunchTemplateId=ltID)
+                    response = boto3_ec2.delete_launch_template(LaunchTemplateId=ltID)
                     if 'LaunchTemplate' not in response:
                         mistake = True
                     else:
@@ -694,16 +734,17 @@ class AWSProvisioner(AbstractProvisioner):
             removed = False
             for attempt in old_retry(timeout=300, predicate=expectedShutdownErrors):
                 with attempt:
-                    for sg in self.aws.boto2(self._region, 'ec2').get_all_security_groups():
+                    security_groups: List[SecurityGroupTypeDef] = boto3_ec2.describe_security_groups()["SecurityGroups"]
+                    for security_group in security_groups:
                         # TODO: If we terminate the leader and the workers but
                         # miss the security group, we won't find it now because
                         # we won't have vpcId set.
-                        if sg.name == self.clusterName and vpcId and sg.vpc_id == vpcId:
+                        if security_group.get("GroupName") == self.clusterName and vpcId and security_group.get("VpcId") == vpcId:
                             try:
-                                self.aws.boto2(self._region, 'ec2').delete_security_group(group_id=sg.id)
+                                boto3_ec2.delete_security_group(GroupId=security_group["GroupId"])
                                 removed = True
-                            except BotoServerError as e:
-                                if e.error_code == 'InvalidGroup.NotFound':
+                            except ClientError as e:
+                                if get_error_code(e) == 'InvalidGroup.NotFound':
                                     pass
                                 else:
                                     raise
@@ -777,10 +818,9 @@ class AWSProvisioner(AbstractProvisioner):
 
         return spot_bid
 
-    def addNodes(self, nodeTypes: Set[str], numNodes, preemptible, spotBid=None) -> int:
+    def addNodes(self, nodeTypes: Set[str], numNodes: int, preemptible: bool, spotBid: Optional[float]=None) -> int:
         # Grab the AWS connection we need
-        ec2 = self.aws.boto2(self._region, 'ec2')
-
+        boto3_ec2 = get_client(service_name='ec2', region_name=self._region)
         assert self._leaderPrivateIP
 
         if preemptible:
@@ -792,7 +832,7 @@ class AWSProvisioner(AbstractProvisioner):
         node_type = next(iter(nodeTypes))
         type_info = E2Instances[node_type]
         root_vol_size = self._nodeStorageOverrides.get(node_type, self._nodeStorage)
-        bdm = self._getBoto2BlockDeviceMapping(type_info,
+        bdm = self._getBoto3BlockDeviceMapping(type_info,
                                                rootVolSize=root_vol_size)
 
         # Pick a zone and subnet_id to launch into
@@ -803,7 +843,7 @@ class AWSProvisioner(AbstractProvisioner):
             # We're allowed to pick from any of these zones.
             zone_options = list(self._worker_subnets_by_zone.keys())
 
-            zone = get_best_aws_zone(spotBid, type_info.name, ec2, zone_options)
+            zone = get_best_aws_zone(spotBid, type_info.name, boto3_ec2, zone_options)
         else:
             # We don't need to ever do any balancing across zones for on-demand
             # instances. Just pick a zone.
@@ -814,6 +854,9 @@ class AWSProvisioner(AbstractProvisioner):
                 # The workers aren't allowed in the leader's zone.
                 # Pick an arbitrary zone we can use.
                 zone = next(iter(self._worker_subnets_by_zone.keys()))
+        if zone is None:
+            logger.exception("Could not find a valid zone. Make sure TOIL_AWS_ZONE is set or spot bids are not too low.")
+            raise NoSuchZoneException()
         if self._leader_subnet in self._worker_subnets_by_zone.get(zone, []):
             # The leader's subnet is an option for this zone, so use it.
             subnet_id = self._leader_subnet
@@ -822,21 +865,40 @@ class AWSProvisioner(AbstractProvisioner):
             subnet_id = next(iter(self._worker_subnets_by_zone[zone]))
 
         keyPath = self._sseKey if self._sseKey else None
-        userData = self._getIgnitionUserData('worker', keyPath, preemptible, self._architecture)
+        userData: str = self._getIgnitionUserData('worker', keyPath, preemptible, self._architecture)
+        userDataBytes: bytes = b""
         if isinstance(userData, str):
             # Spot-market provisioning requires bytes for user data.
-            userData = userData.encode('utf-8')
-
-        kwargs = {'key_name': self._keyName,
-                  'security_group_ids': self._getSecurityGroupIDs(),
-                  'instance_type': type_info.name,
-                  'user_data': userData,
-                  'block_device_map': bdm,
-                  'instance_profile_arn': self._leaderProfileArn,
-                  'placement': zone,
-                  'subnet_id': subnet_id}
-
-        instancesLaunched = []
+            userDataBytes = userData.encode('utf-8')
+
+        spot_kwargs = {'KeyName': self._keyName,
+                       'LaunchSpecification': {
+                           'SecurityGroupIds': self._getSecurityGroupIDs(),
+                           'InstanceType': type_info.name,
+                           'UserData': userDataBytes,
+                           'BlockDeviceMappings': bdm,
+                           'IamInstanceProfile': {
+                               'Arn': self._leaderProfileArn
+                           },
+                           'Placement': {
+                               'AvailabilityZone': zone
+                           },
+                           'SubnetId': subnet_id}
+                       }
+        on_demand_kwargs = {'KeyName': self._keyName,
+                            'SecurityGroupIds': self._getSecurityGroupIDs(),
+                            'InstanceType': type_info.name,
+                            'UserData': userDataBytes,
+                            'BlockDeviceMappings': bdm,
+                            'IamInstanceProfile': {
+                                'Arn': self._leaderProfileArn
+                            },
+                            'Placement': {
+                                'AvailabilityZone': zone
+                            },
+                            'SubnetId': subnet_id}
+
+        instancesLaunched: List[InstanceTypeDef] = []
 
         for attempt in old_retry(predicate=awsRetryPredicate):
             with attempt:
@@ -845,41 +907,45 @@ class AWSProvisioner(AbstractProvisioner):
                 # every request in this method
                 if not preemptible:
                     logger.debug('Launching %s non-preemptible nodes', numNodes)
-                    instancesLaunched = create_ondemand_instances(ec2,
+                    instancesLaunched = create_ondemand_instances(boto3_ec2=boto3_ec2,
                                                                   image_id=self._discoverAMI(),
-                                                                  spec=kwargs, num_instances=numNodes)
+                                                                  spec=on_demand_kwargs, num_instances=numNodes)
                 else:
                     logger.debug('Launching %s preemptible nodes', numNodes)
                     # force generator to evaluate
-                    instancesLaunched = list(create_spot_instances(ec2=ec2,
-                                                                   price=spotBid,
-                                                                   image_id=self._discoverAMI(),
-                                                                   tags={_TAG_KEY_TOIL_CLUSTER_NAME: self.clusterName},
-                                                                   spec=kwargs,
-                                                                   num_instances=numNodes,
-                                                                   tentative=True)
-                                             )
+                    generatedInstancesLaunched: List[DescribeInstancesResultTypeDef] = list(create_spot_instances(boto3_ec2=boto3_ec2,
+                                                                                                                  price=spotBid,
+                                                                                                                  image_id=self._discoverAMI(),
+                                                                                                                  tags={_TAG_KEY_TOIL_CLUSTER_NAME: self.clusterName},
+                                                                                                                  spec=spot_kwargs,
+                                                                                                                  num_instances=numNodes,
+                                                                                                                  tentative=True)
+                                                                                            )
                     # flatten the list
-                    instancesLaunched = [item for sublist in instancesLaunched for item in sublist]
+                    flatten_reservations: List[ReservationTypeDef] = [reservation for subdict in generatedInstancesLaunched for reservation in subdict["Reservations"] for key, value in subdict.items()]
+                    # get a flattened list of all requested instances, as before instancesLaunched is a dict of reservations which is a dict of instance requests
+                    instancesLaunched = [instance for instances in flatten_reservations for instance in instances['Instances']]
 
         for attempt in old_retry(predicate=awsRetryPredicate):
             with attempt:
-                wait_instances_running(ec2, instancesLaunched)
+                list(wait_instances_running(boto3_ec2, instancesLaunched))  # ensure all instances are running
+
+        increase_instance_hop_limit(boto3_ec2, instancesLaunched)
 
         self._tags[_TAG_KEY_TOIL_NODE_TYPE] = 'worker'
-        AWSProvisioner._addTags(instancesLaunched, self._tags)
+        AWSProvisioner._addTags(boto3_ec2, instancesLaunched, self._tags)
         if self._sseKey:
             for i in instancesLaunched:
                 self._waitForIP(i)
-                node = Node(publicIP=i.ip_address, privateIP=i.private_ip_address, name=i.id,
-                            launchTime=i.launch_time, nodeType=i.instance_type, preemptible=preemptible,
-                            tags=i.tags)
+                node = Node(publicIP=i['PublicIpAddress'], privateIP=i['PrivateIpAddress'], name=i['InstanceId'],
+                            launchTime=i['LaunchTime'], nodeType=i['InstanceType'], preemptible=preemptible,
+                            tags=collapse_tags(i['Tags']))
                 node.waitForNode('toil_worker')
                 node.coreRsync([self._sseKey, ':' + self._sseKey], applianceName='toil_worker')
         logger.debug('Launched %s new instance(s)', numNodes)
         return len(instancesLaunched)
 
-    def addManagedNodes(self, nodeTypes: Set[str], minNodes, maxNodes, preemptible, spotBid=None) -> None:
+    def addManagedNodes(self, nodeTypes: Set[str], minNodes: int, maxNodes: int, preemptible: bool, spotBid: Optional[float] = None) -> None:
 
         if self.clusterType != 'kubernetes':
             raise ManagedNodesNotSupportedException("Managed nodes only supported for Kubernetes clusters")
@@ -901,17 +967,17 @@ class AWSProvisioner(AbstractProvisioner):
 
     def getProvisionedWorkers(self, instance_type: Optional[str] = None, preemptible: Optional[bool] = None) -> List[Node]:
         assert self._leaderPrivateIP
-        entireCluster = self._get_nodes_in_cluster(instance_type=instance_type)
+        entireCluster = self._get_nodes_in_cluster_boto3(instance_type=instance_type)
         logger.debug('All nodes in cluster: %s', entireCluster)
-        workerInstances = [i for i in entireCluster if i.private_ip_address != self._leaderPrivateIP]
+        workerInstances: List[InstanceTypeDef] = [i for i in entireCluster if i["PrivateIpAddress"] != self._leaderPrivateIP]
         logger.debug('All workers found in cluster: %s', workerInstances)
         if preemptible is not None:
-            workerInstances = [i for i in workerInstances if preemptible == (i.spot_instance_request_id is not None)]
+            workerInstances = [i for i in workerInstances if preemptible == (i["SpotInstanceRequestId"] is not None)]
             logger.debug('%spreemptible workers found in cluster: %s', 'non-' if not preemptible else '', workerInstances)
-        workerInstances = awsFilterImpairedNodes(workerInstances, self.aws.boto2(self._region, 'ec2'))
-        return [Node(publicIP=i.ip_address, privateIP=i.private_ip_address,
-                     name=i.id, launchTime=i.launch_time, nodeType=i.instance_type,
-                     preemptible=i.spot_instance_request_id is not None, tags=i.tags)
+        workerInstances = awsFilterImpairedNodes(workerInstances, self.aws.client(self._region, 'ec2'))
+        return [Node(publicIP=i["PublicIpAddress"], privateIP=i["PrivateIpAddress"],
+                     name=i["InstanceId"], launchTime=i["LaunchTime"], nodeType=i["InstanceType"],
+                     preemptible=i["SpotInstanceRequestId"] is not None, tags=collapse_tags(i["Tags"]))
                 for i in workerInstances]
 
     @memoize
@@ -952,37 +1018,65 @@ class AWSProvisioner(AbstractProvisioner):
         denamespaced = '/' + '_'.join(s.replace('_', '/') for s in namespaced_name.split('__'))
         return denamespaced.startswith(self._toNameSpace())
 
+    def _getLeaderInstanceBoto3(self) -> InstanceTypeDef:
+        """
+        Get the Boto 3 instance for the cluster's leader.
+        :return: InstanceTypeDef
+        """
+        # Tags are stored differently in Boto 3
+        instances: List[InstanceTypeDef] = self._get_nodes_in_cluster_boto3(include_stopped_nodes=True)
+        instances.sort(key=lambda x: x["LaunchTime"])
+        try:
+            leader = instances[0]  # assume leader was launched first
+        except IndexError:
+            raise NoSuchClusterException(self.clusterName)
+        if leader.get("Tags") is not None:
+            tag_value = next(item["Value"] for item in leader["Tags"] if item["Key"] == _TAG_KEY_TOIL_NODE_TYPE)
+        else:
+            tag_value = None
+        if (tag_value or 'leader') != 'leader':
+            raise InvalidClusterStateException(
+                'Invalid cluster state! The first launched instance appears not to be the leader '
+                'as it is missing the "leader" tag. The safest recovery is to destroy the cluster '
+                'and restart the job. Incorrect Leader ID: %s' % leader["InstanceId"]
+            )
+        return leader
 
-    def _getLeaderInstance(self) -> Boto2Instance:
+    def _getLeaderInstance(self) -> InstanceTypeDef:
         """
         Get the Boto 2 instance for the cluster's leader.
         """
-        instances = self._get_nodes_in_cluster(include_stopped_nodes=True)
-        instances.sort(key=lambda x: x.launch_time)
+        instances = self._get_nodes_in_cluster_boto3(include_stopped_nodes=True)
+        instances.sort(key=lambda x: x["LaunchTime"])
         try:
-            leader = instances[0]  # assume leader was launched first
+            leader: InstanceTypeDef = instances[0]  # assume leader was launched first
         except IndexError:
             raise NoSuchClusterException(self.clusterName)
-        if (leader.tags.get(_TAG_KEY_TOIL_NODE_TYPE) or 'leader') != 'leader':
+        tagged_node_type: str = 'leader'
+        for tag in leader["Tags"]:
+            # If a tag specifying node type exists,
+            if tag.get("Key") is not None and tag["Key"] == _TAG_KEY_TOIL_NODE_TYPE:
+                tagged_node_type = tag["Value"]
+        if tagged_node_type != 'leader':
             raise InvalidClusterStateException(
                 'Invalid cluster state! The first launched instance appears not to be the leader '
                 'as it is missing the "leader" tag. The safest recovery is to destroy the cluster '
-                'and restart the job. Incorrect Leader ID: %s' % leader.id
+                'and restart the job. Incorrect Leader ID: %s' % leader["InstanceId"]
             )
         return leader
 
-    def getLeader(self, wait=False) -> Node:
+    def getLeader(self, wait: bool = False) -> Node:
         """
         Get the leader for the cluster as a Toil Node object.
         """
-        leader = self._getLeaderInstance()
+        leader: InstanceTypeDef = self._getLeaderInstanceBoto3()
 
-        leaderNode = Node(publicIP=leader.ip_address, privateIP=leader.private_ip_address,
-                          name=leader.id, launchTime=leader.launch_time, nodeType=None,
-                          preemptible=False, tags=leader.tags)
+        leaderNode = Node(publicIP=leader["PublicIpAddress"], privateIP=leader["PrivateIpAddress"],
+                          name=leader["InstanceId"], launchTime=leader["LaunchTime"], nodeType=None,
+                          preemptible=False, tags=collapse_tags(leader["Tags"]))
         if wait:
             logger.debug("Waiting for toil_leader to enter 'running' state...")
-            wait_instances_running(self.aws.boto2(self._region, 'ec2'), [leader])
+            wait_instances_running(self.aws.client(self._region, 'ec2'), [leader])
             logger.debug('... toil_leader is running')
             self._waitForIP(leader)
             leaderNode.waitForNode('toil_leader')
@@ -991,17 +1085,20 @@ class AWSProvisioner(AbstractProvisioner):
 
     @classmethod
     @awsRetry
-    def _addTag(cls, instance: Boto2Instance, key: str, value: str):
-        instance.add_tag(key, value)
+    def _addTag(cls, boto3_ec2: EC2Client, instance: InstanceTypeDef, key: str, value: str) -> None:
+        if instance.get('Tags') is None:
+            instance['Tags'] = []
+        new_tag: TagTypeDef = {"Key": key, "Value": value}
+        boto3_ec2.create_tags(Resources=[instance["InstanceId"]], Tags=[new_tag])
 
     @classmethod
-    def _addTags(cls, instances: List[Boto2Instance], tags: Dict[str, str]):
+    def _addTags(cls, boto3_ec2: EC2Client, instances: List[InstanceTypeDef], tags: Dict[str, str]) -> None:
         for instance in instances:
             for key, value in tags.items():
-                cls._addTag(instance, key, value)
+                cls._addTag(boto3_ec2, instance, key, value)
 
     @classmethod
-    def _waitForIP(cls, instance: Boto2Instance):
+    def _waitForIP(cls, instance: InstanceTypeDef) -> None:
         """
         Wait until the instances has a public IP address assigned to it.
 
@@ -1010,32 +1107,32 @@ class AWSProvisioner(AbstractProvisioner):
         logger.debug('Waiting for ip...')
         while True:
             time.sleep(a_short_time)
-            instance.update()
-            if instance.ip_address or instance.public_dns_name or instance.private_ip_address:
+            if instance.get("PublicIpAddress") or instance.get("PublicDnsName") or instance.get("PrivateIpAddress"):
                 logger.debug('...got ip')
                 break
 
-    def _terminateInstances(self, instances: List[Boto2Instance]):
-        instanceIDs = [x.id for x in instances]
+    def _terminateInstances(self, instances: List[InstanceTypeDef]) -> None:
+        instanceIDs = [x["InstanceId"] for x in instances]
         self._terminateIDs(instanceIDs)
         logger.info('... Waiting for instance(s) to shut down...')
         for instance in instances:
-            wait_transition(instance, {'pending', 'running', 'shutting-down', 'stopping', 'stopped'}, 'terminated')
+            wait_transition(self.aws.client(region=self._region, service_name="ec2"), instance, {'pending', 'running', 'shutting-down', 'stopping', 'stopped'}, 'terminated')
         logger.info('Instance(s) terminated.')
 
     @awsRetry
-    def _terminateIDs(self, instanceIDs: List[str]):
+    def _terminateIDs(self, instanceIDs: List[str]) -> None:
         logger.info('Terminating instance(s): %s', instanceIDs)
-        self.aws.boto2(self._region, 'ec2').terminate_instances(instance_ids=instanceIDs)
+        boto3_ec2 = self.aws.client(region=self._region, service_name="ec2")
+        boto3_ec2.terminate_instances(InstanceIds=instanceIDs)
         logger.info('Instance(s) terminated.')
 
     @awsRetry
-    def _deleteRoles(self, names: List[str]):
+    def _deleteRoles(self, names: List[str]) -> None:
         """
         Delete all the given named IAM roles.
         Detatches but does not delete associated instance profiles.
         """
-
+        boto3_iam = self.aws.client(region=self._region, service_name="iam")
         for role_name in names:
             for profile_name in self._getRoleInstanceProfileNames(role_name):
                 # We can't delete either the role or the profile while they
@@ -1043,60 +1140,64 @@ class AWSProvisioner(AbstractProvisioner):
 
                 for attempt in old_retry(timeout=300, predicate=expectedShutdownErrors):
                     with attempt:
-                        self.aws.client(self._region, 'iam').remove_role_from_instance_profile(InstanceProfileName=profile_name,
-                                                                                               RoleName=role_name)
+                        boto3_iam.remove_role_from_instance_profile(InstanceProfileName=profile_name,
+                                                                    RoleName=role_name)
             # We also need to drop all inline policies
             for policy_name in self._getRoleInlinePolicyNames(role_name):
                 for attempt in old_retry(timeout=300, predicate=expectedShutdownErrors):
                     with attempt:
-                        self.aws.client(self._region, 'iam').delete_role_policy(PolicyName=policy_name,
-                                                                                RoleName=role_name)
+                        boto3_iam.delete_role_policy(PolicyName=policy_name,
+                                                     RoleName=role_name)
 
             for attempt in old_retry(timeout=300, predicate=expectedShutdownErrors):
                 with attempt:
-                    self.aws.client(self._region, 'iam').delete_role(RoleName=role_name)
+                    boto3_iam.delete_role(RoleName=role_name)
                     logger.debug('... Successfully deleted IAM role %s', role_name)
 
-
     @awsRetry
-    def _deleteInstanceProfiles(self, names: List[str]):
+    def _deleteInstanceProfiles(self, names: List[str]) -> None:
         """
         Delete all the given named IAM instance profiles.
         All roles must already be detached.
         """
-
+        boto3_iam = self.aws.client(region=self._region, service_name="iam")
         for profile_name in names:
             for attempt in old_retry(timeout=300, predicate=expectedShutdownErrors):
                 with attempt:
-                    self.aws.client(self._region, 'iam').delete_instance_profile(InstanceProfileName=profile_name)
+                    boto3_iam.delete_instance_profile(InstanceProfileName=profile_name)
                     logger.debug('... Succesfully deleted instance profile %s', profile_name)
 
     @classmethod
-    def _getBoto2BlockDeviceMapping(cls, type_info: InstanceType, rootVolSize: int = 50) -> Boto2BlockDeviceMapping:
+    def _getBoto3BlockDeviceMapping(cls, type_info: InstanceType, rootVolSize: int = 50) -> List[BlockDeviceMappingTypeDef]:
         # determine number of ephemeral drives via cgcloud-lib (actually this is moved into toil's lib
         bdtKeys = [''] + [f'/dev/xvd{c}' for c in string.ascii_lowercase[1:]]
-        bdm = Boto2BlockDeviceMapping()
+        bdm_list: List[BlockDeviceMappingTypeDef] = []
         # Change root volume size to allow for bigger Docker instances
-        root_vol = Boto2BlockDeviceType(delete_on_termination=True)
-        root_vol.size = rootVolSize
-        bdm["/dev/xvda"] = root_vol
+        root_vol: EbsBlockDeviceTypeDef = {"DeleteOnTermination": True,
+                                           "VolumeSize": rootVolSize}
+        bdm: BlockDeviceMappingTypeDef = {"DeviceName": "/dev/xvda", "Ebs": root_vol}
+        bdm_list.append(bdm)
         # The first disk is already attached for us so start with 2nd.
         # Disk count is weirdly a float in our instance database, so make it an int here.
         for disk in range(1, int(type_info.disks) + 1):
-            bdm[bdtKeys[disk]] = Boto2BlockDeviceType(
-                ephemeral_name=f'ephemeral{disk - 1}')  # ephemeral counts start at 0
+            bdm = {}
+            bdm["DeviceName"] = bdtKeys[disk]
+            bdm["VirtualName"] = f"ephemeral{disk - 1}"  # ephemeral counts start at 0
+            bdm["Ebs"] = root_vol  # default
+            # bdm["Ebs"] = root_vol.update({"VirtualName": f"ephemeral{disk - 1}"})
+            bdm_list.append(bdm)
 
-        logger.debug('Device mapping: %s', bdm)
-        return bdm
+        logger.debug('Device mapping: %s', bdm_list)
+        return bdm_list
 
     @classmethod
-    def _getBoto3BlockDeviceMappings(cls, type_info: InstanceType, rootVolSize: int = 50) -> List[dict]:
+    def _getBoto3BlockDeviceMappings(cls, type_info: InstanceType, rootVolSize: int = 50) -> List[BlockDeviceMappingTypeDef]:
         """
         Get block device mappings for the root volume for a worker.
         """
 
         # Start with the root
-        bdms = [{
+        bdms: List[BlockDeviceMappingTypeDef] = [{
             'DeviceName': '/dev/xvda',
             'Ebs': {
                 'DeleteOnTermination': True,
@@ -1121,96 +1222,98 @@ class AWSProvisioner(AbstractProvisioner):
         return bdms
 
     @awsRetry
-    def _get_nodes_in_cluster(self, instance_type: Optional[str] = None, include_stopped_nodes=False) -> List[Boto2Instance]:
+    def _get_nodes_in_cluster_boto3(self, instance_type: Optional[str] = None, include_stopped_nodes: bool = False) -> List[InstanceTypeDef]:
         """
-        Get Boto2 instance objects for all nodes in the cluster.
+        Get Boto3 instance objects for all nodes in the cluster.
         """
+        boto3_ec2: EC2Client = self.aws.client(region=self._region, service_name='ec2')
+        instance_filter: FilterTypeDef = {'Name': 'instance.group-name', 'Values': [self.clusterName]}
+        describe_response: DescribeInstancesResultTypeDef = boto3_ec2.describe_instances(Filters=[instance_filter])
+        all_instances: List[InstanceTypeDef] = []
+        for reservation in describe_response['Reservations']:
+            instances = reservation['Instances']
+            all_instances.extend(instances)
 
-        all_instances = self.aws.boto2(self._region, 'ec2').get_only_instances(filters={'instance.group-name': self.clusterName})
+        # all_instances = self.aws.boto2(self._region, 'ec2').get_only_instances(filters={'instance.group-name': self.clusterName})
 
-        def instanceFilter(i):
+        def instanceFilter(i: InstanceTypeDef) -> bool:
             # filter by type only if nodeType is true
-            rightType = not instance_type or i.instance_type == instance_type
-            rightState = i.state == 'running' or i.state == 'pending'
+            rightType = not instance_type or i['InstanceType'] == instance_type
+            rightState = i['State']['Name'] == 'running' or i['State']['Name'] == 'pending'
             if include_stopped_nodes:
-                rightState = rightState or i.state == 'stopping' or i.state == 'stopped'
+                rightState = rightState or i['State']['Name'] == 'stopping' or i['State']['Name'] == 'stopped'
             return rightType and rightState
 
         return [i for i in all_instances if instanceFilter(i)]
 
-    def _filter_nodes_in_cluster(self, instance_type: Optional[str] = None, preemptible: bool = False) -> List[Boto2Instance]:
-        """
-        Get Boto2 instance objects for the nodes in the cluster filtered by preemptability.
-        """
-
-        instances = self._get_nodes_in_cluster(instance_type, include_stopped_nodes=False)
-
-        if preemptible:
-            return [i for i in instances if i.spot_instance_request_id is not None]
-
-        return [i for i in instances if i.spot_instance_request_id is None]
-
     def _getSpotRequestIDs(self) -> List[str]:
         """
         Get the IDs of all spot requests associated with the cluster.
         """
 
         # Grab the connection we need to use for this operation.
-        ec2 = self.aws.boto2(self._region, 'ec2')
+        ec2: EC2Client = self.aws.client(self._region, 'ec2')
 
-        requests = ec2.get_all_spot_instance_requests()
-        tags = ec2.get_all_tags({'tag:': {_TAG_KEY_TOIL_CLUSTER_NAME: self.clusterName}})
-        idsToCancel = [tag.id for tag in tags]
-        return [request for request in requests if request.id in idsToCancel]
+        requests: List[SpotInstanceRequestTypeDef] = ec2.describe_spot_instance_requests()["SpotInstanceRequests"]
+        tag_filter: FilterTypeDef = {"Name": "tag:" + _TAG_KEY_TOIL_CLUSTER_NAME, "Values": [self.clusterName]}
+        tags: List[TagDescriptionTypeDef] = ec2.describe_tags(Filters=[tag_filter])["Tags"]
+        idsToCancel = [tag["ResourceId"] for tag in tags]
+        return [request["SpotInstanceRequestId"] for request in requests if request["InstanceId"] in idsToCancel]
 
     def _createSecurityGroups(self) -> List[str]:
         """
         Create security groups for the cluster. Returns a list of their IDs.
         """
+        def group_not_found(e: ClientError) -> bool:
+            retry = (get_error_status(e) == 400 and 'does not exist in default VPC' in get_error_body(e))
+            return retry
 
         # Grab the connection we need to use for this operation.
         # The VPC connection can do anything the EC2 one can do, but also look at subnets.
-        vpc = self.aws.boto2(self._region, 'vpc')
+        boto3_ec2: EC2Client = self.aws.client(region=self._region, service_name="ec2")
 
-        def groupNotFound(e):
-            retry = (e.status == 400 and 'does not exist in default VPC' in e.body)
-            return retry
-        # Security groups need to belong to the same VPC as the leader. If we
-        # put the leader in a particular non-default subnet, it may be in a
-        # particular non-default VPC, which we need to know about.
-        vpcId = None
+        vpc_id = None
         if self._leader_subnet:
-            subnets = vpc.get_all_subnets(subnet_ids=[self._leader_subnet])
+            subnets = boto3_ec2.describe_subnets(SubnetIds=[self._leader_subnet])["Subnets"]
             if len(subnets) > 0:
-                vpcId = subnets[0].vpc_id
-        # security group create/get. ssh + all ports open within the group
+                vpc_id = subnets[0]["VpcId"]
         try:
-            web = vpc.create_security_group(self.clusterName,
-                                            'Toil appliance security group', vpc_id=vpcId)
-        except EC2ResponseError as e:
-            if e.status == 400 and 'already exists' in e.body:
-                pass  # group exists- nothing to do
+            # Security groups need to belong to the same VPC as the leader. If we
+            # put the leader in a particular non-default subnet, it may be in a
+            # particular non-default VPC, which we need to know about.
+            other = {"GroupName": self.clusterName, "Description": "Toil appliance security group"}
+            if vpc_id is not None:
+                other["VpcId"] = vpc_id
+            # mypy stubs don't explicitly state kwargs even though documentation allows it, and mypy gets confused
+            web_response: CreateSecurityGroupResultTypeDef = boto3_ec2.create_security_group(**other)  # type: ignore[arg-type]
+        except ClientError as e:
+            if get_error_status(e) == 400 and 'already exists' in get_error_body(e):
+                pass
             else:
                 raise
         else:
-            for attempt in old_retry(predicate=groupNotFound, timeout=300):
-                with attempt:
-                    # open port 22 for ssh-ing
-                    web.authorize(ip_protocol='tcp', from_port=22, to_port=22, cidr_ip='0.0.0.0/0')
-                    # TODO: boto2 doesn't support IPv6 here but we need to.
-            for attempt in old_retry(predicate=groupNotFound, timeout=300):
-                with attempt:
-                    # the following authorizes all TCP access within the web security group
-                    web.authorize(ip_protocol='tcp', from_port=0, to_port=65535, src_group=web)
-            for attempt in old_retry(predicate=groupNotFound, timeout=300):
+            for attempt in old_retry(predicate=group_not_found, timeout=300):
                 with attempt:
-                    # We also want to open up UDP, both for user code and for the RealtimeLogger
-                    web.authorize(ip_protocol='udp', from_port=0, to_port=65535, src_group=web)
+                    ip_permissions: List[IpPermissionTypeDef] = [{"IpProtocol": "tcp",
+                                                                  "FromPort": 22,
+                                                                  "ToPort": 22,
+                                                                  "IpRanges": [
+                                                                      {"CidrIp": "0.0.0.0/0"}
+                                                                  ],
+                                                                  "Ipv6Ranges": [{"CidrIpv6": "::/0"}]}]
+                    for protocol in ("tcp", "udp"):
+                        ip_permissions.append({"IpProtocol": protocol,
+                                               "FromPort": 0,
+                                               "ToPort": 65535,
+                                               "UserIdGroupPairs":
+                                                   [{"GroupId": web_response["GroupId"],
+                                                     "GroupName": self.clusterName}]})
+                    boto3_ec2.authorize_security_group_ingress(IpPermissions=ip_permissions, GroupName=self.clusterName, GroupId=web_response["GroupId"])
         out = []
-        for sg in vpc.get_all_security_groups():
-            if sg.name == self.clusterName and (vpcId is None or sg.vpc_id == vpcId):
-                out.append(sg)
-        return [sg.id for sg in out]
+        for sg in boto3_ec2.describe_security_groups()["SecurityGroups"]:
+            if sg["GroupName"] == self.clusterName and (vpc_id is None or sg["VpcId"] == vpc_id):
+                out.append(sg["GroupId"])
+        return out
 
     @awsRetry
     def _getSecurityGroupIDs(self) -> List[str]:
@@ -1222,13 +1325,13 @@ class AWSProvisioner(AbstractProvisioner):
 
         # Depending on if we enumerated them on the leader or locally, we might
         # know the required security groups by name, ID, or both.
-        sgs = [sg for sg in self.aws.boto2(self._region, 'ec2').get_all_security_groups()
-               if (sg.name in self._leaderSecurityGroupNames or
-                   sg.id in self._leaderSecurityGroupIDs)]
-        return [sg.id for sg in sgs]
+        boto3_ec2 = self.aws.client(region=self._region, service_name='ec2')
+        return [sg["GroupId"] for sg in boto3_ec2.describe_security_groups()["SecurityGroups"]
+                if (sg["GroupName"] in self._leaderSecurityGroupNames or
+                    sg["GroupId"] in self._leaderSecurityGroupIDs)]
 
     @awsRetry
-    def _get_launch_template_ids(self, filters: Optional[List[Dict[str, List[str]]]] = None) -> List[str]:
+    def _get_launch_template_ids(self, filters: Optional[List[FilterTypeDef]] = None) -> List[str]:
         """
         Find all launch templates associated with the cluster.
 
@@ -1236,10 +1339,10 @@ class AWSProvisioner(AbstractProvisioner):
         """
 
         # Grab the connection we need to use for this operation.
-        ec2 = self.aws.client(self._region, 'ec2')
+        ec2: EC2Client = self.aws.client(self._region, 'ec2')
 
         # How do we match the right templates?
-        combined_filters = [{'Name': 'tag:' + _TAG_KEY_TOIL_CLUSTER_NAME, 'Values': [self.clusterName]}]
+        combined_filters: List[FilterTypeDef] = [{'Name': 'tag:' + _TAG_KEY_TOIL_CLUSTER_NAME, 'Values': [self.clusterName]}]
 
         if filters:
             # Add any user-specified filters
@@ -1254,7 +1357,7 @@ class AWSProvisioner(AbstractProvisioner):
             allTemplateIDs += [item['LaunchTemplateId'] for item in response.get('LaunchTemplates', [])]
             if 'NextToken' in response:
                 # There are more pages. Get the next one, supplying the token.
-                response = ec2.describe_launch_templates(Filters=filters,
+                response = ec2.describe_launch_templates(Filters=filters or [],
                                                          NextToken=response['NextToken'],
                                                          MaxResults=200)
             else:
@@ -1286,10 +1389,10 @@ class AWSProvisioner(AbstractProvisioner):
         lt_name = self._name_worker_launch_template(instance_type, preemptible=preemptible)
 
         # How do we match the right templates?
-        filters = [{'Name': 'launch-template-name', 'Values': [lt_name]}]
+        filters: List[FilterTypeDef] = [{'Name': 'launch-template-name', 'Values': [lt_name]}]
 
         # Get the templates
-        templates = self._get_launch_template_ids(filters=filters)
+        templates: List[str] = self._get_launch_template_ids(filters=filters)
 
         if len(templates) > 1:
             # There shouldn't ever be multiple templates with our reserved name
@@ -1305,7 +1408,7 @@ class AWSProvisioner(AbstractProvisioner):
                     # writes). Recurse to try again, because now it exists.
                     logger.info('Waiting %f seconds for template %s to be available', backoff, lt_name)
                     time.sleep(backoff)
-                    return self._get_worker_launch_template(instance_type, preemptible=preemptible, backoff=backoff*2)
+                    return self._get_worker_launch_template(instance_type, preemptible=preemptible, backoff=backoff * 2)
                 else:
                     raise
         else:
@@ -1345,7 +1448,7 @@ class AWSProvisioner(AbstractProvisioner):
 
         assert self._leaderPrivateIP
         type_info = E2Instances[instance_type]
-        rootVolSize=self._nodeStorageOverrides.get(instance_type, self._nodeStorage)
+        rootVolSize = self._nodeStorageOverrides.get(instance_type, self._nodeStorage)
         bdms = self._getBoto3BlockDeviceMappings(type_info, rootVolSize=rootVolSize)
 
         keyPath = self._sseKey if self._sseKey else None
@@ -1377,16 +1480,16 @@ class AWSProvisioner(AbstractProvisioner):
         """
 
         # Grab the connection we need to use for this operation.
-        autoscaling = self.aws.client(self._region, 'autoscaling')
+        autoscaling: AutoScalingClient = self.aws.client(self._region, 'autoscaling')
 
         # AWS won't filter ASGs server-side for us in describe_auto_scaling_groups.
         # So we search instances of applied tags for the ASGs they are on.
         # The ASGs tagged with our cluster are our ASGs.
         # The filtering is on different fields of the tag object itself.
-        filters = [{'Name': 'key',
-                    'Values': [_TAG_KEY_TOIL_CLUSTER_NAME]},
-                   {'Name': 'value',
-                    'Values': [self.clusterName]}]
+        filters: List[FilterTypeDef] = [{'Name': 'key',
+                                         'Values': [_TAG_KEY_TOIL_CLUSTER_NAME]},
+                                        {'Name': 'value',
+                                         'Values': [self.clusterName]}]
 
         matchedASGs = []
         # Get the first page with no NextToken
@@ -1461,7 +1564,7 @@ class AWSProvisioner(AbstractProvisioner):
         if self.clusterType == 'kubernetes':
             # We also need to tag it with Kubernetes autoscaler info (empty tags)
             tags['k8s.io/cluster-autoscaler/' + self.clusterName] = ''
-            assert(self.clusterName != 'enabled')
+            assert (self.clusterName != 'enabled')
             tags['k8s.io/cluster-autoscaler/enabled'] = ''
             tags['k8s.io/cluster-autoscaler/node-template/resources/ephemeral-storage'] = f'{min_gigs}G'
 
@@ -1481,7 +1584,7 @@ class AWSProvisioner(AbstractProvisioner):
 
         return asg_name
 
-    def _boto2_pager(self, requestor_callable: Callable, result_attribute_name: str) -> Iterable[Dict[str, Any]]:
+    def _boto2_pager(self, requestor_callable: Callable[[...], Any], result_attribute_name: str) -> Iterable[Dict[str, Any]]:  # type: ignore[misc]
         """
         Yield all the results from calling the given Boto 2 method and paging
         through all the results using the "marker" field. Results are to be
@@ -1489,33 +1592,13 @@ class AWSProvisioner(AbstractProvisioner):
         """
         marker = None
         while True:
-            result = requestor_callable(marker=marker)
+            result = requestor_callable(marker=marker)  # type: ignore[call-arg]
             yield from getattr(result, result_attribute_name)
             if result.is_truncated == 'true':
                 marker = result.marker
             else:
                 break
 
-    def _pager(self, requestor_callable: Callable, result_attribute_name: str, **kwargs) -> Iterable[Dict[str, Any]]:
-        """
-        Yield all the results from calling the given Boto 3 method with the
-        given keyword arguments, paging through the results using the Marker or
-        NextToken, and fetching out and looping over the list in the response
-        with the given attribute name.
-        """
-
-        # Recover the Boto3 client, and the name of the operation
-        client = requestor_callable.__self__
-        op_name = requestor_callable.__name__
-
-        # grab a Boto 3 built-in paginator. See
-        # <https://boto3.amazonaws.com/v1/documentation/api/latest/guide/paginators.html>
-        paginator = client.get_paginator(op_name)
-
-        for page in paginator.paginate(**kwargs):
-            # Invoke it and go through the pages, yielding from them
-            yield from page.get(result_attribute_name, [])
-
     @awsRetry
     def _getRoleNames(self) -> List[str]:
         """
@@ -1523,10 +1606,12 @@ class AWSProvisioner(AbstractProvisioner):
         """
 
         results = []
-        for result in self._boto2_pager(self.aws.boto2(self._region, 'iam').list_roles, 'roles'):
+        boto3_iam = self.aws.client(self._region, 'iam')
+        for result in boto3_pager(boto3_iam.list_roles, 'Roles'):
             # For each Boto2 role object
             # Grab out the name
-            name = result['role_name']
+            cast(RoleTypeDef, result)
+            name = result['RoleName']
             if self._is_our_namespaced_name(name):
                 # If it looks like ours, it is ours.
                 results.append(name)
@@ -1539,11 +1624,12 @@ class AWSProvisioner(AbstractProvisioner):
         """
 
         results = []
-        for result in self._boto2_pager(self.aws.boto2(self._region, 'iam').list_instance_profiles,
-                                        'instance_profiles'):
-            # For each Boto2 role object
+        boto3_iam = self.aws.client(self._region, 'iam')
+        for result in boto3_pager(boto3_iam.list_instance_profiles,
+                                  'InstanceProfiles'):
             # Grab out the name
-            name = result['instance_profile_name']
+            cast(InstanceProfileTypeDef, result)
+            name = result['InstanceProfileName']
             if self._is_our_namespaced_name(name):
                 # If it looks like ours, it is ours.
                 results.append(name)
@@ -1558,9 +1644,9 @@ class AWSProvisioner(AbstractProvisioner):
         """
 
         # Grab the connection we need to use for this operation.
-        iam = self.aws.client(self._region, 'iam')
+        boto3_iam: IAMClient = self.aws.client(self._region, 'iam')
 
-        return [item['InstanceProfileName'] for item in self._pager(iam.list_instance_profiles_for_role,
+        return [item['InstanceProfileName'] for item in boto3_pager(boto3_iam.list_instance_profiles_for_role,
                                                                     'InstanceProfiles',
                                                                     RoleName=role_name)]
 
@@ -1575,11 +1661,11 @@ class AWSProvisioner(AbstractProvisioner):
         """
 
         # Grab the connection we need to use for this operation.
-        iam = self.aws.client(self._region, 'iam')
+        boto3_iam: IAMClient = self.aws.client(self._region, 'iam')
 
         # TODO: we don't currently use attached policies.
 
-        return [item['PolicyArn'] for item in self._pager(iam.list_attached_role_policies,
+        return [item['PolicyArn'] for item in boto3_pager(boto3_iam.list_attached_role_policies,
                                                           'AttachedPolicies',
                                                           RoleName=role_name)]
 
@@ -1591,20 +1677,18 @@ class AWSProvisioner(AbstractProvisioner):
         """
 
         # Grab the connection we need to use for this operation.
-        iam = self.aws.client(self._region, 'iam')
+        boto3_iam: IAMClient = self.aws.client(self._region, 'iam')
 
-        return list(self._pager(iam.list_role_policies,
-                                'PolicyNames',
-                                RoleName=role_name))
+        return list(boto3_pager(boto3_iam.list_role_policies, 'PolicyNames', RoleName=role_name))
 
-    def full_policy(self, resource: str) -> dict:
+    def full_policy(self, resource: str) -> Dict[str, Any]:
         """
         Produce a dict describing the JSON form of a full-access-granting AWS
         IAM policy for the service with the given name (e.g. 's3').
         """
         return dict(Version="2012-10-17", Statement=[dict(Effect="Allow", Resource="*", Action=f"{resource}:*")])
 
-    def kubernetes_policy(self) -> dict:
+    def kubernetes_policy(self) -> Dict[str, Any]:
         """
         Get the Kubernetes policy grants not provided by the full grants on EC2
         and IAM. See
@@ -1671,45 +1755,42 @@ class AWSProvisioner(AbstractProvisioner):
         """
 
         # Grab the connection we need to use for this operation.
-        iam = self.aws.boto2(self._region, 'iam')
+        boto3_iam: IAMClient = self.aws.client(self._region, 'iam')
 
         # Make sure we can tell our roles apart from roles for other clusters
         aws_role_name = self._namespace_name(local_role_name)
         try:
             # Make the role
             logger.debug('Creating IAM role %s...', aws_role_name)
-            iam.create_role(aws_role_name, assume_role_policy_document=json.dumps({
+            assume_role_policy_document = json.dumps({
                 "Version": "2012-10-17",
                 "Statement": [{
                     "Effect": "Allow",
                     "Principal": {"Service": ["ec2.amazonaws.com"]},
                     "Action": ["sts:AssumeRole"]}
-                ]}))
+                ]})
+            boto3_iam.create_role(RoleName=aws_role_name, AssumeRolePolicyDocument=assume_role_policy_document)
             logger.debug('Created new IAM role')
-        except BotoServerError as e:
-            if e.status == 409 and e.error_code == 'EntityAlreadyExists':
+        except ClientError as e:
+            if get_error_status(e) == 409 and get_error_code(e) == 'EntityAlreadyExists':
                 logger.debug('IAM role already exists. Reusing.')
             else:
                 raise
 
         # Delete superfluous policies
-        policy_names = set(iam.list_role_policies(aws_role_name).policy_names)
+        policy_names = set(boto3_iam.list_role_policies(RoleName=aws_role_name)["PolicyNames"])
         for policy_name in policy_names.difference(set(list(policies.keys()))):
-            iam.delete_role_policy(aws_role_name, policy_name)
+            boto3_iam.delete_role_policy(RoleName=aws_role_name, PolicyName=policy_name)
 
         # Create expected policies
         for policy_name, policy in policies.items():
             current_policy = None
             try:
-                current_policy = json.loads(unquote(
-                    iam.get_role_policy(aws_role_name, policy_name).policy_document))
-            except BotoServerError as e:
-                if e.status == 404 and e.error_code == 'NoSuchEntity':
-                    pass
-                else:
-                    raise
+                current_policy = boto3_iam.get_role_policy(RoleName=aws_role_name, PolicyName=policy_name)["PolicyDocument"]
+            except boto3_iam.exceptions.NoSuchEntityException:
+                pass
             if current_policy != policy:
-                iam.put_role_policy(aws_role_name, policy_name, json.dumps(policy))
+                boto3_iam.put_role_policy(RoleName=aws_role_name, PolicyName=policy_name, PolicyDocument=json.dumps(policy))
 
         # Now the role has the right policies so it is ready.
         return aws_role_name
@@ -1724,7 +1805,7 @@ class AWSProvisioner(AbstractProvisioner):
         """
 
         # Grab the connection we need to use for this operation.
-        iam = self.aws.boto2(self._region, 'iam')
+        boto3_iam: IAMClient = self.aws.client(self._region, 'iam')
 
         policy = dict(iam_full=self.full_policy('iam'), ec2_full=self.full_policy('ec2'),
                       s3_full=self.full_policy('s3'), sbd_full=self.full_policy('sdb'))
@@ -1735,45 +1816,41 @@ class AWSProvisioner(AbstractProvisioner):
         iamRoleName = self._setup_iam_ec2_role(_INSTANCE_PROFILE_ROLE_NAME, policy)
 
         try:
-            profile = iam.get_instance_profile(iamRoleName)
-            logger.debug("Have preexisting instance profile: %s", profile.get_instance_profile_response.get_instance_profile_result.instance_profile)
-        except BotoServerError as e:
-            if e.status == 404:
-                profile = iam.create_instance_profile(iamRoleName)
-                profile = profile.create_instance_profile_response.create_instance_profile_result
-                logger.debug("Created new instance profile: %s", profile.instance_profile)
-            else:
-                raise
+            profile_result = boto3_iam.get_instance_profile(InstanceProfileName=iamRoleName)
+            profile: InstanceProfileTypeDef = profile_result["InstanceProfile"]
+            logger.debug("Have preexisting instance profile: %s", profile)
+        except boto3_iam.exceptions.NoSuchEntityException:
+            profile_result = boto3_iam.create_instance_profile(InstanceProfileName=iamRoleName)
+            profile = profile_result["InstanceProfile"]
+            logger.debug("Created new instance profile: %s", profile)
         else:
-            profile = profile.get_instance_profile_response.get_instance_profile_result
-        profile = profile.instance_profile
+            profile = profile_result["InstanceProfile"]
 
-        profile_arn = profile.arn
+        profile_arn: str = profile["Arn"]
 
         # Now we have the profile ARN, but we want to make sure it really is
         # visible by name in a different session.
         wait_until_instance_profile_arn_exists(profile_arn)
 
-        if len(profile.roles) > 1:
+        if len(profile["Roles"]) > 1:
             # This is too many roles. We probably grabbed something we should
             # not have by mistake, and this is some important profile for
             # something else.
             raise RuntimeError(f'Did not expect instance profile {profile_arn} to contain '
                                f'more than one role; is it really a Toil-managed profile?')
-        elif len(profile.roles) == 1:
-            # this should be profile.roles[0].role_name
-            if profile.roles.member.role_name == iamRoleName:
+        elif len(profile["Roles"]) == 1:
+            if profile["Roles"][0]["RoleName"] == iamRoleName:
                 return profile_arn
             else:
                 # Drop this wrong role and use the fallback code for 0 roles
-                iam.remove_role_from_instance_profile(iamRoleName,
-                                                      profile.roles.member.role_name)
+                boto3_iam.remove_role_from_instance_profile(InstanceProfileName=iamRoleName,
+                                                            RoleName=profile["Roles"][0]["RoleName"])
 
         # If we get here, we had 0 roles on the profile, or we had 1 but we removed it.
-        for attempt in old_retry(predicate=lambda err: err.status == 404):
+        for attempt in old_retry(predicate=lambda err: get_error_status(err) == 404):
             with attempt:
                 # Put the IAM role on the profile
-                iam.add_role_to_instance_profile(profile.instance_profile_name, iamRoleName)
+                boto3_iam.add_role_to_instance_profile(InstanceProfileName=profile["InstanceProfileName"], RoleName=iamRoleName)
                 logger.debug("Associated role %s with profile", iamRoleName)
 
         return profile_arn