toil 6.1.0a1__py3-none-any.whl → 7.0.0__py3-none-any.whl

toil/test/cwl/seqtk_seq.cwl

@@ -21,4 +21,4 @@ hints:
     packages:
     - package: seqtk
       version:
-      - r93
+      - '1.4'

toil/test/docs/scriptsTest.py

@@ -4,6 +4,9 @@ import shutil
 import subprocess
 import sys
 import unittest
+import pytest
+
+from typing import List
 
 pkg_root = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))  # noqa
 sys.path.insert(0, pkg_root)  # noqa
@@ -21,12 +24,6 @@ class ToilDocumentationTest(ToilTest):
 
     def tearDown(self) -> None:
         super(ToilTest, self).tearDown()
-        # src/toil/test/docs/scripts/cwlExampleFiles/sample_1_output.txt
-        output_files = ["sample_1_output.txt", "sample_2_output.txt", "sample_3_output.txt"]
-        for output in output_files:
-            output_file = os.path.join(self.directory, 'scripts/cwlExampleFiles', output)
-            if os.path.exists(output_file):
-                os.remove(output_file)
 
         jobstores = ['/mnt/ephemeral/workspace/toil-pull-requests/toilWorkflowRun']
         for jobstore in jobstores:
@@ -36,9 +33,9 @@ class ToilDocumentationTest(ToilTest):
         unittest.TestCase.tearDown(self)
 
     """Just check the exit code"""
-    def checkExitCode(self, script):
+    def checkExitCode(self, script, extra_args: List[str] = []):
         program = os.path.join(self.directory, "scripts", script)
-        process = subprocess.Popen([python, program, "file:my-jobstore", "--clean=always"],
+        process = subprocess.Popen([python, program, "file:my-jobstore", "--clean=always"] + extra_args,
                                    stdout=subprocess.PIPE, stderr=subprocess.PIPE)
         stdout, stderr = process.communicate()
         if isinstance(stdout, bytes):
@@ -65,13 +62,11 @@ class ToilDocumentationTest(ToilTest):
         n = re.search(pattern, outerr)
         self.assertNotEqual(n, None, f"Pattern:\n{expectedPattern}\nOutput:\n{outerr}")
 
-    @needs_cwl
-    def testCwlexample(self):
-        self.checkExitCode("tutorial_cwlexample.py")
-
-    def testDiscoverfiles(self):
-        self.checkExitCode("tutorial_discoverfiles.py")
+    def testStats(self):
+        # This script asks for 4 cores but we might need to run the tests in only 3.
+        self.checkExitCode("tutorial_stats.py", ["--scale=0.5"])
 
+    @pytest.mark.timeout(1200)
     def testDynamic(self):
         self.checkExitCode("tutorial_dynamic.py")
 
@@ -117,6 +112,7 @@ class ToilDocumentationTest(ToilTest):
                                   "second or third.*Hello world, I have a message: second or third.*Hello world,"
                                   " I have a message: last")
 
+    @pytest.mark.timeout(1200)
     def testPromises2(self):
         self.checkExpectedOut("tutorial_promises2.py",
                               "['00000', '00001', '00010', '00011', '00100', '00101', '00110', '00111',"
@@ -146,5 +142,6 @@ class ToilDocumentationTest(ToilTest):
     def testStaging(self):
         self.checkExitCode("tutorial_staging.py")
 
+
 if __name__ == "__main__":
     unittest.main()

toil/test/jobStores/jobStoreTest.py

@@ -40,7 +40,6 @@ from toil.job import Job, JobDescription, TemporaryID
 from toil.jobStores.abstractJobStore import (NoSuchFileException,
                                              NoSuchJobException)
 from toil.jobStores.fileJobStore import FileJobStore
-from toil.lib.aws.utils import create_s3_bucket, get_object_for_url
 from toil.lib.io import mkdtemp
 from toil.lib.memoize import memoize
 from toil.lib.retry import retry
@@ -130,8 +129,7 @@ class AbstractJobStoreTest:
             # Requirements for jobs to be created.
             self.arbitraryRequirements = {'memory': 1, 'disk': 2, 'cores': 1, 'preemptible': False}
             # Function to make an arbitrary new job
-            self.arbitraryJob = lambda: JobDescription(command='command',
-                                                       jobName='arbitrary',
+            self.arbitraryJob = lambda: JobDescription(jobName='arbitrary',
                                                        requirements=self.arbitraryRequirements)
 
             self.parentJobReqs = dict(memory=12, cores=34, disk=35, preemptible=True)
@@ -144,9 +142,9 @@ class AbstractJobStoreTest:
             super().tearDown()
 
         def testInitialState(self):
-            """Ensure proper handling of nonexistant files."""
-            self.assertFalse(self.jobstore_initialized.job_exists('nonexistantFile'))
-            self.assertRaises(NoSuchJobException, self.jobstore_initialized.load_job, 'nonexistantFile')
+            """Ensure proper handling of nonexistent files."""
+            self.assertFalse(self.jobstore_initialized.job_exists('nonexistentFile'))
+            self.assertRaises(NoSuchJobException, self.jobstore_initialized.load_job, 'nonexistentFile')
 
         def testJobCreation(self):
             """
@@ -159,8 +157,7 @@ class AbstractJobStoreTest:
             jobstore = self.jobstore_initialized
 
             # Create a job and verify its existence/properties
-            job = JobDescription(command='parent1',
-                                 requirements=self.parentJobReqs,
+            job = JobDescription(requirements=self.parentJobReqs,
                                  jobName='test1', unitName='onParent')
             self.assertTrue(isinstance(job.jobStoreID, TemporaryID))
             jobstore.assign_job_id(job)
@@ -170,7 +167,6 @@ class AbstractJobStoreTest:
             self.assertEqual(created, job)
 
             self.assertTrue(jobstore.job_exists(job.jobStoreID))
-            self.assertEqual(job.command, 'parent1')
             self.assertEqual(job.memory, self.parentJobReqs['memory'])
             self.assertEqual(job.cores, self.parentJobReqs['cores'])
             self.assertEqual(job.disk, self.parentJobReqs['disk'])
@@ -195,8 +191,7 @@ class AbstractJobStoreTest:
             """Tests that a job created via one JobStore instance can be loaded from another."""
 
             # Create a job on the first jobstore.
-            jobDesc1 = JobDescription(command='jobstore1',
-                                      requirements=self.parentJobReqs,
+            jobDesc1 = JobDescription(requirements=self.parentJobReqs,
                                       jobName='test1', unitName='onJS1')
             self.jobstore_initialized.assign_job_id(jobDesc1)
             self.jobstore_initialized.create_job(jobDesc1)
@@ -204,16 +199,14 @@ class AbstractJobStoreTest:
             # Load it from the second jobstore
             jobDesc2 = self.jobstore_resumed_noconfig.load_job(jobDesc1.jobStoreID)
 
-            self.assertEqual(jobDesc1.command, jobDesc2.command)
+            self.assertEqual(jobDesc1._body, jobDesc2._body)
 
         def testChildLoadingEquality(self):
             """Test that loading a child job operates as expected."""
-            job = JobDescription(command='parent1',
-                                 requirements=self.parentJobReqs,
+            job = JobDescription(requirements=self.parentJobReqs,
                                  jobName='test1', unitName='onParent')
 
-            childJob = JobDescription(command='child1',
-                                      requirements=self.childJobReqs1,
+            childJob = JobDescription(requirements=self.childJobReqs1,
                                       jobName='test2', unitName='onChild1')
             self.jobstore_initialized.assign_job_id(job)
             self.jobstore_initialized.assign_job_id(childJob)
@@ -222,7 +215,7 @@ class AbstractJobStoreTest:
             job.addChild(childJob.jobStoreID)
             self.jobstore_initialized.update_job(job)
 
-            self.assertEqual(self.jobstore_initialized.load_job(list(job.allSuccessors())[0]).command, childJob.command)
+            self.assertEqual(self.jobstore_initialized.load_job(list(job.allSuccessors())[0])._body, childJob._body)
 
         def testPersistantFilesToDelete(self):
             """
@@ -237,8 +230,7 @@ class AbstractJobStoreTest:
             """
 
             # Create a job.
-            job = JobDescription(command='job1',
-                                 requirements=self.parentJobReqs,
+            job = JobDescription(requirements=self.parentJobReqs,
                                  jobName='test1', unitName='onJS1')
 
             self.jobstore_initialized.assign_job_id(job)
@@ -252,16 +244,13 @@ class AbstractJobStoreTest:
             jobstore1 = self.jobstore_initialized
             jobstore2 = self.jobstore_resumed_noconfig
 
-            job1 = JobDescription(command='parent1',
-                                 requirements=self.parentJobReqs,
+            job1 = JobDescription(requirements=self.parentJobReqs,
                                  jobName='test1', unitName='onParent')
 
-            childJob1 = JobDescription(command='child1',
-                                      requirements=self.childJobReqs1,
+            childJob1 = JobDescription(requirements=self.childJobReqs1,
                                       jobName='test2', unitName='onChild1')
 
-            childJob2 = JobDescription(command='child2',
-                                      requirements=self.childJobReqs2,
+            childJob2 = JobDescription(requirements=self.childJobReqs2,
                                       jobName='test3', unitName='onChild2')
 
             jobstore1.assign_job_id(job1)
@@ -298,20 +287,17 @@ class AbstractJobStoreTest:
             """Tests the consequences of deleting jobs."""
             # A local jobstore object for testing.
             jobstore = self.jobstore_initialized
-            job = JobDescription(command='job1',
-                                 requirements=self.parentJobReqs,
+            job = JobDescription(requirements=self.parentJobReqs,
                                  jobName='test1', unitName='onJob')
             # Create job
             jobstore.assign_job_id(job)
             jobstore.create_job(job)
 
             # Create child Jobs
-            child1 = JobDescription(command='child1',
-                                    requirements=self.childJobReqs1,
+            child1 = JobDescription(requirements=self.childJobReqs1,
                                     jobName='test2', unitName='onChild1')
 
-            child2 = JobDescription(command='job1',
-                                    requirements=self.childJobReqs2,
+            child2 = JobDescription(requirements=self.childJobReqs2,
                                     jobName='test3', unitName='onChild2')
 
             # Add children to parent.
@@ -323,9 +309,8 @@ class AbstractJobStoreTest:
             job.addChild(child2.jobStoreID)
             jobstore.update_job(job)
 
-            # Get it ready to run children
-            job.command = None
-            jobstore.update_job(job)
+            # Parent must have no body to start on children
+            assert not job.has_body()
 
             # Go get the children
             childJobs = [jobstore.load_job(childID) for childID in job.nextSuccessors()]
@@ -422,8 +407,7 @@ class AbstractJobStoreTest:
             jobstore2 = self.jobstore_resumed_noconfig
 
             # Create jobNodeOnJS1
-            jobOnJobStore1 = JobDescription(command='job1',
-                                            requirements=self.parentJobReqs,
+            jobOnJobStore1 = JobDescription(requirements=self.parentJobReqs,
                                             jobName='test1', unitName='onJobStore1')
 
             # First recreate job
@@ -490,12 +474,11 @@ class AbstractJobStoreTest:
                     pass
 
         def testStatsAndLogging(self):
-            """Tests behavior of reading and writting stats and logging."""
+            """Tests behavior of reading and writing stats and logging."""
             jobstore1 = self.jobstore_initialized
             jobstore2 = self.jobstore_resumed_noconfig
 
-            jobOnJobStore1 = JobDescription(command='job1',
-                                            requirements=self.parentJobReqs,
+            jobOnJobStore1 = JobDescription(requirements=self.parentJobReqs,
                                             jobName='test1', unitName='onJobStore1')
 
             jobstore1.assign_job_id(jobOnJobStore1)
@@ -548,14 +531,16 @@ class AbstractJobStoreTest:
             jobNames = ['testStatsAndLogging_writeLogFiles']
             jobLogList = ['string', b'bytes', '', b'newline\n']
             config = self._createConfig()
-            setattr(config, 'writeLogs', '.')
+            setattr(config, 'writeLogs', self._createTempDir())
             setattr(config, 'writeLogsGzip', None)
             StatsAndLogging.writeLogFiles(jobNames, jobLogList, config)
-            jobLogFile = os.path.join(config.writeLogs, jobNames[0] + '000.log')
+            jobLogFile = os.path.join(config.writeLogs, jobNames[0] + '_000.log')
+            # The log directory should get exactly one file, names after this
+            # easy job name with no replacements needed.
+            self.assertEqual(os.listdir(config.writeLogs), [os.path.basename(jobLogFile)])
             self.assertTrue(os.path.isfile(jobLogFile))
             with open(jobLogFile) as f:
                 self.assertEqual(f.read(), 'string\nbytes\n\nnewline\n')
-            os.remove(jobLogFile)
 
         def testBatchCreate(self):
             """Test creation of many jobs."""
@@ -564,8 +549,7 @@ class AbstractJobStoreTest:
             jobs = []
             with jobstore.batch():
                 for i in range(100):
-                    overlargeJob = JobDescription(command='overlarge',
-                                                  requirements=jobRequirements,
+                    overlargeJob = JobDescription(requirements=jobRequirements,
                                                   jobName='test-overlarge', unitName='onJobStore')
                     jobstore.assign_job_id(overlargeJob)
                     jobstore.create_job(overlargeJob)
@@ -667,7 +651,7 @@ class AbstractJobStoreTest:
                 :param int size: the size of the file to test importing/exporting with
                 """
                 # Prepare test file in other job store
-                self.jobstore_initialized.partSize = cls.mpTestPartSize
+                self.jobstore_initialized.part_size = cls.mpTestPartSize
                 self.jobstore_initialized.moveExports = moveExports
 
                 # Test assumes imports are not linked
@@ -723,7 +707,7 @@ class AbstractJobStoreTest:
                        to import from or export to
                 """
                 # Prepare test file in other job store
-                self.jobstore_initialized.partSize = cls.mpTestPartSize
+                self.jobstore_initialized.part_size = cls.mpTestPartSize
                 other = otherCls('testSharedFiles')
                 store = other._externalStore()
 
@@ -1311,7 +1295,6 @@ class AWSJobStoreTest(AbstractJobStoreTest.Test):
         failed to be created.  We simulate a failed jobstore bucket creation by using a bucket in a
         different region with the same name.
         """
-        from boto.sdb import connect_to_region
         from botocore.exceptions import ClientError
 
         from toil.jobStores.aws.jobStore import BucketLocationConflictException
@@ -1351,13 +1334,16 @@ class AWSJobStoreTest(AbstractJobStoreTest.Test):
             except BucketLocationConflictException:
                 # Catch the expected BucketLocationConflictException and ensure that the bound
                 # domains don't exist in SDB.
-                sdb = connect_to_region(self.awsRegion())
+                sdb = establish_boto3_session().client(region_name=self.awsRegion(), service_name="sdb")
                 next_token = None
                 allDomainNames = []
                 while True:
-                    domains = sdb.get_all_domains(max_domains=100, next_token=next_token)
-                    allDomainNames.extend([x.name for x in domains])
-                    next_token = domains.next_token
+                    if next_token is None:
+                        domains = sdb.list_domains(MaxNumberOfDomains=100)
+                    else:
+                        domains = sdb.list_domains(MaxNumberOfDomains=100, NextToken=next_token)
+                    allDomainNames.extend(domains["DomainNames"])
+                    next_token = domains.get("NextToken")
                     if next_token is None:
                         break
                 self.assertFalse([d for d in allDomainNames if testJobStoreUUID in d])
@@ -1393,8 +1379,7 @@ class AWSJobStoreTest(AbstractJobStoreTest.Test):
     def testOverlargeJob(self):
         jobstore = self.jobstore_initialized
         jobRequirements = dict(memory=12, cores=34, disk=35, preemptible=True)
-        overlargeJob = JobDescription(command='overlarge',
-                                      requirements=jobRequirements,
+        overlargeJob = JobDescription(requirements=jobRequirements,
                                       jobName='test-overlarge', unitName='onJobStore')
 
         # Make the pickled size of the job larger than 256K
@@ -1463,6 +1448,7 @@ class AWSJobStoreTest(AbstractJobStoreTest.Test):
 
     def _hashTestFile(self, url: str) -> str:
         from toil.jobStores.aws.jobStore import AWSJobStore
+        from toil.lib.aws.utils import get_object_for_url
         str(AWSJobStore)  # to prevent removal of that import
         key = get_object_for_url(urlparse.urlparse(url), existing=True)
         contents = key.get().get('Body').read()
@@ -1471,7 +1457,7 @@ class AWSJobStoreTest(AbstractJobStoreTest.Test):
     def _createExternalStore(self):
         """A S3.Bucket instance is returned"""
         from toil.jobStores.aws.jobStore import establish_boto3_session
-        from toil.lib.aws.utils import retry_s3
+        from toil.lib.aws.utils import retry_s3, create_s3_bucket
 
         resource = establish_boto3_session().resource(
             "s3", region_name=self.awsRegion()

toil/test/lib/aws/test_iam.py

@@ -15,7 +15,7 @@ import json
 import logging
 
 import boto3
-from moto import mock_iam
+from moto import mock_aws
 
 from toil.lib.aws import iam
 from toil.test import ToilTest
@@ -46,7 +46,7 @@ class IAMTest(ToilTest):
         assert iam.permission_matches_any("iam:*", ["*"]) is True
         assert iam.permission_matches_any("ec2:*", ['iam:*']) is False
 
-    @mock_iam
+    @mock_aws
     def test_get_policy_permissions(self):
         mock_iam = boto3.client("iam")
 

toil/test/lib/test_ec2.py

@@ -20,7 +20,6 @@ from toil.lib.aws.ami import (aws_marketplace_flatcar_ami_search,
                               feed_flatcar_ami_release,
                               flatcar_release_feed_amis,
                               get_flatcar_ami)
-from toil.lib.aws.session import establish_boto3_session
 from toil.test import ToilTest, needs_aws_ec2, needs_online
 
 logger = logging.getLogger(__name__)
@@ -59,6 +58,7 @@ class FlatcarFeedTest(ToilTest):
 class AMITest(ToilTest):
     @classmethod
     def setUpClass(cls):
+        from toil.lib.aws.session import establish_boto3_session
         session = establish_boto3_session(region_name='us-west-2')
         cls.ec2_client = session.client('ec2')
 

toil/test/options/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (C) 2015-2021 Regents of the University of California
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

toil/test/options/options.py

@@ -0,0 +1,37 @@
+from configargparse import ArgParser
+
+from toil.common import addOptions, Toil
+from toil.test import ToilTest
+
+
+class OptionsTest(ToilTest):
+    """
+    Class to test functionality of all Toil options
+    """
+    def test_default_caching_slurm(self):
+        """
+        Test to ensure that caching will be set to false when running on Slurm
+        :return:
+        """
+        parser = ArgParser()
+        addOptions(parser, jobstore_as_flag=True, wdl=False, cwl=False)
+        test_args = ["--jobstore=example-jobstore", "--batchSystem=slurm"]
+        options = parser.parse_args(test_args)
+        with Toil(options) as toil:
+            caching_value = toil.config.caching
+        self.assertEqual(caching_value, False)
+
+    def test_caching_option_priority(self):
+        """
+        Test to ensure that the --caching option takes priority over the default_caching() return value
+        :return:
+        """
+        parser = ArgParser()
+        addOptions(parser, jobstore_as_flag=True, wdl=False, cwl=False)
+        # the kubernetes batchsystem (and I think all batchsystems including singlemachine) return False
+        # for default_caching
+        test_args = ["--jobstore=example-jobstore", "--batchSystem=kubernetes", "--caching=True"]
+        options = parser.parse_args(test_args)
+        with Toil(options) as toil:
+            caching_value = toil.config.caching
+        self.assertEqual(caching_value, True)

toil/test/provisioners/aws/awsProvisionerTest.py

@@ -19,9 +19,13 @@ import time
 from abc import abstractmethod
 from inspect import getsource
 from textwrap import dedent
+from typing import Optional, List
 from uuid import uuid4
 
+import botocore.exceptions
 import pytest
+from mypy_boto3_ec2 import EC2Client
+from mypy_boto3_ec2.type_defs import EbsInstanceBlockDeviceTypeDef, InstanceTypeDef, InstanceBlockDeviceMappingTypeDef, FilterTypeDef, DescribeVolumesResultTypeDef, VolumeTypeDef
 
 from toil.provisioners import cluster_factory
 from toil.provisioners.aws.awsProvisioner import AWSProvisioner
@@ -113,15 +117,20 @@ class AbstractAWSAutoscaleTest(AbstractClusterTest):
     def rsyncUtil(self, src, dest):
         subprocess.check_call(['toil', 'rsync-cluster', '--insecure', '-p=aws', '-z', self.zone, self.clusterName] + [src, dest])
 
-    def getRootVolID(self):
-        instances = self.cluster._get_nodes_in_cluster()
-        instances.sort(key=lambda x: x.launch_time)
-        leader = instances[0]  # assume leader was launched first
-
-        from boto.ec2.blockdevicemapping import BlockDeviceType
-        rootBlockDevice = leader.block_device_mapping["/dev/xvda"]
-        assert isinstance(rootBlockDevice, BlockDeviceType)
-        return rootBlockDevice.volume_id
+    def getRootVolID(self) -> str:
+        instances: List[InstanceTypeDef] = self.cluster._get_nodes_in_cluster_boto3()
+        instances.sort(key=lambda x: x.get("LaunchTime"))
+        leader: InstanceTypeDef = instances[0]  # assume leader was launched first
+
+        bdm: Optional[List[InstanceBlockDeviceMappingTypeDef]] = leader.get("BlockDeviceMappings")
+        assert bdm is not None
+        root_block_device: Optional[EbsInstanceBlockDeviceTypeDef] = None
+        for device in bdm:
+            if device["DeviceName"] == "/dev/xvda":
+                root_block_device = device["Ebs"]
+        assert root_block_device is not None  # There should be a device named "/dev/xvda"
+        assert root_block_device.get("VolumeId") is not None
+        return root_block_device["VolumeId"]
 
     @abstractmethod
     def _getScript(self):
@@ -191,21 +200,20 @@ class AbstractAWSAutoscaleTest(AbstractClusterTest):
 
         assert len(self.cluster._getRoleNames()) == 1
 
-        from boto.exception import EC2ResponseError
         volumeID = self.getRootVolID()
         self.cluster.destroyCluster()
+        boto3_ec2: EC2Client = self.aws.client(region=self.region, service_name="ec2")
+        volume_filter: FilterTypeDef = {"Name": "volume-id", "Values": [volumeID]}
+        volumes: Optional[List[VolumeTypeDef]] = None
         for attempt in range(6):
             # https://github.com/BD2KGenomics/toil/issues/1567
             # retry this for up to 1 minute until the volume disappears
-            try:
-                self.boto2_ec2.get_all_volumes(volume_ids=[volumeID])
-                time.sleep(10)
-            except EC2ResponseError as e:
-                if e.status == 400 and 'InvalidVolume.NotFound' in e.code:
-                    break
-                else:
-                    raise
-        else:
+            volumes = boto3_ec2.describe_volumes(Filters=[volume_filter])["Volumes"]
+            if len(volumes) == 0:
+                # None are left, so they have been properly deleted
+                break
+            time.sleep(10)
+        if volumes is None or len(volumes) > 0:
             self.fail('Volume with ID %s was not cleaned up properly' % volumeID)
 
         assert len(self.cluster._getRoleNames()) == 0
@@ -246,16 +254,19 @@ class AWSAutoscaleTest(AbstractAWSAutoscaleTest):
         # add arguments to test that we can specify leader storage
         self.createClusterUtil(args=['--leaderStorage', str(self.requestedLeaderStorage)])
 
-    def getRootVolID(self):
+    def getRootVolID(self) -> str:
         """
         Adds in test to check that EBS volume is build with adequate size.
         Otherwise is functionally equivalent to parent.
         :return: volumeID
         """
         volumeID = super().getRootVolID()
-        rootVolume = self.boto2_ec2.get_all_volumes(volume_ids=[volumeID])[0]
+        boto3_ec2: EC2Client = self.aws.client(region=self.region, service_name="ec2")
+        volume_filter: FilterTypeDef = {"Name": "volume-id", "Values": [volumeID]}
+        volumes: DescribeVolumesResultTypeDef = boto3_ec2.describe_volumes(Filters=[volume_filter])
+        root_volume: VolumeTypeDef = volumes["Volumes"][0]  # should be first
         # test that the leader is given adequate storage
-        self.assertGreaterEqual(rootVolume.size, self.requestedLeaderStorage)
+        self.assertGreaterEqual(root_volume["Size"], self.requestedLeaderStorage)
         return volumeID
 
     @integrative
@@ -290,8 +301,6 @@ class AWSStaticAutoscaleTest(AWSAutoscaleTest):
         self.requestedNodeStorage = 20
 
     def launchCluster(self):
-        from boto.ec2.blockdevicemapping import BlockDeviceType
-
         from toil.lib.ec2 import wait_instances_running
         self.createClusterUtil(args=['--leaderStorage', str(self.requestedLeaderStorage),
                                      '--nodeTypes', ",".join(self.instanceTypes),
@@ -303,8 +312,8 @@ class AWSStaticAutoscaleTest(AWSAutoscaleTest):
         # visible to EC2 read requests immediately after the create returns,
         # which is the last thing that starting the cluster does.
         time.sleep(10)
-        nodes = self.cluster._get_nodes_in_cluster()
-        nodes.sort(key=lambda x: x.launch_time)
+        nodes: List[InstanceTypeDef] = self.cluster._get_nodes_in_cluster_boto3()
+        nodes.sort(key=lambda x: x.get("LaunchTime"))
         # assuming that leader is first
         workers = nodes[1:]
         # test that two worker nodes were created
@@ -312,11 +321,22 @@ class AWSStaticAutoscaleTest(AWSAutoscaleTest):
         # test that workers have expected storage size
         # just use the first worker
         worker = workers[0]
-        worker = next(wait_instances_running(self.boto2_ec2, [worker]))
-        rootBlockDevice = worker.block_device_mapping["/dev/xvda"]
-        self.assertTrue(isinstance(rootBlockDevice, BlockDeviceType))
-        rootVolume = self.boto2_ec2.get_all_volumes(volume_ids=[rootBlockDevice.volume_id])[0]
-        self.assertGreaterEqual(rootVolume.size, self.requestedNodeStorage)
+        boto3_ec2: EC2Client = self.aws.client(region=self.region, service_name="ec2")
+
+        worker: InstanceTypeDef = next(wait_instances_running(boto3_ec2, [worker]))
+
+        bdm: Optional[List[InstanceBlockDeviceMappingTypeDef]] = worker.get("BlockDeviceMappings")
+        assert bdm is not None
+        root_block_device: Optional[EbsInstanceBlockDeviceTypeDef] = None
+        for device in bdm:
+            if device["DeviceName"] == "/dev/xvda":
+                root_block_device = device["Ebs"]
+        assert root_block_device is not None
+        assert root_block_device.get("VolumeId") is not None  # TypedDicts cannot have runtime type checks
+
+        volume_filter: FilterTypeDef = {"Name": "volume-id", "Values": [root_block_device["VolumeId"]]}
+        root_volume: VolumeTypeDef = boto3_ec2.describe_volumes(Filters=[volume_filter])["Volumes"][0]  # should be first
+        self.assertGreaterEqual(root_volume.get("Size"), self.requestedNodeStorage)
 
     def _runScript(self, toilOptions):
         # Autoscale even though we have static nodes
@@ -337,9 +357,6 @@ class AWSManagedAutoscaleTest(AWSAutoscaleTest):
         self.requestedNodeStorage = 20
 
     def launchCluster(self):
-        from boto.ec2.blockdevicemapping import BlockDeviceType  # noqa
-
-        from toil.lib.ec2 import wait_instances_running  # noqa
         self.createClusterUtil(args=['--leaderStorage', str(self.requestedLeaderStorage),
                                      '--nodeTypes', ",".join(self.instanceTypes),
                                      '--workers', ",".join([f'0-{c}' for c in self.numWorkers]),