toil 6.1.0__py3-none-any.whl → 7.0.0__py3-none-any.whl

toil/test/cwl/cwlTest.py

@@ -23,15 +23,14 @@ import sys
 import unittest
 import uuid
 import zipfile
+
 from functools import partial
 from io import StringIO
 from pathlib import Path
 from typing import (TYPE_CHECKING,
-                    Any,
                     Callable,
                     Dict,
                     List,
-                    MutableMapping,
                     Optional,
                     cast)
 from unittest.mock import Mock, call
@@ -51,7 +50,6 @@ from toil.cwl.utils import (DirectoryStructure,
                             download_structure,
                             visit_cwl_class_and_reduce,
                             visit_top_cwl_class)
-from toil.exceptions import FailedJobsException
 from toil.fileStores import FileID
 from toil.fileStores.abstractFileStore import AbstractFileStore
 from toil.lib.threading import cpu_count
@@ -74,7 +72,6 @@ from toil.test import (ToilTest,
                        needs_torque,
                        needs_wes_server,
                        slow)
-from toil.test.provisioners.clusterTest import AbstractClusterTest
 
 log = logging.getLogger(__name__)
 CONFORMANCE_TEST_TIMEOUT = 10000
@@ -505,7 +502,7 @@ class CWLWorkflowTest(ToilTest):
             "src/toil/test/cwl/seqtk_seq.cwl",
             "src/toil/test/cwl/seqtk_seq_job.json",
             self._expected_seqtk_output(self.outDir),
-            main_args=["--default-container", "quay.io/biocontainers/seqtk:r93--0"],
+            main_args=["--default-container", "quay.io/biocontainers/seqtk:1.4--he4a0461_1"],
             out_name="output1",
         )
 
@@ -540,7 +537,6 @@ class CWLWorkflowTest(ToilTest):
         """
         log.info("Running CWL Test Restart.  Expecting failure, then success.")
         from toil.cwl import cwltoil
-        from toil.jobStores.abstractJobStore import NoSuchJobStoreException
 
         outDir = self._createTempDir()
         cwlDir = os.path.join(self._projectRootPath(), "src", "toil", "test", "cwl")
@@ -570,18 +566,19 @@ class CWLWorkflowTest(ToilTest):
         # Force a failure by trying to use an incorrect version of `rev` from the PATH
         os.environ["PATH"] = path_with_bogus_rev()
         try:
-            cwltoil.main(cmd)
+            subprocess.check_output(["toil-cwl-runner"] + cmd, env=os.environ.copy(), stderr=subprocess.STDOUT)
             self.fail("Expected problem job with incorrect PATH did not fail")
-        except FailedJobsException:
+        except subprocess.CalledProcessError:
             pass
         # Finish the job with a correct PATH
         os.environ["PATH"] = orig_path
-        cwltoil.main(["--restart"] + cmd)
+        cmd.insert(0, "--restart")
+        cwltoil.main(cmd)
         # Should fail because previous job completed successfully
         try:
-            cwltoil.main(["--restart"] + cmd)
+            subprocess.check_output(["toil-cwl-runner"] + cmd, env=os.environ.copy(), stderr=subprocess.STDOUT)
             self.fail("Restart with missing directory did not fail")
-        except NoSuchJobStoreException:
+        except subprocess.CalledProcessError:
             pass
 
     @needs_aws_s3
@@ -1168,86 +1165,6 @@ class CWLv12Test(ToilTest):
         )
 
 
-@needs_aws_ec2
-@needs_fetchable_appliance
-@slow
-class CWLOnARMTest(AbstractClusterTest):
-    """
-    Run the CWL 1.2 conformance tests on ARM specifically.
-    """
-
-    def __init__(self, methodName: str) -> None:
-        super().__init__(methodName=methodName)
-        self.clusterName = "cwl-test-" + str(uuid.uuid4())
-        self.leaderNodeType = "t4g.2xlarge"
-        self.clusterType = "kubernetes"
-        # We need to be running in a directory which Flatcar and the Toil Appliance both have
-        self.cwl_test_dir = "/tmp/toil/cwlTests"
-
-    def setUp(self) -> None:
-        super().setUp()
-        self.jobStore = f"aws:{self.awsRegion()}:cluster-{uuid.uuid4()}"
-
-    @needs_env_var("CI_COMMIT_SHA", "a git commit sha")
-    def test_cwl_on_arm(self) -> None:
-        # Make a cluster
-        self.launchCluster()
-        # get the leader so we know the IP address - we don't need to wait since create cluster
-        # already ensures the leader is running
-        self.cluster = cluster_factory(
-            provisioner="aws", zone=self.zone, clusterName=self.clusterName
-        )
-        self.leader = self.cluster.getLeader()
-
-        commit = os.environ["CI_COMMIT_SHA"]
-        self.sshUtil(
-            [
-                "bash",
-                "-c",
-                f"mkdir -p {self.cwl_test_dir} && cd {self.cwl_test_dir} && git clone https://github.com/DataBiosphere/toil.git",
-            ]
-        )
-
-        # We use CI_COMMIT_SHA to retrieve the Toil version needed to run the CWL tests
-        self.sshUtil(
-            ["bash", "-c", f"cd {self.cwl_test_dir}/toil && git checkout {commit}"]
-        )
-
-        # --never-download prevents silent upgrades to pip, wheel and setuptools
-        self.sshUtil(
-            [
-                "bash",
-                "-c",
-                f"virtualenv --system-site-packages --never-download {self.venvDir}",
-            ]
-        )
-        self.sshUtil(
-            [
-                "bash",
-                "-c",
-                f". .{self.venvDir}/bin/activate && cd {self.cwl_test_dir}/toil && make prepare && make develop extras=[all]",
-            ]
-        )
-
-        # Runs the CWLv12Test on an ARM instance
-        self.sshUtil(
-            [
-                "bash",
-                "-c",
-                f". .{self.venvDir}/bin/activate && cd {self.cwl_test_dir}/toil && pytest --log-cli-level DEBUG -r s src/toil/test/cwl/cwlTest.py::CWLv12Test::test_run_conformance",
-            ]
-        )
-
-        # We know if it succeeds it should save a junit XML for us to read.
-        # Bring it back to be an artifact.
-        self.rsync_util(
-            f":{self.cwl_test_dir}/toil/conformance-1.2.junit.xml",
-            os.path.join(
-                self._projectRootPath(),
-                "arm-conformance-1.2.junit.xml"
-            )
-        )
-
 @needs_cwl
 @pytest.mark.cwl_small_log_dir
 def test_workflow_echo_string_scatter_stderr_log_dir(tmp_path: Path) -> None:

toil/test/cwl/seqtk_seq.cwl

@@ -21,4 +21,4 @@ hints:
     packages:
     - package: seqtk
       version:
-      - r93
+      - '1.4'

toil/test/docs/scriptsTest.py

@@ -4,6 +4,9 @@ import shutil
 import subprocess
 import sys
 import unittest
+import pytest
+
+from typing import List
 
 pkg_root = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))  # noqa
 sys.path.insert(0, pkg_root)  # noqa
@@ -21,12 +24,6 @@ class ToilDocumentationTest(ToilTest):
 
     def tearDown(self) -> None:
         super(ToilTest, self).tearDown()
-        # src/toil/test/docs/scripts/cwlExampleFiles/sample_1_output.txt
-        output_files = ["sample_1_output.txt", "sample_2_output.txt", "sample_3_output.txt"]
-        for output in output_files:
-            output_file = os.path.join(self.directory, 'scripts/cwlExampleFiles', output)
-            if os.path.exists(output_file):
-                os.remove(output_file)
 
         jobstores = ['/mnt/ephemeral/workspace/toil-pull-requests/toilWorkflowRun']
         for jobstore in jobstores:
@@ -36,9 +33,9 @@ class ToilDocumentationTest(ToilTest):
         unittest.TestCase.tearDown(self)
 
     """Just check the exit code"""
-    def checkExitCode(self, script):
+    def checkExitCode(self, script, extra_args: List[str] = []):
         program = os.path.join(self.directory, "scripts", script)
-        process = subprocess.Popen([python, program, "file:my-jobstore", "--clean=always"],
+        process = subprocess.Popen([python, program, "file:my-jobstore", "--clean=always"] + extra_args,
                                    stdout=subprocess.PIPE, stderr=subprocess.PIPE)
         stdout, stderr = process.communicate()
         if isinstance(stdout, bytes):
@@ -65,13 +62,11 @@ class ToilDocumentationTest(ToilTest):
         n = re.search(pattern, outerr)
         self.assertNotEqual(n, None, f"Pattern:\n{expectedPattern}\nOutput:\n{outerr}")
 
-    @needs_cwl
-    def testCwlexample(self):
-        self.checkExitCode("tutorial_cwlexample.py")
-
     def testStats(self):
-        self.checkExitCode("tutorial_stats.py")
+        # This script asks for 4 cores but we might need to run the tests in only 3.
+        self.checkExitCode("tutorial_stats.py", ["--scale=0.5"])
 
+    @pytest.mark.timeout(1200)
     def testDynamic(self):
         self.checkExitCode("tutorial_dynamic.py")
 
@@ -117,6 +112,7 @@ class ToilDocumentationTest(ToilTest):
                                   "second or third.*Hello world, I have a message: second or third.*Hello world,"
                                   " I have a message: last")
 
+    @pytest.mark.timeout(1200)
     def testPromises2(self):
         self.checkExpectedOut("tutorial_promises2.py",
                               "['00000', '00001', '00010', '00011', '00100', '00101', '00110', '00111',"
@@ -146,5 +142,6 @@ class ToilDocumentationTest(ToilTest):
     def testStaging(self):
         self.checkExitCode("tutorial_staging.py")
 
+
 if __name__ == "__main__":
     unittest.main()

toil/test/jobStores/jobStoreTest.py

@@ -129,8 +129,7 @@ class AbstractJobStoreTest:
             # Requirements for jobs to be created.
             self.arbitraryRequirements = {'memory': 1, 'disk': 2, 'cores': 1, 'preemptible': False}
             # Function to make an arbitrary new job
-            self.arbitraryJob = lambda: JobDescription(command='command',
-                                                       jobName='arbitrary',
+            self.arbitraryJob = lambda: JobDescription(jobName='arbitrary',
                                                        requirements=self.arbitraryRequirements)
 
             self.parentJobReqs = dict(memory=12, cores=34, disk=35, preemptible=True)
@@ -143,9 +142,9 @@ class AbstractJobStoreTest:
             super().tearDown()
 
         def testInitialState(self):
-            """Ensure proper handling of nonexistant files."""
-            self.assertFalse(self.jobstore_initialized.job_exists('nonexistantFile'))
-            self.assertRaises(NoSuchJobException, self.jobstore_initialized.load_job, 'nonexistantFile')
+            """Ensure proper handling of nonexistent files."""
+            self.assertFalse(self.jobstore_initialized.job_exists('nonexistentFile'))
+            self.assertRaises(NoSuchJobException, self.jobstore_initialized.load_job, 'nonexistentFile')
 
         def testJobCreation(self):
             """
@@ -158,8 +157,7 @@ class AbstractJobStoreTest:
             jobstore = self.jobstore_initialized
 
             # Create a job and verify its existence/properties
-            job = JobDescription(command='parent1',
-                                 requirements=self.parentJobReqs,
+            job = JobDescription(requirements=self.parentJobReqs,
                                  jobName='test1', unitName='onParent')
             self.assertTrue(isinstance(job.jobStoreID, TemporaryID))
             jobstore.assign_job_id(job)
@@ -169,7 +167,6 @@ class AbstractJobStoreTest:
             self.assertEqual(created, job)
 
             self.assertTrue(jobstore.job_exists(job.jobStoreID))
-            self.assertEqual(job.command, 'parent1')
             self.assertEqual(job.memory, self.parentJobReqs['memory'])
             self.assertEqual(job.cores, self.parentJobReqs['cores'])
             self.assertEqual(job.disk, self.parentJobReqs['disk'])
@@ -194,8 +191,7 @@ class AbstractJobStoreTest:
             """Tests that a job created via one JobStore instance can be loaded from another."""
 
             # Create a job on the first jobstore.
-            jobDesc1 = JobDescription(command='jobstore1',
-                                      requirements=self.parentJobReqs,
+            jobDesc1 = JobDescription(requirements=self.parentJobReqs,
                                       jobName='test1', unitName='onJS1')
             self.jobstore_initialized.assign_job_id(jobDesc1)
             self.jobstore_initialized.create_job(jobDesc1)
@@ -203,16 +199,14 @@ class AbstractJobStoreTest:
             # Load it from the second jobstore
             jobDesc2 = self.jobstore_resumed_noconfig.load_job(jobDesc1.jobStoreID)
 
-            self.assertEqual(jobDesc1.command, jobDesc2.command)
+            self.assertEqual(jobDesc1._body, jobDesc2._body)
 
         def testChildLoadingEquality(self):
             """Test that loading a child job operates as expected."""
-            job = JobDescription(command='parent1',
-                                 requirements=self.parentJobReqs,
+            job = JobDescription(requirements=self.parentJobReqs,
                                  jobName='test1', unitName='onParent')
 
-            childJob = JobDescription(command='child1',
-                                      requirements=self.childJobReqs1,
+            childJob = JobDescription(requirements=self.childJobReqs1,
                                       jobName='test2', unitName='onChild1')
             self.jobstore_initialized.assign_job_id(job)
             self.jobstore_initialized.assign_job_id(childJob)
@@ -221,7 +215,7 @@ class AbstractJobStoreTest:
             job.addChild(childJob.jobStoreID)
             self.jobstore_initialized.update_job(job)
 
-            self.assertEqual(self.jobstore_initialized.load_job(list(job.allSuccessors())[0]).command, childJob.command)
+            self.assertEqual(self.jobstore_initialized.load_job(list(job.allSuccessors())[0])._body, childJob._body)
 
         def testPersistantFilesToDelete(self):
             """
@@ -236,8 +230,7 @@ class AbstractJobStoreTest:
             """
 
             # Create a job.
-            job = JobDescription(command='job1',
-                                 requirements=self.parentJobReqs,
+            job = JobDescription(requirements=self.parentJobReqs,
                                  jobName='test1', unitName='onJS1')
 
             self.jobstore_initialized.assign_job_id(job)
@@ -251,16 +244,13 @@ class AbstractJobStoreTest:
             jobstore1 = self.jobstore_initialized
             jobstore2 = self.jobstore_resumed_noconfig
 
-            job1 = JobDescription(command='parent1',
-                                 requirements=self.parentJobReqs,
+            job1 = JobDescription(requirements=self.parentJobReqs,
                                  jobName='test1', unitName='onParent')
 
-            childJob1 = JobDescription(command='child1',
-                                      requirements=self.childJobReqs1,
+            childJob1 = JobDescription(requirements=self.childJobReqs1,
                                       jobName='test2', unitName='onChild1')
 
-            childJob2 = JobDescription(command='child2',
-                                      requirements=self.childJobReqs2,
+            childJob2 = JobDescription(requirements=self.childJobReqs2,
                                       jobName='test3', unitName='onChild2')
 
             jobstore1.assign_job_id(job1)
@@ -297,20 +287,17 @@ class AbstractJobStoreTest:
             """Tests the consequences of deleting jobs."""
             # A local jobstore object for testing.
             jobstore = self.jobstore_initialized
-            job = JobDescription(command='job1',
-                                 requirements=self.parentJobReqs,
+            job = JobDescription(requirements=self.parentJobReqs,
                                  jobName='test1', unitName='onJob')
             # Create job
             jobstore.assign_job_id(job)
             jobstore.create_job(job)
 
             # Create child Jobs
-            child1 = JobDescription(command='child1',
-                                    requirements=self.childJobReqs1,
+            child1 = JobDescription(requirements=self.childJobReqs1,
                                     jobName='test2', unitName='onChild1')
 
-            child2 = JobDescription(command='job1',
-                                    requirements=self.childJobReqs2,
+            child2 = JobDescription(requirements=self.childJobReqs2,
                                     jobName='test3', unitName='onChild2')
 
             # Add children to parent.
@@ -322,9 +309,8 @@ class AbstractJobStoreTest:
             job.addChild(child2.jobStoreID)
             jobstore.update_job(job)
 
-            # Get it ready to run children
-            job.command = None
-            jobstore.update_job(job)
+            # Parent must have no body to start on children
+            assert not job.has_body()
 
             # Go get the children
             childJobs = [jobstore.load_job(childID) for childID in job.nextSuccessors()]
@@ -421,8 +407,7 @@ class AbstractJobStoreTest:
             jobstore2 = self.jobstore_resumed_noconfig
 
             # Create jobNodeOnJS1
-            jobOnJobStore1 = JobDescription(command='job1',
-                                            requirements=self.parentJobReqs,
+            jobOnJobStore1 = JobDescription(requirements=self.parentJobReqs,
                                             jobName='test1', unitName='onJobStore1')
 
             # First recreate job
@@ -489,12 +474,11 @@ class AbstractJobStoreTest:
                     pass
 
         def testStatsAndLogging(self):
-            """Tests behavior of reading and writting stats and logging."""
+            """Tests behavior of reading and writing stats and logging."""
             jobstore1 = self.jobstore_initialized
             jobstore2 = self.jobstore_resumed_noconfig
 
-            jobOnJobStore1 = JobDescription(command='job1',
-                                            requirements=self.parentJobReqs,
+            jobOnJobStore1 = JobDescription(requirements=self.parentJobReqs,
                                             jobName='test1', unitName='onJobStore1')
 
             jobstore1.assign_job_id(jobOnJobStore1)
@@ -565,8 +549,7 @@ class AbstractJobStoreTest:
             jobs = []
             with jobstore.batch():
                 for i in range(100):
-                    overlargeJob = JobDescription(command='overlarge',
-                                                  requirements=jobRequirements,
+                    overlargeJob = JobDescription(requirements=jobRequirements,
                                                   jobName='test-overlarge', unitName='onJobStore')
                     jobstore.assign_job_id(overlargeJob)
                     jobstore.create_job(overlargeJob)
@@ -668,7 +651,7 @@ class AbstractJobStoreTest:
                 :param int size: the size of the file to test importing/exporting with
                 """
                 # Prepare test file in other job store
-                self.jobstore_initialized.partSize = cls.mpTestPartSize
+                self.jobstore_initialized.part_size = cls.mpTestPartSize
                 self.jobstore_initialized.moveExports = moveExports
 
                 # Test assumes imports are not linked
@@ -724,7 +707,7 @@ class AbstractJobStoreTest:
                        to import from or export to
                 """
                 # Prepare test file in other job store
-                self.jobstore_initialized.partSize = cls.mpTestPartSize
+                self.jobstore_initialized.part_size = cls.mpTestPartSize
                 other = otherCls('testSharedFiles')
                 store = other._externalStore()
 
@@ -1312,7 +1295,6 @@ class AWSJobStoreTest(AbstractJobStoreTest.Test):
         failed to be created.  We simulate a failed jobstore bucket creation by using a bucket in a
         different region with the same name.
         """
-        from boto.sdb import connect_to_region
         from botocore.exceptions import ClientError
 
         from toil.jobStores.aws.jobStore import BucketLocationConflictException
@@ -1352,13 +1334,16 @@ class AWSJobStoreTest(AbstractJobStoreTest.Test):
             except BucketLocationConflictException:
                 # Catch the expected BucketLocationConflictException and ensure that the bound
                 # domains don't exist in SDB.
-                sdb = connect_to_region(self.awsRegion())
+                sdb = establish_boto3_session().client(region_name=self.awsRegion(), service_name="sdb")
                 next_token = None
                 allDomainNames = []
                 while True:
-                    domains = sdb.get_all_domains(max_domains=100, next_token=next_token)
-                    allDomainNames.extend([x.name for x in domains])
-                    next_token = domains.next_token
+                    if next_token is None:
+                        domains = sdb.list_domains(MaxNumberOfDomains=100)
+                    else:
+                        domains = sdb.list_domains(MaxNumberOfDomains=100, NextToken=next_token)
+                    allDomainNames.extend(domains["DomainNames"])
+                    next_token = domains.get("NextToken")
                     if next_token is None:
                         break
                 self.assertFalse([d for d in allDomainNames if testJobStoreUUID in d])
@@ -1394,8 +1379,7 @@ class AWSJobStoreTest(AbstractJobStoreTest.Test):
     def testOverlargeJob(self):
         jobstore = self.jobstore_initialized
         jobRequirements = dict(memory=12, cores=34, disk=35, preemptible=True)
-        overlargeJob = JobDescription(command='overlarge',
-                                      requirements=jobRequirements,
+        overlargeJob = JobDescription(requirements=jobRequirements,
                                       jobName='test-overlarge', unitName='onJobStore')
 
         # Make the pickled size of the job larger than 256K

toil/test/lib/aws/test_iam.py

@@ -15,7 +15,7 @@ import json
 import logging
 
 import boto3
-from moto import mock_iam
+from moto import mock_aws
 
 from toil.lib.aws import iam
 from toil.test import ToilTest
@@ -46,7 +46,7 @@ class IAMTest(ToilTest):
         assert iam.permission_matches_any("iam:*", ["*"]) is True
         assert iam.permission_matches_any("ec2:*", ['iam:*']) is False
 
-    @mock_iam
+    @mock_aws
     def test_get_policy_permissions(self):
         mock_iam = boto3.client("iam")
 

toil/test/provisioners/aws/awsProvisionerTest.py

@@ -19,9 +19,13 @@ import time
 from abc import abstractmethod
 from inspect import getsource
 from textwrap import dedent
+from typing import Optional, List
 from uuid import uuid4
 
+import botocore.exceptions
 import pytest
+from mypy_boto3_ec2 import EC2Client
+from mypy_boto3_ec2.type_defs import EbsInstanceBlockDeviceTypeDef, InstanceTypeDef, InstanceBlockDeviceMappingTypeDef, FilterTypeDef, DescribeVolumesResultTypeDef, VolumeTypeDef
 
 from toil.provisioners import cluster_factory
 from toil.provisioners.aws.awsProvisioner import AWSProvisioner
@@ -113,15 +117,20 @@ class AbstractAWSAutoscaleTest(AbstractClusterTest):
     def rsyncUtil(self, src, dest):
         subprocess.check_call(['toil', 'rsync-cluster', '--insecure', '-p=aws', '-z', self.zone, self.clusterName] + [src, dest])
 
-    def getRootVolID(self):
-        instances = self.cluster._get_nodes_in_cluster()
-        instances.sort(key=lambda x: x.launch_time)
-        leader = instances[0]  # assume leader was launched first
-
-        from boto.ec2.blockdevicemapping import BlockDeviceType
-        rootBlockDevice = leader.block_device_mapping["/dev/xvda"]
-        assert isinstance(rootBlockDevice, BlockDeviceType)
-        return rootBlockDevice.volume_id
+    def getRootVolID(self) -> str:
+        instances: List[InstanceTypeDef] = self.cluster._get_nodes_in_cluster_boto3()
+        instances.sort(key=lambda x: x.get("LaunchTime"))
+        leader: InstanceTypeDef = instances[0]  # assume leader was launched first
+
+        bdm: Optional[List[InstanceBlockDeviceMappingTypeDef]] = leader.get("BlockDeviceMappings")
+        assert bdm is not None
+        root_block_device: Optional[EbsInstanceBlockDeviceTypeDef] = None
+        for device in bdm:
+            if device["DeviceName"] == "/dev/xvda":
+                root_block_device = device["Ebs"]
+        assert root_block_device is not None  # There should be a device named "/dev/xvda"
+        assert root_block_device.get("VolumeId") is not None
+        return root_block_device["VolumeId"]
 
     @abstractmethod
     def _getScript(self):
@@ -191,21 +200,20 @@ class AbstractAWSAutoscaleTest(AbstractClusterTest):
 
         assert len(self.cluster._getRoleNames()) == 1
 
-        from boto.exception import EC2ResponseError
         volumeID = self.getRootVolID()
         self.cluster.destroyCluster()
+        boto3_ec2: EC2Client = self.aws.client(region=self.region, service_name="ec2")
+        volume_filter: FilterTypeDef = {"Name": "volume-id", "Values": [volumeID]}
+        volumes: Optional[List[VolumeTypeDef]] = None
         for attempt in range(6):
             # https://github.com/BD2KGenomics/toil/issues/1567
             # retry this for up to 1 minute until the volume disappears
-            try:
-                self.boto2_ec2.get_all_volumes(volume_ids=[volumeID])
-                time.sleep(10)
-            except EC2ResponseError as e:
-                if e.status == 400 and 'InvalidVolume.NotFound' in e.code:
-                    break
-                else:
-                    raise
-        else:
+            volumes = boto3_ec2.describe_volumes(Filters=[volume_filter])["Volumes"]
+            if len(volumes) == 0:
+                # None are left, so they have been properly deleted
+                break
+            time.sleep(10)
+        if volumes is None or len(volumes) > 0:
             self.fail('Volume with ID %s was not cleaned up properly' % volumeID)
 
         assert len(self.cluster._getRoleNames()) == 0
@@ -246,16 +254,19 @@ class AWSAutoscaleTest(AbstractAWSAutoscaleTest):
         # add arguments to test that we can specify leader storage
         self.createClusterUtil(args=['--leaderStorage', str(self.requestedLeaderStorage)])
 
-    def getRootVolID(self):
+    def getRootVolID(self) -> str:
         """
         Adds in test to check that EBS volume is build with adequate size.
         Otherwise is functionally equivalent to parent.
         :return: volumeID
         """
         volumeID = super().getRootVolID()
-        rootVolume = self.boto2_ec2.get_all_volumes(volume_ids=[volumeID])[0]
+        boto3_ec2: EC2Client = self.aws.client(region=self.region, service_name="ec2")
+        volume_filter: FilterTypeDef = {"Name": "volume-id", "Values": [volumeID]}
+        volumes: DescribeVolumesResultTypeDef = boto3_ec2.describe_volumes(Filters=[volume_filter])
+        root_volume: VolumeTypeDef = volumes["Volumes"][0]  # should be first
         # test that the leader is given adequate storage
-        self.assertGreaterEqual(rootVolume.size, self.requestedLeaderStorage)
+        self.assertGreaterEqual(root_volume["Size"], self.requestedLeaderStorage)
         return volumeID
 
     @integrative
@@ -290,8 +301,6 @@ class AWSStaticAutoscaleTest(AWSAutoscaleTest):
         self.requestedNodeStorage = 20
 
     def launchCluster(self):
-        from boto.ec2.blockdevicemapping import BlockDeviceType
-
         from toil.lib.ec2 import wait_instances_running
         self.createClusterUtil(args=['--leaderStorage', str(self.requestedLeaderStorage),
                                      '--nodeTypes', ",".join(self.instanceTypes),
@@ -303,8 +312,8 @@ class AWSStaticAutoscaleTest(AWSAutoscaleTest):
         # visible to EC2 read requests immediately after the create returns,
         # which is the last thing that starting the cluster does.
         time.sleep(10)
-        nodes = self.cluster._get_nodes_in_cluster()
-        nodes.sort(key=lambda x: x.launch_time)
+        nodes: List[InstanceTypeDef] = self.cluster._get_nodes_in_cluster_boto3()
+        nodes.sort(key=lambda x: x.get("LaunchTime"))
         # assuming that leader is first
         workers = nodes[1:]
         # test that two worker nodes were created
@@ -312,11 +321,22 @@ class AWSStaticAutoscaleTest(AWSAutoscaleTest):
         # test that workers have expected storage size
         # just use the first worker
         worker = workers[0]
-        worker = next(wait_instances_running(self.boto2_ec2, [worker]))
-        rootBlockDevice = worker.block_device_mapping["/dev/xvda"]
-        self.assertTrue(isinstance(rootBlockDevice, BlockDeviceType))
-        rootVolume = self.boto2_ec2.get_all_volumes(volume_ids=[rootBlockDevice.volume_id])[0]
-        self.assertGreaterEqual(rootVolume.size, self.requestedNodeStorage)
+        boto3_ec2: EC2Client = self.aws.client(region=self.region, service_name="ec2")
+
+        worker: InstanceTypeDef = next(wait_instances_running(boto3_ec2, [worker]))
+
+        bdm: Optional[List[InstanceBlockDeviceMappingTypeDef]] = worker.get("BlockDeviceMappings")
+        assert bdm is not None
+        root_block_device: Optional[EbsInstanceBlockDeviceTypeDef] = None
+        for device in bdm:
+            if device["DeviceName"] == "/dev/xvda":
+                root_block_device = device["Ebs"]
+        assert root_block_device is not None
+        assert root_block_device.get("VolumeId") is not None  # TypedDicts cannot have runtime type checks
+
+        volume_filter: FilterTypeDef = {"Name": "volume-id", "Values": [root_block_device["VolumeId"]]}
+        root_volume: VolumeTypeDef = boto3_ec2.describe_volumes(Filters=[volume_filter])["Volumes"][0]  # should be first
+        self.assertGreaterEqual(root_volume.get("Size"), self.requestedNodeStorage)
 
     def _runScript(self, toilOptions):
         # Autoscale even though we have static nodes
@@ -337,9 +357,6 @@ class AWSManagedAutoscaleTest(AWSAutoscaleTest):
         self.requestedNodeStorage = 20
 
     def launchCluster(self):
-        from boto.ec2.blockdevicemapping import BlockDeviceType  # noqa
-
-        from toil.lib.ec2 import wait_instances_running  # noqa
         self.createClusterUtil(args=['--leaderStorage', str(self.requestedLeaderStorage),
                                      '--nodeTypes', ",".join(self.instanceTypes),
                                      '--workers', ",".join([f'0-{c}' for c in self.numWorkers]),