toil 9.0.0__py3-none-any.whl → 9.1.1__py3-none-any.whl

toil/test/lib/aws/test_iam.py

@@ -28,9 +28,9 @@ logging.basicConfig(level=logging.DEBUG)
 class IAMTest(ToilTest):
     """Check that given permissions and associated functions perform correctly"""
 
-    def test_permissions_iam(self):
+    def test_permissions_iam(self) -> None:
         granted_perms = {
-            "*": {"Action": ["ec2:*", "iam:*", "s3:*", "sdb:*"], "NotAction": []}
+            "*": {"Action": ["ec2:*", "iam:*", "s3:*"], "NotAction": []}
         }
         assert (
             iam.policy_permissions_allow(
@@ -46,8 +46,8 @@ class IAMTest(ToilTest):
             is True
         )
 
-    def test_negative_permissions_iam(self):
-        granted_perms = {"*": {"Action": ["ec2:*", "s3:*", "sdb:*"], "NotAction": []}}
+    def test_negative_permissions_iam(self) -> None:
+        granted_perms = {"*": {"Action": ["ec2:*", "s3:*"], "NotAction": []}}
         assert (
             iam.policy_permissions_allow(
                 granted_perms, iam.CLUSTER_LAUNCHING_PERMISSIONS
@@ -62,7 +62,7 @@ class IAMTest(ToilTest):
             is False
         )
 
-    def test_wildcard_handling(self):
+    def test_wildcard_handling(self) -> None:
         assert iam.permission_matches_any("iam:CreateRole", ["iam:Create**"]) is True
         assert iam.permission_matches_any("iam:GetUser", ["iam:???????"]) is True
         assert iam.permission_matches_any("iam:ListRoleTags", ["iam:*?*Tags"]) is True
@@ -71,7 +71,7 @@ class IAMTest(ToilTest):
 
     @mock_aws
     @needs_aws_s3  # mock is incomplete, this avoid 'botocore.exceptions.NoCredentialsError: Unable to locate credentials'
-    def test_get_policy_permissions(self):
+    def test_get_policy_permissions(self) -> None:
         mock_iam = boto3.client("iam")
 
         # username that moto pretends we have from client.get_user()
@@ -167,7 +167,7 @@ class IAMTest(ToilTest):
         assert notactions_set == set()
 
     @needs_aws_s3
-    def test_create_delete_iam_role(self):
+    def test_create_delete_iam_role(self) -> None:
         region = "us-west-2"
         role_name = f'test{str(uuid4()).replace("-", "")}'
         with self.subTest("Create role w/policies."):

toil/test/lib/aws/test_s3.py

@@ -18,67 +18,64 @@ from typing import TYPE_CHECKING, Optional
 
 from toil.jobStores.aws.jobStore import AWSJobStore
 from toil.lib.aws.session import establish_boto3_session
-from toil.lib.aws.utils import create_s3_bucket, get_bucket_region
+from toil.lib.aws.utils import create_s3_bucket, delete_s3_bucket, get_bucket_region
 from toil.test import ToilTest, needs_aws_s3
 
 logger = logging.getLogger(__name__)
 logging.basicConfig(level=logging.DEBUG)
 
+if TYPE_CHECKING:
+    from mypy_boto3_s3 import S3ServiceResource
+    from mypy_boto3_s3.service_resource import Bucket
 
 @needs_aws_s3
 class S3Test(ToilTest):
     """Confirm the workarounds for us-east-1."""
 
-    if TYPE_CHECKING:
-        from mypy_boto3_s3 import S3ServiceResource
-        from mypy_boto3_s3.service_resource import Bucket
-
     s3_resource: Optional["S3ServiceResource"]
-    bucket: Optional["Bucket"]
 
     @classmethod
     def setUpClass(cls) -> None:
         super().setUpClass()
         session = establish_boto3_session(region_name="us-east-1")
         cls.s3_resource = session.resource("s3", region_name="us-east-1")
-        cls.bucket = None
 
     def test_create_bucket(self) -> None:
         """Test bucket creation for us-east-1."""
         bucket_name = f"toil-s3test-{uuid.uuid4()}"
         assert self.s3_resource
-        S3Test.bucket = create_s3_bucket(self.s3_resource, bucket_name, "us-east-1")
-        S3Test.bucket.wait_until_exists()
-        owner_tag = os.environ.get("TOIL_OWNER_TAG")
-        if owner_tag:
-            bucket_tagging = self.s3_resource.BucketTagging(bucket_name)
-            bucket_tagging.put(
-                Tagging={"TagSet": [{"Key": "Owner", "Value": owner_tag}]}
-            )
-        self.assertEqual(get_bucket_region(bucket_name), "us-east-1")
+        bucket: Optional["Bucket"] = None
+        try:
+            bucket = create_s3_bucket(self.s3_resource, bucket_name, "us-east-1")
+            bucket.wait_until_exists()
+            owner_tag = os.environ.get("TOIL_OWNER_TAG")
+            if owner_tag:
+                bucket_tagging = self.s3_resource.BucketTagging(bucket_name)
+                bucket_tagging.put(
+                    Tagging={"TagSet": [{"Key": "Owner", "Value": owner_tag}]}
+                )
+            self.assertEqual(get_bucket_region(bucket_name), "us-east-1")
 
-        # Make sure all the bucket location getting strategies work on a bucket we created
-        self.assertEqual(
-            get_bucket_region(bucket_name, only_strategies={1}), "us-east-1"
-        )
-        self.assertEqual(
-            get_bucket_region(bucket_name, only_strategies={2}), "us-east-1"
-        )
-        self.assertEqual(
-            get_bucket_region(bucket_name, only_strategies={3}), "us-east-1"
-        )
+            # Make sure all the bucket location getting strategies work on a bucket we created
+            self.assertEqual(
+                get_bucket_region(bucket_name, only_strategies={1}), "us-east-1"
+            )
+            self.assertEqual(
+                get_bucket_region(bucket_name, only_strategies={2}), "us-east-1"
+            )
+            self.assertEqual(
+                get_bucket_region(bucket_name, only_strategies={3}), "us-east-1"
+            )
+        finally:
+            # Clean up the bucket if we managed to make it
+            if bucket is not None:
+                delete_s3_bucket(self.s3_resource, bucket_name)
 
     def test_get_bucket_location_public_bucket(self) -> None:
         """
-        Test getting buket location for a bucket we don't own.
+        Test getting bucket location for a bucket we don't own.
         """
 
         bucket_name = "spacenet-dataset"
         # This bucket happens to live in us-east-1
         self.assertEqual(get_bucket_region(bucket_name), "us-east-1")
-
-    @classmethod
-    def tearDownClass(cls) -> None:
-        if cls.bucket:
-            AWSJobStore._delete_bucket(cls.bucket)
-        super().tearDownClass()

toil/test/lib/aws/test_utils.py

@@ -27,35 +27,35 @@ class TagGenerationTest(ToilTest):
     Test for tag generation from environment variables
     """
 
-    def test_build_tag(self):
+    def test_build_tag(self) -> None:
         environment = dict()
         environment["TOIL_OWNER_TAG"] = "😀"
-        environment["TOIL_AWS_TAGS"] = None
+        environment["TOIL_AWS_TAGS"] = ""
         tag_dict = build_tag_dict_from_env(environment)
         assert tag_dict == {"Owner": "😀"}
 
-    def test_empty_aws_tags(self):
+    def test_empty_aws_tags(self) -> None:
         environment = dict()
-        environment["TOIL_OWNER_TAG"] = None
+        environment["TOIL_OWNER_TAG"] = ""
         environment["TOIL_AWS_TAGS"] = "{}"
         tag_dict = build_tag_dict_from_env(environment)
         assert tag_dict == dict()
 
-    def test_incorrect_json_object(self):
+    def test_incorrect_json_object(self) -> None:
         with pytest.raises(SystemExit):
             environment = dict()
-            environment["TOIL_OWNER_TAG"] = None
+            environment["TOIL_OWNER_TAG"] = ""
             environment["TOIL_AWS_TAGS"] = "231"
             tag_dict = build_tag_dict_from_env(environment)
 
-    def test_incorrect_json_emoji(self):
+    def test_incorrect_json_emoji(self) -> None:
         with pytest.raises(SystemExit):
             environment = dict()
-            environment["TOIL_OWNER_TAG"] = None
+            environment["TOIL_OWNER_TAG"] = ""
             environment["TOIL_AWS_TAGS"] = "😀"
             tag_dict = build_tag_dict_from_env(environment)
 
-    def test_build_tag_with_tags(self):
+    def test_build_tag_with_tags(self) -> None:
         environment = dict()
         environment["TOIL_OWNER_TAG"] = "😀"
         environment["TOIL_AWS_TAGS"] = '{"1": "2", " ":")"}'

toil/test/provisioners/aws/awsProvisionerTest.py

@@ -11,6 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import datetime
 import logging
 import os
 import subprocess
@@ -24,13 +25,22 @@ from uuid import uuid4
 
 import pytest
 
+import toil.lib.aws.session
+
+from toil.lib.aws import zone_to_region
 from toil.provisioners import cluster_factory
 from toil.provisioners.aws.awsProvisioner import AWSProvisioner
+from toil.provisioners.aws import (
+    _get_spot_history,
+    get_aws_zone_from_spot_market,
+    get_best_aws_zone,
+)
 from toil.test import (
     ToilTest,
     get_data,
     integrative,
     needs_aws_ec2,
+    needs_aws_s3,
     needs_fetchable_appliance,
     needs_mesos,
     slow,
@@ -53,10 +63,35 @@ if TYPE_CHECKING:
 
 log = logging.getLogger(__name__)
 
+@pytest.fixture
+def aws_zone():
+    """
+    Supply an appropriate AWS zone to work in to tests that need one.
+    """
+    zone = get_best_aws_zone()
+    assert (
+        zone is not None
+    ), "Could not determine AWS availability zone to test in; is TOIL_AWS_ZONE set?"
+    return zone
+
+@pytest.fixture
+def aws_region(aws_zone):
+    """
+    Supply an appropriate AWS region to work in to tests that need one.
+    """
+    return zone_to_region(aws_zone)
+
+@pytest.fixture
+def ec2_client(aws_region):
+    """
+    Supply an AWS EC2 client tests that need one.
+    """
+    return toil.lib.aws.session.client("ec2", aws_region)
+
 
-class AWSProvisionerBenchTest(ToilTest):
+class TestAWSProvisionerBenchTest:
     """
-    Tests for the AWS provisioner that don't actually provision anything.
+    Tests for the AWS provisioner that don't actually provision instances.
     """
 
     # Needs to talk to EC2 for image discovery
@@ -71,7 +106,8 @@ class AWSProvisionerBenchTest(ToilTest):
             assert ami.startswith("ami-")
 
     @needs_aws_ec2
-    def test_read_write_global_files(self):
+    @needs_aws_s3
+    def test_read_write_global_files(self, aws_zone):
         """
         Make sure the `_write_file_to_cloud()` and `_read_file_from_cloud()`
         functions of the AWS provisioner work as intended.
@@ -79,7 +115,7 @@ class AWSProvisionerBenchTest(ToilTest):
         provisioner = AWSProvisioner(
             f"aws-provisioner-test-{uuid4()}",
             "mesos",
-            "us-west-2a",
+            aws_zone,
             50,
             None,
             None,
@@ -90,13 +126,30 @@ class AWSProvisionerBenchTest(ToilTest):
 
         try:
             url = provisioner._write_file_to_cloud(key, contents=contents)
-            self.assertTrue(url.startswith("s3://"))
+            assert url.startswith("s3://")
 
-            self.assertEqual(contents, provisioner._read_file_from_cloud(key))
+            assert provisioner._read_file_from_cloud(key) == contents
         finally:
             # the cluster was never launched, but we need to clean up the s3 bucket
             provisioner.destroyCluster()
 
+    @needs_aws_ec2
+    def test_get_spot_history(self, ec2_client) -> None:
+        """
+        Make sure that we can download spot price history from AWS.
+        """
+        history = _get_spot_history(ec2_client, "t3.large")
+        # We should have 7 days of history, newest first.
+
+    @needs_aws_ec2
+    def test_get_aws_zone_from_spot_market(self, ec2_client) -> None:
+        """
+        Make sure that we can process spot price history to pick a zone.
+        """
+        zone_options = ["us-west-2a", "af-south-1c"]
+        zone_choice = get_aws_zone_from_spot_market(0.01, "t3.large", ec2_client, zone_options)
+        assert zone_choice in zone_options
+
 
 @needs_aws_ec2
 @needs_fetchable_appliance

toil/test/src/autoDeploymentTest.py

@@ -4,7 +4,6 @@ import time
 from contextlib import contextmanager
 
 from toil.exceptions import FailedJobsException
-from toil.lib.iterables import concat
 from toil.test import ApplianceTestSupport, needs_local_appliance, needs_mesos, slow
 from toil.version import exactPython
 
@@ -85,7 +84,7 @@ class AutoDeploymentTest(ApplianceTestSupport):
                 "--defaultMemory=10M",
                 "/data/jobstore",
             ]
-            command = concat(pythonArgs, toilArgs)
+            command = pythonArgs + toilArgs
             self.assertRaises(
                 subprocess.CalledProcessError, leader.runOnAppliance, *command
             )
@@ -96,7 +95,7 @@ class AutoDeploymentTest(ApplianceTestSupport):
                 path=self.sitePackages, packagePath="foo.bar", script=userScript
             )
             # ... and restart Toil.
-            command = concat(pythonArgs, "--restart", toilArgs)
+            command = pythonArgs + ["--restart"] + toilArgs
             leader.runOnAppliance(*command)
 
     def testSplitRootPackages(self):