toil 8.2.0__py3-none-any.whl → 9.1.0__py3-none-any.whl

toil/test/provisioners/aws/awsProvisionerTest.py

@@ -11,6 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import datetime
 import logging
 import os
 import subprocess
@@ -24,13 +25,22 @@ from uuid import uuid4
 
 import pytest
 
+import toil.lib.aws.session
+
+from toil.lib.aws import zone_to_region
 from toil.provisioners import cluster_factory
 from toil.provisioners.aws.awsProvisioner import AWSProvisioner
+from toil.provisioners.aws import (
+    _get_spot_history,
+    get_aws_zone_from_spot_market,
+    get_best_aws_zone,
+)
 from toil.test import (
     ToilTest,
     get_data,
     integrative,
     needs_aws_ec2,
+    needs_aws_s3,
     needs_fetchable_appliance,
     needs_mesos,
     slow,
@@ -53,10 +63,35 @@ if TYPE_CHECKING:
 
 log = logging.getLogger(__name__)
 
+@pytest.fixture
+def aws_zone():
+    """
+    Supply an appropriate AWS zone to work in to tests that need one.
+    """
+    zone = get_best_aws_zone()
+    assert (
+        zone is not None
+    ), "Could not determine AWS availability zone to test in; is TOIL_AWS_ZONE set?"
+    return zone
+
+@pytest.fixture
+def aws_region(aws_zone):
+    """
+    Supply an appropriate AWS region to work in to tests that need one.
+    """
+    return zone_to_region(aws_zone)
+
+@pytest.fixture
+def ec2_client(aws_region):
+    """
+    Supply an AWS EC2 client tests that need one.
+    """
+    return toil.lib.aws.session.client("ec2", aws_region)
+
 
-class AWSProvisionerBenchTest(ToilTest):
+class TestAWSProvisionerBenchTest:
     """
-    Tests for the AWS provisioner that don't actually provision anything.
+    Tests for the AWS provisioner that don't actually provision instances.
     """
 
     # Needs to talk to EC2 for image discovery
@@ -71,7 +106,8 @@ class AWSProvisionerBenchTest(ToilTest):
             assert ami.startswith("ami-")
 
     @needs_aws_ec2
-    def test_read_write_global_files(self):
+    @needs_aws_s3
+    def test_read_write_global_files(self, aws_zone):
         """
         Make sure the `_write_file_to_cloud()` and `_read_file_from_cloud()`
         functions of the AWS provisioner work as intended.
@@ -79,7 +115,7 @@ class AWSProvisionerBenchTest(ToilTest):
         provisioner = AWSProvisioner(
             f"aws-provisioner-test-{uuid4()}",
             "mesos",
-            "us-west-2a",
+            aws_zone,
             50,
             None,
             None,
@@ -90,13 +126,30 @@ class AWSProvisionerBenchTest(ToilTest):
 
         try:
             url = provisioner._write_file_to_cloud(key, contents=contents)
-            self.assertTrue(url.startswith("s3://"))
+            assert url.startswith("s3://")
 
-            self.assertEqual(contents, provisioner._read_file_from_cloud(key))
+            assert provisioner._read_file_from_cloud(key) == contents
         finally:
             # the cluster was never launched, but we need to clean up the s3 bucket
             provisioner.destroyCluster()
 
+    @needs_aws_ec2
+    def test_get_spot_history(self, ec2_client) -> None:
+        """
+        Make sure that we can download spot price history from AWS.
+        """
+        history = _get_spot_history(ec2_client, "t3.large")
+        # We should have 7 days of history, newest first.
+
+    @needs_aws_ec2
+    def test_get_aws_zone_from_spot_market(self, ec2_client) -> None:
+        """
+        Make sure that we can process spot price history to pick a zone.
+        """
+        zone_options = ["us-west-2a", "af-south-1c"]
+        zone_choice = get_aws_zone_from_spot_market(0.01, "t3.large", ec2_client, zone_options)
+        assert zone_choice in zone_options
+
 
 @needs_aws_ec2
 @needs_fetchable_appliance
@@ -526,7 +579,7 @@ class AWSAutoscaleTestMultipleNodeTypes(AbstractAWSAutoscaleTest):
         runCommand = [
             self.python(),
             self.script(),
-            "--fileToSort=/home/s3am/bin/asadmin",
+            "--fileToSort=/etc/passwd",
             "--sortMemory=0.6G",
             "--mergeMemory=3.0G",
         ]

toil/test/provisioners/clusterTest.py

@@ -31,6 +31,8 @@ from toil.test import (
     slow,
 )
 
+from toil.test.cwl.cwlTest import TestCWLv12Conformance
+
 log = logging.getLogger(__name__)
 
 
@@ -237,6 +239,17 @@ class CWLOnARMTest(AbstractClusterTest):
 
     @needs_env_var("CI_COMMIT_SHA", "a git commit sha")
     def test_cwl_on_arm(self) -> None:
+        # Import the test we want to run remotely, so we know right away if it exists.
+        test_class = TestCWLv12Conformance
+        test_method = test_class.test_run_conformance
+
+        # Work out how to describe it as a pytest test spec.
+        # __qualname__ gives classname.methodname
+        test_name = test_method.__qualname__.replace(".", "::")
+        # The module path is the file path under src, with dots.
+        test_path = test_class.__module__.replace(".", "/")
+        test_spec = f"src/{test_path}.py::{test_name}"
+
         # Make a cluster
         self.launchCluster()
         # get the leader so we know the IP address - we don't need to wait since create cluster
@@ -276,12 +289,12 @@ class CWLOnARMTest(AbstractClusterTest):
             ]
         )
 
-        # Runs the TestCWLv12 on an ARM instance
+        # Runs the test on an ARM instance
         self.sshUtil(
             [
                 "bash",
                 "-c",
-                f". .{self.venvDir}/bin/activate && cd {self.cwl_test_dir}/toil && pytest --log-cli-level DEBUG -r s src/toil/test/cwl/cwlTest.py::TestCWLv12::test_run_conformance",
+                f". .{self.venvDir}/bin/activate && cd {self.cwl_test_dir}/toil && pytest --log-cli-level DEBUG -r s {test_spec}",
             ]
         )
 

toil/test/provisioners/gceProvisionerTest.py

@@ -341,7 +341,7 @@ class GCEAutoscaleTestMultipleNodeTypes(AbstractGCEAutoscaleTest):
         runCommand = [
             "/home/venv/bin/python",
             "/home/sort.py",
-            "--fileToSort=/home/s3am/bin/asadmin",
+            "--fileToSort=/etc/passwd",
             "--sortMemory=0.6G",
             "--mergeMemory=3.0G",
         ]

toil/test/server/serverTest.py

@@ -33,7 +33,7 @@ except ImportError:
     # extra wasn't installed. We'll then skip them all.
     pass
 
-from toil.test import ToilTest, needs_aws_s3, needs_celery_broker, needs_server
+from toil.test import ToilTest, needs_aws_s3, needs_celery_broker, needs_cwl, needs_server, integrative
 
 logger = logging.getLogger(__name__)
 logging.basicConfig(level=logging.INFO)
@@ -185,6 +185,7 @@ class FileStateStoreURLTest(hidden.AbstractStateStoreTest):
 
 
 @needs_aws_s3
+@integrative
 class BucketUsingTest(ToilTest):
     """
     Base class for tests that need a bucket.
@@ -268,6 +269,35 @@ class AWSStateStoreTest(hidden.AbstractStateStoreTest, BucketUsingTest):
         self.assertEqual(obj.content_length, len("testvalue"))
 
 
+# Problem: httpx (which the Connexion test client uses the API of)
+# automatically decides whether to send posts in
+# application/x-www-form-urlencoded or multipart/form-data format
+# based on whether they are uploading any files. But the GA4GH WES
+# API run workflow endpoint only accepts multipart/form-data. It takes
+# workflow_attachment as an array of binary strings officially, but this is
+# actually how Swagger takes multiple-file upload fields. See
+# <https://swagger.io/docs/specification/v2_0/file-upload/#multiple-upload>.
+# (httpx doesn't seem to support actually sending multiple files to one field
+# either, but if we send just one file it ends up as the only value in
+# Swagger's array.)
+#
+# The apparently-official workaround for forcing multipart encoding is to
+# construct an empty dict-saped data structure that is truthy, and pass that as
+# your file list. See
+# <https://github.com/encode/httpx/discussions/2399#discussioncomment-3814186>
+class TrueDict(dict):
+    """
+    Dict that is truthy even when empty.
+
+    Used as a workaround to set httpx post request encoding as recommended in
+    <https://github.com/encode/httpx/discussions/2399#discussioncomment-3814186>.
+    """
+    def __bool__(self) -> bool:
+        """
+        Always say the object is truthy.
+        """
+        return True
+
 @needs_server
 class AbstractToilWESServerTest(ToilTest):
     """
@@ -296,11 +326,12 @@ class AbstractToilWESServerTest(ToilTest):
         )
 
         # Make the FlaskApp
-        server_app = create_app(args)
+        self.app = create_app(args)
 
-        # Fish out the actual Flask
-        self.app: Flask = server_app.app
-        self.app.testing = True
+        # Neither <https://flask.palletsprojects.com/en/stable/testing/> nor
+        # <https://connexion.readthedocs.io/en/latest/testing.html#testing>
+        # suggests setting a testing flag on the Connexion app or its internal
+        # Flask app, so we don't.
 
         self.example_cwl = textwrap.dedent(
             """
@@ -345,7 +376,6 @@ class AbstractToilWESServerTest(ToilTest):
         """
         rv = client.get(f"/ga4gh/wes/v1/runs/{run_id}")
         self.assertEqual(rv.status_code, 200)
-        self.assertTrue(rv.is_json)
         return rv
 
     def _check_successful_log(self, client: "FlaskClient", run_id: str) -> None:
@@ -354,15 +384,16 @@ class AbstractToilWESServerTest(ToilTest):
         The workflow should succeed, it should have some tasks, and they should have all succeeded.
         """
         rv = self._fetch_run_log(client, run_id)
+        rv_json = rv.json()
         logger.debug("Log info: %s", rv.json)
-        run_log = rv.json.get("run_log")
+        run_log = rv_json.get("run_log")
         self.assertEqual(type(run_log), dict)
         if "exit_code" in run_log:
             # The workflow succeeded if it has an exit code
             self.assertEqual(run_log["exit_code"], 0)
         # The workflow is complete
-        self.assertEqual(rv.json.get("state"), "COMPLETE")
-        task_logs = rv.json.get("task_logs")
+        self.assertEqual(rv_json.get("state"), "COMPLETE")
+        task_logs = rv_json.get("task_logs")
         # There are tasks reported
         self.assertEqual(type(task_logs), list)
         self.assertGreater(len(task_logs), 0)
@@ -375,7 +406,7 @@ class AbstractToilWESServerTest(ToilTest):
         """Report the log for the given workflow run."""
         rv = self._fetch_run_log(client, run_id)
         logger.debug(f"Report log response: {rv.json}")
-        run_log = rv.json.get("run_log")
+        run_log = rv.json().get("run_log")
         self.assertEqual(type(run_log), dict)
         self.assertEqual(type(run_log.get("stdout")), str)
         self.assertEqual(type(run_log.get("stderr")), str)
@@ -395,12 +426,13 @@ class AbstractToilWESServerTest(ToilTest):
         logger.info("Fetch %s", url)
         rv = client.get(url)
         self.assertEqual(rv.status_code, 200)
-        logger.info("Got %s:\n%s", url, rv.data.decode("utf-8"))
+        logger.info("Got %s:\n%s", url, rv.content.decode("utf-8"))
 
     def _start_slow_workflow(self, client: "FlaskClient") -> str:
         """
         Start a slow workflow and return its ID.
         """
+
         rv = client.post(
             "/ga4gh/wes/v1/runs",
             data={
@@ -408,15 +440,18 @@ class AbstractToilWESServerTest(ToilTest):
                 "workflow_type": "CWL",
                 "workflow_type_version": "v1.0",
                 "workflow_params": json.dumps({"delay": "5"}),
-                "workflow_attachment": [
-                    (BytesIO(self.slow_cwl.encode()), "slow.cwl"),
-                ],
+            },
+            files={
+                "workflow_attachment": (
+                    "slow.cwl",
+                    BytesIO(self.slow_cwl.encode()),
+                    "application/octet-stream",
+                ),
             },
         )
         # workflow is submitted successfully
         self.assertEqual(rv.status_code, 200)
-        self.assertTrue(rv.is_json)
-        run_id = rv.json.get("run_id")
+        run_id = rv.json().get("run_id")
         self.assertIsNotNone(run_id)
 
         return run_id
@@ -428,11 +463,11 @@ class AbstractToilWESServerTest(ToilTest):
 
         rv = client.get(f"/ga4gh/wes/v1/runs/{run_id}/status")
         self.assertEqual(rv.status_code, 200)
-        self.assertTrue(rv.is_json)
-        self.assertIn("run_id", rv.json)
-        self.assertEqual(rv.json.get("run_id"), run_id)
-        self.assertIn("state", rv.json)
-        state = rv.json.get("state")
+        rv_json = rv.json()
+        self.assertIn("run_id", rv_json)
+        self.assertEqual(rv_json.get("run_id"), run_id)
+        self.assertIn("state", rv_json)
+        state = rv_json.get("state")
         self.assertIn(
             state,
             [
@@ -492,7 +527,10 @@ class ToilWESServerBenchTest(AbstractToilWESServerTest):
         """Test the homepage endpoint."""
         with self.app.test_client() as client:
             rv = client.get("/")
-        self.assertEqual(rv.status_code, 302)
+        # The client will follow the redirect and populate the url on the response
+        self.assertEqual(rv.url.path, "/ga4gh/wes/v1/service-info")
+        # We see the final 200 OK status code
+        self.assertEqual(rv.status_code, 200)
 
     def test_health(self) -> None:
         """Test the health check endpoint."""
@@ -505,7 +543,7 @@ class ToilWESServerBenchTest(AbstractToilWESServerTest):
         with self.app.test_client() as client:
             rv = client.get("/ga4gh/wes/v1/service-info")
         self.assertEqual(rv.status_code, 200)
-        service_info = json.loads(rv.data)
+        service_info = rv.json()
 
         self.assertIn("version", service_info)
         self.assertIn("workflow_type_versions", service_info)
@@ -519,7 +557,7 @@ class ToilWESServerBenchTest(AbstractToilWESServerTest):
         self.assertIn("system_state_counts", service_info)
         self.assertIn("tags", service_info)
 
-
+@needs_cwl
 class ToilWESServerWorkflowTest(AbstractToilWESServerTest):
     """
     Tests of the WES server running workflows.
@@ -548,11 +586,10 @@ class ToilWESServerWorkflowTest(AbstractToilWESServerTest):
                 {"message": "Hello, world!"} if include_message else {}
             )
         with self.app.test_client() as client:
-            rv = client.post("/ga4gh/wes/v1/runs", data=post_data)
+            rv = client.post("/ga4gh/wes/v1/runs", data=post_data, files=TrueDict())
             # workflow is submitted successfully
             self.assertEqual(rv.status_code, 200)
-            self.assertTrue(rv.is_json)
-            run_id = rv.json.get("run_id")
+            run_id = rv.json().get("run_id")
             self.assertIsNotNone(run_id)
 
             # Check status
@@ -571,11 +608,11 @@ class ToilWESServerWorkflowTest(AbstractToilWESServerTest):
                     "workflow_type_version": "v1.0",
                     "workflow_params": "{}",
                 },
+                files=TrueDict(),
             )
             self.assertEqual(rv.status_code, 400)
-            self.assertTrue(rv.is_json)
             self.assertEqual(
-                rv.json.get("msg"),
+                rv.json().get("msg"),
                 "Relative 'workflow_url' but missing 'workflow_attachment'",
             )
 
@@ -589,20 +626,24 @@ class ToilWESServerWorkflowTest(AbstractToilWESServerTest):
                     "workflow_type": "CWL",
                     "workflow_type_version": "v1.0",
                     "workflow_params": json.dumps({"message": "Hello, world!"}),
-                    "workflow_attachment": [
-                        (BytesIO(self.example_cwl.encode()), "example.cwl"),
-                    ],
+                },
+                files={
+                    "workflow_attachment": (
+                        "example.cwl",
+                        BytesIO(self.example_cwl.encode()),
+                        "application/octet-stream",
+                    ),
                 },
             )
             # workflow is submitted successfully
             self.assertEqual(rv.status_code, 200)
-            self.assertTrue(rv.is_json)
-            run_id = rv.json.get("run_id")
+            run_id = rv.json().get("run_id")
             self.assertIsNotNone(run_id)
 
             # Check status
             self._wait_for_success(client, run_id)
 
+    @integrative
     def test_run_workflow_https_url(self) -> None:
         """Test run example CWL workflow from the Internet."""
         with self.app.test_client() as client:
@@ -614,11 +655,11 @@ class ToilWESServerWorkflowTest(AbstractToilWESServerTest):
                     "workflow_type_version": "v1.2",
                     "workflow_params": json.dumps({"message": "Hello, world!"}),
                 },
+                files=TrueDict(),
             )
             # workflow is submitted successfully
             self.assertEqual(rv.status_code, 200)
-            self.assertTrue(rv.is_json)
-            run_id = rv.json.get("run_id")
+            run_id = rv.json().get("run_id")
             self.assertIsNotNone(run_id)
 
             # Check status
@@ -730,6 +771,7 @@ class ToilWESServerWorkflowTest(AbstractToilWESServerTest):
 
 
 @needs_celery_broker
+@integrative
 class ToilWESServerCeleryWorkflowTest(ToilWESServerWorkflowTest):
     """
     End-to-end workflow-running tests against Celery.

toil/test/src/autoDeploymentTest.py

@@ -4,7 +4,6 @@ import time
 from contextlib import contextmanager
 
 from toil.exceptions import FailedJobsException
-from toil.lib.iterables import concat
 from toil.test import ApplianceTestSupport, needs_local_appliance, needs_mesos, slow
 from toil.version import exactPython
 
@@ -85,7 +84,7 @@ class AutoDeploymentTest(ApplianceTestSupport):
                 "--defaultMemory=10M",
                 "/data/jobstore",
             ]
-            command = concat(pythonArgs, toilArgs)
+            command = pythonArgs + toilArgs
             self.assertRaises(
                 subprocess.CalledProcessError, leader.runOnAppliance, *command
             )
@@ -96,7 +95,7 @@ class AutoDeploymentTest(ApplianceTestSupport):
                 path=self.sitePackages, packagePath="foo.bar", script=userScript
             )
             # ... and restart Toil.
-            command = concat(pythonArgs, "--restart", toilArgs)
+            command = pythonArgs + ["--restart"] + toilArgs
             leader.runOnAppliance(*command)
 
     def testSplitRootPackages(self):