toil 8.1.0b1__py3-none-any.whl → 9.0.0__py3-none-any.whl

toil/test/server/serverTest.py

@@ -33,7 +33,7 @@ except ImportError:
     # extra wasn't installed. We'll then skip them all.
     pass
 
-from toil.test import ToilTest, needs_aws_s3, needs_celery_broker, needs_server
+from toil.test import ToilTest, needs_aws_s3, needs_celery_broker, needs_cwl, needs_server, integrative
 
 logger = logging.getLogger(__name__)
 logging.basicConfig(level=logging.INFO)
@@ -185,6 +185,7 @@ class FileStateStoreURLTest(hidden.AbstractStateStoreTest):
 
 
 @needs_aws_s3
+@integrative
 class BucketUsingTest(ToilTest):
     """
     Base class for tests that need a bucket.
@@ -268,6 +269,35 @@ class AWSStateStoreTest(hidden.AbstractStateStoreTest, BucketUsingTest):
         self.assertEqual(obj.content_length, len("testvalue"))
 
 
+# Problem: httpx (which the Connexion test client uses the API of)
+# automatically decides whether to send posts in
+# application/x-www-form-urlencoded or multipart/form-data format
+# based on whether they are uploading any files. But the GA4GH WES
+# API run workflow endpoint only accepts multipart/form-data. It takes
+# workflow_attachment as an array of binary strings officially, but this is
+# actually how Swagger takes multiple-file upload fields. See
+# <https://swagger.io/docs/specification/v2_0/file-upload/#multiple-upload>.
+# (httpx doesn't seem to support actually sending multiple files to one field
+# either, but if we send just one file it ends up as the only value in
+# Swagger's array.)
+#
+# The apparently-official workaround for forcing multipart encoding is to
+# construct an empty dict-saped data structure that is truthy, and pass that as
+# your file list. See
+# <https://github.com/encode/httpx/discussions/2399#discussioncomment-3814186>
+class TrueDict(dict):
+    """
+    Dict that is truthy even when empty.
+
+    Used as a workaround to set httpx post request encoding as recommended in
+    <https://github.com/encode/httpx/discussions/2399#discussioncomment-3814186>.
+    """
+    def __bool__(self) -> bool:
+        """
+        Always say the object is truthy.
+        """
+        return True
+
 @needs_server
 class AbstractToilWESServerTest(ToilTest):
     """
@@ -296,11 +326,12 @@ class AbstractToilWESServerTest(ToilTest):
         )
 
         # Make the FlaskApp
-        server_app = create_app(args)
+        self.app = create_app(args)
 
-        # Fish out the actual Flask
-        self.app: Flask = server_app.app
-        self.app.testing = True
+        # Neither <https://flask.palletsprojects.com/en/stable/testing/> nor
+        # <https://connexion.readthedocs.io/en/latest/testing.html#testing>
+        # suggests setting a testing flag on the Connexion app or its internal
+        # Flask app, so we don't.
 
         self.example_cwl = textwrap.dedent(
             """
@@ -345,7 +376,6 @@ class AbstractToilWESServerTest(ToilTest):
         """
         rv = client.get(f"/ga4gh/wes/v1/runs/{run_id}")
         self.assertEqual(rv.status_code, 200)
-        self.assertTrue(rv.is_json)
         return rv
 
     def _check_successful_log(self, client: "FlaskClient", run_id: str) -> None:
@@ -354,15 +384,16 @@ class AbstractToilWESServerTest(ToilTest):
         The workflow should succeed, it should have some tasks, and they should have all succeeded.
         """
         rv = self._fetch_run_log(client, run_id)
+        rv_json = rv.json()
         logger.debug("Log info: %s", rv.json)
-        run_log = rv.json.get("run_log")
+        run_log = rv_json.get("run_log")
         self.assertEqual(type(run_log), dict)
         if "exit_code" in run_log:
             # The workflow succeeded if it has an exit code
             self.assertEqual(run_log["exit_code"], 0)
         # The workflow is complete
-        self.assertEqual(rv.json.get("state"), "COMPLETE")
-        task_logs = rv.json.get("task_logs")
+        self.assertEqual(rv_json.get("state"), "COMPLETE")
+        task_logs = rv_json.get("task_logs")
         # There are tasks reported
         self.assertEqual(type(task_logs), list)
         self.assertGreater(len(task_logs), 0)
@@ -375,7 +406,7 @@ class AbstractToilWESServerTest(ToilTest):
         """Report the log for the given workflow run."""
         rv = self._fetch_run_log(client, run_id)
         logger.debug(f"Report log response: {rv.json}")
-        run_log = rv.json.get("run_log")
+        run_log = rv.json().get("run_log")
         self.assertEqual(type(run_log), dict)
         self.assertEqual(type(run_log.get("stdout")), str)
         self.assertEqual(type(run_log.get("stderr")), str)
@@ -395,12 +426,13 @@ class AbstractToilWESServerTest(ToilTest):
         logger.info("Fetch %s", url)
         rv = client.get(url)
         self.assertEqual(rv.status_code, 200)
-        logger.info("Got %s:\n%s", url, rv.data.decode("utf-8"))
+        logger.info("Got %s:\n%s", url, rv.content.decode("utf-8"))
 
     def _start_slow_workflow(self, client: "FlaskClient") -> str:
         """
         Start a slow workflow and return its ID.
         """
+
         rv = client.post(
             "/ga4gh/wes/v1/runs",
             data={
@@ -408,15 +440,18 @@ class AbstractToilWESServerTest(ToilTest):
                 "workflow_type": "CWL",
                 "workflow_type_version": "v1.0",
                 "workflow_params": json.dumps({"delay": "5"}),
-                "workflow_attachment": [
-                    (BytesIO(self.slow_cwl.encode()), "slow.cwl"),
-                ],
+            },
+            files={
+                "workflow_attachment": (
+                    "slow.cwl",
+                    BytesIO(self.slow_cwl.encode()),
+                    "application/octet-stream",
+                ),
             },
         )
         # workflow is submitted successfully
         self.assertEqual(rv.status_code, 200)
-        self.assertTrue(rv.is_json)
-        run_id = rv.json.get("run_id")
+        run_id = rv.json().get("run_id")
         self.assertIsNotNone(run_id)
 
         return run_id
@@ -428,11 +463,11 @@ class AbstractToilWESServerTest(ToilTest):
 
         rv = client.get(f"/ga4gh/wes/v1/runs/{run_id}/status")
         self.assertEqual(rv.status_code, 200)
-        self.assertTrue(rv.is_json)
-        self.assertIn("run_id", rv.json)
-        self.assertEqual(rv.json.get("run_id"), run_id)
-        self.assertIn("state", rv.json)
-        state = rv.json.get("state")
+        rv_json = rv.json()
+        self.assertIn("run_id", rv_json)
+        self.assertEqual(rv_json.get("run_id"), run_id)
+        self.assertIn("state", rv_json)
+        state = rv_json.get("state")
         self.assertIn(
             state,
             [
@@ -492,7 +527,10 @@ class ToilWESServerBenchTest(AbstractToilWESServerTest):
         """Test the homepage endpoint."""
         with self.app.test_client() as client:
             rv = client.get("/")
-        self.assertEqual(rv.status_code, 302)
+        # The client will follow the redirect and populate the url on the response
+        self.assertEqual(rv.url.path, "/ga4gh/wes/v1/service-info")
+        # We see the final 200 OK status code
+        self.assertEqual(rv.status_code, 200)
 
     def test_health(self) -> None:
         """Test the health check endpoint."""
@@ -505,7 +543,7 @@ class ToilWESServerBenchTest(AbstractToilWESServerTest):
         with self.app.test_client() as client:
             rv = client.get("/ga4gh/wes/v1/service-info")
         self.assertEqual(rv.status_code, 200)
-        service_info = json.loads(rv.data)
+        service_info = rv.json()
 
         self.assertIn("version", service_info)
         self.assertIn("workflow_type_versions", service_info)
@@ -519,7 +557,7 @@ class ToilWESServerBenchTest(AbstractToilWESServerTest):
         self.assertIn("system_state_counts", service_info)
         self.assertIn("tags", service_info)
 
-
+@needs_cwl
 class ToilWESServerWorkflowTest(AbstractToilWESServerTest):
     """
     Tests of the WES server running workflows.
@@ -548,11 +586,10 @@ class ToilWESServerWorkflowTest(AbstractToilWESServerTest):
                 {"message": "Hello, world!"} if include_message else {}
             )
         with self.app.test_client() as client:
-            rv = client.post("/ga4gh/wes/v1/runs", data=post_data)
+            rv = client.post("/ga4gh/wes/v1/runs", data=post_data, files=TrueDict())
             # workflow is submitted successfully
             self.assertEqual(rv.status_code, 200)
-            self.assertTrue(rv.is_json)
-            run_id = rv.json.get("run_id")
+            run_id = rv.json().get("run_id")
             self.assertIsNotNone(run_id)
 
             # Check status
@@ -571,11 +608,11 @@ class ToilWESServerWorkflowTest(AbstractToilWESServerTest):
                     "workflow_type_version": "v1.0",
                     "workflow_params": "{}",
                 },
+                files=TrueDict(),
             )
             self.assertEqual(rv.status_code, 400)
-            self.assertTrue(rv.is_json)
             self.assertEqual(
-                rv.json.get("msg"),
+                rv.json().get("msg"),
                 "Relative 'workflow_url' but missing 'workflow_attachment'",
             )
 
@@ -589,20 +626,24 @@ class ToilWESServerWorkflowTest(AbstractToilWESServerTest):
                     "workflow_type": "CWL",
                     "workflow_type_version": "v1.0",
                     "workflow_params": json.dumps({"message": "Hello, world!"}),
-                    "workflow_attachment": [
-                        (BytesIO(self.example_cwl.encode()), "example.cwl"),
-                    ],
+                },
+                files={
+                    "workflow_attachment": (
+                        "example.cwl",
+                        BytesIO(self.example_cwl.encode()),
+                        "application/octet-stream",
+                    ),
                 },
             )
             # workflow is submitted successfully
             self.assertEqual(rv.status_code, 200)
-            self.assertTrue(rv.is_json)
-            run_id = rv.json.get("run_id")
+            run_id = rv.json().get("run_id")
             self.assertIsNotNone(run_id)
 
             # Check status
             self._wait_for_success(client, run_id)
 
+    @integrative
     def test_run_workflow_https_url(self) -> None:
         """Test run example CWL workflow from the Internet."""
         with self.app.test_client() as client:
@@ -614,11 +655,11 @@ class ToilWESServerWorkflowTest(AbstractToilWESServerTest):
                     "workflow_type_version": "v1.2",
                     "workflow_params": json.dumps({"message": "Hello, world!"}),
                 },
+                files=TrueDict(),
             )
             # workflow is submitted successfully
             self.assertEqual(rv.status_code, 200)
-            self.assertTrue(rv.is_json)
-            run_id = rv.json.get("run_id")
+            run_id = rv.json().get("run_id")
             self.assertIsNotNone(run_id)
 
             # Check status
@@ -730,6 +771,7 @@ class ToilWESServerWorkflowTest(AbstractToilWESServerTest):
 
 
 @needs_celery_broker
+@integrative
 class ToilWESServerCeleryWorkflowTest(ToilWESServerWorkflowTest):
     """
     End-to-end workflow-running tests against Celery.

toil/test/sort/sort.py

@@ -23,6 +23,7 @@ from configargparse import ArgumentParser
 
 from toil.common import Toil
 from toil.job import Job
+from toil.lib.misc import StrPath
 from toil.realtimeLogger import RealtimeLogger
 
 defaultLines = 1000
@@ -207,7 +208,9 @@ def getMidPoint(file, fileStart, fileEnd):
     return len(line) + fileStart - 1
 
 
-def makeFileToSort(fileName, lines=defaultLines, lineLen=defaultLineLen):
+def makeFileToSort(
+    fileName: StrPath, lines: int = defaultLines, lineLen: int = defaultLineLen
+) -> None:
     with open(fileName, "w") as f:
         for _ in range(lines):
             line = (

toil/test/src/busTest.py

@@ -14,7 +14,9 @@
 
 import logging
 import os
+from pathlib import Path
 from threading import Thread, current_thread
+from typing import NoReturn
 
 from toil.batchSystems.abstractBatchSystem import BatchJobExitReason
 from toil.bus import (
@@ -26,18 +28,17 @@ from toil.bus import (
 from toil.common import Toil
 from toil.exceptions import FailedJobsException
 from toil.job import Job
-from toil.test import ToilTest, get_temp_file
 
 logger = logging.getLogger(__name__)
 
 
-class MessageBusTest(ToilTest):
+class TestMessageBus:
 
-    def test_enum_ints_in_file(self) -> None:
+    def test_enum_ints_in_file(self, tmp_path: Path) -> None:
         """
         Make sure writing bus messages to files works with enums.
         """
-        bus_file = get_temp_file()
+        bus_file = tmp_path / "bus"
 
         bus = MessageBus()
         # Connect the handler and hold the result to protect it from GC
@@ -73,7 +74,7 @@ class MessageBusTest(ToilTest):
             # Message should always arrive in the main thread.
             nonlocal message_count
             logger.debug("Got message: %s", received)
-            self.assertEqual(current_thread(), main_thread)
+            assert current_thread() == main_thread
             message_count += 1
 
         bus.subscribe(JobIssuedMessage, handler)
@@ -101,28 +102,27 @@ class MessageBusTest(ToilTest):
             t.join()
 
         # We should ge tone message per thread, plus our own
-        self.assertEqual(box.count(JobIssuedMessage), 11)
+        assert box.count(JobIssuedMessage) == 11
         # And having polled for those, our handler should have run
-        self.assertEqual(message_count, 11)
+        assert message_count == 11
 
-    def test_restart_without_bus_path(self) -> None:
+    def test_restart_without_bus_path(self, tmp_path: Path) -> None:
         """
         Test the ability to restart a workflow when the message bus path used
         by the previous attempt is gone.
         """
-        temp_dir = self._createTempDir(purpose="tempDir")
-        job_store = self._getTestJobStorePath()
+        temp_dir = tmp_path / "tempDir"
+        temp_dir.mkdir()
+        job_store = tmp_path / "jobstore"
 
-        bus_holder_dir = os.path.join(temp_dir, "bus_holder")
-        os.mkdir(bus_holder_dir)
+        bus_holder_dir = temp_dir / "bus_holder"
+        bus_holder_dir.mkdir()
 
         start_options = Job.Runner.getDefaultOptions(job_store)
         start_options.logLevel = "DEBUG"
         start_options.retryCount = 0
         start_options.clean = "never"
-        start_options.write_messages = os.path.abspath(
-            os.path.join(bus_holder_dir, "messagebus.txt")
-        )
+        start_options.write_messages = str(bus_holder_dir / "messagebus.txt")
 
         root = Job.wrapJobFn(failing_job_fn)
 
@@ -137,7 +137,7 @@ class MessageBusTest(ToilTest):
 
         # Get rid of the bus
         os.unlink(start_options.write_messages)
-        os.rmdir(bus_holder_dir)
+        bus_holder_dir.rmdir()
 
         logger.info("Making second attempt")
 
@@ -158,7 +158,7 @@ class MessageBusTest(ToilTest):
         logger.info("Second attempt successfully failed")
 
 
-def failing_job_fn(job: Job) -> None:
+def failing_job_fn(job: Job) -> NoReturn:
     """
     This function is guaranteed to fail.
     """