toil 6.1.0a1__py3-none-any.whl → 8.0.0__py3-none-any.whl

toil/test/cwl/cwlTest.py

@@ -26,10 +26,13 @@ import zipfile
 from functools import partial
 from io import StringIO
 from pathlib import Path
-from typing import Dict, List, MutableMapping, Optional
+from typing import TYPE_CHECKING, Callable, Optional, cast
 from unittest.mock import Mock, call
 from urllib.request import urlretrieve
 
+if TYPE_CHECKING:
+    from cwltool.utils import CWLObjectType
+
 import pytest
 
 pkg_root = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))  # noqa
@@ -37,33 +40,33 @@ sys.path.insert(0, pkg_root)  # noqa
 
 from schema_salad.exceptions import ValidationException
 
-from toil.cwl.utils import (download_structure,
-                            visit_cwl_class_and_reduce,
-                            visit_top_cwl_class)
-from toil.exceptions import FailedJobsException
+from toil.cwl.utils import (
+    DirectoryStructure,
+    download_structure,
+    visit_cwl_class_and_reduce,
+    visit_top_cwl_class,
+)
 from toil.fileStores import FileID
 from toil.fileStores.abstractFileStore import AbstractFileStore
 from toil.lib.threading import cpu_count
-from toil.provisioners import cluster_factory
-from toil.test import (ToilTest,
-                       needs_aws_ec2,
-                       needs_aws_s3,
-                       needs_cwl,
-                       needs_docker,
-                       needs_docker_cuda,
-                       needs_env_var,
-                       needs_fetchable_appliance,
-                       needs_gridengine,
-                       needs_kubernetes,
-                       needs_local_cuda,
-                       needs_lsf,
-                       needs_mesos,
-                       needs_online,
-                       needs_slurm,
-                       needs_torque,
-                       needs_wes_server,
-                       slow)
-from toil.test.provisioners.clusterTest import AbstractClusterTest
+from toil.test import (
+    ToilTest,
+    needs_aws_s3,
+    needs_cwl,
+    needs_docker,
+    needs_docker_cuda,
+    needs_gridengine,
+    needs_kubernetes,
+    needs_local_cuda,
+    needs_lsf,
+    needs_mesos,
+    needs_online,
+    needs_singularity_or_docker,
+    needs_slurm,
+    needs_torque,
+    needs_wes_server,
+    slow,
+)
 
 log = logging.getLogger(__name__)
 CONFORMANCE_TEST_TIMEOUT = 10000
@@ -74,14 +77,14 @@ def run_conformance_tests(
     yml: str,
     runner: Optional[str] = None,
     caching: bool = False,
-    batchSystem: str = None,
-    selected_tests: str = None,
-    selected_tags: str = None,
-    skipped_tests: str = None,
-    extra_args: Optional[List[str]] = None,
+    batchSystem: Optional[str] = None,
+    selected_tests: Optional[str] = None,
+    selected_tags: Optional[str] = None,
+    skipped_tests: Optional[str] = None,
+    extra_args: Optional[list[str]] = None,
     must_support_all_features: bool = False,
     junit_file: Optional[str] = None,
-):
+) -> None:
     """
     Run the CWL conformance tests.
 
@@ -139,7 +142,7 @@ def run_conformance_tests(
             "--relax-path-checks",
             # Defaults to 20s but we can't start hundreds of nodejs processes that fast on our CI potatoes
             "--eval-timeout=600",
-            f"--caching={caching}"
+            f"--caching={caching}",
         ]
 
         if extra_args:
@@ -173,30 +176,53 @@ def run_conformance_tests(
         cmd.extend(["--"] + args_passed_directly_to_runner)
 
         log.info("Running: '%s'", "' '".join(cmd))
+        output_lines: list[str] = []
         try:
-            output = subprocess.check_output(cmd, cwd=workDir, stderr=subprocess.STDOUT)
+            child = subprocess.Popen(
+                cmd, cwd=workDir, stdout=subprocess.PIPE, stderr=subprocess.STDOUT
+            )
+
+            if child.stdout is not None:
+                for line_bytes in child.stdout:
+                    # Pass through all the logs
+                    line_text = line_bytes.decode("utf-8", errors="replace").rstrip()
+                    output_lines.append(line_text)
+                    log.info(line_text)
+
+            # Once it's done writing, amke sure it succeeded.
+            child.wait()
+            log.info("CWL tests finished with exit code %s", child.returncode)
+            if child.returncode != 0:
+                # Act like check_output and raise an error.
+                raise subprocess.CalledProcessError(child.returncode, " ".join(cmd))
         finally:
             if job_store_override:
                 # Clean up the job store we used for all the tests, if it is still there.
                 subprocess.run(["toil", "clean", job_store_override])
 
     except subprocess.CalledProcessError as e:
+        log.info("CWL test runner return code was unsuccessful")
         only_unsupported = False
         # check output -- if we failed but only have unsupported features, we're okay
         p = re.compile(
             r"(?P<failures>\d+) failures, (?P<unsupported>\d+) unsupported features"
         )
 
-        error_log = e.output.decode("utf-8")
-        for line in error_log.split("\n"):
-            m = p.search(line)
+        for line_text in output_lines:
+            m = p.search(line_text)
             if m:
                 if int(m.group("failures")) == 0 and int(m.group("unsupported")) > 0:
                     only_unsupported = True
                     break
         if (not only_unsupported) or must_support_all_features:
-            print(error_log)
+            log.error(
+                "CWL tests gave unacceptable output:\n%s", "\n".join(output_lines)
+            )
             raise e
+        log.info("Unsuccessful return code is OK")
+
+
+TesterFuncType = Callable[[str, str, "CWLObjectType"], None]
 
 
 @needs_cwl
@@ -207,43 +233,49 @@ class CWLWorkflowTest(ToilTest):
     inputs.
     """
 
-    def setUp(self):
+    def setUp(self) -> None:
         """Runs anew before each test to create farm fresh temp dirs."""
         self.outDir = f"/tmp/toil-cwl-test-{str(uuid.uuid4())}"
         os.makedirs(self.outDir)
         self.rootDir = self._projectRootPath()
+        self.jobStoreDir = f"./jobstore-{str(uuid.uuid4())}"
 
-    def tearDown(self):
+    def tearDown(self) -> None:
         """Clean up outputs."""
         if os.path.exists(self.outDir):
             shutil.rmtree(self.outDir)
+        if os.path.exists(self.jobStoreDir):
+            shutil.rmtree(self.jobStoreDir)
         unittest.TestCase.tearDown(self)
 
-    def test_cwl_cmdline_input(self):
+    def test_cwl_cmdline_input(self) -> None:
         """
         Test that running a CWL workflow with inputs specified on the command line passes.
         """
         from toil.cwl import cwltoil
+
         cwlfile = "src/toil/test/cwl/conditional_wf.cwl"
         args = [cwlfile, "--message", "str", "--sleep", "2"]
         st = StringIO()
         # If the workflow runs, it must have had options
         cwltoil.main(args, stdout=st)
 
-    def _tester(self, cwlfile, jobfile, expect, main_args=[], out_name="output", output_here=False):
+    def _tester(
+        self,
+        cwlfile: str,
+        jobfile: str,
+        expect: "CWLObjectType",
+        main_args: list[str] = [],
+        out_name: str = "output",
+        output_here: bool = False,
+    ) -> None:
         from toil.cwl import cwltoil
 
         st = StringIO()
         main_args = main_args[:]
         if not output_here:
             # Don't just dump output in the working directory.
-            main_args.extend(
-                [
-                    "--logDebug",
-                    "--outdir",
-                    self.outDir
-                ]
-            )
+            main_args.extend(["--logDebug", "--outdir", self.outDir])
         main_args.extend(
             [
                 os.path.join(self.rootDir, cwlfile),
@@ -258,13 +290,20 @@ class CWLWorkflowTest(ToilTest):
         self.assertEqual(out, expect)
 
         for k, v in expect.items():
-            if isinstance(v, dict) and "class" in v and v["class"] == "File" and "path" in v:
+            if (
+                isinstance(v, dict)
+                and "class" in v
+                and v["class"] == "File"
+                and "path" in v
+            ):
                 # This is a top-level output file.
                 # None of our output files should be executable.
                 self.assertTrue(os.path.exists(v["path"]))
                 self.assertFalse(os.stat(v["path"]).st_mode & stat.S_IXUSR)
 
-    def _debug_worker_tester(self, cwlfile, jobfile, expect):
+    def _debug_worker_tester(
+        self, cwlfile: str, jobfile: str, expect: "CWLObjectType"
+    ) -> None:
         from toil.cwl import cwltoil
 
         st = StringIO()
@@ -284,21 +323,21 @@ class CWLWorkflowTest(ToilTest):
         out["output"].pop("nameroot", None)
         self.assertEqual(out, expect)
 
-    def revsort(self, cwl_filename, tester_fn):
+    def revsort(self, cwl_filename: str, tester_fn: TesterFuncType) -> None:
         tester_fn(
             "src/toil/test/cwl/" + cwl_filename,
             "src/toil/test/cwl/revsort-job.json",
             self._expected_revsort_output(self.outDir),
         )
 
-    def revsort_no_checksum(self, cwl_filename, tester_fn):
+    def revsort_no_checksum(self, cwl_filename: str, tester_fn: TesterFuncType) -> None:
         tester_fn(
             "src/toil/test/cwl/" + cwl_filename,
             "src/toil/test/cwl/revsort-job.json",
             self._expected_revsort_nochecksum_output(self.outDir),
         )
 
-    def download(self, inputs, tester_fn):
+    def download(self, inputs: str, tester_fn: TesterFuncType) -> None:
         input_location = os.path.join("src/toil/test/cwl", inputs)
         tester_fn(
             "src/toil/test/cwl/download.cwl",
@@ -306,7 +345,7 @@ class CWLWorkflowTest(ToilTest):
             self._expected_download_output(self.outDir),
         )
 
-    def load_contents(self, inputs, tester_fn):
+    def load_contents(self, inputs: str, tester_fn: TesterFuncType) -> None:
         input_location = os.path.join("src/toil/test/cwl", inputs)
         tester_fn(
             "src/toil/test/cwl/load_contents.cwl",
@@ -314,7 +353,7 @@ class CWLWorkflowTest(ToilTest):
             self._expected_load_contents_output(self.outDir),
         )
 
-    def download_directory(self, inputs, tester_fn):
+    def download_directory(self, inputs: str, tester_fn: TesterFuncType) -> None:
         input_location = os.path.join("src/toil/test/cwl", inputs)
         tester_fn(
             "src/toil/test/cwl/download_directory.cwl",
@@ -322,7 +361,7 @@ class CWLWorkflowTest(ToilTest):
             self._expected_download_output(self.outDir),
         )
 
-    def download_subdirectory(self, inputs, tester_fn):
+    def download_subdirectory(self, inputs: str, tester_fn: TesterFuncType) -> None:
         input_location = os.path.join("src/toil/test/cwl", inputs)
         tester_fn(
             "src/toil/test/cwl/download_subdirectory.cwl",
@@ -330,7 +369,7 @@ class CWLWorkflowTest(ToilTest):
             self._expected_download_output(self.outDir),
         )
 
-    def test_mpi(self):
+    def test_mpi(self) -> None:
         from toil.cwl import cwltoil
 
         stdout = StringIO()
@@ -355,7 +394,7 @@ class CWLWorkflowTest(ToilTest):
         self.assertTrue(isinstance(two_pids[1], int))
 
     @needs_aws_s3
-    def test_s3_as_secondary_file(self):
+    def test_s3_as_secondary_file(self) -> None:
         from toil.cwl import cwltoil
 
         stdout = StringIO()
@@ -374,29 +413,52 @@ class CWLWorkflowTest(ToilTest):
         with open(out["output"]["location"][len("file://") :]) as f:
             self.assertEqual(f.read().strip(), "When is s4 coming out?")
 
-    def test_run_revsort(self):
+    def test_run_revsort(self) -> None:
         self.revsort("revsort.cwl", self._tester)
 
-    def test_run_revsort_nochecksum(self):
+    def test_run_revsort_nochecksum(self) -> None:
         self.revsort_no_checksum(
             "revsort.cwl", partial(self._tester, main_args=["--no-compute-checksum"])
         )
 
-    def test_run_revsort2(self):
+    def test_run_revsort_no_container(self) -> None:
+        self.revsort(
+            "revsort.cwl", partial(self._tester, main_args=["--no-container"])
+        )
+
+    def test_run_revsort2(self) -> None:
         self.revsort("revsort2.cwl", self._tester)
 
-    def test_run_revsort_debug_worker(self):
+    def test_run_revsort_debug_worker(self) -> None:
         self.revsort("revsort.cwl", self._debug_worker_tester)
 
-    def test_run_colon_output(self):
+    def test_run_colon_output(self) -> None:
         self._tester(
             "src/toil/test/cwl/colon_test_output.cwl",
             "src/toil/test/cwl/colon_test_output_job.yaml",
             self._expected_colon_output(self.outDir),
             out_name="result",
         )
+    
+    @pytest.mark.integrative
+    @needs_singularity_or_docker
+    def test_run_dockstore_trs(self) -> None:
+        from toil.cwl import cwltoil
+
+        stdout = StringIO()
+        main_args = [
+            "--outdir",
+            self.outDir,
+            "#workflow/github.com/dockstore-testing/md5sum-checker",
+            "https://raw.githubusercontent.com/dockstore-testing/md5sum-checker/refs/heads/master/md5sum/md5sum-input-cwl.json"
+        ]
+        cwltoil.main(main_args, stdout=stdout)
+        out = json.loads(stdout.getvalue())
+        with open(out.get("output_file", {}).get("location")[len("file://") :]) as f:
+            computed_hash = f.read().strip()
+        self.assertEqual(computed_hash, "00579a00e3e7fa0674428ac7049423e2")
 
-    def test_glob_dir_bypass_file_store(self):
+    def test_glob_dir_bypass_file_store(self) -> None:
         self.maxDiff = 1000
         try:
             # We need to output to the current directory to make sure that
@@ -406,7 +468,7 @@ class CWLWorkflowTest(ToilTest):
                 "src/toil/test/cwl/empty.json",
                 self._expected_glob_dir_output(os.getcwd()),
                 main_args=["--bypass-file-store"],
-                output_here=True 
+                output_here=True,
             )
         finally:
             # Clean up anything we made in the current directory.
@@ -415,59 +477,128 @@ class CWLWorkflowTest(ToilTest):
             except FileNotFoundError:
                 pass
 
+    def test_required_input_condition_protection(self) -> None:
+        # This doesn't run containerized
+        self._tester(
+            "src/toil/test/cwl/not_run_required_input.cwl",
+            "src/toil/test/cwl/empty.json",
+            {},
+        )
+
+    @needs_slurm
+    def test_slurm_node_memory(self) -> None:
+        pass
+
+        # Run the workflow. This will either finish quickly and tell us the
+        # memory we got, or take a long time because it requested a whole
+        # node's worth of memory and no nodes are free right now. We need to
+        # support both.
+
+        # And if we run out of time we need to stop the workflow gracefully and
+        # cancel the Slurm jobs.
+
+        main_args = [
+            f"--jobStore={self.jobStoreDir}",
+            # Avoid racing to toil kill before the jobstore is removed
+            "--clean=never",
+            "--batchSystem=slurm",
+            "--no-cwl-default-ram",
+            "--slurmDefaultAllMem=True",
+            "--outdir",
+            self.outDir,
+            os.path.join(self.rootDir, "src/toil/test/cwl/measure_default_memory.cwl"),
+        ]
+        try:
+            log.debug("Start test workflow")
+            child = subprocess.Popen(
+                ["toil-cwl-runner"] + main_args, stdout=subprocess.PIPE
+            )
+            output, _ = child.communicate(timeout=60)
+        except subprocess.TimeoutExpired:
+            # The job didn't finish quickly; presumably waiting for a full node.
+            # Stop the workflow
+            log.debug("Workflow might be waiting for a full node. Stop it.")
+            subprocess.check_call(["toil", "kill", self.jobStoreDir])
+            # Wait another little bit for it to clean up, making sure to collect output in case it is blocked on writing
+            child.communicate(timeout=20)
+            # Kill it off in case it is still running
+            child.kill()
+            # Reap it
+            child.wait()
+            # The test passes
+        else:
+            out = json.loads(output)
+            log.debug("Workflow output: %s", out)
+            memory_string = out["memory"]
+            log.debug("Observed memory: %s", memory_string)
+            # If there's no memory limit enforced, Slurm will return "unlimited".
+            # Set result to something sensible.
+            if memory_string.strip() == "unlimited":
+                result = 4 * 1024 * 1024
+            else:
+                result = int(memory_string)
+            # We should see more than the CWL default or the Toil default, assuming Slurm nodes of reasonable size (3 GiB).
+            self.assertGreater(result, 3 * 1024 * 1024)
+
     @needs_aws_s3
-    def test_download_s3(self):
+    def test_download_s3(self) -> None:
         self.download("download_s3.json", self._tester)
 
-    def test_download_http(self):
+    def test_download_http(self) -> None:
         self.download("download_http.json", self._tester)
 
-    def test_download_https(self):
+    def test_download_https(self) -> None:
         self.download("download_https.json", self._tester)
 
-    def test_download_https_reference(self):
-        self.download("download_https.json", partial(self._tester, main_args=["--reference-inputs"]))
+    def test_download_https_reference(self) -> None:
+        self.download(
+            "download_https.json",
+            partial(self._tester, main_args=["--reference-inputs"]),
+        )
 
-    def test_download_file(self):
+    def test_download_file(self) -> None:
         self.download("download_file.json", self._tester)
 
     @needs_aws_s3
-    def test_download_directory_s3(self):
+    def test_download_directory_s3(self) -> None:
         self.download_directory("download_directory_s3.json", self._tester)
 
     @needs_aws_s3
-    def test_download_directory_s3_reference(self):
-        self.download_directory("download_directory_s3.json", partial(self._tester, main_args=["--reference-inputs"]))
+    def test_download_directory_s3_reference(self) -> None:
+        self.download_directory(
+            "download_directory_s3.json",
+            partial(self._tester, main_args=["--reference-inputs"]),
+        )
 
-    def test_download_directory_file(self):
+    def test_download_directory_file(self) -> None:
         self.download_directory("download_directory_file.json", self._tester)
 
     @needs_aws_s3
-    def test_download_subdirectory_s3(self):
+    def test_download_subdirectory_s3(self) -> None:
         self.download_subdirectory("download_subdirectory_s3.json", self._tester)
 
-    def test_download_subdirectory_file(self):
+    def test_download_subdirectory_file(self) -> None:
         self.download_subdirectory("download_subdirectory_file.json", self._tester)
 
     # We also want to make sure we can run a bare tool with loadContents on the inputs, which requires accessing the input data early in the leader.
 
     @needs_aws_s3
-    def test_load_contents_s3(self):
+    def test_load_contents_s3(self) -> None:
         self.load_contents("download_s3.json", self._tester)
 
-    def test_load_contents_http(self):
+    def test_load_contents_http(self) -> None:
         self.load_contents("download_http.json", self._tester)
 
-    def test_load_contents_https(self):
+    def test_load_contents_https(self) -> None:
         self.load_contents("download_https.json", self._tester)
 
-    def test_load_contents_file(self):
+    def test_load_contents_file(self) -> None:
         self.load_contents("download_file.json", self._tester)
 
     @slow
     @pytest.mark.integrative
-    @unittest.skip
-    def test_bioconda(self):
+    @unittest.skip("Fails too often due to remote service")
+    def test_bioconda(self) -> None:
         self._tester(
             "src/toil/test/cwl/seqtk_seq.cwl",
             "src/toil/test/cwl/seqtk_seq_job.json",
@@ -476,10 +607,23 @@ class CWLWorkflowTest(ToilTest):
             out_name="output1",
         )
 
+    @needs_docker
+    def test_default_args(self) -> None:
+        self._tester(
+            "src/toil/test/cwl/seqtk_seq.cwl",
+            "src/toil/test/cwl/seqtk_seq_job.json",
+            self._expected_seqtk_output(self.outDir),
+            main_args=[
+                "--default-container",
+                "quay.io/biocontainers/seqtk:1.4--he4a0461_1",
+            ],
+            out_name="output1",
+        )
+
     @needs_docker
     @pytest.mark.integrative
-    @unittest.skip
-    def test_biocontainers(self):
+    @unittest.skip("Fails too often due to remote service")
+    def test_biocontainers(self) -> None:
         self._tester(
             "src/toil/test/cwl/seqtk_seq.cwl",
             "src/toil/test/cwl/seqtk_seq_job.json",
@@ -491,7 +635,7 @@ class CWLWorkflowTest(ToilTest):
     @needs_docker
     @needs_docker_cuda
     @needs_local_cuda
-    def test_cuda(self):
+    def test_cuda(self) -> None:
         self._tester(
             "src/toil/test/cwl/nvidia_smi.cwl",
             "src/toil/test/cwl/empty.json",
@@ -500,14 +644,13 @@ class CWLWorkflowTest(ToilTest):
         )
 
     @slow
-    def test_restart(self):
+    def test_restart(self) -> None:
         """
         Enable restarts with toil-cwl-runner -- run failing test, re-run correct test.
         Only implemented for single machine.
         """
         log.info("Running CWL Test Restart.  Expecting failure, then success.")
         from toil.cwl import cwltoil
-        from toil.jobStores.abstractJobStore import NoSuchJobStoreException
 
         outDir = self._createTempDir()
         cwlDir = os.path.join(self._projectRootPath(), "src", "toil", "test", "cwl")
@@ -529,7 +672,7 @@ class CWLWorkflowTest(ToilTest):
         ][-1]
         os.symlink(os.path.join(cal_path, "date"), f'{os.path.join(outDir, "rev")}')
 
-        def path_with_bogus_rev():
+        def path_with_bogus_rev() -> str:
             # append to the front of the PATH so that we check there first
             return f"{outDir}:" + os.environ["PATH"]
 
@@ -537,22 +680,31 @@ class CWLWorkflowTest(ToilTest):
         # Force a failure by trying to use an incorrect version of `rev` from the PATH
         os.environ["PATH"] = path_with_bogus_rev()
         try:
-            cwltoil.main(cmd)
+            subprocess.check_output(
+                ["toil-cwl-runner"] + cmd,
+                env=os.environ.copy(),
+                stderr=subprocess.STDOUT,
+            )
             self.fail("Expected problem job with incorrect PATH did not fail")
-        except FailedJobsException:
+        except subprocess.CalledProcessError:
             pass
         # Finish the job with a correct PATH
         os.environ["PATH"] = orig_path
-        cwltoil.main(["--restart"] + cmd)
+        cmd.insert(0, "--restart")
+        cwltoil.main(cmd)
         # Should fail because previous job completed successfully
         try:
-            cwltoil.main(["--restart"] + cmd)
+            subprocess.check_output(
+                ["toil-cwl-runner"] + cmd,
+                env=os.environ.copy(),
+                stderr=subprocess.STDOUT,
+            )
             self.fail("Restart with missing directory did not fail")
-        except NoSuchJobStoreException:
+        except subprocess.CalledProcessError:
             pass
 
     @needs_aws_s3
-    def test_streamable(self, extra_args: List[str] = None):
+    def test_streamable(self, extra_args: Optional[list[str]] = None) -> None:
         """
         Test that a file with 'streamable'=True is a named pipe.
         This is a CWL1.2 feature.
@@ -585,13 +737,13 @@ class CWLWorkflowTest(ToilTest):
             self.assertEqual(f.read().strip(), "When is s4 coming out?")
 
     @needs_aws_s3
-    def test_streamable_reference(self):
+    def test_streamable_reference(self) -> None:
         """
         Test that a streamable file is a stream even when passed around by URI.
         """
         self.test_streamable(extra_args=["--reference-inputs"])
 
-    def test_preemptible(self):
+    def test_preemptible(self) -> None:
         """
         Tests that the http://arvados.org/cwl#UsePreemptible extension is supported.
         """
@@ -615,7 +767,7 @@ class CWLWorkflowTest(ToilTest):
         with open(out[out_name]["location"][len("file://") :]) as f:
             self.assertEqual(f.read().strip(), "hello")
 
-    def test_preemptible_expression(self):
+    def test_preemptible_expression(self) -> None:
         """
         Tests that the http://arvados.org/cwl#UsePreemptible extension is validated.
         """
@@ -636,10 +788,9 @@ class CWLWorkflowTest(ToilTest):
         except ValidationException as e:
             # Make sure we chastise the user appropriately.
             assert "expressions are not allowed" in str(e)
-        
 
     @staticmethod
-    def _expected_seqtk_output(outDir):
+    def _expected_seqtk_output(outDir: str) -> "CWLObjectType":
         path = os.path.join(outDir, "out")
         loc = "file://" + path
         return {
@@ -654,7 +805,7 @@ class CWLWorkflowTest(ToilTest):
         }
 
     @staticmethod
-    def _expected_revsort_output(outDir):
+    def _expected_revsort_output(outDir: str) -> "CWLObjectType":
         path = os.path.join(outDir, "output.txt")
         loc = "file://" + path
         return {
@@ -669,7 +820,7 @@ class CWLWorkflowTest(ToilTest):
         }
 
     @staticmethod
-    def _expected_revsort_nochecksum_output(outDir):
+    def _expected_revsort_nochecksum_output(outDir: str) -> "CWLObjectType":
         path = os.path.join(outDir, "output.txt")
         loc = "file://" + path
         return {
@@ -683,7 +834,7 @@ class CWLWorkflowTest(ToilTest):
         }
 
     @staticmethod
-    def _expected_download_output(outDir):
+    def _expected_download_output(outDir: str) -> "CWLObjectType":
         path = os.path.join(outDir, "output.txt")
         loc = "file://" + path
         return {
@@ -693,12 +844,12 @@ class CWLWorkflowTest(ToilTest):
                 "size": 0,
                 "class": "File",
                 "checksum": "sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709",
-                "path": path
+                "path": path,
             }
         }
 
     @staticmethod
-    def _expected_glob_dir_output(out_dir):
+    def _expected_glob_dir_output(out_dir: str) -> "CWLObjectType":
         dir_path = os.path.join(out_dir, "shouldmake")
         dir_loc = "file://" + dir_path
         file_path = os.path.join(dir_path, "test.txt")
@@ -720,14 +871,14 @@ class CWLWorkflowTest(ToilTest):
                         "checksum": "sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709",
                         "size": 0,
                         "nameroot": "test",
-                        "nameext": ".txt"
+                        "nameext": ".txt",
                     }
-                ]
+                ],
             }
         }
 
     @classmethod
-    def _expected_load_contents_output(cls, out_dir):
+    def _expected_load_contents_output(cls, out_dir: str) -> "CWLObjectType":
         """
         Generate the putput we expect from load_contents.cwl, when sending
         output files to the given directory.
@@ -737,7 +888,7 @@ class CWLWorkflowTest(ToilTest):
         return expected
 
     @staticmethod
-    def _expected_colon_output(outDir):
+    def _expected_colon_output(outDir: str) -> "CWLObjectType":
         path = os.path.join(outDir, "A:Gln2Cys_result")
         loc = "file://" + os.path.join(outDir, "A%3AGln2Cys_result")
         return {
@@ -755,13 +906,13 @@ class CWLWorkflowTest(ToilTest):
                         "size": 1111,
                         "nameroot": "whale",
                         "nameext": ".txt",
-                        "path": f"{path}/whale.txt"
+                        "path": f"{path}/whale.txt",
                     }
                 ],
             }
         }
 
-    def _expected_streaming_output(self, outDir):
+    def _expected_streaming_output(self, outDir: str) -> "CWLObjectType":
         path = os.path.join(outDir, "output.txt")
         loc = "file://" + path
         return {
@@ -783,7 +934,7 @@ class CWLv10Test(ToilTest):
     Run the CWL 1.0 conformance tests in various environments.
     """
 
-    def setUp(self):
+    def setUp(self) -> None:
         """Runs anew before each test to create farm fresh temp dirs."""
         self.outDir = f"/tmp/toil-cwl-test-{str(uuid.uuid4())}"
         os.makedirs(self.outDir)
@@ -804,7 +955,7 @@ class CWLv10Test(ToilTest):
             shutil.move("common-workflow-language-%s" % testhash, self.cwlSpec)
             os.remove("spec.zip")
 
-    def tearDown(self):
+    def tearDown(self) -> None:
         """Clean up outputs."""
         if os.path.exists(self.outDir):
             shutil.rmtree(self.outDir)
@@ -812,99 +963,106 @@ class CWLv10Test(ToilTest):
 
     @slow
     @pytest.mark.timeout(CONFORMANCE_TEST_TIMEOUT)
-    def test_run_conformance_with_caching(self):
+    def test_run_conformance_with_caching(self) -> None:
         self.test_run_conformance(caching=True)
 
     @slow
     @pytest.mark.timeout(CONFORMANCE_TEST_TIMEOUT)
     def test_run_conformance(
-        self, batchSystem=None, caching=False, selected_tests=None
-    ):
+        self,
+        batchSystem: Optional[str] = None,
+        caching: bool = False,
+        selected_tests: Optional[str] = None,
+        skipped_tests: Optional[str] = None,
+        extra_args: Optional[list[str]] = None,
+    ) -> None:
         run_conformance_tests(
             workDir=self.workDir,
             yml="conformance_test_v1.0.yaml",
             caching=caching,
             batchSystem=batchSystem,
             selected_tests=selected_tests,
+            skipped_tests=skipped_tests,
+            extra_args=extra_args,
         )
 
     @slow
     @needs_lsf
-    @unittest.skip
-    def test_lsf_cwl_conformance(self, **kwargs):
-        return self.test_run_conformance(batchSystem="lsf", **kwargs)
+    @unittest.skip("Not run")
+    def test_lsf_cwl_conformance(self, caching: bool = False) -> None:
+        self.test_run_conformance(batchSystem="lsf", caching=caching)
 
     @slow
     @needs_slurm
-    @unittest.skip
-    def test_slurm_cwl_conformance(self, **kwargs):
-        return self.test_run_conformance(batchSystem="slurm", **kwargs)
+    @unittest.skip("Not run")
+    def test_slurm_cwl_conformance(self, caching: bool = False) -> None:
+        self.test_run_conformance(batchSystem="slurm", caching=caching)
 
     @slow
     @needs_torque
-    @unittest.skip
-    def test_torque_cwl_conformance(self, **kwargs):
-        return self.test_run_conformance(batchSystem="torque", **kwargs)
+    @unittest.skip("Not run")
+    def test_torque_cwl_conformance(self, caching: bool = False) -> None:
+        self.test_run_conformance(batchSystem="torque", caching=caching)
 
     @slow
     @needs_gridengine
-    @unittest.skip
-    def test_gridengine_cwl_conformance(self, **kwargs):
-        return self.test_run_conformance(batchSystem="grid_engine", **kwargs)
+    @unittest.skip("Not run")
+    def test_gridengine_cwl_conformance(self, caching: bool = False) -> None:
+        self.test_run_conformance(batchSystem="grid_engine", caching=caching)
 
     @slow
     @needs_mesos
-    @unittest.skip
-    def test_mesos_cwl_conformance(self, **kwargs):
-        return self.test_run_conformance(batchSystem="mesos", **kwargs)
+    @unittest.skip("Not run")
+    def test_mesos_cwl_conformance(self, caching: bool = False) -> None:
+        self.test_run_conformance(batchSystem="mesos", caching=caching)
 
     @slow
     @needs_kubernetes
-    def test_kubernetes_cwl_conformance(self, **kwargs):
-        return self.test_run_conformance(
+    def test_kubernetes_cwl_conformance(self, caching: bool = False) -> None:
+        self.test_run_conformance(
+            caching=caching,
             batchSystem="kubernetes",
             extra_args=["--retryCount=3"],
             # This test doesn't work with
             # Singularity; see
             # https://github.com/common-workflow-language/cwltool/blob/7094ede917c2d5b16d11f9231fe0c05260b51be6/conformance-test.sh#L99-L117
             skipped_tests="docker_entrypoint",
-            **kwargs,
         )
 
     @slow
     @needs_lsf
-    @unittest.skip
-    def test_lsf_cwl_conformance_with_caching(self):
-        return self.test_lsf_cwl_conformance(caching=True)
+    @unittest.skip("Not run")
+    def test_lsf_cwl_conformance_with_caching(self) -> None:
+        self.test_lsf_cwl_conformance(caching=True)
 
     @slow
     @needs_slurm
-    @unittest.skip
-    def test_slurm_cwl_conformance_with_caching(self):
-        return self.test_slurm_cwl_conformance(caching=True)
+    @unittest.skip("Not run")
+    def test_slurm_cwl_conformance_with_caching(self) -> None:
+        self.test_slurm_cwl_conformance(caching=True)
 
     @slow
     @needs_torque
-    @unittest.skip
-    def test_torque_cwl_conformance_with_caching(self):
-        return self.test_torque_cwl_conformance(caching=True)
+    @unittest.skip("Not run")
+    def test_torque_cwl_conformance_with_caching(self) -> None:
+        self.test_torque_cwl_conformance(caching=True)
 
     @slow
     @needs_gridengine
-    @unittest.skip
-    def test_gridengine_cwl_conformance_with_caching(self):
-        return self.test_gridengine_cwl_conformance(caching=True)
+    @unittest.skip("Not run")
+    def test_gridengine_cwl_conformance_with_caching(self) -> None:
+        self.test_gridengine_cwl_conformance(caching=True)
 
     @slow
     @needs_mesos
-    @unittest.skip
-    def test_mesos_cwl_conformance_with_caching(self):
-        return self.test_mesos_cwl_conformance(caching=True)
+    @unittest.skip("Not run")
+    def test_mesos_cwl_conformance_with_caching(self) -> None:
+        self.test_mesos_cwl_conformance(caching=True)
 
     @slow
     @needs_kubernetes
-    def test_kubernetes_cwl_conformance_with_caching(self):
-        return self.test_kubernetes_cwl_conformance(caching=True)
+    def test_kubernetes_cwl_conformance_with_caching(self) -> None:
+        self.test_kubernetes_cwl_conformance(caching=True)
 
 
 @needs_cwl
@@ -914,8 +1072,12 @@ class CWLv11Test(ToilTest):
     Run the CWL 1.1 conformance tests in various environments.
     """
 
+    rootDir: str
+    cwlSpec: str
+    test_yaml: str
+
     @classmethod
-    def setUpClass(cls):
+    def setUpClass(cls) -> None:
         """Runs anew before each test."""
         cls.rootDir = cls._projectRootPath()
         cls.cwlSpec = os.path.join(cls.rootDir, "src/toil/test/cwl/spec_v11")
@@ -929,37 +1091,50 @@ class CWLv11Test(ToilTest):
         )
         p.communicate()
 
-    def tearDown(self):
+    def tearDown(self) -> None:
         """Clean up outputs."""
         unittest.TestCase.tearDown(self)
 
     @slow
     @pytest.mark.timeout(CONFORMANCE_TEST_TIMEOUT)
-    def test_run_conformance(self, **kwargs):
-        run_conformance_tests(workDir=self.cwlSpec, yml=self.test_yaml, **kwargs)
+    def test_run_conformance(
+        self,
+        caching: bool = False,
+        batchSystem: Optional[str] = None,
+        skipped_tests: Optional[str] = None,
+        extra_args: Optional[list[str]] = None,
+    ) -> None:
+        run_conformance_tests(
+            workDir=self.cwlSpec,
+            yml=self.test_yaml,
+            caching=caching,
+            batchSystem=batchSystem,
+            skipped_tests=skipped_tests,
+            extra_args=extra_args,
+        )
 
     @slow
     @pytest.mark.timeout(CONFORMANCE_TEST_TIMEOUT)
-    def test_run_conformance_with_caching(self):
+    def test_run_conformance_with_caching(self) -> None:
         self.test_run_conformance(caching=True)
 
     @slow
     @needs_kubernetes
-    def test_kubernetes_cwl_conformance(self, **kwargs):
-        return self.test_run_conformance(
+    def test_kubernetes_cwl_conformance(self, caching: bool = False) -> None:
+        self.test_run_conformance(
             batchSystem="kubernetes",
             extra_args=["--retryCount=3"],
             # These tests don't work with
             # Singularity; see
             # https://github.com/common-workflow-language/cwltool/blob/7094ede917c2d5b16d11f9231fe0c05260b51be6/conformance-test.sh#L99-L117
             skipped_tests="docker_entrypoint,stdin_shorcut",
-            **kwargs,
+            caching=caching,
         )
 
     @slow
     @needs_kubernetes
-    def test_kubernetes_cwl_conformance_with_caching(self):
-        return self.test_kubernetes_cwl_conformance(caching=True)
+    def test_kubernetes_cwl_conformance_with_caching(self) -> None:
+        self.test_kubernetes_cwl_conformance(caching=True)
 
 
 @needs_cwl
@@ -969,8 +1144,12 @@ class CWLv12Test(ToilTest):
     Run the CWL 1.2 conformance tests in various environments.
     """
 
+    rootDir: str
+    cwlSpec: str
+    test_yaml: str
+
     @classmethod
-    def setUpClass(cls):
+    def setUpClass(cls) -> None:
         """Runs anew before each test."""
         cls.rootDir = cls._projectRootPath()
         cls.cwlSpec = os.path.join(cls.rootDir, "src/toil/test/cwl/spec_v12")
@@ -984,52 +1163,73 @@ class CWLv12Test(ToilTest):
         )
         p.communicate()
 
-    def tearDown(self):
+    def tearDown(self) -> None:
         """Clean up outputs."""
         unittest.TestCase.tearDown(self)
 
     @slow
     @pytest.mark.timeout(CONFORMANCE_TEST_TIMEOUT)
-    def test_run_conformance(self, **kwargs):
-        if "junit_file" not in kwargs:
-            kwargs["junit_file"] = os.path.join(
-                self.rootDir, "conformance-1.2.junit.xml"
-            )
-        run_conformance_tests(workDir=self.cwlSpec, yml=self.test_yaml, **kwargs)
+    def test_run_conformance(
+        self,
+        runner: Optional[str] = None,
+        caching: bool = False,
+        batchSystem: Optional[str] = None,
+        selected_tests: Optional[str] = None,
+        skipped_tests: Optional[str] = None,
+        extra_args: Optional[list[str]] = None,
+        must_support_all_features: bool = False,
+        junit_file: Optional[str] = None,
+    ) -> None:
+        if junit_file is None:
+            junit_file = os.path.join(self.rootDir, "conformance-1.2.junit.xml")
+        run_conformance_tests(
+            workDir=self.cwlSpec,
+            yml=self.test_yaml,
+            runner=runner,
+            caching=caching,
+            batchSystem=batchSystem,
+            selected_tests=selected_tests,
+            skipped_tests=skipped_tests,
+            extra_args=extra_args,
+            must_support_all_features=must_support_all_features,
+            junit_file=junit_file,
+        )
 
     @slow
     @pytest.mark.timeout(CONFORMANCE_TEST_TIMEOUT)
-    def test_run_conformance_with_caching(self):
+    def test_run_conformance_with_caching(self) -> None:
         self.test_run_conformance(
             caching=True,
-            junit_file = os.path.join(
-                self.rootDir, "caching-conformance-1.2.junit.xml"
-            )
+            junit_file=os.path.join(self.rootDir, "caching-conformance-1.2.junit.xml"),
         )
 
     @slow
     @pytest.mark.timeout(CONFORMANCE_TEST_TIMEOUT)
-    def test_run_conformance_with_in_place_update(self):
+    def test_run_conformance_with_in_place_update(self) -> None:
         """
         Make sure that with --bypass-file-store we properly support in place
         update on a single node, and that this doesn't break any other
         features.
         """
         self.test_run_conformance(
-            extra_args=["--bypass-file-store"], must_support_all_features=True,
-            junit_file = os.path.join(
+            extra_args=["--bypass-file-store"],
+            must_support_all_features=True,
+            junit_file=os.path.join(
                 self.rootDir, "in-place-update-conformance-1.2.junit.xml"
-            )
+            ),
         )
 
     @slow
     @needs_kubernetes
-    def test_kubernetes_cwl_conformance(self, **kwargs):
-        if "junit_file" not in kwargs:
-            kwargs["junit_file"] = os.path.join(
+    def test_kubernetes_cwl_conformance(
+        self, caching: bool = False, junit_file: Optional[str] = None
+    ) -> None:
+        if junit_file is None:
+            junit_file = os.path.join(
                 self.rootDir, "kubernetes-conformance-1.2.junit.xml"
             )
-        return self.test_run_conformance(
+        self.test_run_conformance(
+            caching=caching,
             batchSystem="kubernetes",
             extra_args=["--retryCount=3"],
             # This test doesn't work with
@@ -1038,13 +1238,13 @@ class CWLv12Test(ToilTest):
             # and
             # https://github.com/common-workflow-language/cwltool/issues/1441#issuecomment-826747975
             skipped_tests="docker_entrypoint",
-            **kwargs,
+            junit_file=junit_file,
         )
 
     @slow
     @needs_kubernetes
-    def test_kubernetes_cwl_conformance_with_caching(self):
-        return self.test_kubernetes_cwl_conformance(
+    def test_kubernetes_cwl_conformance_with_caching(self) -> None:
+        self.test_kubernetes_cwl_conformance(
             caching=True,
             junit_file=os.path.join(
                 self.rootDir, "kubernetes-caching-conformance-1.2.junit.xml"
@@ -1053,7 +1253,7 @@ class CWLv12Test(ToilTest):
 
     @slow
     @needs_wes_server
-    def test_wes_server_cwl_conformance(self):
+    def test_wes_server_cwl_conformance(self) -> None:
         """
         Run the CWL conformance tests via WES. TOIL_WES_ENDPOINT must be
         specified. If the WES server requires authentication, set TOIL_WES_USER
@@ -1078,96 +1278,16 @@ class CWLv12Test(ToilTest):
         # 1. `cwltool --print-deps` doesn't seem to include secondary files from the default
         #     e.g.: https://github.com/common-workflow-language/cwl-v1.2/blob/1.2.1_proposed/tests/mixed-versions/wf-v10.cwl#L4-L10
 
-        return self.test_run_conformance(
+        self.test_run_conformance(
             runner="toil-wes-cwl-runner",
             selected_tests="1-309,313-337",
             extra_args=extra_args,
         )
 
 
-@needs_aws_ec2
-@needs_fetchable_appliance
-@slow
-class CWLOnARMTest(AbstractClusterTest):
-    """
-    Run the CWL 1.2 conformance tests on ARM specifically.
-    """
-
-    def __init__(self, methodName):
-        super().__init__(methodName=methodName)
-        self.clusterName = "cwl-test-" + str(uuid.uuid4())
-        self.leaderNodeType = "t4g.2xlarge"
-        self.clusterType = "kubernetes"
-        # We need to be running in a directory which Flatcar and the Toil Appliance both have
-        self.cwl_test_dir = "/tmp/toil/cwlTests"
-
-    def setUp(self):
-        super().setUp()
-        self.jobStore = f"aws:{self.awsRegion()}:cluster-{uuid.uuid4()}"
-
-    @needs_env_var("CI_COMMIT_SHA", "a git commit sha")
-    def test_cwl_on_arm(self):
-        # Make a cluster
-        self.launchCluster()
-        # get the leader so we know the IP address - we don't need to wait since create cluster
-        # already ensures the leader is running
-        self.cluster = cluster_factory(
-            provisioner="aws", zone=self.zone, clusterName=self.clusterName
-        )
-        self.leader = self.cluster.getLeader()
-
-        commit = os.environ["CI_COMMIT_SHA"]
-        self.sshUtil(
-            [
-                "bash",
-                "-c",
-                f"mkdir -p {self.cwl_test_dir} && cd {self.cwl_test_dir} && git clone https://github.com/DataBiosphere/toil.git",
-            ]
-        )
-
-        # We use CI_COMMIT_SHA to retrieve the Toil version needed to run the CWL tests
-        self.sshUtil(
-            ["bash", "-c", f"cd {self.cwl_test_dir}/toil && git checkout {commit}"]
-        )
-
-        # --never-download prevents silent upgrades to pip, wheel and setuptools
-        self.sshUtil(
-            [
-                "bash",
-                "-c",
-                f"virtualenv --system-site-packages --never-download {self.venvDir}",
-            ]
-        )
-        self.sshUtil(
-            [
-                "bash",
-                "-c",
-                f". .{self.venvDir}/bin/activate && cd {self.cwl_test_dir}/toil && make prepare && make develop extras=[all]",
-            ]
-        )
-
-        # Runs the CWLv12Test on an ARM instance
-        self.sshUtil(
-            [
-                "bash",
-                "-c",
-                f". .{self.venvDir}/bin/activate && cd {self.cwl_test_dir}/toil && pytest --log-cli-level DEBUG -r s src/toil/test/cwl/cwlTest.py::CWLv12Test::test_run_conformance",
-            ]
-        )
-
-        # We know if it succeeds it should save a junit XML for us to read.
-        # Bring it back to be an artifact.
-        self.rsync_util(
-            f":{self.cwl_test_dir}/toil/conformance-1.2.junit.xml",
-            os.path.join(
-                self._projectRootPath(),
-                "arm-conformance-1.2.junit.xml"
-            )
-        )
-
 @needs_cwl
 @pytest.mark.cwl_small_log_dir
-def test_workflow_echo_string_scatter_stderr_log_dir(tmp_path: Path):
+def test_workflow_echo_string_scatter_stderr_log_dir(tmp_path: Path) -> None:
     log_dir = tmp_path / "cwl-logs"
     job_store = "test_workflow_echo_string_scatter_stderr_log_dir"
     toil = "toil-cwl-runner"
@@ -1272,9 +1392,12 @@ def test_log_dir_echo_stderr(tmp_path: Path) -> None:
     assert output == "hello\n"
 
 
+# TODO: It's not clear how this test tests filename conflict resolution; it
+# seems like it runs a python script to copy some files and makes sure the
+# workflow doesn't fail.
 @needs_cwl
 @pytest.mark.cwl_small_log_dir
-def test_filename_conflict_resolution(tmp_path: Path):
+def test_filename_conflict_resolution(tmp_path: Path) -> None:
     out_dir = tmp_path / "cwl-out-dir"
     toil = "toil-cwl-runner"
     options = [
@@ -1294,10 +1417,31 @@ def test_filename_conflict_resolution(tmp_path: Path):
     assert b"Finished toil run successfully" in stderr
     assert p.returncode == 0
 
+
+@needs_cwl
+@pytest.mark.cwl_small_log_dir
+def test_filename_conflict_resolution_3_or_more(tmp_path: Path) -> None:
+    out_dir = tmp_path / "cwl-out-dir"
+    toil = "toil-cwl-runner"
+    options = [
+        f"--outdir={out_dir}",
+        "--clean=always",
+    ]
+    cwl = os.path.join(os.path.dirname(__file__), "scatter_duplicate_outputs.cwl")
+    cmd = [toil] + options + [cwl]
+    p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    stdout, stderr = p.communicate()
+    assert b"Finished toil run successfully" in stderr
+    assert p.returncode == 0
+    assert (
+        len(os.listdir(out_dir)) == 9
+    ), "All 9 files made by the scatter should be in the directory"
+
+
 @needs_cwl
 @needs_docker
 @pytest.mark.cwl_small_log_dir
-def test_filename_conflict_detection(tmp_path: Path):
+def test_filename_conflict_detection(tmp_path: Path) -> None:
     """
     Make sure we don't just stage files over each other when using a container.
     """
@@ -1316,10 +1460,11 @@ def test_filename_conflict_detection(tmp_path: Path):
     assert b"File staging conflict" in stderr
     assert p.returncode != 0
 
+
 @needs_cwl
 @needs_docker
 @pytest.mark.cwl_small_log_dir
-def test_filename_conflict_detection_at_root(tmp_path: Path):
+def test_filename_conflict_detection_at_root(tmp_path: Path) -> None:
     """
     Make sure we don't just stage files over each other.
 
@@ -1343,7 +1488,7 @@ def test_filename_conflict_detection_at_root(tmp_path: Path):
 
 @needs_cwl
 @pytest.mark.cwl_small
-def test_pick_value_with_one_null_value(caplog):
+def test_pick_value_with_one_null_value(caplog: pytest.LogCaptureFixture) -> None:
     """
     Make sure toil-cwl-runner does not false log a warning when pickValue is
     used but outputSource only contains one null value. See: #3991.
@@ -1357,12 +1502,15 @@ def test_pick_value_with_one_null_value(caplog):
     with caplog.at_level(logging.WARNING, logger="toil.cwl.cwltoil"):
         cwltoil.main(args)
         for line in caplog.messages:
-            assert "You had a conditional step that did not run, but you did not use pickValue to handle the skipped input." not in line
+            assert (
+                "You had a conditional step that did not run, but you did not use pickValue to handle the skipped input."
+                not in line
+            )
 
 
 @needs_cwl
 @pytest.mark.cwl_small
-def test_workflow_echo_string():
+def test_workflow_echo_string() -> None:
     toil = "toil-cwl-runner"
     jobstore = f"--jobStore=file:explicit-local-jobstore-{uuid.uuid4()}"
     option_1 = "--strict-memory-limit"
@@ -1372,14 +1520,18 @@ def test_workflow_echo_string():
     cmd = [toil, jobstore, option_1, option_2, option_3, cwl]
     p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
     stdout, stderr = p.communicate()
-    assert stdout.decode("utf-8").strip() == "{}", f"Got wrong output: {stdout}\nWith error: {stderr}"
-    assert b"Finished toil run successfully" in stderr
+    stdout2 = stdout.decode("utf-8")
+    stderr2 = stderr.decode("utf-8")
+    assert (
+        stdout2.strip() == "{}"
+    ), f"Got wrong output: {stdout2}\nWith error: {stderr2}"
+    assert "Finished toil run successfully" in stderr2
     assert p.returncode == 0
 
 
 @needs_cwl
 @pytest.mark.cwl_small
-def test_workflow_echo_string_scatter_capture_stdout():
+def test_workflow_echo_string_scatter_capture_stdout() -> None:
     toil = "toil-cwl-runner"
     jobstore = f"--jobStore=file:explicit-local-jobstore-{uuid.uuid4()}"
     option_1 = "--strict-memory-limit"
@@ -1391,6 +1543,8 @@ def test_workflow_echo_string_scatter_capture_stdout():
     cmd = [toil, jobstore, option_1, option_2, option_3, cwl]
     p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
     stdout, stderr = p.communicate()
+    log.debug("Workflow standard output: %s", stdout)
+    assert len(stdout) > 0
     outputs = json.loads(stdout)
     out_list = outputs["list_out"]
     assert len(out_list) == 2, f"outList shoud have two file elements {out_list}"
@@ -1412,7 +1566,7 @@ def test_workflow_echo_string_scatter_capture_stdout():
 
 @needs_cwl
 @pytest.mark.cwl_small
-def test_visit_top_cwl_class():
+def test_visit_top_cwl_class() -> None:
     structure = {
         "class": "Directory",
         "listing": [
@@ -1438,7 +1592,7 @@ def test_visit_top_cwl_class():
 
     counter = 0
 
-    def increment(thing: Dict) -> None:
+    def increment(thing: "CWLObjectType") -> None:
         """
         Make sure we are at something CWL object like, and count it.
         """
@@ -1463,7 +1617,7 @@ def test_visit_top_cwl_class():
 
 @needs_cwl
 @pytest.mark.cwl_small
-def test_visit_cwl_class_and_reduce():
+def test_visit_cwl_class_and_reduce() -> None:
     structure = {
         "class": "Directory",
         "listing": [
@@ -1489,7 +1643,7 @@ def test_visit_cwl_class_and_reduce():
 
     down_count = 0
 
-    def op_down(thing: MutableMapping) -> int:
+    def op_down(thing: "CWLObjectType") -> int:
         """
         Grab the ID of the thing we are at, and count what we visit going
         down.
@@ -1501,7 +1655,7 @@ def test_visit_cwl_class_and_reduce():
     up_count = 0
     up_child_count = 0
 
-    def op_up(thing: MutableMapping, down_value: int, child_results: List[str]) -> str:
+    def op_up(thing: "CWLObjectType", down_value: int, child_results: list[str]) -> str:
         """
         Check the down return value and the up return values, and count
         what we visit going up and what child relationships we have.
@@ -1524,7 +1678,7 @@ def test_visit_cwl_class_and_reduce():
 
 @needs_cwl
 @pytest.mark.cwl_small
-def test_download_structure(tmp_path) -> None:
+def test_download_structure(tmp_path: Path) -> None:
     """
     Make sure that download_structure makes the right calls to what it thinks is the file store.
     """
@@ -1534,7 +1688,7 @@ def test_download_structure(tmp_path) -> None:
     fid2 = FileID("adifferentfile", 1000, True)
 
     # And what directory structure it would be in
-    structure = {
+    structure: DirectoryStructure = {
         "dir1": {
             "dir2": {
                 "f1": "toilfile:" + fid1.pack(),
@@ -1555,9 +1709,9 @@ def test_download_structure(tmp_path) -> None:
     # These will be populated.
     # TODO: This cache seems unused. Remove it?
     # This maps filesystem path to CWL URI
-    index = {}
+    index: dict[str, str] = {}
     # This maps CWL URI to filesystem path
-    existing = {}
+    existing: dict[str, str] = {}
 
     # Do the download
     download_structure(file_store, index, existing, structure, to_dir)
@@ -1573,11 +1727,16 @@ def test_download_structure(tmp_path) -> None:
     assert os.path.join(to_dir, "dir1/dir2/f1again") in index
     assert os.path.join(to_dir, "anotherfile") in index
     assert (
-        index[os.path.join(to_dir, "dir1/dir2/f1")] == structure["dir1"]["dir2"]["f1"]
+        index[os.path.join(to_dir, "dir1/dir2/f1")]
+        == cast(
+            DirectoryStructure, cast(DirectoryStructure, structure["dir1"])["dir2"]
+        )["f1"]
     )
     assert (
         index[os.path.join(to_dir, "dir1/dir2/f1again")]
-        == structure["dir1"]["dir2"]["f1again"]
+        == cast(
+            DirectoryStructure, cast(DirectoryStructure, structure["dir1"])["dir2"]
+        )["f1again"]
     )
     assert index[os.path.join(to_dir, "anotherfile")] == structure["anotherfile"]
 
@@ -1605,3 +1764,53 @@ def test_download_structure(tmp_path) -> None:
         ],
         any_order=True,
     )
+
+
+@needs_cwl
+@pytest.mark.timeout(300)
+def test_import_on_workers() -> None:
+    args = [
+        "src/toil/test/cwl/download.cwl",
+        "src/toil/test/cwl/download_file.json",
+        "--runImportsOnWorkers",
+        "--importWorkersDisk=10MiB",
+        "--realTimeLogging=True",
+        "--logLevel=INFO",
+        "--logColors=False",
+    ]
+    from toil.cwl import cwltoil
+
+    detector = ImportWorkersMessageHandler()
+
+    # Set up a log message detector to the root logger
+    logging.getLogger().addHandler(detector)
+
+    cwltoil.main(args)
+
+    assert detector.detected is True
+
+
+# StreamHandler is generic, _typeshed doesn't exist at runtime, do a bit of typing trickery, see https://github.com/python/typeshed/issues/5680
+if TYPE_CHECKING:
+    from _typeshed import SupportsWrite
+
+    _stream_handler = logging.StreamHandler[SupportsWrite[str]]
+else:
+    _stream_handler = logging.StreamHandler
+
+
+class ImportWorkersMessageHandler(_stream_handler):
+    """
+    Detect the import workers log message and set a flag.
+    """
+
+    def __init__(self) -> None:
+        self.detected = False  # Have we seen the message we want?
+
+        super().__init__(sys.stderr)
+
+    def emit(self, record: logging.LogRecord) -> None:
+        if (record.msg % record.args).startswith(
+            "Issued job 'CWLImportJob' CWLImportJob"
+        ):
+            self.detected = True