toil 8.0.0__py3-none-any.whl → 8.2.0__py3-none-any.whl

toil/test/cwl/cwlTest.py

@@ -12,6 +12,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+from collections.abc import Generator
 import json
 import logging
 import os
@@ -20,7 +21,6 @@ import shutil
 import stat
 import subprocess
 import sys
-import unittest
 import uuid
 import zipfile
 from functools import partial
@@ -50,22 +50,23 @@ from toil.fileStores import FileID
 from toil.fileStores.abstractFileStore import AbstractFileStore
 from toil.lib.threading import cpu_count
 from toil.test import (
-    ToilTest,
-    needs_aws_s3,
-    needs_cwl,
-    needs_docker,
-    needs_docker_cuda,
-    needs_gridengine,
-    needs_kubernetes,
-    needs_local_cuda,
-    needs_lsf,
-    needs_mesos,
-    needs_online,
-    needs_singularity_or_docker,
-    needs_slurm,
-    needs_torque,
-    needs_wes_server,
-    slow,
+    get_data,
+)
+from toil.test import (
+    pslow as slow,
+    pneeds_docker as needs_docker,
+    pneeds_cwl as needs_cwl,
+    pneeds_aws_s3 as needs_aws_s3,
+    pneeds_docker_cuda as needs_docker_cuda,
+    pneeds_gridengine as needs_gridengine,
+    pneeds_kubernetes as needs_kubernetes,
+    pneeds_local_cuda as needs_local_cuda,
+    pneeds_lsf as needs_lsf,
+    pneeds_mesos as needs_mesos,
+    pneeds_online as needs_online,
+    pneeds_slurm as needs_slurm,
+    pneeds_torque as needs_torque,
+    pneeds_wes_server as needs_wes_server,
 )
 
 log = logging.getLogger(__name__)
@@ -222,72 +223,58 @@ def run_conformance_tests(
         log.info("Unsuccessful return code is OK")
 
 
-TesterFuncType = Callable[[str, str, "CWLObjectType"], None]
+TesterFuncType = Callable[[Path, Path, "CWLObjectType", Path], None]
 
 
 @needs_cwl
-class CWLWorkflowTest(ToilTest):
+@pytest.mark.cwl
+class TestCWLWorkflow:
     """
     CWL tests included in Toil that don't involve the whole CWL conformance
     test suite. Tests Toil-specific functions like URL types supported for
     inputs.
     """
 
-    def setUp(self) -> None:
-        """Runs anew before each test to create farm fresh temp dirs."""
-        self.outDir = f"/tmp/toil-cwl-test-{str(uuid.uuid4())}"
-        os.makedirs(self.outDir)
-        self.rootDir = self._projectRootPath()
-        self.jobStoreDir = f"./jobstore-{str(uuid.uuid4())}"
-
-    def tearDown(self) -> None:
-        """Clean up outputs."""
-        if os.path.exists(self.outDir):
-            shutil.rmtree(self.outDir)
-        if os.path.exists(self.jobStoreDir):
-            shutil.rmtree(self.jobStoreDir)
-        unittest.TestCase.tearDown(self)
-
     def test_cwl_cmdline_input(self) -> None:
         """
         Test that running a CWL workflow with inputs specified on the command line passes.
         """
         from toil.cwl import cwltoil
 
-        cwlfile = "src/toil/test/cwl/conditional_wf.cwl"
-        args = [cwlfile, "--message", "str", "--sleep", "2"]
-        st = StringIO()
-        # If the workflow runs, it must have had options
-        cwltoil.main(args, stdout=st)
+        with get_data("test/cwl/conditional_wf.cwl") as cwlfile:
+            args = [str(cwlfile), "--message", "str", "--sleep", "2"]
+            st = StringIO()
+            # If the workflow runs, it must have had options
+            cwltoil.main(args, stdout=st)
 
     def _tester(
         self,
-        cwlfile: str,
-        jobfile: str,
+        cwlfile: Path,
+        jobfile: Path,
         expect: "CWLObjectType",
-        main_args: list[str] = [],
+        outdir: Path,
         out_name: str = "output",
-        output_here: bool = False,
+        main_args: Optional[list[str]] = None,
     ) -> None:
         from toil.cwl import cwltoil
 
         st = StringIO()
-        main_args = main_args[:]
-        if not output_here:
-            # Don't just dump output in the working directory.
-            main_args.extend(["--logDebug", "--outdir", self.outDir])
-        main_args.extend(
+        real_main_args = main_args or []
+        real_main_args.extend(
             [
-                os.path.join(self.rootDir, cwlfile),
-                os.path.join(self.rootDir, jobfile),
+                "--logDebug",
+                "--outdir",
+                str(outdir),
+                str(cwlfile),
+                str(jobfile),
             ]
         )
-        cwltoil.main(main_args, stdout=st)
+        cwltoil.main(real_main_args, stdout=st)
         out = json.loads(st.getvalue())
         out.get(out_name, {}).pop("http://commonwl.org/cwltool#generation", None)
         out.get(out_name, {}).pop("nameext", None)
         out.get(out_name, {}).pop("nameroot", None)
-        self.assertEqual(out, expect)
+        assert out == expect
 
         for k, v in expect.items():
             if (
@@ -298,11 +285,11 @@ class CWLWorkflowTest(ToilTest):
             ):
                 # This is a top-level output file.
                 # None of our output files should be executable.
-                self.assertTrue(os.path.exists(v["path"]))
-                self.assertFalse(os.stat(v["path"]).st_mode & stat.S_IXUSR)
+                assert os.path.exists(v["path"]) is True
+                assert (os.stat(v["path"]).st_mode & stat.S_IXUSR) == 0
 
     def _debug_worker_tester(
-        self, cwlfile: str, jobfile: str, expect: "CWLObjectType"
+        self, cwlfile: Path, jobfile: Path, expect: "CWLObjectType", outdir: Path
     ) -> None:
         from toil.cwl import cwltoil
 
@@ -311,9 +298,9 @@ class CWLWorkflowTest(ToilTest):
             [
                 "--debugWorker",
                 "--outdir",
-                self.outDir,
-                os.path.join(self.rootDir, cwlfile),
-                os.path.join(self.rootDir, jobfile),
+                str(outdir),
+                str(cwlfile),
+                str(jobfile),
             ],
             stdout=st,
         )
@@ -321,172 +308,216 @@ class CWLWorkflowTest(ToilTest):
         out["output"].pop("http://commonwl.org/cwltool#generation", None)
         out["output"].pop("nameext", None)
         out["output"].pop("nameroot", None)
-        self.assertEqual(out, expect)
-
-    def revsort(self, cwl_filename: str, tester_fn: TesterFuncType) -> None:
-        tester_fn(
-            "src/toil/test/cwl/" + cwl_filename,
-            "src/toil/test/cwl/revsort-job.json",
-            self._expected_revsort_output(self.outDir),
-        )
-
-    def revsort_no_checksum(self, cwl_filename: str, tester_fn: TesterFuncType) -> None:
-        tester_fn(
-            "src/toil/test/cwl/" + cwl_filename,
-            "src/toil/test/cwl/revsort-job.json",
-            self._expected_revsort_nochecksum_output(self.outDir),
-        )
-
-    def download(self, inputs: str, tester_fn: TesterFuncType) -> None:
-        input_location = os.path.join("src/toil/test/cwl", inputs)
-        tester_fn(
-            "src/toil/test/cwl/download.cwl",
-            input_location,
-            self._expected_download_output(self.outDir),
-        )
+        assert out == expect
 
-    def load_contents(self, inputs: str, tester_fn: TesterFuncType) -> None:
-        input_location = os.path.join("src/toil/test/cwl", inputs)
-        tester_fn(
-            "src/toil/test/cwl/load_contents.cwl",
-            input_location,
-            self._expected_load_contents_output(self.outDir),
-        )
-
-    def download_directory(self, inputs: str, tester_fn: TesterFuncType) -> None:
-        input_location = os.path.join("src/toil/test/cwl", inputs)
-        tester_fn(
-            "src/toil/test/cwl/download_directory.cwl",
-            input_location,
-            self._expected_download_output(self.outDir),
-        )
-
-    def download_subdirectory(self, inputs: str, tester_fn: TesterFuncType) -> None:
-        input_location = os.path.join("src/toil/test/cwl", inputs)
-        tester_fn(
-            "src/toil/test/cwl/download_subdirectory.cwl",
-            input_location,
-            self._expected_download_output(self.outDir),
-        )
-
-    def test_mpi(self) -> None:
+    def revsort(
+        self, cwl_filename: str, tester_fn: TesterFuncType, out_dir: Path
+    ) -> None:
+        with get_data(f"test/cwl/{cwl_filename}") as cwl_file:
+            with get_data("test/cwl/revsort-job.json") as job_file:
+                tester_fn(
+                    cwl_file, job_file, self._expected_revsort_output(out_dir), out_dir
+                )
+
+    def revsort_no_checksum(
+        self, cwl_filename: str, tester_fn: TesterFuncType, out_dir: Path
+    ) -> None:
+        with get_data(f"test/cwl/{cwl_filename}") as cwl_file:
+            with get_data("test/cwl/revsort-job.json") as job_file:
+                tester_fn(
+                    cwl_file,
+                    job_file,
+                    self._expected_revsort_nochecksum_output(out_dir),
+                    out_dir,
+                )
+
+    def download(self, inputs: str, tester_fn: TesterFuncType, out_dir: Path) -> None:
+        with get_data(f"test/cwl/{inputs}") as input_location:
+            with get_data("test/cwl/download.cwl") as cwl_file:
+                tester_fn(
+                    cwl_file,
+                    input_location,
+                    self._expected_download_output(out_dir),
+                    out_dir,
+                )
+
+    def load_contents(
+        self, inputs: str, tester_fn: TesterFuncType, out_dir: Path
+    ) -> None:
+        with get_data(f"test/cwl/{inputs}") as input_location:
+            with get_data("test/cwl/load_contents.cwl") as cwl_file:
+                tester_fn(
+                    cwl_file,
+                    input_location,
+                    self._expected_load_contents_output(out_dir),
+                    out_dir,
+                )
+
+    def download_directory(
+        self, inputs: str, tester_fn: TesterFuncType, out_dir: Path
+    ) -> None:
+        with get_data(f"test/cwl/{inputs}") as input_location:
+            with get_data("test/cwl/download_directory.cwl") as cwl_file:
+                tester_fn(
+                    cwl_file,
+                    input_location,
+                    self._expected_download_output(out_dir),
+                    out_dir,
+                )
+
+    def download_subdirectory(
+        self, inputs: str, tester_fn: TesterFuncType, out_dir: Path
+    ) -> None:
+        with get_data(f"test/cwl/{inputs}") as input_location:
+            with get_data("test/cwl/download_subdirectory.cwl") as cwl_file:
+                tester_fn(
+                    cwl_file,
+                    input_location,
+                    self._expected_download_output(out_dir),
+                    out_dir,
+                )
+
+    def test_mpi(self, tmp_path: Path) -> None:
         from toil.cwl import cwltoil
 
         stdout = StringIO()
-        main_args = [
-            "--outdir",
-            self.outDir,
-            "--enable-dev",
-            "--enable-ext",
-            "--mpi-config-file",
-            os.path.join(self.rootDir, "src/toil/test/cwl/mock_mpi/fake_mpi.yml"),
-            os.path.join(self.rootDir, "src/toil/test/cwl/mpi_simple.cwl"),
-        ]
-        path = os.environ["PATH"]
-        os.environ["PATH"] = f"{path}:{self.rootDir}/src/toil/test/cwl/mock_mpi/"
-        cwltoil.main(main_args, stdout=stdout)
-        os.environ["PATH"] = path
-        out = json.loads(stdout.getvalue())
-        with open(out.get("pids", {}).get("location")[len("file://") :]) as f:
-            two_pids = [int(i) for i in f.read().split()]
-        self.assertEqual(len(two_pids), 2)
-        self.assertTrue(isinstance(two_pids[0], int))
-        self.assertTrue(isinstance(two_pids[1], int))
+        with get_data("test/cwl/mock_mpi/fake_mpi.yml") as mpi_config_file:
+            with get_data("test/cwl/mpi_simple.cwl") as cwl_file:
+                with get_data("test/cwl/mock_mpi/fake_mpi_run.py") as fake_mpi_run:
+                    main_args = [
+                        "--logDebug",
+                        "--outdir",
+                        str(tmp_path),
+                        "--enable-dev",
+                        "--enable-ext",
+                        "--mpi-config-file",
+                        str(mpi_config_file),
+                        str(cwl_file),
+                    ]
+                    path = os.environ["PATH"]
+                    os.environ["PATH"] = f"{path}:{fake_mpi_run.parent}"
+                    cwltoil.main(main_args, stdout=stdout)
+                    os.environ["PATH"] = path
+                    stdout_text = stdout.getvalue()
+                    assert "pids" in stdout_text
+                    out = json.loads(stdout_text)
+                    with open(
+                        out.get("pids", {}).get("location")[len("file://") :]
+                    ) as f:
+                        two_pids = [int(i) for i in f.read().split()]
+                    assert len(two_pids) == 2
+                    assert isinstance(two_pids[0], int)
+                    assert isinstance(two_pids[1], int)
 
     @needs_aws_s3
-    def test_s3_as_secondary_file(self) -> None:
+    @pytest.mark.aws_s3
+    @pytest.mark.online
+    def test_s3_as_secondary_file(self, tmp_path: Path) -> None:
         from toil.cwl import cwltoil
 
         stdout = StringIO()
-        main_args = [
-            "--outdir",
-            self.outDir,
-            os.path.join(self.rootDir, "src/toil/test/cwl/s3_secondary_file.cwl"),
-            os.path.join(self.rootDir, "src/toil/test/cwl/s3_secondary_file.json"),
-        ]
-        cwltoil.main(main_args, stdout=stdout)
-        out = json.loads(stdout.getvalue())
-        self.assertEqual(
-            out["output"]["checksum"], "sha1$d14dd02e354918b4776b941d154c18ebc15b9b38"
-        )
-        self.assertEqual(out["output"]["size"], 24)
-        with open(out["output"]["location"][len("file://") :]) as f:
-            self.assertEqual(f.read().strip(), "When is s4 coming out?")
+        with get_data("test/cwl/s3_secondary_file.cwl") as cwl_file:
+            with get_data("test/cwl/s3_secondary_file.json") as inputs_file:
+                main_args = ["--outdir", str(tmp_path), str(cwl_file), str(inputs_file)]
+                cwltoil.main(main_args, stdout=stdout)
+                out = json.loads(stdout.getvalue())
+                assert (
+                    out["output"]["checksum"]
+                    == "sha1$d14dd02e354918b4776b941d154c18ebc15b9b38"
+                )
+
+                assert out["output"]["size"] == 24
+                with open(out["output"]["location"][len("file://") :]) as f:
+                    assert f.read().strip() == "When is s4 coming out?"
 
-    def test_run_revsort(self) -> None:
-        self.revsort("revsort.cwl", self._tester)
+    @needs_docker
+    @pytest.mark.docker
+    @pytest.mark.online
+    def test_run_revsort(self, tmp_path: Path) -> None:
+        self.revsort("revsort.cwl", self._tester, tmp_path)
 
-    def test_run_revsort_nochecksum(self) -> None:
+    @needs_docker
+    @pytest.mark.docker
+    @pytest.mark.online
+    def test_run_revsort_nochecksum(self, tmp_path: Path) -> None:
         self.revsort_no_checksum(
-            "revsort.cwl", partial(self._tester, main_args=["--no-compute-checksum"])
+            "revsort.cwl",
+            partial(self._tester, main_args=["--no-compute-checksum"]),
+            tmp_path,
         )
 
-    def test_run_revsort_no_container(self) -> None:
+    def test_run_revsort_no_container(self, tmp_path: Path) -> None:
         self.revsort(
-            "revsort.cwl", partial(self._tester, main_args=["--no-container"])
+            "revsort.cwl", partial(self._tester, main_args=["--no-container"]), tmp_path
         )
 
-    def test_run_revsort2(self) -> None:
-        self.revsort("revsort2.cwl", self._tester)
+    @needs_docker
+    @pytest.mark.docker
+    @pytest.mark.online
+    def test_run_revsort2(self, tmp_path: Path) -> None:
+        self.revsort("revsort2.cwl", self._tester, tmp_path)
 
-    def test_run_revsort_debug_worker(self) -> None:
-        self.revsort("revsort.cwl", self._debug_worker_tester)
+    @needs_docker
+    @pytest.mark.docker
+    @pytest.mark.online
+    def test_run_revsort_debug_worker(self, tmp_path: Path) -> None:
+        self.revsort("revsort.cwl", self._debug_worker_tester, tmp_path)
+
+    @needs_docker
+    @pytest.mark.docker
+    @pytest.mark.online
+    def test_run_colon_output(self, tmp_path: Path) -> None:
+        with get_data("test/cwl/colon_test_output.cwl") as cwl_file:
+            with get_data("test/cwl/colon_test_output_job.yaml") as inputs_file:
+                self._tester(
+                    cwl_file,
+                    inputs_file,
+                    self._expected_colon_output(tmp_path),
+                    tmp_path,
+                    out_name="result",
+                )
 
-    def test_run_colon_output(self) -> None:
-        self._tester(
-            "src/toil/test/cwl/colon_test_output.cwl",
-            "src/toil/test/cwl/colon_test_output_job.yaml",
-            self._expected_colon_output(self.outDir),
-            out_name="result",
-        )
-    
     @pytest.mark.integrative
-    @needs_singularity_or_docker
-    def test_run_dockstore_trs(self) -> None:
+    @needs_docker
+    @pytest.mark.docker
+    @pytest.mark.online
+    def test_run_dockstore_trs(self, tmp_path: Path) -> None:
         from toil.cwl import cwltoil
 
         stdout = StringIO()
         main_args = [
             "--outdir",
-            self.outDir,
-            "#workflow/github.com/dockstore-testing/md5sum-checker",
-            "https://raw.githubusercontent.com/dockstore-testing/md5sum-checker/refs/heads/master/md5sum/md5sum-input-cwl.json"
+            str(tmp_path),
+            "#workflow/github.com/dockstore-testing/md5sum-checker:master",
+            "https://raw.githubusercontent.com/dockstore-testing/md5sum-checker/refs/heads/master/md5sum/md5sum-input-cwl.json",
         ]
         cwltoil.main(main_args, stdout=stdout)
         out = json.loads(stdout.getvalue())
         with open(out.get("output_file", {}).get("location")[len("file://") :]) as f:
             computed_hash = f.read().strip()
-        self.assertEqual(computed_hash, "00579a00e3e7fa0674428ac7049423e2")
+        assert computed_hash == "00579a00e3e7fa0674428ac7049423e2"
 
-    def test_glob_dir_bypass_file_store(self) -> None:
+    def test_glob_dir_bypass_file_store(self, tmp_path: Path) -> None:
         self.maxDiff = 1000
-        try:
-            # We need to output to the current directory to make sure that
-            # works.
-            self._tester(
-                "src/toil/test/cwl/glob_dir.cwl",
-                "src/toil/test/cwl/empty.json",
-                self._expected_glob_dir_output(os.getcwd()),
-                main_args=["--bypass-file-store"],
-                output_here=True,
-            )
-        finally:
-            # Clean up anything we made in the current directory.
-            try:
-                shutil.rmtree(os.path.join(os.getcwd(), "shouldmake"))
-            except FileNotFoundError:
-                pass
-
-    def test_required_input_condition_protection(self) -> None:
+        with get_data("test/cwl/glob_dir.cwl") as cwl_file:
+            with get_data("test/cwl/empty.json") as inputs_file:
+                self._tester(
+                    cwl_file,
+                    inputs_file,
+                    self._expected_glob_dir_output(tmp_path),
+                    tmp_path,
+                    main_args=["--bypass-file-store"],
+                )
+
+    def test_required_input_condition_protection(self, tmp_path: Path) -> None:
         # This doesn't run containerized
-        self._tester(
-            "src/toil/test/cwl/not_run_required_input.cwl",
-            "src/toil/test/cwl/empty.json",
-            {},
-        )
+        with get_data("test/cwl/not_run_required_input.cwl") as cwl_file:
+            with get_data("test/cwl/empty.json") as inputs_file:
+                self._tester(cwl_file, inputs_file, {}, tmp_path)
 
     @needs_slurm
-    def test_slurm_node_memory(self) -> None:
+    @pytest.mark.slurm
+    def test_slurm_node_memory(self, tmp_path: Path) -> None:
         pass
 
         # Run the workflow. This will either finish quickly and tell us the
@@ -497,28 +528,29 @@ class CWLWorkflowTest(ToilTest):
         # And if we run out of time we need to stop the workflow gracefully and
         # cancel the Slurm jobs.
 
-        main_args = [
-            f"--jobStore={self.jobStoreDir}",
-            # Avoid racing to toil kill before the jobstore is removed
-            "--clean=never",
-            "--batchSystem=slurm",
-            "--no-cwl-default-ram",
-            "--slurmDefaultAllMem=True",
-            "--outdir",
-            self.outDir,
-            os.path.join(self.rootDir, "src/toil/test/cwl/measure_default_memory.cwl"),
-        ]
         try:
-            log.debug("Start test workflow")
-            child = subprocess.Popen(
-                ["toil-cwl-runner"] + main_args, stdout=subprocess.PIPE
-            )
-            output, _ = child.communicate(timeout=60)
+            with get_data("test/cwl/measure_default_memory.cwl") as cwl_file:
+                main_args = [
+                    f"--jobStore={str(tmp_path / 'jobStoreDir')}",
+                    # Avoid racing to toil kill before the jobstore is removed
+                    "--clean=never",
+                    "--batchSystem=slurm",
+                    "--no-cwl-default-ram",
+                    "--slurmDefaultAllMem=True",
+                    "--outdir",
+                    str(tmp_path / "outdir"),
+                    str(cwl_file),
+                ]
+                log.debug("Start test workflow")
+                child = subprocess.Popen(
+                    ["toil-cwl-runner"] + main_args, stdout=subprocess.PIPE
+                )
+                output, _ = child.communicate(timeout=60)
         except subprocess.TimeoutExpired:
             # The job didn't finish quickly; presumably waiting for a full node.
             # Stop the workflow
             log.debug("Workflow might be waiting for a full node. Stop it.")
-            subprocess.check_call(["toil", "kill", self.jobStoreDir])
+            subprocess.check_call(["toil", "kill", str(tmp_path / "jobStoreDir")])
             # Wait another little bit for it to clean up, making sure to collect output in case it is blocked on writing
             child.communicate(timeout=20)
             # Kill it off in case it is still running
@@ -538,113 +570,151 @@ class CWLWorkflowTest(ToilTest):
             else:
                 result = int(memory_string)
             # We should see more than the CWL default or the Toil default, assuming Slurm nodes of reasonable size (3 GiB).
-            self.assertGreater(result, 3 * 1024 * 1024)
+            assert result > (3 * 1024 * 1024)
 
     @needs_aws_s3
-    def test_download_s3(self) -> None:
-        self.download("download_s3.json", self._tester)
+    @pytest.mark.aws_s3
+    @pytest.mark.online
+    def test_download_s3(self, tmp_path: Path) -> None:
+        self.download("download_s3.json", self._tester, tmp_path)
 
-    def test_download_http(self) -> None:
-        self.download("download_http.json", self._tester)
+    def test_download_http(self, tmp_path: Path) -> None:
+        self.download("download_http.json", self._tester, tmp_path)
 
-    def test_download_https(self) -> None:
-        self.download("download_https.json", self._tester)
+    def test_download_https(self, tmp_path: Path) -> None:
+        self.download("download_https.json", self._tester, tmp_path)
 
-    def test_download_https_reference(self) -> None:
+    def test_download_https_reference(self, tmp_path: Path) -> None:
         self.download(
             "download_https.json",
             partial(self._tester, main_args=["--reference-inputs"]),
+            tmp_path,
         )
 
-    def test_download_file(self) -> None:
-        self.download("download_file.json", self._tester)
+    def test_download_file(self, tmp_path: Path) -> None:
+        self.download("download_file.json", self._tester, tmp_path)
 
     @needs_aws_s3
-    def test_download_directory_s3(self) -> None:
-        self.download_directory("download_directory_s3.json", self._tester)
+    @pytest.mark.aws_s3
+    @pytest.mark.online
+    def test_download_directory_s3(self, tmp_path: Path) -> None:
+        self.download_directory("download_directory_s3.json", self._tester, tmp_path)
 
     @needs_aws_s3
-    def test_download_directory_s3_reference(self) -> None:
+    @pytest.mark.aws_s3
+    @pytest.mark.online
+    def test_download_directory_s3_reference(self, tmp_path: Path) -> None:
         self.download_directory(
             "download_directory_s3.json",
             partial(self._tester, main_args=["--reference-inputs"]),
+            tmp_path,
         )
 
-    def test_download_directory_file(self) -> None:
-        self.download_directory("download_directory_file.json", self._tester)
+    def test_download_directory_file(self, tmp_path: Path) -> None:
+        self.download_directory("download_directory_file.json", self._tester, tmp_path)
 
     @needs_aws_s3
-    def test_download_subdirectory_s3(self) -> None:
-        self.download_subdirectory("download_subdirectory_s3.json", self._tester)
+    @pytest.mark.aws_s3
+    @pytest.mark.online
+    def test_download_subdirectory_s3(self, tmp_path: Path) -> None:
+        self.download_subdirectory(
+            "download_subdirectory_s3.json", self._tester, tmp_path
+        )
 
-    def test_download_subdirectory_file(self) -> None:
-        self.download_subdirectory("download_subdirectory_file.json", self._tester)
+    def test_download_subdirectory_file(self, tmp_path: Path) -> None:
+        self.download_subdirectory(
+            "download_subdirectory_file.json", self._tester, tmp_path
+        )
 
     # We also want to make sure we can run a bare tool with loadContents on the inputs, which requires accessing the input data early in the leader.
 
     @needs_aws_s3
-    def test_load_contents_s3(self) -> None:
-        self.load_contents("download_s3.json", self._tester)
+    @pytest.mark.aws_s3
+    @pytest.mark.online
+    def test_load_contents_s3(self, tmp_path: Path) -> None:
+        self.load_contents("download_s3.json", self._tester, tmp_path)
 
-    def test_load_contents_http(self) -> None:
-        self.load_contents("download_http.json", self._tester)
+    def test_load_contents_http(self, tmp_path: Path) -> None:
+        self.load_contents("download_http.json", self._tester, tmp_path)
 
-    def test_load_contents_https(self) -> None:
-        self.load_contents("download_https.json", self._tester)
+    def test_load_contents_https(self, tmp_path: Path) -> None:
+        self.load_contents("download_https.json", self._tester, tmp_path)
 
-    def test_load_contents_file(self) -> None:
-        self.load_contents("download_file.json", self._tester)
+    def test_load_contents_file(self, tmp_path: Path) -> None:
+        self.load_contents("download_file.json", self._tester, tmp_path)
 
     @slow
     @pytest.mark.integrative
-    @unittest.skip("Fails too often due to remote service")
-    def test_bioconda(self) -> None:
-        self._tester(
-            "src/toil/test/cwl/seqtk_seq.cwl",
-            "src/toil/test/cwl/seqtk_seq_job.json",
-            self._expected_seqtk_output(self.outDir),
-            main_args=["--beta-conda-dependencies"],
-            out_name="output1",
-        )
+    @pytest.mark.slow
+    @pytest.mark.skip("Fails too often due to remote service")
+    def test_bioconda(self, tmp_path: Path) -> None:
+        with get_data("test/cwl/seqtk_seq.cwl") as cwl_file:
+            with get_data("test/cwl/seqtk_seq_job.json") as inputs_file:
+                self._tester(
+                    cwl_file,
+                    inputs_file,
+                    self._expected_seqtk_output(tmp_path),
+                    tmp_path,
+                    main_args=["--beta-conda-dependencies"],
+                    out_name="output1",
+                )
 
     @needs_docker
-    def test_default_args(self) -> None:
-        self._tester(
-            "src/toil/test/cwl/seqtk_seq.cwl",
-            "src/toil/test/cwl/seqtk_seq_job.json",
-            self._expected_seqtk_output(self.outDir),
-            main_args=[
-                "--default-container",
-                "quay.io/biocontainers/seqtk:1.4--he4a0461_1",
-            ],
-            out_name="output1",
-        )
+    @pytest.mark.docker
+    @pytest.mark.online
+    def test_default_args(self, tmp_path: Path) -> None:
+        with get_data("test/cwl/seqtk_seq.cwl") as cwl_file:
+            with get_data("test/cwl/seqtk_seq_job.json") as inputs_file:
+                self._tester(
+                    cwl_file,
+                    inputs_file,
+                    self._expected_seqtk_output(tmp_path),
+                    tmp_path,
+                    main_args=[
+                        "--default-container",
+                        "quay.io/biocontainers/seqtk:1.4--he4a0461_1",
+                    ],
+                    out_name="output1",
+                )
 
     @needs_docker
+    @pytest.mark.docker
     @pytest.mark.integrative
-    @unittest.skip("Fails too often due to remote service")
-    def test_biocontainers(self) -> None:
-        self._tester(
-            "src/toil/test/cwl/seqtk_seq.cwl",
-            "src/toil/test/cwl/seqtk_seq_job.json",
-            self._expected_seqtk_output(self.outDir),
-            main_args=["--beta-use-biocontainers"],
-            out_name="output1",
-        )
+    @pytest.mark.online
+    @pytest.mark.skip(reason="Fails too often due to remote service")
+    def test_biocontainers(self, tmp_path: Path) -> None:
+        with get_data("test/cwl/seqtk_seq.cwl") as cwl_file:
+            with get_data("test/cwl/seqtk_seq_job.json") as inputs_file:
+                self._tester(
+                    cwl_file,
+                    inputs_file,
+                    self._expected_seqtk_output(tmp_path),
+                    tmp_path,
+                    main_args=["--beta-use-biocontainers"],
+                    out_name="output1",
+                )
 
     @needs_docker
     @needs_docker_cuda
     @needs_local_cuda
-    def test_cuda(self) -> None:
-        self._tester(
-            "src/toil/test/cwl/nvidia_smi.cwl",
-            "src/toil/test/cwl/empty.json",
-            {},
-            out_name="result",
-        )
+    @pytest.mark.docker
+    @pytest.mark.online
+    @pytest.mark.docker_cuda
+    @pytest.mark.local_cuda
+    def test_cuda(self, tmp_path: Path) -> None:
+        with get_data("test/cwl/nvidia_smi.cwl") as cwl_file:
+            with get_data("test/cwl/empty.json") as inputs_file:
+                self._tester(
+                    cwl_file,
+                    inputs_file,
+                    {},
+                    tmp_path,
+                    out_name="result",
+                )
 
     @slow
-    def test_restart(self) -> None:
+    @pytest.mark.slow
+    def test_restart(self, tmp_path: Path) -> None:
         """
         Enable restarts with toil-cwl-runner -- run failing test, re-run correct test.
         Only implemented for single machine.
@@ -652,151 +722,203 @@ class CWLWorkflowTest(ToilTest):
         log.info("Running CWL Test Restart.  Expecting failure, then success.")
         from toil.cwl import cwltoil
 
-        outDir = self._createTempDir()
-        cwlDir = os.path.join(self._projectRootPath(), "src", "toil", "test", "cwl")
-        cmd = [
-            "--outdir",
-            outDir,
-            "--jobStore",
-            os.path.join(outDir, "jobStore"),
-            "--no-container",
-            os.path.join(cwlDir, "revsort.cwl"),
-            os.path.join(cwlDir, "revsort-job.json"),
-        ]
+        outDir = tmp_path / "outDir"
+        outDir.mkdir()
+        jobStore = tmp_path / "jobStore"
+        with get_data("test/cwl/revsort.cwl") as cwl_file:
+            with get_data("test/cwl/revsort-job.json") as job_file:
+                cmd = [
+                    "--outdir",
+                    str(outDir),
+                    "--jobStore",
+                    str(jobStore),
+                    "--no-container",
+                    str(cwl_file),
+                    str(job_file),
+                ]
+
+                # create a fake rev bin that actually points to the "date" binary
+                cal_path = [
+                    d
+                    for d in os.environ["PATH"].split(":")
+                    if os.path.exists(os.path.join(d, "date"))
+                ][-1]
+                os.symlink(
+                    os.path.realpath(os.path.join(cal_path, "date")), outDir / "rev"
+                )
+
+                def path_with_bogus_rev() -> str:
+                    # append to the front of the PATH so that we check there first
+                    return f"{str(outDir)}:" + os.environ["PATH"]
+
+                orig_path = os.environ["PATH"]
+                # Force a failure by trying to use an incorrect version of `rev` from the PATH
+                os.environ["PATH"] = path_with_bogus_rev()
+                try:
+                    subprocess.check_output(
+                        ["toil-cwl-runner"] + cmd,
+                        env=os.environ.copy(),
+                        stderr=subprocess.STDOUT,
+                    )
+                    pytest.fail("Expected problem job with incorrect PATH did not fail")
+                except subprocess.CalledProcessError:
+                    pass
+                # Finish the job with a correct PATH
+                os.environ["PATH"] = orig_path
+                cmd.insert(0, "--restart")
+                cwltoil.main(cmd)
+                # Should fail because previous job completed successfully
+                try:
+                    subprocess.check_output(
+                        ["toil-cwl-runner"] + cmd,
+                        env=os.environ.copy(),
+                        stderr=subprocess.STDOUT,
+                    )
+                    pytest.fail("Restart with missing directory did not fail")
+                except subprocess.CalledProcessError:
+                    pass
+
+    def test_caching(self, tmp_path: Path) -> None:
+        log.info("Running CWL caching test.")
+        from toil.cwl import cwltoil
 
-        # create a fake rev bin that actually points to the "date" binary
-        cal_path = [
-            d
-            for d in os.environ["PATH"].split(":")
-            if os.path.exists(os.path.join(d, "date"))
-        ][-1]
-        os.symlink(os.path.join(cal_path, "date"), f'{os.path.join(outDir, "rev")}')
-
-        def path_with_bogus_rev() -> str:
-            # append to the front of the PATH so that we check there first
-            return f"{outDir}:" + os.environ["PATH"]
-
-        orig_path = os.environ["PATH"]
-        # Force a failure by trying to use an incorrect version of `rev` from the PATH
-        os.environ["PATH"] = path_with_bogus_rev()
-        try:
-            subprocess.check_output(
-                ["toil-cwl-runner"] + cmd,
-                env=os.environ.copy(),
-                stderr=subprocess.STDOUT,
-            )
-            self.fail("Expected problem job with incorrect PATH did not fail")
-        except subprocess.CalledProcessError:
-            pass
-        # Finish the job with a correct PATH
-        os.environ["PATH"] = orig_path
-        cmd.insert(0, "--restart")
-        cwltoil.main(cmd)
-        # Should fail because previous job completed successfully
-        try:
-            subprocess.check_output(
-                ["toil-cwl-runner"] + cmd,
-                env=os.environ.copy(),
-                stderr=subprocess.STDOUT,
-            )
-            self.fail("Restart with missing directory did not fail")
-        except subprocess.CalledProcessError:
-            pass
+        outDir = tmp_path / "outDir"
+        cacheDir = tmp_path / "cacheDir"
+        log_path = outDir / "log"
+
+        with get_data("test/cwl/revsort.cwl") as cwl_file:
+            with get_data("test/cwl/revsort-job.json") as job_file:
+                cmd = [
+                    "--outdir",
+                    str(outDir),
+                    "--jobStore",
+                    str(tmp_path / "jobStore"),
+                    "--clean=always",
+                    "--no-container",
+                    "--cachedir",
+                    str(cacheDir),
+                    str(cwl_file),
+                    str(job_file),
+                ]
+                st = StringIO()
+                ret = cwltoil.main(cmd, stdout=st)
+                assert ret == 0
+                # cwltool hashes certain steps into directories, ensure it exists
+                # since cwltool caches per task and revsort has 2 cwl tasks, there should be 2 directories and 2 status files
+                assert sum(1 for _ in cacheDir.iterdir()) == 4
+
+                # Rerun the workflow to ensure there is a cache hit and that we don't rerun the tools
+                st = StringIO()
+                cmd = [
+                    "--writeLogsFromAllJobs=True",
+                    "--writeLogs",
+                    str(log_path),
+                ] + cmd
+                ret = cwltoil.main(cmd, stdout=st)
+                assert ret == 0
+
+                # Ensure all of the worker logs are using their cached outputs
+                for file in log_path.iterdir():
+                    assert "Using cached output" in file.read_text(encoding="utf-8")
 
     @needs_aws_s3
-    def test_streamable(self, extra_args: Optional[list[str]] = None) -> None:
+    @pytest.mark.aws_s3
+    @pytest.mark.online
+    def test_streamable(
+        self, tmp_path: Path, extra_args: Optional[list[str]] = None
+    ) -> None:
         """
         Test that a file with 'streamable'=True is a named pipe.
         This is a CWL1.2 feature.
         """
-        cwlfile = "src/toil/test/cwl/stream.cwl"
-        jobfile = "src/toil/test/cwl/stream.json"
-        out_name = "output"
-        jobstore = f"--jobStore=aws:us-west-1:toil-stream-{uuid.uuid4()}"
-        from toil.cwl import cwltoil
-
         st = StringIO()
-        args = [
-            "--logDebug",
-            "--outdir",
-            self.outDir,
-            jobstore,
-            os.path.join(self.rootDir, cwlfile),
-            os.path.join(self.rootDir, jobfile),
-        ]
-        if extra_args:
-            args = extra_args + args
-        log.info("Run CWL run: %s", " ".join(args))
-        cwltoil.main(args, stdout=st)
+        outDir = tmp_path / "outDir"
+        with get_data("test/cwl/stream.cwl") as cwlfile:
+            with get_data("test/cwl/stream.json") as jobfile:
+                out_name = "output"
+                jobstore = f"--jobStore=aws:us-west-1:toil-stream-{uuid.uuid4()}"
+                from toil.cwl import cwltoil
+
+                args = [
+                    "--logDebug",
+                    "--outdir",
+                    str(outDir),
+                    jobstore,
+                    str(cwlfile),
+                    str(jobfile),
+                ]
+                if extra_args:
+                    args = extra_args + args
+                log.info("Run CWL run: %s", " ".join(args))
+                cwltoil.main(args, stdout=st)
         out = json.loads(st.getvalue())
         out[out_name].pop("http://commonwl.org/cwltool#generation", None)
         out[out_name].pop("nameext", None)
         out[out_name].pop("nameroot", None)
-        self.assertEqual(out, self._expected_streaming_output(self.outDir))
+        assert out == self._expected_streaming_output(outDir)
         with open(out[out_name]["location"][len("file://") :]) as f:
-            self.assertEqual(f.read().strip(), "When is s4 coming out?")
+            assert f.read().strip() == "When is s4 coming out?"
 
     @needs_aws_s3
-    def test_streamable_reference(self) -> None:
+    @pytest.mark.aws_s3
+    @pytest.mark.online
+    def test_streamable_reference(self, tmp_path: Path) -> None:
         """
         Test that a streamable file is a stream even when passed around by URI.
         """
-        self.test_streamable(extra_args=["--reference-inputs"])
+        self.test_streamable(tmp_path=tmp_path, extra_args=["--reference-inputs"])
 
-    def test_preemptible(self) -> None:
+    def test_preemptible(self, tmp_path: Path) -> None:
         """
         Tests that the http://arvados.org/cwl#UsePreemptible extension is supported.
         """
-        cwlfile = "src/toil/test/cwl/preemptible.cwl"
-        jobfile = "src/toil/test/cwl/empty.json"
-        out_name = "output"
         from toil.cwl import cwltoil
 
         st = StringIO()
-        args = [
-            "--outdir",
-            self.outDir,
-            os.path.join(self.rootDir, cwlfile),
-            os.path.join(self.rootDir, jobfile),
-        ]
-        cwltoil.main(args, stdout=st)
+        out_name = "output"
+        with get_data("test/cwl/preemptible.cwl") as cwlfile:
+            with get_data("test/cwl/empty.json") as jobfile:
+                args = [
+                    "--outdir",
+                    str(tmp_path / "outDir"),
+                    str(cwlfile),
+                    str(jobfile),
+                ]
+                cwltoil.main(args, stdout=st)
         out = json.loads(st.getvalue())
         out[out_name].pop("http://commonwl.org/cwltool#generation", None)
         out[out_name].pop("nameext", None)
         out[out_name].pop("nameroot", None)
         with open(out[out_name]["location"][len("file://") :]) as f:
-            self.assertEqual(f.read().strip(), "hello")
+            assert f.read().strip() == "hello"
 
-    def test_preemptible_expression(self) -> None:
+    def test_preemptible_expression(self, tmp_path: Path) -> None:
         """
         Tests that the http://arvados.org/cwl#UsePreemptible extension is validated.
         """
-        cwlfile = "src/toil/test/cwl/preemptible_expression.cwl"
-        jobfile = "src/toil/test/cwl/preemptible_expression.json"
         from toil.cwl import cwltoil
 
         st = StringIO()
-        args = [
-            "--outdir",
-            self.outDir,
-            os.path.join(self.rootDir, cwlfile),
-            os.path.join(self.rootDir, jobfile),
-        ]
-        try:
-            cwltoil.main(args, stdout=st)
-            raise RuntimeError("Did not raise correct exception")
-        except ValidationException as e:
-            # Make sure we chastise the user appropriately.
-            assert "expressions are not allowed" in str(e)
+        with get_data("test/cwl/preemptible_expression.cwl") as cwlfile:
+            with get_data("test/cwl/preemptible_expression.json") as jobfile:
+                args = [
+                    "--outdir",
+                    str(tmp_path),
+                    str(cwlfile),
+                    str(jobfile),
+                ]
+                with pytest.raises(
+                    ValidationException, match=re.escape("expressions are not allowed")
+                ):
+                    cwltoil.main(args, stdout=st)
 
     @staticmethod
-    def _expected_seqtk_output(outDir: str) -> "CWLObjectType":
-        path = os.path.join(outDir, "out")
-        loc = "file://" + path
+    def _expected_seqtk_output(outDir: Path) -> "CWLObjectType":
+        path = outDir / "out"
         return {
             "output1": {
-                "location": loc,
-                "path": path,
+                "location": path.as_uri(),
+                "path": str(path),
                 "checksum": "sha1$322e001e5a99f19abdce9f02ad0f02a17b5066c2",
                 "basename": "out",
                 "class": "File",
@@ -805,13 +927,12 @@ class CWLWorkflowTest(ToilTest):
         }
 
     @staticmethod
-    def _expected_revsort_output(outDir: str) -> "CWLObjectType":
-        path = os.path.join(outDir, "output.txt")
-        loc = "file://" + path
+    def _expected_revsort_output(outDir: Path) -> "CWLObjectType":
+        path = outDir / "output.txt"
         return {
             "output": {
-                "location": loc,
-                "path": path,
+                "location": path.as_uri(),
+                "path": str(path),
                 "basename": "output.txt",
                 "size": 1111,
                 "class": "File",
@@ -820,13 +941,12 @@ class CWLWorkflowTest(ToilTest):
         }
 
     @staticmethod
-    def _expected_revsort_nochecksum_output(outDir: str) -> "CWLObjectType":
-        path = os.path.join(outDir, "output.txt")
-        loc = "file://" + path
+    def _expected_revsort_nochecksum_output(outDir: Path) -> "CWLObjectType":
+        path = outDir / "output.txt"
         return {
             "output": {
-                "location": loc,
-                "path": path,
+                "location": path.as_uri(),
+                "path": str(path),
                 "basename": "output.txt",
                 "size": 1111,
                 "class": "File",
@@ -834,30 +954,29 @@ class CWLWorkflowTest(ToilTest):
         }
 
     @staticmethod
-    def _expected_download_output(outDir: str) -> "CWLObjectType":
-        path = os.path.join(outDir, "output.txt")
-        loc = "file://" + path
+    def _expected_download_output(outDir: Path) -> "CWLObjectType":
+        path = outDir / "output.txt"
         return {
             "output": {
-                "location": loc,
+                "location": path.as_uri(),
                 "basename": "output.txt",
                 "size": 0,
                 "class": "File",
                 "checksum": "sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709",
-                "path": path,
+                "path": str(path),
             }
         }
 
     @staticmethod
-    def _expected_glob_dir_output(out_dir: str) -> "CWLObjectType":
-        dir_path = os.path.join(out_dir, "shouldmake")
-        dir_loc = "file://" + dir_path
-        file_path = os.path.join(dir_path, "test.txt")
-        file_loc = os.path.join(dir_loc, "test.txt")
+    def _expected_glob_dir_output(out_dir: Path) -> "CWLObjectType":
+        dir_path = out_dir / "shouldmake"
+        dir_loc = dir_path.as_uri()
+        file_path = dir_path / "test.txt"
+        file_loc = file_path.as_uri()
         return {
             "shouldmake": {
                 "location": dir_loc,
-                "path": dir_path,
+                "path": str(dir_path),
                 "basename": "shouldmake",
                 "nameroot": "shouldmake",
                 "nameext": "",
@@ -866,7 +985,7 @@ class CWLWorkflowTest(ToilTest):
                     {
                         "class": "File",
                         "location": file_loc,
-                        "path": file_path,
+                        "path": str(file_path),
                         "basename": "test.txt",
                         "checksum": "sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709",
                         "size": 0,
@@ -878,7 +997,7 @@ class CWLWorkflowTest(ToilTest):
         }
 
     @classmethod
-    def _expected_load_contents_output(cls, out_dir: str) -> "CWLObjectType":
+    def _expected_load_contents_output(cls, out_dir: Path) -> "CWLObjectType":
         """
         Generate the putput we expect from load_contents.cwl, when sending
         output files to the given directory.
@@ -888,13 +1007,15 @@ class CWLWorkflowTest(ToilTest):
         return expected
 
     @staticmethod
-    def _expected_colon_output(outDir: str) -> "CWLObjectType":
-        path = os.path.join(outDir, "A:Gln2Cys_result")
-        loc = "file://" + os.path.join(outDir, "A%3AGln2Cys_result")
+    def _expected_colon_output(outDir: Path) -> "CWLObjectType":
+        path = outDir / "A:Gln2Cys_result"
+        loc = "file://" + os.path.join(
+            outDir, "A%3AGln2Cys_result"
+        )  # not using .as_uri to ensure the expected escaping
         return {
             "result": {
                 "location": loc,
-                "path": path,
+                "path": str(path),
                 "basename": "A:Gln2Cys_result",
                 "class": "Directory",
                 "listing": [
@@ -912,13 +1033,12 @@ class CWLWorkflowTest(ToilTest):
             }
         }
 
-    def _expected_streaming_output(self, outDir: str) -> "CWLObjectType":
-        path = os.path.join(outDir, "output.txt")
-        loc = "file://" + path
+    def _expected_streaming_output(self, outDir: Path) -> "CWLObjectType":
+        path = outDir / "output.txt"
         return {
             "output": {
-                "location": loc,
-                "path": path,
+                "location": path.as_uri(),
+                "path": str(path),
                 "basename": "output.txt",
                 "size": 24,
                 "class": "File",
@@ -926,50 +1046,100 @@ class CWLWorkflowTest(ToilTest):
             }
         }
 
+    @needs_docker
+    @pytest.mark.docker
+    @pytest.mark.online
+    def test_missing_import(self, tmp_path: Path) -> None:
+        with get_data("test/cwl/revsort.cwl") as cwl_file:
+            with get_data("test/cwl/revsort-job-missing.json") as inputs_file:
+                cmd = [
+                    "toil-cwl-runner",
+                    f"--outdir={str(tmp_path)}",
+                    "--clean=always",
+                    str(cwl_file),
+                    str(inputs_file),
+                ]
+                p = subprocess.run(cmd, capture_output=True, text=True)
+                # Make sure that the missing file is mentioned in the log so the user knows
+                assert p.returncode == 1, p.stderr
+                assert "missing.txt" in p.stderr
+
+    @needs_aws_s3
+    @pytest.mark.aws_s3
+    @pytest.mark.online
+    def test_optional_secondary_files_exists(self, tmp_path: Path) -> None:
+        from toil.cwl import cwltoil
+
+        with get_data("test/cwl/optional-file.cwl") as cwlfile:
+            with get_data("test/cwl/optional-file-exists.json") as jobfile:
+                args = [str(cwlfile), str(jobfile), f"--outdir={str(tmp_path)}"]
+                ret = cwltoil.main(args)
+                assert ret == 0
+                assert (tmp_path / "wdl_templates_old.zip").exists()
+
+    @needs_aws_s3
+    @pytest.mark.aws_s3
+    @pytest.mark.online
+    def test_optional_secondary_files_missing(self, tmp_path: Path) -> None:
+        from toil.cwl import cwltoil
+
+        with get_data("test/cwl/optional-file.cwl") as cwlfile:
+            with get_data("test/cwl/optional-file-missing.json") as jobfile:
+                args = [str(cwlfile), str(jobfile), f"--outdir={str(tmp_path)}"]
+                ret = cwltoil.main(args)
+                assert ret == 0
+                assert not (tmp_path / "hello_old.zip").exists()
+
+
+@pytest.fixture(scope="function")
+def cwl_v1_0_spec(tmp_path: Path) -> Generator[Path]:
+    # The latest cwl git commit hash from https://github.com/common-workflow-language/common-workflow-language.
+    # Update it to get the latest tests.
+    testhash = (
+        "6a955874ade22080b8ef962b4e0d6e408112c1ef"  # Date:   Tue Dec 16 2020 8:43pm PST
+    )
+    url = (
+        "https://github.com/common-workflow-language/common-workflow-language/archive/%s.zip"
+        % testhash
+    )
+    urlretrieve(url, "spec.zip")
+    with zipfile.ZipFile("spec.zip", "r") as z:
+        z.extractall()
+    shutil.move("common-workflow-language-%s" % testhash, str(tmp_path))
+    os.remove("spec.zip")
+    try:
+        yield tmp_path / ("common-workflow-language-%s" % testhash)
+    finally:
+        pass  # no cleanup
+
 
 @needs_cwl
 @needs_online
-class CWLv10Test(ToilTest):
+@pytest.mark.cwl
+@pytest.mark.online
+class TestCWLv10:
     """
     Run the CWL 1.0 conformance tests in various environments.
     """
 
-    def setUp(self) -> None:
-        """Runs anew before each test to create farm fresh temp dirs."""
-        self.outDir = f"/tmp/toil-cwl-test-{str(uuid.uuid4())}"
-        os.makedirs(self.outDir)
-        self.rootDir = self._projectRootPath()
-        self.cwlSpec = os.path.join(self.rootDir, "src/toil/test/cwl/spec")
-        self.workDir = os.path.join(self.cwlSpec, "v1.0")
-        # The latest cwl git commit hash from https://github.com/common-workflow-language/common-workflow-language.
-        # Update it to get the latest tests.
-        testhash = "6a955874ade22080b8ef962b4e0d6e408112c1ef"  # Date:   Tue Dec 16 2020 8:43pm PST
-        url = (
-            "https://github.com/common-workflow-language/common-workflow-language/archive/%s.zip"
-            % testhash
-        )
-        if not os.path.exists(self.cwlSpec):
-            urlretrieve(url, "spec.zip")
-            with zipfile.ZipFile("spec.zip", "r") as z:
-                z.extractall()
-            shutil.move("common-workflow-language-%s" % testhash, self.cwlSpec)
-            os.remove("spec.zip")
-
-    def tearDown(self) -> None:
-        """Clean up outputs."""
-        if os.path.exists(self.outDir):
-            shutil.rmtree(self.outDir)
-        unittest.TestCase.tearDown(self)
-
     @slow
+    @needs_docker
+    @pytest.mark.slow
+    @pytest.mark.docker
+    @pytest.mark.online
     @pytest.mark.timeout(CONFORMANCE_TEST_TIMEOUT)
-    def test_run_conformance_with_caching(self) -> None:
-        self.test_run_conformance(caching=True)
+    def test_run_conformance_with_caching(self, cwl_v1_0_spec: Path) -> None:
+        self.test_run_conformance(cwl_v1_0_spec, caching=True)
 
     @slow
+    @needs_docker
+    @pytest.mark.slow
+    @pytest.mark.docker
+    @pytest.mark.online
     @pytest.mark.timeout(CONFORMANCE_TEST_TIMEOUT)
     def test_run_conformance(
         self,
+        cwl_v1_0_spec: Path,
         batchSystem: Optional[str] = None,
         caching: bool = False,
         selected_tests: Optional[str] = None,
@@ -977,8 +1147,8 @@ class CWLv10Test(ToilTest):
         extra_args: Optional[list[str]] = None,
     ) -> None:
         run_conformance_tests(
-            workDir=self.workDir,
-            yml="conformance_test_v1.0.yaml",
+            workDir=str(cwl_v1_0_spec / "v1.0"),
+            yml=str(cwl_v1_0_spec / "v1.0" / "conformance_test_v1.0.yaml"),
             caching=caching,
             batchSystem=batchSystem,
             selected_tests=selected_tests,
@@ -988,38 +1158,66 @@ class CWLv10Test(ToilTest):
 
     @slow
     @needs_lsf
-    @unittest.skip("Not run")
-    def test_lsf_cwl_conformance(self, caching: bool = False) -> None:
-        self.test_run_conformance(batchSystem="lsf", caching=caching)
+    @pytest.mark.slow
+    @pytest.mark.lsf
+    @pytest.mark.skip("Not run")
+    def test_lsf_cwl_conformance(
+        self, cwl_v1_0_spec: Path, caching: bool = False
+    ) -> None:
+        self.test_run_conformance(cwl_v1_0_spec, batchSystem="lsf", caching=caching)
 
     @slow
     @needs_slurm
-    @unittest.skip("Not run")
-    def test_slurm_cwl_conformance(self, caching: bool = False) -> None:
-        self.test_run_conformance(batchSystem="slurm", caching=caching)
+    @pytest.mark.slow
+    @pytest.mark.slurm
+    @pytest.mark.skip("Not run")
+    def test_slurm_cwl_conformance(
+        self, cwl_v1_0_spec: Path, caching: bool = False
+    ) -> None:
+        self.test_run_conformance(cwl_v1_0_spec, batchSystem="slurm", caching=caching)
 
     @slow
     @needs_torque
-    @unittest.skip("Not run")
-    def test_torque_cwl_conformance(self, caching: bool = False) -> None:
-        self.test_run_conformance(batchSystem="torque", caching=caching)
+    @pytest.mark.slow
+    @pytest.mark.torque
+    @pytest.mark.skip("Not run")
+    def test_torque_cwl_conformance(
+        self, cwl_v1_0_spec: Path, caching: bool = False
+    ) -> None:
+        self.test_run_conformance(cwl_v1_0_spec, batchSystem="torque", caching=caching)
 
     @slow
     @needs_gridengine
-    @unittest.skip("Not run")
-    def test_gridengine_cwl_conformance(self, caching: bool = False) -> None:
-        self.test_run_conformance(batchSystem="grid_engine", caching=caching)
+    @pytest.mark.slow
+    @pytest.mark.gridengine
+    @pytest.mark.skip("Not run")
+    def test_gridengine_cwl_conformance(
+        self, cwl_v1_0_spec: Path, caching: bool = False
+    ) -> None:
+        self.test_run_conformance(
+            cwl_v1_0_spec, batchSystem="grid_engine", caching=caching
+        )
 
     @slow
     @needs_mesos
-    @unittest.skip("Not run")
-    def test_mesos_cwl_conformance(self, caching: bool = False) -> None:
-        self.test_run_conformance(batchSystem="mesos", caching=caching)
+    @pytest.mark.slow
+    @pytest.mark.mesos
+    @pytest.mark.skip("Not run")
+    def test_mesos_cwl_conformance(
+        self, cwl_v1_0_spec: Path, caching: bool = False
+    ) -> None:
+        self.test_run_conformance(cwl_v1_0_spec, batchSystem="mesos", caching=caching)
 
     @slow
     @needs_kubernetes
-    def test_kubernetes_cwl_conformance(self, caching: bool = False) -> None:
+    @pytest.mark.slow
+    @pytest.mark.kubernetes
+    @pytest.mark.online
+    def test_kubernetes_cwl_conformance(
+        self, cwl_v1_0_spec: Path, caching: bool = False
+    ) -> None:
         self.test_run_conformance(
+            cwl_v1_0_spec,
             caching=caching,
             batchSystem="kubernetes",
             extra_args=["--retryCount=3"],
@@ -1031,82 +1229,98 @@ class CWLv10Test(ToilTest):
 
     @slow
     @needs_lsf
-    @unittest.skip("Not run")
-    def test_lsf_cwl_conformance_with_caching(self) -> None:
-        self.test_lsf_cwl_conformance(caching=True)
+    @pytest.mark.slow
+    @pytest.mark.lsf
+    @pytest.mark.skip(reason="Not run")
+    def test_lsf_cwl_conformance_with_caching(self, cwl_v1_0_spec: Path) -> None:
+        self.test_lsf_cwl_conformance(cwl_v1_0_spec, caching=True)
 
     @slow
     @needs_slurm
-    @unittest.skip("Not run")
-    def test_slurm_cwl_conformance_with_caching(self) -> None:
-        self.test_slurm_cwl_conformance(caching=True)
+    @pytest.mark.slow
+    @pytest.mark.slurm
+    @pytest.mark.skip(reason="Not run")
+    def test_slurm_cwl_conformance_with_caching(self, cwl_v1_0_spec: Path) -> None:
+        self.test_slurm_cwl_conformance(cwl_v1_0_spec, caching=True)
 
     @slow
     @needs_torque
-    @unittest.skip("Not run")
-    def test_torque_cwl_conformance_with_caching(self) -> None:
-        self.test_torque_cwl_conformance(caching=True)
+    @pytest.mark.slow
+    @pytest.mark.torque
+    @pytest.mark.skip(reason="Not run")
+    def test_torque_cwl_conformance_with_caching(self, cwl_v1_0_spec: Path) -> None:
+        self.test_torque_cwl_conformance(cwl_v1_0_spec, caching=True)
 
     @slow
     @needs_gridengine
-    @unittest.skip("Not run")
-    def test_gridengine_cwl_conformance_with_caching(self) -> None:
-        self.test_gridengine_cwl_conformance(caching=True)
+    @pytest.mark.slow
+    @pytest.mark.gridengine
+    @pytest.mark.skip(reason="Not run")
+    def test_gridengine_cwl_conformance_with_caching(self, cwl_v1_0_spec: Path) -> None:
+        self.test_gridengine_cwl_conformance(cwl_v1_0_spec, caching=True)
 
     @slow
     @needs_mesos
-    @unittest.skip("Not run")
-    def test_mesos_cwl_conformance_with_caching(self) -> None:
-        self.test_mesos_cwl_conformance(caching=True)
+    @pytest.mark.slow
+    @pytest.mark.mesos
+    @pytest.mark.skip(reason="Not run")
+    def test_mesos_cwl_conformance_with_caching(self, cwl_v1_0_spec: Path) -> None:
+        self.test_mesos_cwl_conformance(cwl_v1_0_spec, caching=True)
 
     @slow
     @needs_kubernetes
-    def test_kubernetes_cwl_conformance_with_caching(self) -> None:
-        self.test_kubernetes_cwl_conformance(caching=True)
+    @pytest.mark.slow
+    @pytest.mark.kubernetes
+    @pytest.mark.online
+    def test_kubernetes_cwl_conformance_with_caching(self, cwl_v1_0_spec: Path) -> None:
+        self.test_kubernetes_cwl_conformance(cwl_v1_0_spec, caching=True)
+
+
+@pytest.fixture(scope="function")
+def cwl_v1_1_spec(tmp_path: Path) -> Generator[Path]:
+    # The latest cwl git commit hash from https://github.com/common-workflow-language/cwl-v1.1
+    # Update it to get the latest tests.
+    testhash = "664835e83eb5e57eee18a04ce7b05fb9d70d77b7"
+    url = (
+        "https://github.com/common-workflow-language/cwl-v1.1/archive/%s.zip" % testhash
+    )
+    urlretrieve(url, "spec.zip")
+    with zipfile.ZipFile("spec.zip", "r") as z:
+        z.extractall()
+    shutil.move("cwl-v1.1-%s" % testhash, str(tmp_path))
+    os.remove("spec.zip")
+    try:
+        yield tmp_path / ("cwl-v1.1-%s" % testhash)
+    finally:
+        pass  # no cleanup
 
 
 @needs_cwl
 @needs_online
-class CWLv11Test(ToilTest):
+@pytest.mark.cwl
+@pytest.mark.online
+class TestCWLv11:
     """
     Run the CWL 1.1 conformance tests in various environments.
     """
 
-    rootDir: str
-    cwlSpec: str
-    test_yaml: str
-
-    @classmethod
-    def setUpClass(cls) -> None:
-        """Runs anew before each test."""
-        cls.rootDir = cls._projectRootPath()
-        cls.cwlSpec = os.path.join(cls.rootDir, "src/toil/test/cwl/spec_v11")
-        cls.test_yaml = os.path.join(cls.cwlSpec, "conformance_tests.yaml")
-        # TODO: Use a commit zip in case someone decides to rewrite master's history?
-        url = "https://github.com/common-workflow-language/cwl-v1.1.git"
-        commit = "664835e83eb5e57eee18a04ce7b05fb9d70d77b7"
-        p = subprocess.Popen(
-            f"git clone {url} {cls.cwlSpec} && cd {cls.cwlSpec} && git checkout {commit}",
-            shell=True,
-        )
-        p.communicate()
-
-    def tearDown(self) -> None:
-        """Clean up outputs."""
-        unittest.TestCase.tearDown(self)
-
     @slow
+    @needs_docker
+    @pytest.mark.slow
+    @pytest.mark.docker
+    @pytest.mark.online
     @pytest.mark.timeout(CONFORMANCE_TEST_TIMEOUT)
     def test_run_conformance(
         self,
+        cwl_v1_1_spec: Path,
         caching: bool = False,
         batchSystem: Optional[str] = None,
         skipped_tests: Optional[str] = None,
         extra_args: Optional[list[str]] = None,
     ) -> None:
         run_conformance_tests(
-            workDir=self.cwlSpec,
-            yml=self.test_yaml,
+            workDir=str(cwl_v1_1_spec),
+            yml=str(cwl_v1_1_spec / "conformance_tests.yaml"),
             caching=caching,
             batchSystem=batchSystem,
             skipped_tests=skipped_tests,
@@ -1114,14 +1328,24 @@ class CWLv11Test(ToilTest):
         )
 
     @slow
+    @needs_docker
+    @pytest.mark.slow
+    @pytest.mark.docker
+    @pytest.mark.online
     @pytest.mark.timeout(CONFORMANCE_TEST_TIMEOUT)
-    def test_run_conformance_with_caching(self) -> None:
-        self.test_run_conformance(caching=True)
+    def test_run_conformance_with_caching(self, cwl_v1_1_spec: Path) -> None:
+        self.test_run_conformance(cwl_v1_1_spec, caching=True)
 
     @slow
     @needs_kubernetes
-    def test_kubernetes_cwl_conformance(self, caching: bool = False) -> None:
+    @pytest.mark.slow
+    @pytest.mark.kubernetes
+    @pytest.mark.online
+    def test_kubernetes_cwl_conformance(
+        self, cwl_v1_1_spec: Path, caching: bool = False
+    ) -> None:
         self.test_run_conformance(
+            cwl_v1_1_spec,
             batchSystem="kubernetes",
             extra_args=["--retryCount=3"],
             # These tests don't work with
@@ -1133,44 +1357,50 @@ class CWLv11Test(ToilTest):
 
     @slow
     @needs_kubernetes
-    def test_kubernetes_cwl_conformance_with_caching(self) -> None:
-        self.test_kubernetes_cwl_conformance(caching=True)
+    @pytest.mark.slow
+    @pytest.mark.kubernetes
+    @pytest.mark.online
+    def test_kubernetes_cwl_conformance_with_caching(self, cwl_v1_1_spec: Path) -> None:
+        self.test_kubernetes_cwl_conformance(cwl_v1_1_spec, caching=True)
+
+
+@pytest.fixture(scope="function")
+def cwl_v1_2_spec(tmp_path: Path) -> Generator[Path]:
+    # The latest cwl git commit hash from https://github.com/common-workflow-language/cwl-v1.2
+    # Update it to get the latest tests.
+    testhash = "0d538a0dbc5518f3c6083ce4571926f65cb84f76"
+    url = (
+        "https://github.com/common-workflow-language/cwl-v1.2/archive/%s.zip" % testhash
+    )
+    urlretrieve(url, "spec.zip")
+    with zipfile.ZipFile("spec.zip", "r") as z:
+        z.extractall()
+    shutil.move("cwl-v1.2-%s" % testhash, str(tmp_path))
+    os.remove("spec.zip")
+    try:
+        yield tmp_path / ("cwl-v1.2-%s" % testhash)
+    finally:
+        pass  # no cleanup
 
 
 @needs_cwl
 @needs_online
-class CWLv12Test(ToilTest):
+@pytest.mark.cwl
+@pytest.mark.online
+class TestCWLv12:
     """
     Run the CWL 1.2 conformance tests in various environments.
     """
 
-    rootDir: str
-    cwlSpec: str
-    test_yaml: str
-
-    @classmethod
-    def setUpClass(cls) -> None:
-        """Runs anew before each test."""
-        cls.rootDir = cls._projectRootPath()
-        cls.cwlSpec = os.path.join(cls.rootDir, "src/toil/test/cwl/spec_v12")
-        cls.test_yaml = os.path.join(cls.cwlSpec, "conformance_tests.yaml")
-        # TODO: Use a commit zip in case someone decides to rewrite master's history?
-        url = "https://github.com/common-workflow-language/cwl-v1.2.git"
-        commit = "0d538a0dbc5518f3c6083ce4571926f65cb84f76"
-        p = subprocess.Popen(
-            f"git clone {url} {cls.cwlSpec} && cd {cls.cwlSpec} && git checkout {commit}",
-            shell=True,
-        )
-        p.communicate()
-
-    def tearDown(self) -> None:
-        """Clean up outputs."""
-        unittest.TestCase.tearDown(self)
-
     @slow
+    @needs_docker
+    @pytest.mark.slow
+    @pytest.mark.docker
+    @pytest.mark.online
     @pytest.mark.timeout(CONFORMANCE_TEST_TIMEOUT)
     def test_run_conformance(
         self,
+        cwl_v1_2_spec: Path,
         runner: Optional[str] = None,
         caching: bool = False,
         batchSystem: Optional[str] = None,
@@ -1181,10 +1411,10 @@ class CWLv12Test(ToilTest):
         junit_file: Optional[str] = None,
     ) -> None:
         if junit_file is None:
-            junit_file = os.path.join(self.rootDir, "conformance-1.2.junit.xml")
+            junit_file = os.path.abspath("conformance-1.2.junit.xml")
         run_conformance_tests(
-            workDir=self.cwlSpec,
-            yml=self.test_yaml,
+            workDir=str(cwl_v1_2_spec),
+            yml=str(cwl_v1_2_spec / "conformance_tests.yaml"),
             runner=runner,
             caching=caching,
             batchSystem=batchSystem,
@@ -1196,39 +1426,65 @@ class CWLv12Test(ToilTest):
         )
 
     @slow
+    @needs_docker
+    @pytest.mark.slow
+    @pytest.mark.docker
+    @pytest.mark.online
     @pytest.mark.timeout(CONFORMANCE_TEST_TIMEOUT)
-    def test_run_conformance_with_caching(self) -> None:
+    def test_run_conformance_with_caching(self, cwl_v1_2_spec: Path) -> None:
         self.test_run_conformance(
+            cwl_v1_2_spec,
             caching=True,
-            junit_file=os.path.join(self.rootDir, "caching-conformance-1.2.junit.xml"),
+            junit_file=os.path.abspath("caching-conformance-1.2.junit.xml"),
+        )
+
+    @slow
+    @needs_docker
+    @pytest.mark.slow
+    @pytest.mark.docker
+    @pytest.mark.timeout(CONFORMANCE_TEST_TIMEOUT)
+    def test_run_conformance_with_task_caching(
+        self, cwl_v1_2_spec: Path, tmp_path: Path
+    ) -> None:
+        self.test_run_conformance(
+            cwl_v1_2_spec,
+            junit_file=os.path.abspath("task-caching-conformance-1.2.junit.xml"),
+            extra_args=["--cachedir", str(tmp_path / "task_cache")],
         )
 
     @slow
+    @needs_docker
+    @pytest.mark.slow
+    @pytest.mark.docker
     @pytest.mark.timeout(CONFORMANCE_TEST_TIMEOUT)
-    def test_run_conformance_with_in_place_update(self) -> None:
+    def test_run_conformance_with_in_place_update(self, cwl_v1_2_spec: Path) -> None:
         """
         Make sure that with --bypass-file-store we properly support in place
         update on a single node, and that this doesn't break any other
         features.
         """
         self.test_run_conformance(
+            cwl_v1_2_spec,
             extra_args=["--bypass-file-store"],
             must_support_all_features=True,
-            junit_file=os.path.join(
-                self.rootDir, "in-place-update-conformance-1.2.junit.xml"
-            ),
+            junit_file=os.path.abspath("in-place-update-conformance-1.2.junit.xml"),
         )
 
     @slow
     @needs_kubernetes
+    @pytest.mark.slow
+    @pytest.mark.kubernetes
+    @pytest.mark.online
     def test_kubernetes_cwl_conformance(
-        self, caching: bool = False, junit_file: Optional[str] = None
+        self,
+        cwl_v1_2_spec: Path,
+        caching: bool = False,
+        junit_file: Optional[str] = None,
     ) -> None:
         if junit_file is None:
-            junit_file = os.path.join(
-                self.rootDir, "kubernetes-conformance-1.2.junit.xml"
-            )
+            junit_file = os.path.abspath("kubernetes-conformance-1.2.junit.xml")
         self.test_run_conformance(
+            cwl_v1_2_spec,
             caching=caching,
             batchSystem="kubernetes",
             extra_args=["--retryCount=3"],
@@ -1243,17 +1499,22 @@ class CWLv12Test(ToilTest):
 
     @slow
     @needs_kubernetes
-    def test_kubernetes_cwl_conformance_with_caching(self) -> None:
+    @pytest.mark.slow
+    @pytest.mark.kubernetes
+    @pytest.mark.online
+    def test_kubernetes_cwl_conformance_with_caching(self, cwl_v1_2_spec: Path) -> None:
         self.test_kubernetes_cwl_conformance(
+            cwl_v1_2_spec,
             caching=True,
-            junit_file=os.path.join(
-                self.rootDir, "kubernetes-caching-conformance-1.2.junit.xml"
-            ),
+            junit_file=os.path.abspath("kubernetes-caching-conformance-1.2.junit.xml"),
         )
 
     @slow
     @needs_wes_server
-    def test_wes_server_cwl_conformance(self) -> None:
+    @pytest.mark.slow
+    @pytest.mark.wes_server
+    @pytest.mark.online
+    def test_wes_server_cwl_conformance(self, cwl_v1_2_spec: Path) -> None:
         """
         Run the CWL conformance tests via WES. TOIL_WES_ENDPOINT must be
         specified. If the WES server requires authentication, set TOIL_WES_USER
@@ -1264,7 +1525,7 @@ class CWLv12Test(ToilTest):
         TOIL_WES_ENDPOINT=http://localhost:8080 \
         TOIL_WES_USER=test \
         TOIL_WES_PASSWORD=password \
-        python -m pytest src/toil/test/cwl/cwlTest.py::CWLv12Test::test_wes_server_cwl_conformance -vv --log-level INFO --log-cli-level INFO
+        python -m pytest src/toil/test/cwl/cwlTest.py::TestCWLv12::test_wes_server_cwl_conformance -vv --log-level INFO --log-cli-level INFO
         """
         endpoint = os.environ.get("TOIL_WES_ENDPOINT")
         extra_args = [f"--wes_endpoint={endpoint}"]
@@ -1279,6 +1540,7 @@ class CWLv12Test(ToilTest):
         #     e.g.: https://github.com/common-workflow-language/cwl-v1.2/blob/1.2.1_proposed/tests/mixed-versions/wf-v10.cwl#L4-L10
 
         self.test_run_conformance(
+            cwl_v1_2_spec,
             runner="toil-wes-cwl-runner",
             selected_tests="1-309,313-337",
             extra_args=extra_args,
@@ -1286,183 +1548,162 @@ class CWLv12Test(ToilTest):
 
 
 @needs_cwl
+@pytest.mark.cwl
 @pytest.mark.cwl_small_log_dir
 def test_workflow_echo_string_scatter_stderr_log_dir(tmp_path: Path) -> None:
     log_dir = tmp_path / "cwl-logs"
-    job_store = "test_workflow_echo_string_scatter_stderr_log_dir"
-    toil = "toil-cwl-runner"
-    jobstore = f"--jobStore={job_store}"
-    option_1 = "--strict-memory-limit"
-    option_2 = "--force-docker-pull"
-    option_3 = "--clean=always"
-    option_4 = f"--log-dir={log_dir}"
-    cwl = os.path.join(
-        os.path.dirname(__file__), "echo_string_scatter_capture_stdout.cwl"
-    )
-    cmd = [toil, jobstore, option_1, option_2, option_3, option_4, cwl]
-    p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-    stdout, stderr = p.communicate()
-    outputs = json.loads(stdout)
-    out_list = outputs["list_out"]
-    assert len(out_list) == 2, f"outList shoud have two file elements {out_list}"
-    out_base = outputs["list_out"][0]
-    # This is a test on the scatter functionality and stdout.
-    # Each value of scatter should generate a separate file in the output.
-    for index, file in enumerate(out_list):
-        if index > 0:
-            new_file_loc = out_base["location"] + f"_{index + 1}"
-        else:
-            new_file_loc = out_base["location"]
-        assert (
-            new_file_loc == file["location"]
-        ), f"Toil should have detected conflicts for these stdout files {new_file_loc} and {file}"
+    with get_data("test/cwl/echo_string_scatter_capture_stdout.cwl") as cwl_file:
+        cmd = [
+            "toil-cwl-runner",
+            f"--jobStore={tmp_path / 'jobstore'}",
+            "--strict-memory-limit",
+            f"--log-dir={log_dir}",
+            str(cwl_file),
+        ]
+        p = subprocess.run(cmd, capture_output=True, text=True)
+        outputs = json.loads(p.stdout)
+        out_list = outputs["list_out"]
+        assert len(out_list) == 2, f"outList shoud have two file elements {out_list}"
+        out_base = outputs["list_out"][0]
+        # This is a test on the scatter functionality and stdout.
+        # Each value of scatter should generate a separate file in the output.
+        for index, file in enumerate(out_list):
+            if index > 0:
+                new_file_loc = out_base["location"] + f"_{index + 1}"
+            else:
+                new_file_loc = out_base["location"]
+            assert (
+                new_file_loc == file["location"]
+            ), f"Toil should have detected conflicts for these stdout files {new_file_loc} and {file}"
 
-    assert b"Finished toil run successfully" in stderr
-    assert p.returncode == 0
+        assert "Finished toil run successfully" in p.stderr
+        assert p.returncode == 0
 
-    assert log_dir.exists()
-    scatter_0 = log_dir / "echo-test-scatter.0.scatter"
-    scatter_1 = log_dir / "echo-test-scatter.1.scatter"
-    list_0 = log_dir / "echo-test-scatter.0.list"
-    list_1 = log_dir / "echo-test-scatter.1.list"
-    assert scatter_0.exists()
-    assert scatter_1.exists()
-    assert list_0.exists()
-    assert list_1.exists()
+        assert log_dir.exists()
+        scatter_0 = log_dir / "echo-test-scatter.0.scatter"
+        scatter_1 = log_dir / "echo-test-scatter.1.scatter"
+        list_0 = log_dir / "echo-test-scatter.0.list"
+        list_1 = log_dir / "echo-test-scatter.1.list"
+        assert scatter_0.exists()
+        assert scatter_1.exists()
+        assert list_0.exists()
+        assert list_1.exists()
 
 
 @needs_cwl
+@pytest.mark.cwl
 @pytest.mark.cwl_small_log_dir
 def test_log_dir_echo_no_output(tmp_path: Path) -> None:
     log_dir = tmp_path / "cwl-logs"
-    job_store = "test_log_dir_echo_no_output"
-    toil = "toil-cwl-runner"
-    jobstore = f"--jobStore={job_store}"
-    option_1 = "--strict-memory-limit"
-    option_2 = "--force-docker-pull"
-    option_3 = "--clean=always"
-    option_4 = f"--log-dir={log_dir}"
-    cwl = os.path.join(os.path.dirname(__file__), "echo-stdout-log-dir.cwl")
-    cmd = [toil, jobstore, option_1, option_2, option_3, option_4, cwl]
-    p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-    stdout, stderr = p.communicate()
-
-    tmp_path = log_dir
-
-    assert log_dir.exists()
-    assert len(list(tmp_path.iterdir())) == 1
-
-    subdir = next(tmp_path.iterdir())
-    assert subdir.name == "echo"
-    assert subdir.is_dir()
-    assert len(list(subdir.iterdir())) == 1
-    result = next(subdir.iterdir())
-    assert result.name == "out.txt"
-    output = open(result).read()
-    assert "hello" in output
+    job_store = tmp_path / "test_log_dir_echo_no_output"
+    with get_data("test/cwl/echo-stdout-log-dir.cwl") as cwl_file:
+        cmd = [
+            "toil-cwl-runner",
+            f"--jobStore={job_store}",
+            "--strict-memory-limit",
+            f"--log-dir={str(log_dir)}",
+            str(cwl_file),
+        ]
+        subprocess.run(cmd)
+
+        assert log_dir.exists()
+        assert sum(1 for _ in log_dir.iterdir()) == 1
+
+        subdir = next(log_dir.iterdir())
+        assert subdir.name == "echo"
+        assert subdir.is_dir()
+        assert sum(1 for _ in subdir.iterdir()) == 1
+        result = next(subdir.iterdir())
+        assert result.name == "out.txt"
+        assert "hello" in result.read_text()
 
 
 @needs_cwl
+@pytest.mark.cwl
 @pytest.mark.cwl_small_log_dir
 def test_log_dir_echo_stderr(tmp_path: Path) -> None:
     log_dir = tmp_path / "cwl-logs"
+    log_dir.mkdir()
+    with get_data("test/cwl/echo-stderr.cwl") as cwl_file:
+        cmd = [
+            "toil-cwl-runner",
+            f"--jobStore={str(tmp_path / 'test_log_dir_echo_stderr')}",
+            "--strict-memory-limit",
+            "--force-docker-pull",
+            "--clean=always",
+            f"--log-dir={str(log_dir)}",
+            str(cwl_file),
+        ]
+        subprocess.run(cmd)
+        tmp_path = log_dir
 
-    job_store = "test_log_dir_echo_stderr"
-    toil = "toil-cwl-runner"
-    jobstore = f"--jobStore={job_store}"
-    option_1 = "--strict-memory-limit"
-    option_2 = "--force-docker-pull"
-    option_3 = "--clean=always"
-    option_4 = f"--log-dir={log_dir}"
-    cwl = os.path.join(os.path.dirname(__file__), "echo-stderr.cwl")
-    cmd = [toil, jobstore, option_1, option_2, option_3, option_4, cwl]
-    p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-    stdout, stderr = p.communicate()
-    tmp_path = log_dir
-
-    assert len(list(tmp_path.iterdir())) == 1
-
-    subdir = next(tmp_path.iterdir())
-    assert subdir.name == "echo-stderr.cwl"
-    assert subdir.is_dir()
-    assert len(list(subdir.iterdir())) == 1
-    result = next(subdir.iterdir())
-    assert result.name == "out.txt"
-    output = open(result).read()
-    assert output == "hello\n"
+        assert len(list(tmp_path.iterdir())) == 1
+
+        subdir = next(tmp_path.iterdir())
+        assert subdir.name == "echo-stderr.cwl"
+        assert subdir.is_dir()
+        assert len(list(subdir.iterdir())) == 1
+        result = next(subdir.iterdir())
+        assert result.name == "out.txt"
+        output = open(result).read()
+        assert output == "hello\n"
 
 
 # TODO: It's not clear how this test tests filename conflict resolution; it
 # seems like it runs a python script to copy some files and makes sure the
 # workflow doesn't fail.
 @needs_cwl
+@pytest.mark.cwl
 @pytest.mark.cwl_small_log_dir
 def test_filename_conflict_resolution(tmp_path: Path) -> None:
-    out_dir = tmp_path / "cwl-out-dir"
-    toil = "toil-cwl-runner"
-    options = [
-        f"--outdir={out_dir}",
-        "--clean=always",
-    ]
-    cwl = os.path.join(
-        os.path.dirname(__file__), "test_filename_conflict_resolution.cwl"
-    )
-    input = os.path.join(
-        os.path.dirname(__file__), "test_filename_conflict_resolution.ms"
-    )
-    cwl_inputs = ["--msin", input]
-    cmd = [toil] + options + [cwl] + cwl_inputs
-    p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-    stdout, stderr = p.communicate()
-    assert b"Finished toil run successfully" in stderr
-    assert p.returncode == 0
+    with get_data("test/cwl/test_filename_conflict_resolution.cwl") as cwl_file:
+        with get_data("test/cwl/test_filename_conflict_resolution.ms") as msin:
+            cmd = [
+                "toil-cwl-runner",
+                f"--outdir={tmp_path}",
+                str(cwl_file),
+                "--msin",
+                str(msin),
+            ]
+            p = subprocess.run(cmd, capture_output=True, text=True)
+            assert "Finished toil run successfully" in p.stderr
+            assert p.returncode == 0
 
 
 @needs_cwl
+@pytest.mark.cwl
 @pytest.mark.cwl_small_log_dir
 def test_filename_conflict_resolution_3_or_more(tmp_path: Path) -> None:
-    out_dir = tmp_path / "cwl-out-dir"
-    toil = "toil-cwl-runner"
-    options = [
-        f"--outdir={out_dir}",
-        "--clean=always",
-    ]
-    cwl = os.path.join(os.path.dirname(__file__), "scatter_duplicate_outputs.cwl")
-    cmd = [toil] + options + [cwl]
-    p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-    stdout, stderr = p.communicate()
-    assert b"Finished toil run successfully" in stderr
-    assert p.returncode == 0
-    assert (
-        len(os.listdir(out_dir)) == 9
-    ), "All 9 files made by the scatter should be in the directory"
+    with get_data("test/cwl/scatter_duplicate_outputs.cwl") as cwl_file:
+        cmd = ["toil-cwl-runner", f"--outdir={tmp_path}", str(cwl_file)]
+        p = subprocess.run(cmd, capture_output=True, text=True)
+        assert "Finished toil run successfully" in p.stderr
+        assert p.returncode == 0
+        assert (
+            sum(1 for _ in tmp_path.iterdir()) == 9
+        ), f"All 9 files made by the scatter should be in the directory: {tmp_path}"
 
 
 @needs_cwl
 @needs_docker
+@pytest.mark.cwl
+@pytest.mark.docker
 @pytest.mark.cwl_small_log_dir
 def test_filename_conflict_detection(tmp_path: Path) -> None:
     """
     Make sure we don't just stage files over each other when using a container.
     """
-    out_dir = tmp_path / "cwl-out-dir"
-    toil = "toil-cwl-runner"
-    options = [
-        f"--outdir={out_dir}",
-        "--clean=always",
-    ]
-    cwl = os.path.join(
-        os.path.dirname(__file__), "test_filename_conflict_detection.cwl"
-    )
-    cmd = [toil] + options + [cwl]
-    p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-    stdout, stderr = p.communicate()
-    assert b"File staging conflict" in stderr
-    assert p.returncode != 0
+    with get_data("test/cwl/test_filename_conflict_detection.cwl") as cwl_file:
+        cmd = ["toil-cwl-runner", f"--outdir={tmp_path}", str(cwl_file)]
+        p = subprocess.run(cmd, capture_output=True, text=True)
+        assert "File staging conflict" in p.stderr
+        assert p.returncode != 0
 
 
 @needs_cwl
 @needs_docker
+@pytest.mark.cwl
+@pytest.mark.docker
 @pytest.mark.cwl_small_log_dir
 def test_filename_conflict_detection_at_root(tmp_path: Path) -> None:
     """
@@ -1470,101 +1711,89 @@ def test_filename_conflict_detection_at_root(tmp_path: Path) -> None:
 
     Specifically, when using a container and the files are at the root of the work dir.
     """
-    out_dir = tmp_path / "cwl-out-dir"
-    toil = "toil-cwl-runner"
-    options = [
-        f"--outdir={out_dir}",
-        "--clean=always",
-    ]
-    cwl = os.path.join(
-        os.path.dirname(__file__), "test_filename_conflict_detection_at_root.cwl"
-    )
-    cmd = [toil] + options + [cwl]
-    p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-    stdout, stderr = p.communicate()
-    assert b"File staging conflict" in stderr
-    assert p.returncode != 0
+    with get_data("test/cwl/test_filename_conflict_detection_at_root.cwl") as cwl_file:
+        cmd = ["toil-cwl-runner", f"--outdir={tmp_path}", str(cwl_file)]
+        p = subprocess.run(cmd, capture_output=True, text=True)
+        assert "File staging conflict" in p.stderr
+        assert p.returncode != 0
 
 
 @needs_cwl
+@pytest.mark.cwl
 @pytest.mark.cwl_small
-def test_pick_value_with_one_null_value(caplog: pytest.LogCaptureFixture) -> None:
+def test_pick_value_with_one_null_value(
+    caplog: pytest.LogCaptureFixture, tmp_path: Path
+) -> None:
     """
     Make sure toil-cwl-runner does not false log a warning when pickValue is
     used but outputSource only contains one null value. See: #3991.
     """
     from toil.cwl import cwltoil
 
-    cwl_file = os.path.join(os.path.dirname(__file__), "conditional_wf.cwl")
-    job_file = os.path.join(os.path.dirname(__file__), "conditional_wf.yaml")
-    args = [cwl_file, job_file]
-
-    with caplog.at_level(logging.WARNING, logger="toil.cwl.cwltoil"):
-        cwltoil.main(args)
-        for line in caplog.messages:
-            assert (
-                "You had a conditional step that did not run, but you did not use pickValue to handle the skipped input."
-                not in line
-            )
+    with get_data("test/cwl/conditional_wf.cwl") as cwl_file:
+        with get_data("test/cwl/conditional_wf.yaml") as job_file:
+            with caplog.at_level(logging.WARNING, logger="toil.cwl.cwltoil"):
+                cwltoil.main([f"--outdir={tmp_path}", str(cwl_file), str(job_file)])
+                for line in caplog.messages:
+                    assert (
+                        "You had a conditional step that did not run, but you did not use pickValue to handle the skipped input."
+                        not in line
+                    )
 
 
 @needs_cwl
+@pytest.mark.cwl
 @pytest.mark.cwl_small
-def test_workflow_echo_string() -> None:
-    toil = "toil-cwl-runner"
-    jobstore = f"--jobStore=file:explicit-local-jobstore-{uuid.uuid4()}"
-    option_1 = "--strict-memory-limit"
-    option_2 = "--force-docker-pull"
-    option_3 = "--clean=always"
-    cwl = os.path.join(os.path.dirname(__file__), "echo_string.cwl")
-    cmd = [toil, jobstore, option_1, option_2, option_3, cwl]
-    p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-    stdout, stderr = p.communicate()
-    stdout2 = stdout.decode("utf-8")
-    stderr2 = stderr.decode("utf-8")
-    assert (
-        stdout2.strip() == "{}"
-    ), f"Got wrong output: {stdout2}\nWith error: {stderr2}"
-    assert "Finished toil run successfully" in stderr2
-    assert p.returncode == 0
+def test_workflow_echo_string(tmp_path: Path) -> None:
+    with get_data("test/cwl/echo_string.cwl") as cwl_file:
+        cmd = [
+            "toil-cwl-runner",
+            f"--jobStore=file:{tmp_path / 'jobstore'}",
+            "--strict-memory-limit",
+            str(cwl_file),
+        ]
+        p = subprocess.run(cmd, capture_output=True, text=True)
+        assert (
+            p.stdout.strip() == "{}"
+        ), f"Got wrong output: {p.stdout}\nWith error: {p.stderr}"
+        assert "Finished toil run successfully" in p.stderr
+        assert p.returncode == 0
 
 
 @needs_cwl
+@pytest.mark.cwl
 @pytest.mark.cwl_small
-def test_workflow_echo_string_scatter_capture_stdout() -> None:
-    toil = "toil-cwl-runner"
-    jobstore = f"--jobStore=file:explicit-local-jobstore-{uuid.uuid4()}"
-    option_1 = "--strict-memory-limit"
-    option_2 = "--force-docker-pull"
-    option_3 = "--clean=always"
-    cwl = os.path.join(
-        os.path.dirname(__file__), "echo_string_scatter_capture_stdout.cwl"
-    )
-    cmd = [toil, jobstore, option_1, option_2, option_3, cwl]
-    p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-    stdout, stderr = p.communicate()
-    log.debug("Workflow standard output: %s", stdout)
-    assert len(stdout) > 0
-    outputs = json.loads(stdout)
-    out_list = outputs["list_out"]
-    assert len(out_list) == 2, f"outList shoud have two file elements {out_list}"
-    out_base = outputs["list_out"][0]
-    # This is a test on the scatter functionality and stdout.
-    # Each value of scatter should generate a separate file in the output.
-    for index, file in enumerate(out_list):
-        if index > 0:
-            new_file_loc = out_base["location"] + f"_{index + 1}"
-        else:
-            new_file_loc = out_base["location"]
-        assert (
-            new_file_loc == file["location"]
-        ), f"Toil should have detected conflicts for these stdout files {new_file_loc} and {file}"
+def test_workflow_echo_string_scatter_capture_stdout(tmp_path: Path) -> None:
+    with get_data("test/cwl/echo_string_scatter_capture_stdout.cwl") as cwl_file:
+        cmd = [
+            "toil-cwl-runner",
+            f"--jobStore=file:{tmp_path / 'jobStore'}",
+            "--strict-memory-limit",
+            str(cwl_file),
+        ]
+        p = subprocess.run(cmd, capture_output=True, text=True)
+        assert len(p.stdout) > 0
+        outputs = json.loads(p.stdout)
+        out_list = outputs["list_out"]
+        assert len(out_list) == 2, f"outList shoud have two file elements {out_list}"
+        out_base = outputs["list_out"][0]
+        # This is a test on the scatter functionality and stdout.
+        # Each value of scatter should generate a separate file in the output.
+        for index, file in enumerate(out_list):
+            if index > 0:
+                new_file_loc = out_base["location"] + f"_{index + 1}"
+            else:
+                new_file_loc = out_base["location"]
+            assert (
+                new_file_loc == file["location"]
+            ), f"Toil should have detected conflicts for these stdout files {new_file_loc} and {file}"
 
-    assert b"Finished toil run successfully" in stderr
-    assert p.returncode == 0
+        assert "Finished toil run successfully" in p.stderr
+        assert p.returncode == 0
 
 
 @needs_cwl
+@pytest.mark.cwl
 @pytest.mark.cwl_small
 def test_visit_top_cwl_class() -> None:
     structure = {
@@ -1616,6 +1845,7 @@ def test_visit_top_cwl_class() -> None:
 
 
 @needs_cwl
+@pytest.mark.cwl
 @pytest.mark.cwl_small
 def test_visit_cwl_class_and_reduce() -> None:
     structure = {
@@ -1677,6 +1907,7 @@ def test_visit_cwl_class_and_reduce() -> None:
 
 
 @needs_cwl
+@pytest.mark.cwl
 @pytest.mark.cwl_small
 def test_download_structure(tmp_path: Path) -> None:
     """
@@ -1701,7 +1932,7 @@ def test_download_structure(tmp_path: Path) -> None:
     }
 
     # Say where to put it on the filesystem
-    to_dir = str(tmp_path)
+    to_dir = tmp_path
 
     # Make a fake file store
     file_store = Mock(AbstractFileStore)
@@ -1714,7 +1945,7 @@ def test_download_structure(tmp_path: Path) -> None:
     existing: dict[str, str] = {}
 
     # Do the download
-    download_structure(file_store, index, existing, structure, to_dir)
+    download_structure(file_store, index, existing, structure, str(to_dir))
 
     # Check the results
     # 3 files should be made
@@ -1723,61 +1954,53 @@ def test_download_structure(tmp_path: Path) -> None:
     assert len(existing) == 2
 
     # Make sure that the index contents (path to URI) are correct
-    assert os.path.join(to_dir, "dir1/dir2/f1") in index
-    assert os.path.join(to_dir, "dir1/dir2/f1again") in index
-    assert os.path.join(to_dir, "anotherfile") in index
+    assert str(to_dir / "dir1/dir2/f1") in index
+    assert str(to_dir / "dir1/dir2/f1again") in index
+    assert str(to_dir / "anotherfile") in index
     assert (
-        index[os.path.join(to_dir, "dir1/dir2/f1")]
+        index[str(to_dir / "dir1/dir2/f1")]
         == cast(
             DirectoryStructure, cast(DirectoryStructure, structure["dir1"])["dir2"]
         )["f1"]
     )
     assert (
-        index[os.path.join(to_dir, "dir1/dir2/f1again")]
+        index[str(to_dir / "dir1/dir2/f1again")]
         == cast(
             DirectoryStructure, cast(DirectoryStructure, structure["dir1"])["dir2"]
         )["f1again"]
     )
-    assert index[os.path.join(to_dir, "anotherfile")] == structure["anotherfile"]
+    assert index[str(to_dir / "anotherfile")] == structure["anotherfile"]
 
     # And the existing contents (URI to path)
     assert "toilfile:" + fid1.pack() in existing
     assert "toilfile:" + fid2.pack() in existing
     assert existing["toilfile:" + fid1.pack()] in [
-        os.path.join(to_dir, "dir1/dir2/f1"),
-        os.path.join(to_dir, "dir1/dir2/f1again"),
+        str(to_dir / "dir1/dir2/f1"),
+        str(to_dir / "dir1/dir2/f1again"),
     ]
-    assert existing["toilfile:" + fid2.pack()] == os.path.join(to_dir, "anotherfile")
+    assert existing["toilfile:" + fid2.pack()] == str(to_dir / "anotherfile")
 
     # The directory structure should be created for real
-    assert os.path.isdir(os.path.join(to_dir, "dir1")) is True
-    assert os.path.isdir(os.path.join(to_dir, "dir1/dir2")) is True
-    assert os.path.isdir(os.path.join(to_dir, "dir1/dir2/dir2sub")) is True
-    assert os.path.isdir(os.path.join(to_dir, "dir1/dir3")) is True
+    assert (to_dir / "dir1").is_dir()
+    assert (to_dir / "dir1/dir2").is_dir()
+    assert (to_dir / "dir1/dir2/dir2sub").is_dir()
+    assert (to_dir / "dir1/dir3").is_dir()
 
     # The file store should have been asked to do the download
     file_store.readGlobalFile.assert_has_calls(
         [
-            call(fid1, os.path.join(to_dir, "dir1/dir2/f1"), symlink=False),
-            call(fid1, os.path.join(to_dir, "dir1/dir2/f1again"), symlink=False),
-            call(fid2, os.path.join(to_dir, "anotherfile"), symlink=False),
+            call(fid1, str(to_dir / "dir1/dir2/f1"), symlink=False),
+            call(fid1, str(to_dir / "dir1/dir2/f1again"), symlink=False),
+            call(fid2, str(to_dir / "anotherfile"), symlink=False),
         ],
         any_order=True,
     )
 
 
 @needs_cwl
+@pytest.mark.cwl
 @pytest.mark.timeout(300)
 def test_import_on_workers() -> None:
-    args = [
-        "src/toil/test/cwl/download.cwl",
-        "src/toil/test/cwl/download_file.json",
-        "--runImportsOnWorkers",
-        "--importWorkersDisk=10MiB",
-        "--realTimeLogging=True",
-        "--logLevel=INFO",
-        "--logColors=False",
-    ]
     from toil.cwl import cwltoil
 
     detector = ImportWorkersMessageHandler()
@@ -1785,9 +2008,21 @@ def test_import_on_workers() -> None:
     # Set up a log message detector to the root logger
     logging.getLogger().addHandler(detector)
 
-    cwltoil.main(args)
+    with get_data("test/cwl/download.cwl") as cwl_file:
+        with get_data("test/cwl/directory/directory/file.txt") as file_path:
+            args = [
+                "--runImportsOnWorkers",
+                "--importWorkersDisk=10MiB",
+                "--realTimeLogging=True",
+                "--logLevel=INFO",
+                "--logColors=False",
+                str(cwl_file),
+                "--input",
+                str(file_path),
+            ]
+            cwltoil.main(args)
 
-    assert detector.detected is True
+        assert detector.detected is True
 
 
 # StreamHandler is generic, _typeshed doesn't exist at runtime, do a bit of typing trickery, see https://github.com/python/typeshed/issues/5680