toil 8.1.0b1__py3-none-any.whl → 8.2.0__py3-none-any.whl

toil/test/utils/ABCWorkflowDebug/debugWorkflow.py

@@ -0,0 +1,204 @@
+import logging
+import os
+import subprocess
+import sys
+
+from toil.common import Toil
+from toil.job import Job
+from toil.lib.io import mkdtemp
+from toil.test import get_data
+from toil.version import python
+
+logger = logging.getLogger(__name__)
+
+"""
+This workflow's purpose is to create files and jobs for viewing using stats,
+status, and printDot() in toilDebugTest.py.  It's intended for future use in a
+debugging tutorial containing a broken job.  It is also a minor integration test.
+"""
+
+
+def initialize_jobs(job):
+    """
+    Stub function used to start a toil workflow since toil workflows can only
+    start with one job (but afterwards can run many in parallel).
+    """
+    job.fileStore.log_to_leader("""initialize_jobs""")
+
+
+def writeA(job, mkFile):
+    """Runs a program, and writes a string 'A' into A.txt using mkFile.py."""
+    job.fileStore.log_to_leader("""writeA""")
+
+    # temp folder for the run
+    tempDir = job.fileStore.getLocalTempDir()
+
+    # import files
+    mkFile_fs = job.fileStore.readGlobalFile(
+        mkFile[0], userPath=os.path.join(tempDir, mkFile[1])
+    )
+
+    # make a file (A.txt) and writes a string 'A' into it using 'mkFile.py'
+    content = "A"
+    cmd = python + " " + mkFile_fs + " " + "A.txt" + " " + content
+    this_process = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE)
+    this_process.wait()
+
+    # get the output file and return it as a tuple of location + name
+    output_filename = "A.txt"
+    output_file = job.fileStore.writeGlobalFile(output_filename)
+    A1 = (output_file, output_filename)
+    rvDict = {"A1": A1}
+    return rvDict
+
+
+def writeB(job, mkFile, B_file):
+    """
+    Runs a program, extracts a string 'B' from an existing file, B_file.txt, and
+    writes it into B.txt using mkFile.py.
+    """
+    job.fileStore.log_to_leader("""writeB""")
+
+    # temp folder for the run
+    tempDir = job.fileStore.getLocalTempDir()
+
+    # import files
+    mkFile_fs = job.fileStore.readGlobalFile(
+        mkFile[0], userPath=os.path.join(tempDir, mkFile[1])
+    )
+    B_file_fs = job.fileStore.readGlobalFile(
+        B_file[0], userPath=os.path.join(tempDir, B_file[1])
+    )
+
+    # make a file (B.txt) and write the contents of 'B_file.txt' into it using 'mkFile.py'
+    with open(B_file_fs) as f:
+        file_contents = ""
+        for line in f:
+            file_contents = file_contents + line
+
+    cmd = python + " " + mkFile_fs + " " + "B.txt" + " " + file_contents
+    this_process = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE)
+    this_process.wait()
+
+    # get the output file and return it as a tuple of location + name
+    output_filename = "B.txt"
+    output_file = job.fileStore.writeGlobalFile(output_filename)
+    B1 = (output_file, output_filename)
+    rvDict = {"B1": B1}
+    return rvDict
+
+
+def writeC(job):
+    """Creates/writes a file, C.txt, containing the string 'C'."""
+    job.fileStore.log_to_leader("""writeC""")
+
+    # temp folder for the run
+    tempDir = job.fileStore.getLocalTempDir()
+
+    # get the output file and return it as a tuple of location + name
+    output_filename = os.path.join(tempDir, "C.txt")
+    with open(output_filename, "w") as f:
+        f.write("C")
+    output_file = job.fileStore.writeGlobalFile(output_filename)
+    C1 = (output_file, "C.txt")
+    rvDict = {"C1": C1}
+    return rvDict
+
+
+def writeABC(job, A_dict, B_dict, C_dict, filepath: str) -> None:
+    """Takes 3 files (specified as dictionaries) and writes their contents to ABC.txt."""
+    job.fileStore.log_to_leader("""writeABC""")
+
+    # temp folder for the run
+    tempDir = job.fileStore.getLocalTempDir()
+
+    # import files
+    A_fs = job.fileStore.readGlobalFile(
+        A_dict["A1"][0], userPath=os.path.join(tempDir, A_dict["A1"][1])
+    )
+    B_fs = job.fileStore.readGlobalFile(
+        B_dict["B1"][0], userPath=os.path.join(tempDir, B_dict["B1"][1])
+    )
+    C_fs = job.fileStore.readGlobalFile(
+        C_dict["C1"][0], userPath=os.path.join(tempDir, C_dict["C1"][1])
+    )
+
+    file_contents = ""
+    with open(A_fs) as f:
+        for line in f:
+            file_contents = file_contents + line
+
+    with open(B_fs) as f:
+        for line in f:
+            file_contents = file_contents + line
+
+    with open(C_fs) as f:
+        for line in f:
+            file_contents = file_contents + line
+
+    with open(os.path.join(tempDir, "ABC.txt"), "w") as f:
+        f.write(file_contents)
+
+    # get the output file and return it as a tuple of location + name
+    output_filename = os.path.join(tempDir, "ABC.txt")
+    output_file = job.fileStore.writeGlobalFile(output_filename)
+    job.fileStore.export_file(output_file, "file://" + filepath)
+
+
+def finalize_jobs(job, num):
+    """Does nothing but should be recorded in stats, status, and printDot()."""
+    job.fileStore.log_to_leader("""finalize_jobs""")
+
+
+def broken_job(job, num):
+    """A job that will always fail.  To be used for a tutorial."""
+    job.fileStore.log_to_leader("""broken_job""")
+    file = toil.importFile(None)
+
+
+if __name__ == "__main__":
+    jobStorePath = sys.argv[1] if len(sys.argv) > 1 else mkdtemp("debugWorkflow")
+    options = Job.Runner.getDefaultOptions(jobStorePath)
+    options.clean = "never"
+    options.stats = True
+    options.logLevel = "INFO"
+    with Toil(options) as toil:
+
+        with get_data("test/utils/ABCWorkflowDebug/B_file.txt") as B_file_path:
+            B_file0 = toil.importFile(B_file_path.as_uri())
+        B_file0_preserveThisFilename = "B_file.txt"
+        B_file = (B_file0, B_file0_preserveThisFilename)
+
+        with get_data("test/utils/ABCWorkflowDebug/mkFile.py") as mkFile_path:
+            file_maker0 = toil.importFile(mkFile_path.as_uri())
+        file_maker0_preserveThisFilename = "mkFile.py"
+        file_maker = (file_maker0, file_maker0_preserveThisFilename)
+
+        job0 = Job.wrapJobFn(initialize_jobs)
+        job1 = Job.wrapJobFn(writeA, file_maker)
+        job2 = Job.wrapJobFn(writeB, file_maker, B_file)
+        job3 = Job.wrapJobFn(writeC)
+        with get_data("test/utils/ABCWorkflowDebug/ABC.txt") as filepath:
+            job4 = Job.wrapJobFn(
+                writeABC, job1.rv(), job2.rv(), job3.rv(), str(filepath)
+            )
+        job5 = Job.wrapJobFn(finalize_jobs, 1)
+        job6 = Job.wrapJobFn(finalize_jobs, 2)
+        job7 = Job.wrapJobFn(finalize_jobs, 3)
+        job8 = Job.wrapJobFn(finalize_jobs, 4)
+
+        # write files 'A.txt', 'B.txt', and 'C.txt'
+        job0.addChild(job1)
+        job1.addChild(job2)
+        job2.addChild(job3)
+
+        # finally use 'A.txt', 'B.txt', and 'C.txt' to write ABC.txt
+        job0.addFollowOn(job4)
+
+        # these jobs do nothing, but should display in status
+        job4.addChild(job5)
+        job4.addChild(job6)
+        job4.addChild(job7)
+        job4.addChild(job8)
+
+        toil.start(job0)

toil/test/utils/ABCWorkflowDebug/mkFile.py

@@ -0,0 +1,16 @@
+from configargparse import ArgumentParser
+
+
+def main():
+    parser = ArgumentParser(description="Creates a file and writes into it.")
+    parser.add_argument("file_name", help="File name to be written to.")
+    parser.add_argument("contents", help="A string to be written into the file.")
+
+    args, unknown_args = parser.parse_known_args()
+
+    with open(args.file_name, "w") as f:
+        f.write(args.contents)
+
+
+if __name__ == "__main__":
+    main()

toil/test/utils/ABCWorkflowDebug/sleep.cwl

@@ -0,0 +1,12 @@
+cwlVersion: v1.0
+class: CommandLineTool
+baseCommand: sleep
+stdout: output.txt
+inputs:
+  message:
+    type: int
+    inputBinding:
+      position: 1
+outputs:
+  output:
+    type: stdout

toil/test/utils/ABCWorkflowDebug/sleep.yaml

@@ -0,0 +1 @@
+message: 100

toil/test/utils/toilDebugTest.py

@@ -13,30 +13,34 @@
 # limitations under the License.
 import logging
 import os
+from pathlib import Path
 import subprocess
-import tempfile
 
 from toil.lib.resources import glob
-from toil.test import ToilTest, needs_wdl, slow
+from toil.test import get_data, pneeds_wdl as needs_wdl, pslow as slow
 from toil.version import python
 
+import pytest
+
 logger = logging.getLogger(__name__)
 
 
-def workflow_debug_jobstore() -> str:
-    job_store_path = os.path.join(tempfile.mkdtemp(), "toilWorkflowRun")
-    subprocess.check_call(
-        [
-            python,
-            os.path.abspath("src/toil/test/utils/ABCWorkflowDebug/debugWorkflow.py"),
-            job_store_path,
-        ]
-    )
+def workflow_debug_jobstore(tmp_path: Path) -> Path:
+    job_store_path = tmp_path / "toilWorkflowRun"
+    with get_data("test/utils/ABCWorkflowDebug/debugWorkflow.py") as debugWorkflow_py:
+        subprocess.check_call(
+            [
+                python,
+                str(debugWorkflow_py),
+                str(job_store_path),
+            ]
+        )
     return job_store_path
 
 
 @slow
-def testJobStoreContents():
+@pytest.mark.slow
+def testJobStoreContents(tmp_path: Path) -> None:
     """
     Test toilDebugFile.printContentsOfJobStore().
 
@@ -46,19 +50,22 @@ def testJobStoreContents():
     """
     contents = ["A.txt", "B.txt", "C.txt", "ABC.txt", "mkFile.py"]
 
-    subprocess.check_call(
-        [
-            python,
-            os.path.abspath("src/toil/utils/toilDebugFile.py"),
-            workflow_debug_jobstore(),
-            "--logDebug",
-            "--listFilesInJobStore=True",
-        ]
-    )
-    jobstoreFileContents = os.path.abspath("jobstore_files.txt")
+    original_path = os.getcwd()
+    os.chdir(tmp_path)
+    with get_data("utils/toilDebugFile.py") as toilDebugFile:
+        subprocess.check_call(
+            [
+                python,
+                str(toilDebugFile),
+                str(workflow_debug_jobstore(tmp_path)),
+                "--logDebug",
+                "--listFilesInJobStore=True",
+            ]
+        )
+    jobstoreFileContents = tmp_path / "jobstore_files.txt"
     files = []
     match = 0
-    with open(jobstoreFileContents) as f:
+    with jobstoreFileContents.open() as f:
         for line in f:
             files.append(line.strip())
     for xfile in files:
@@ -70,10 +77,10 @@ def testJobStoreContents():
     logger.debug(match)
     # C.txt will match twice (once with 'C.txt', and once with 'ABC.txt')
     assert match == 6
-    os.remove(jobstoreFileContents)
+    os.chdir(original_path)
 
 
-def fetchFiles(symLink: bool, jobStoreDir: str, outputDir: str):
+def fetchFiles(symLink: bool, jobStoreDir: Path, outputDir: Path) -> None:
     """
     Fn for testFetchJobStoreFiles() and testFetchJobStoreFilesWSymlinks().
 
@@ -84,21 +91,22 @@ def fetchFiles(symLink: bool, jobStoreDir: str, outputDir: str):
     then delete them.
     """
     contents = ["A.txt", "B.txt", "C.txt", "ABC.txt", "mkFile.py"]
-    cmd = [
-        python,
-        os.path.abspath("src/toil/utils/toilDebugFile.py"),
-        jobStoreDir,
-        "--fetch",
-        "*A.txt",
-        "*B.txt",
-        "*C.txt",
-        "*ABC.txt",
-        "*mkFile.py",
-        f"--localFilePath={outputDir}",
-        f"--useSymlinks={symLink}",
-    ]
-    print(cmd)
-    subprocess.check_call(cmd)
+    with get_data("utils/toilDebugFile.py") as toilDebugFile:
+        cmd = [
+            python,
+            str(toilDebugFile),
+            str(jobStoreDir),
+            "--fetch",
+            "*A.txt",
+            "*B.txt",
+            "*C.txt",
+            "*ABC.txt",
+            "*mkFile.py",
+            f"--localFilePath={outputDir}",
+            f"--useSymlinks={symLink}",
+        ]
+        print(cmd)
+        subprocess.check_call(cmd)
     for xfile in contents:
         matchingFilesFound = glob(glob_pattern="*" + xfile, directoryname=outputDir)
         assert len(matchingFilesFound) >= 1
@@ -109,42 +117,45 @@ def fetchFiles(symLink: bool, jobStoreDir: str, outputDir: str):
 
 
 # expected run time = 4s
-def testFetchJobStoreFiles() -> None:
+def testFetchJobStoreFiles(tmp_path: Path) -> None:
     """Test toilDebugFile.fetchJobStoreFiles() symlinks."""
-    job_store_dir = workflow_debug_jobstore()
-    output_dir = os.path.join(os.path.dirname(job_store_dir), "testoutput")
-    os.makedirs(output_dir, exist_ok=True)
+    job_store_dir = workflow_debug_jobstore(tmp_path)
+    output_dir = tmp_path / "testoutput"
+    output_dir.mkdir()
     for symlink in (True, False):
         fetchFiles(symLink=symlink, jobStoreDir=job_store_dir, outputDir=output_dir)
 
 
-class DebugJobTest(ToilTest):
+class TestDebugJob:
     """
     Test the toil debug-job command.
     """
 
-    def _get_job_store_and_job_id(self):
+    def _get_job_store_and_job_id(self, tmp_path: Path) -> tuple[Path, str]:
         """
         Get a job store and the ID of a failing job within it.
         """
 
         # First make a job store.
-        job_store = os.path.join(self._createTempDir(), "tree")
+        job_store = tmp_path / "tree"
 
         logger.info("Running workflow that always fails")
         try:
             # Run an always-failing workflow
-            subprocess.check_call(
-                [
-                    python,
-                    os.path.abspath("src/toil/test/docs/scripts/example_alwaysfail.py"),
-                    "--retryCount=0",
-                    "--logCritical",
-                    "--disableProgress",
-                    job_store,
-                ],
-                stderr=subprocess.DEVNULL,
-            )
+            with get_data(
+                "test/docs/scripts/example_alwaysfail.py"
+            ) as example_alwaysfail_py:
+                subprocess.check_call(
+                    [
+                        python,
+                        str(example_alwaysfail_py),
+                        "--retryCount=0",
+                        "--logCritical",
+                        "--disableProgress",
+                        str(job_store),
+                    ],
+                    stderr=subprocess.DEVNULL,
+                )
             raise RuntimeError("Failing workflow succeeded!")
         except subprocess.CalledProcessError:
             # Should fail to run
@@ -152,40 +163,38 @@ class DebugJobTest(ToilTest):
 
         # Get the job ID.
         # TODO: This assumes a lot about the FileJobStore. Use the MessageBus instead?
-        job_id = (
-            "kind-explode/"
-            + os.listdir(os.path.join(job_store, "jobs/kind-explode"))[0]
-        )
+        job_id = "kind-explode/" + os.listdir(job_store / "jobs/kind-explode")[0]
 
         return job_store, job_id
 
-    def _get_wdl_job_store_and_job_name(self):
+    def _get_wdl_job_store_and_job_name(self, tmp_path: Path) -> tuple[Path, str]:
         """
         Get a job store and the name of a failed job in it that actually wanted to use some files.
         """
 
         # First make a job store.
-        job_store = os.path.join(self._createTempDir(), "tree")
+        job_store = tmp_path / "tree"
 
         logger.info("Running workflow that always fails")
         # Run an always-failing workflow
-        wf_result = subprocess.run(
-            [
-                "toil-wdl-runner",
-                os.path.abspath(
-                    "src/toil/test/docs/scripts/example_alwaysfail_with_files.wdl"
-                ),
-                "--retryCount=0",
-                "--logDebug",
-                "--disableProgress",
-                "--jobStore",
-                job_store,
-            ],
-            stdout=subprocess.PIPE,
-            stderr=subprocess.STDOUT,
-            encoding="utf-8",
-            errors="replace",
-        )
+        with get_data(
+            "test/docs/scripts/example_alwaysfail_with_files.wdl"
+        ) as wdl_file:
+            wf_result = subprocess.run(
+                [
+                    "toil-wdl-runner",
+                    str(wdl_file),
+                    "--retryCount=0",
+                    "--logDebug",
+                    "--disableProgress",
+                    "--jobStore",
+                    str(job_store),
+                ],
+                stdout=subprocess.PIPE,
+                stderr=subprocess.STDOUT,
+                encoding="utf-8",
+                errors="replace",
+            )
         logger.debug("Always-failing workflow output: %s", wf_result.stdout)
         if wf_result.returncode == 0:
             raise RuntimeError("Failing workflow succeeded!")
@@ -195,58 +204,66 @@ class DebugJobTest(ToilTest):
         # Make sure that the job store we created actually has its job store
         # root job ID file. If it doesn't, we failed during workflow setup and
         # not because of a real failing job.
-        assert os.path.exists(
-            os.path.join(job_store, "files/shared/rootJobStoreID")
-        ), "Failed workflow still needs a root job"
+        assert (
+            job_store / "files/shared/rootJobStoreID"
+        ).exists(), "Failed workflow still needs a root job"
 
         # Get a job name for a job that fails
         job_name = "WDLTaskJob"
 
         return job_store, job_name
 
-    def test_run_job(self):
+    def test_run_job(self, tmp_path: Path) -> None:
         """
         Make sure that we can use toil debug-job to try and run a job in-process.
         """
 
-        job_store, job_id = self._get_job_store_and_job_id()
+        job_store, job_id = self._get_job_store_and_job_id(tmp_path)
 
         logger.info("Trying to rerun job %s", job_id)
 
         # Rerun the job, which should fail again
         output = subprocess.check_output(
-            ["toil", "debug-job", "--logDebug", job_store, job_id],
+            ["toil", "debug-job", "--logDebug", str(job_store), job_id],
             stderr=subprocess.STDOUT,
         )
         # Even if the job fails, the attempt to run it will succeed.
         log = output.decode("utf-8")
         assert "Boom!" in log, f"Did not find the expected exception message in: {log}"
 
-    def test_print_job_info(self):
+    def test_print_job_info(self, tmp_path: Path) -> None:
         """
         Make sure that we can use --printJobInfo to get information on a job from a job store.
         """
 
-        job_store, job_id = self._get_job_store_and_job_id()
+        job_store, job_id = self._get_job_store_and_job_id(tmp_path)
 
         logger.info("Trying to print job info for job %s", job_id)
 
         # Print the job info and make sure that doesn't crash.
         subprocess.check_call(
-            ["toil", "debug-job", "--logDebug", job_store, "--printJobInfo", job_id]
+            [
+                "toil",
+                "debug-job",
+                "--logDebug",
+                str(job_store),
+                "--printJobInfo",
+                job_id,
+            ]
         )
 
     @needs_wdl
-    def test_retrieve_task_directory(self):
+    @pytest.mark.wdl
+    def test_retrieve_task_directory(self, tmp_path: Path) -> None:
         """
         Make sure that we can use --retrieveTaskDirectory to get the input files for a job.
         """
 
-        job_store, job_name = self._get_wdl_job_store_and_job_name()
+        job_store, job_name = self._get_wdl_job_store_and_job_name(tmp_path)
 
         logger.info("Trying to retrieve task dorectory for job %s", job_name)
 
-        dest_dir = os.path.join(self._createTempDir(), "dump")
+        dest_dir = tmp_path / "dump"
 
         # Print the job info and make sure that doesn't crash.
         subprocess.check_call(
@@ -254,18 +271,16 @@ class DebugJobTest(ToilTest):
                 "toil",
                 "debug-job",
                 "--logDebug",
-                job_store,
+                str(job_store),
                 job_name,
                 "--retrieveTaskDirectory",
-                dest_dir,
+                str(dest_dir),
             ]
         )
 
-        first_file = os.path.join(
-            dest_dir,
-            "inside/mnt/miniwdl_task_container/work/_miniwdl_inputs/0/test.txt",
+        first_file = (
+            dest_dir
+            / "inside/mnt/miniwdl_task_container/work/_miniwdl_inputs/0/test.txt"
         )
-        assert os.path.exists(
-            first_file
-        ), "Input file not found in fake container environment"
-        self.assertEqual(open(first_file).read(), "These are the contents\n")
+        assert first_file.exists(), "Input file not found in fake container environment"
+        assert first_file.read_text() == "These are the contents\n"