toil 8.2.0__py3-none-any.whl → 9.1.0__py3-none-any.whl

toil/test/src/fileStoreTest.py

@@ -22,6 +22,7 @@ import os
 import random
 import signal
 import stat
+import tempfile
 import time
 from abc import ABCMeta
 from struct import pack, unpack
@@ -83,15 +84,21 @@ class hidden:
         def setUp(self):
             super().setUp()
             self.work_dir = self._createTempDir()
-            self.options = Job.Runner.getDefaultOptions(self._getTestJobStore())
-            self.options.logLevel = "DEBUG"
-            self.options.realTimeLogging = True
-            self.options.workDir = self.work_dir
-            self.options.clean = "always"
-            self.options.logFile = os.path.join(self.work_dir, "logFile")
-
             self.tmp_dir = self._createTempDir()
 
+        def options(self):
+            """
+            Mint a fresh options object with default settings and a fresh jobstore.
+            """
+            options = Job.Runner.getDefaultOptions(self._getTestJobStore())
+            options.logLevel = "DEBUG"
+            options.realTimeLogging = True
+            options.workDir = self.work_dir
+            options.clean = "always"
+            log_fd, options.logFile = tempfile.mkstemp(dir=self.work_dir, prefix="logFile")
+            os.close(log_fd)
+            return options
+
         def create_file(self, content, executable=False):
             file_path = f"{self.tmp_dir}/{uuid4()}"
 
@@ -124,7 +131,7 @@ class hidden:
             A.addChild(C)
             B.addChild(D)
             C.addChild(D)
-            Job.Runner.startToil(A, self.options)
+            Job.Runner.startToil(A, self.options())
 
         @slow
         def testFileStoreLogging(self):
@@ -154,7 +161,7 @@ class hidden:
 
             F = Job.wrapJobFn(self._accessAndFail, disk="100M")
             try:
-                Job.Runner.startToil(F, self.options)
+                Job.Runner.startToil(F, self.options())
             except FailedJobsException:
                 # We expect this.
                 pass
@@ -190,7 +197,7 @@ class hidden:
                 numIters=30,
                 disk="2G",
             )
-            Job.Runner.startToil(F, self.options)
+            Job.Runner.startToil(F, self.options())
 
         @staticmethod
         def _testFileStoreOperations(job, nonLocalDir, numIters=100):
@@ -300,12 +307,13 @@ class hidden:
                         f"[executable: {executable}]\n"
                         f"[caching: {caching}]\n"
                     ):
-                        self.options.caching = caching
+                        options = self.options()
+                        options.caching = caching
                         read_write_job = Job.wrapJobFn(
                             self._testWriteReadGlobalFilePermissions,
                             executable=executable,
                         )
-                        Job.Runner.startToil(read_write_job, self.options)
+                        Job.Runner.startToil(read_write_job, options)
 
         @staticmethod
         def _testWriteReadGlobalFilePermissions(job, executable):
@@ -341,7 +349,7 @@ class hidden:
                 export_file_job = Job.wrapJobFn(
                     self._testWriteExportFileCompatibility, executable=executable
                 )
-                with Toil(self.options) as toil:
+                with Toil(self.options()) as toil:
                     initialPermissions, fileID = toil.start(export_file_job)
                     dstFile = os.path.join(self._createTempDir(), str(uuid4()))
                     toil.exportFile(fileID, "file://" + dstFile)
@@ -367,16 +375,17 @@ class hidden:
             Ensures that files imported to the leader preserve their executable permissions
             when they are read by the fileStore.
             """
-            with Toil(self.options) as toil:
-                for executable in True, False:
-                    file_path = self.create_file(content="Hello", executable=executable)
-                    initial_permissions = os.stat(file_path).st_mode & stat.S_IXUSR
-                    file_id = toil.importFile(f"file://{file_path}")
-                    for mutable in True, False:
-                        for symlink in True, False:
-                            with self.subTest(
-                                f"Now testing readGlobalFileWith: mutable={mutable} symlink={symlink}"
-                            ):
+            
+            for executable in True, False:
+                file_path = self.create_file(content="Hello", executable=executable)
+                initial_permissions = os.stat(file_path).st_mode & stat.S_IXUSR
+                for mutable in True, False:
+                    for symlink in True, False:
+                        with self.subTest(
+                            f"Now testing readGlobalFileWith: mutable={mutable} symlink={symlink}"
+                        ):
+                            with Toil(self.options()) as toil:
+                                file_id = toil.importFile(f"file://{file_path}")
                                 A = Job.wrapJobFn(
                                     self._testImportReadFileCompatibility,
                                     fileID=file_id,
@@ -401,7 +410,7 @@ class hidden:
             """
             Checks if text mode is compatible with file streams.
             """
-            with Toil(self.options) as toil:
+            with Toil(self.options()) as toil:
                 A = Job.wrapJobFn(self._testReadWriteFileStreamTextMode)
                 toil.start(A)
 
@@ -446,9 +455,10 @@ class hidden:
         :class:toil.fileStores.CachingFileStore.
         """
 
-        def setUp(self):
-            super().setUp()
-            self.options.caching = False
+        def options(self):
+            options = super().options()
+            options.caching = False
+            return options
 
     class AbstractCachingFileStoreTest(AbstractFileStoreTest, metaclass=ABCMeta):
         """
@@ -456,9 +466,10 @@ class hidden:
         :class:toil.fileStores.cachingFileStore.CachingFileStore.
         """
 
-        def setUp(self):
-            super().setUp()
-            self.options.caching = True
+        def options(self):
+            options = super().options()
+            options.caching = True
+            return options
 
         @slow
         @pytest.mark.xfail(reason="Cannot succeed in time on small CI runners")
@@ -470,9 +481,6 @@ class hidden:
             """
             if testingIsAutomatic and self.jobStoreType != "file":
                 self.skipTest("To save time")
-            self.options.retryCount = 10
-            self.options.badWorker = 0.25
-            self.options.badWorkerFailInterval = 0.2
             for test in range(0, 20):
                 E = Job.wrapJobFn(self._uselessFunc)
                 F = Job.wrapJobFn(self._uselessFunc)
@@ -481,7 +489,11 @@ class hidden:
                     jobs[i] = Job.wrapJobFn(self._uselessFunc)
                     E.addChild(jobs[i])
                     jobs[i].addChild(F)
-                Job.Runner.startToil(E, self.options)
+                options = self.options()
+                options.retryCount = 10
+                options.badWorker = 0.25
+                options.badWorkerFailInterval = 0.2
+                Job.Runner.startToil(E, options)
 
         @slow
         def testCacheEvictionPartialEvict(self):
@@ -491,12 +503,6 @@ class hidden:
             A Third Job requests 10MB of disk requiring eviction of the 1st file.  Ensure that the
             behavior is as expected.
             """
-            self._testValidityOfCacheEvictTest()
-
-            # Explicitly set clean to always so even the failed cases get cleaned (This will
-            # overwrite the value set in setUp if it is ever changed in the future)
-            self.options.clean = "always"
-
             self._testCacheEviction(file1MB=20, file2MB=30, diskRequestMB=10)
 
         @slow
@@ -507,12 +513,6 @@ class hidden:
             A Third Job requests 10MB of disk requiring eviction of the 1st file.  Ensure that the
             behavior is as expected.
             """
-            self._testValidityOfCacheEvictTest()
-
-            # Explicitly set clean to always so even the failed cases get cleaned (This will
-            # overwrite the value set in setUp if it is ever changed in the future)
-            self.options.clean = "always"
-
             self._testCacheEviction(file1MB=20, file2MB=30, diskRequestMB=30)
 
         @slow
@@ -523,31 +523,25 @@ class hidden:
             A Third Job requests 10MB of disk requiring eviction of the 1st file.  Ensure that the
             behavior is as expected.
             """
-            self._testValidityOfCacheEvictTest()
-
-            # Explicitly set clean to always so even the failed cases get cleaned (This will
-            # overwrite the value set in setUp if it is ever changed in the future)
-            self.options.clean = "always"
-
             self._testCacheEviction(file1MB=20, file2MB=30, diskRequestMB=60)
 
-        def _testValidityOfCacheEvictTest(self):
+        def _testValidityOfCacheEvictTest(self, options):
             # If the job store and cache are on the same file system, file
             # sizes are accounted for by the job store and are not reflected in
             # the cache hence this test is redundant (caching will be free).
-            if not self.options.jobStore.startswith(("aws", "google")):
-                workDirDev = os.stat(self.options.workDir).st_dev
-                if self.options.jobStore.startswith("file:"):
+            if not options.jobStore.startswith(("aws", "google")):
+                workDirDev = os.stat(options.workDir).st_dev
+                if options.jobStore.startswith("file:"):
                     # Before #4538, options.jobStore would have the raw path while the Config object would prepend the
                     # filesystem to the path (/path/to/file vs file:/path/to/file)
                     # The options namespace and the Config object now have the exact same behavior
                     # which means parse_jobstore will be called with argparse rather than with the config object
                     # so remove the prepended file: scheme
                     jobStoreDev = os.stat(
-                        os.path.dirname(self.options.jobStore[5:])
+                        os.path.dirname(options.jobStore[5:])
                     ).st_dev
                 else:
-                    jobStoreDev = os.stat(os.path.dirname(self.options.jobStore)).st_dev
+                    jobStoreDev = os.stat(os.path.dirname(options.jobStore)).st_dev
                 if workDirDev == jobStoreDev:
                     self.skipTest(
                         "Job store and working directory are on the same filesystem."
@@ -561,7 +555,9 @@ class hidden:
             or results in an error due to lack of space, respectively.  Ensure that the behavior is
             as expected.
             """
-            self.options.retryCount = 0
+            options = self.options()
+            options.retryCount = 0
+            self._testValidityOfCacheEvictTest(options)
             if diskRequestMB > 50:
                 # This can be non int as it will never reach _probeJobReqs
                 expectedResult = "Fail"
@@ -599,9 +595,9 @@ class hidden:
                 D.addChild(E)
                 E.addChild(F)
                 F.addChild(G)
-                Job.Runner.startToil(A, self.options)
+                Job.Runner.startToil(A, options)
             except FailedJobsException as err:
-                with open(self.options.logFile) as f:
+                with open(options.logFile) as f:
                     logContents = f.read()
                 if CacheUnbalancedError.message in logContents:
                     self.assertEqual(expectedResult, "Fail")
@@ -742,8 +738,9 @@ class hidden:
             """
             print("Testing")
             logger.debug("Testing testing 123")
-            self.options.retryCount = 0
-            self.options.logLevel = "DEBUG"
+            options = self.options()
+            options.retryCount = 0
+            options.logLevel = "DEBUG"
             A = Job.wrapJobFn(self._adjustCacheLimit, newTotalMB=1024, disk="1G")
             B = Job.wrapJobFn(self._doubleWriteFileToJobStore, fileMB=850, disk="900M")
             C = Job.wrapJobFn(
@@ -754,7 +751,7 @@ class hidden:
             A.addChild(B)
             B.addChild(C)
             C.addChild(D)
-            Job.Runner.startToil(A, self.options)
+            Job.Runner.startToil(A, options)
 
         @staticmethod
         def _doubleWriteFileToJobStore(job, fileMB):
@@ -809,7 +806,7 @@ class hidden:
                 isLocalFile=False,
                 nonLocalDir=workdir,
             )
-            Job.Runner.startToil(A, self.options)
+            Job.Runner.startToil(A, self.options())
 
         def testWriteLocalFileToJobStore(self):
             """
@@ -817,7 +814,7 @@ class hidden:
             default.  Ensure the file is cached.
             """
             A = Job.wrapJobFn(self._writeFileToJobStoreWithAsserts, isLocalFile=True)
-            Job.Runner.startToil(A, self.options)
+            Job.Runner.startToil(A, self.options())
 
         # readGlobalFile tests
 
@@ -854,7 +851,7 @@ class hidden:
                 fsID=A.rv(),
             )
             A.addChild(B)
-            Job.Runner.startToil(A, self.options)
+            Job.Runner.startToil(A, self.options())
 
         @staticmethod
         def _readFromJobStore(job, isCachedFile, cacheReadFile, fsID, isTest=True):
@@ -927,7 +924,7 @@ class hidden:
                 fsID=A.rv(),
             )
             A.addChild(B)
-            Job.Runner.startToil(A, self.options)
+            Job.Runner.startToil(A, self.options())
 
         @slow
         def testMultipleJobsReadSameCacheHitGlobalFile(self):
@@ -980,7 +977,7 @@ class hidden:
                 )
                 A.addChild(jobs[i])
                 jobs[i].addChild(B)
-            Job.Runner.startToil(A, self.options)
+            Job.Runner.startToil(A, self.options())
             with open(file_name) as y:
                 # At least one job at a time should have been observed.
                 # We can't actually guarantee that any of our jobs will
@@ -1065,7 +1062,7 @@ class hidden:
             # Tests that files written to job store can be immediately exported
             # motivated by https://github.com/BD2KGenomics/toil/issues/1469
             root = Job.wrapJobFn(self._writeExportGlobalFile)
-            Job.Runner.startToil(root, self.options)
+            Job.Runner.startToil(root, self.options())
 
         # Testing for the return of file sizes to the sigma job pool.
         @slow
@@ -1083,7 +1080,7 @@ class hidden:
                 nonLocalDir=workdir,
                 disk="2Gi",
             )
-            Job.Runner.startToil(F, self.options)
+            Job.Runner.startToil(F, self.options())
 
         @slow
         def testReturnFileSizesWithBadWorker(self):
@@ -1092,9 +1089,10 @@ class hidden:
             Read back written and locally deleted files. Ensure that after
             every step that the cache is in a valid state.
             """
-            self.options.retryCount = 20
-            self.options.badWorker = 0.5
-            self.options.badWorkerFailInterval = 0.1
+            options = self.options()
+            options.retryCount = 20
+            options.badWorker = 0.5
+            options.badWorkerFailInterval = 0.1
             workdir = self._createTempDir(purpose="nonLocalDir")
             F = Job.wrapJobFn(
                 self._returnFileTestFn,
@@ -1104,7 +1102,7 @@ class hidden:
                 numIters=30,
                 disk="2Gi",
             )
-            Job.Runner.startToil(F, self.options)
+            Job.Runner.startToil(F, options)
 
         @staticmethod
         def _returnFileTestFn(
@@ -1350,7 +1348,8 @@ class hidden:
             tracking values in the cache state file appropriately.
             """
             workdir = self._createTempDir(purpose="nonLocalDir")
-            self.options.retryCount = 1
+            options = self.options()
+            options.retryCount = 1
             jobDiskBytes = 2 * 1024 * 1024 * 1024
             F = Job.wrapJobFn(
                 self._controlledFailTestFn,
@@ -1360,7 +1359,7 @@ class hidden:
             )
             G = Job.wrapJobFn(self._probeJobReqs, sigmaJob=100, disk="100Mi")
             F.addChild(G)
-            Job.Runner.startToil(F, self.options)
+            Job.Runner.startToil(F, options)
 
         @staticmethod
         def _controlledFailTestFn(job, jobDisk, testDir):
@@ -1420,7 +1419,8 @@ class hidden:
             self._deleteLocallyReadFilesFn(readAsMutable=False)
 
         def _deleteLocallyReadFilesFn(self, readAsMutable):
-            self.options.retryCount = 0
+            options = self.options()
+            options.retryCount = 0
             A = Job.wrapJobFn(
                 self._writeFileToJobStoreWithAsserts, isLocalFile=True, memory="10M"
             )
@@ -1431,7 +1431,7 @@ class hidden:
                 memory="20M",
             )
             A.addChild(B)
-            Job.Runner.startToil(A, self.options)
+            Job.Runner.startToil(A, options)
 
         @staticmethod
         def _removeReadFileFn(job, fileToDelete, readAsMutable):
@@ -1450,20 +1450,20 @@ class hidden:
             outfile = job.fileStore.readGlobalFile(
                 fileToDelete, os.path.join(work_dir, "temp"), mutable=readAsMutable
             )
-            tempfile = os.path.join(work_dir, "tmp.tmp")
+            temp_file = os.path.join(work_dir, "tmp.tmp")
             # The first time we run this loop, processsingReadFile is True and fileToDelete is the
             # file read from the job store.  The second time, processsingReadFile is False and
             # fileToDelete is one that was just written in to the job store. Ensure the correct
             # behaviour is seen in both conditions.
             while True:
-                os.rename(outfile, tempfile)
+                os.rename(outfile, temp_file)
                 try:
                     job.fileStore.deleteLocalFile(fileToDelete)
                 except IllegalDeletionCacheError:
                     job.fileStore.log_to_leader(
                         "Detected a deleted file %s." % fileToDelete
                     )
-                    os.rename(tempfile, outfile)
+                    os.rename(temp_file, outfile)
                 else:
                     # If we are processing the write test, or if we are testing the immutably read
                     # file, we should not reach here.
@@ -1482,10 +1482,11 @@ class hidden:
             """
             Test the deletion capabilities of deleteLocalFile
             """
-            self.options.retryCount = 0
+            options = self.options()
+            options.retryCount = 0
             workdir = self._createTempDir(purpose="nonLocalDir")
             A = Job.wrapJobFn(self._deleteLocalFileFn, nonLocalDir=workdir)
-            Job.Runner.startToil(A, self.options)
+            Job.Runner.startToil(A, options)
 
         @staticmethod
         def _deleteLocalFileFn(job, nonLocalDir):
@@ -1549,8 +1550,9 @@ class hidden:
             Test many simultaneous read attempts on a file created via a stream
             directly to the job store.
             """
-            self.options.retryCount = 0
-            self.options.disableChaining = True
+            options = self.options()
+            options.retryCount = 0
+            options.disableChaining = True
 
             # Make a file
             parent = Job.wrapJobFn(self._createUncachedFileStream)
@@ -1558,7 +1560,7 @@ class hidden:
             for i in range(30):
                 parent.addChildJobFn(self._readFileWithDelay, parent.rv())
 
-            Job.Runner.startToil(parent, self.options)
+            Job.Runner.startToil(parent, options)
 
         @staticmethod
         def _createUncachedFileStream(job):

toil/test/utils/ABCWorkflowDebug/ABC.txt

@@ -0,0 +1 @@
+ABC

toil/test/utils/ABCWorkflowDebug/debugWorkflow.py

@@ -158,6 +158,7 @@ def broken_job(job, num):
 
 if __name__ == "__main__":
     jobStorePath = sys.argv[1] if len(sys.argv) > 1 else mkdtemp("debugWorkflow")
+    tmp: str = mkdtemp("debugWorkflow_tmp")
     options = Job.Runner.getDefaultOptions(jobStorePath)
     options.clean = "never"
     options.stats = True
@@ -178,10 +179,9 @@ if __name__ == "__main__":
         job1 = Job.wrapJobFn(writeA, file_maker)
         job2 = Job.wrapJobFn(writeB, file_maker, B_file)
         job3 = Job.wrapJobFn(writeC)
-        with get_data("test/utils/ABCWorkflowDebug/ABC.txt") as filepath:
-            job4 = Job.wrapJobFn(
-                writeABC, job1.rv(), job2.rv(), job3.rv(), str(filepath)
-            )
+        job4 = Job.wrapJobFn(
+            writeABC, job1.rv(), job2.rv(), job3.rv(), os.path.join(tmp, "ABC.txt")
+        )
         job5 = Job.wrapJobFn(finalize_jobs, 1)
         job6 = Job.wrapJobFn(finalize_jobs, 2)
         job7 = Job.wrapJobFn(finalize_jobs, 3)

toil/test/utils/toilKillTest.py

@@ -47,34 +47,40 @@ class _ToilKillTest:
                     str(input_file),
                 ]
                 kill_cmd = ["toil", "kill", job_store]
-
-                # run the sleep workflow
-                logger.info("Running workflow: %s", " ".join(run_cmd))
-                cwl_process = subprocess.Popen(run_cmd)
-
-                # wait until workflow starts running
-                while True:
-                    assert (
-                        cwl_process.poll() is None
-                    ), "toil-cwl-runner finished too soon"
-                    try:
-                        job_store_real = Toil.resumeJobStore(job_store)
-                        job_store_real.read_leader_pid()
-                        # pid file exists, now wait for the kill flag to exist
-                        if not job_store_real.read_kill_flag():
-                            # kill flag exists to be deleted to kill the leader
-                            break
-                        else:
-                            logger.info("Waiting for kill flag...")
-                    except (NoSuchJobStoreException, NoSuchFileException):
-                        logger.info("Waiting for job store to be openable...")
-                    time.sleep(2)
-
-                # run toil kill
-                subprocess.check_call(kill_cmd)
-
-                # after toil kill succeeds, the workflow should've exited
-                assert cwl_process.poll() is None
+                clean_cmd = ["toil", "clean", job_store]
+
+                try:
+                    # run the sleep workflow
+                    logger.info("Running workflow: %s", " ".join(run_cmd))
+                    cwl_process = subprocess.Popen(run_cmd)
+
+                    # wait until workflow starts running
+                    while True:
+                        assert (
+                            cwl_process.poll() is None
+                        ), "toil-cwl-runner finished too soon"
+                        try:
+                            job_store_real = Toil.resumeJobStore(job_store)
+                            job_store_real.read_leader_pid()
+                            # pid file exists, now wait for the kill flag to exist
+                            if not job_store_real.read_kill_flag():
+                                # kill flag exists to be deleted to kill the leader
+                                break
+                            else:
+                                logger.info("Waiting for kill flag...")
+                        except (NoSuchJobStoreException, NoSuchFileException):
+                            logger.info("Waiting for job store to be openable...")
+                        time.sleep(2)
+
+                    # run toil kill
+                    subprocess.check_call(kill_cmd)
+
+                    # after toil kill succeeds, the workflow should've exited
+                    assert cwl_process.poll() is None
+                finally:
+                    # Clean up the job store since the workflow won't do it
+                    # since it got killed.
+                    subprocess.check_call(clean_cmd)
 
 
 class TestToilKill(_ToilKillTest):
@@ -91,6 +97,7 @@ class TestToilKillWithAWSJobStore(_ToilKillTest):
     """A set of test cases for "toil kill" using the AWS job store."""
 
     @needs_cwl
+    @needs_aws_s3
     def test_cwl_toil_kill(self) -> None:
         """Test "toil kill" on a CWL workflow with a 100 second sleep."""
         self._test_cwl_toil_kill(generate_locator("aws", decoration="testkill"))

toil/test/wdl/md5sum/md5sum-gs.json

@@ -1 +1 @@
-{"ga4ghMd5.inputFile": "gs://broad-public-datasets/NA12878/NA12878.cram.crai"}
+{"ga4ghMd5.inputFile": "gs://gcp-public-data-landsat/LT04/01/003/027/LT04_L1TP_003027_19830202_20170220_01_T1/README.GTF"}

toil/test/wdl/md5sum/md5sum.json

@@ -1 +1 @@
-{"ga4ghMd5.inputFile": "src/toil/test/wdl/md5sum/md5sum.input"}
+{"ga4ghMd5.inputFile": "md5sum.input"}

toil/test/wdl/testfiles/read_file.wdl

@@ -0,0 +1,18 @@
+version 1.0
+
+# Workflow to read a file from a string path
+
+workflow read_file {
+
+    input {
+        String input_string
+    }
+
+    Array[String] the_lines = read_lines(input_string)
+
+    output {
+        Array[String] lines = the_lines
+        File remade_file = write_lines(the_lines)
+    }
+
+}

toil/test/wdl/testfiles/url_to_optional_file.wdl

@@ -3,9 +3,10 @@ version 1.0
 workflow url_to_optional_file {
     input {
         Int http_code = 404
+        String base_url = "https://httpstat.us/"
     }
     
-    File? the_file = "https://httpstat.us/" + http_code
+    File? the_file = base_url + http_code
 
     output {
         File? out_file = the_file