toil 9.1.0__py3-none-any.whl → 9.1.2__py3-none-any.whl

toil/leader.py

@@ -349,14 +349,9 @@ class Leader:
     def create_status_sentinel_file(self, fail: bool) -> None:
         """Create a file in the jobstore indicating failure or success."""
         logName = "failed.log" if fail else "succeeded.log"
-        localLog = os.path.join(os.getcwd(), logName)
-        open(localLog, "w").close()
-        self.jobStore.import_file("file://" + localLog, logName, hardlink=True)
-
-        if os.path.exists(
-            localLog
-        ):  # Bandaid for Jenkins tests failing stochastically and unexplainably.
-            os.remove(localLog)
+        with self.jobStore.write_shared_file_stream(logName) as file_handle:
+            # We just need an empty file, so don't write any content
+            pass
 
     def _handledFailedSuccessor(self, successor_id: str, predecessor_id: str) -> bool:
         """

toil/test/wdl/testfiles/gather.wdl

@@ -0,0 +1,52 @@
+version 1.1
+
+workflow gather {
+    input {
+    }
+
+    scatter(number in [1, 2, 3]) {
+        call task1 {
+            input:
+                number=number
+        }
+    }
+
+    call task2 {
+        input:
+            files = task1.foo
+    }
+
+    output {
+        File outfile = task2.outfile
+    }
+}
+
+task task1 {
+
+    input {
+        Int number
+    }
+
+    command <<<
+        echo ~{number} > foo.txt
+    >>>
+
+    output {
+        File foo = "foo.txt"
+    }
+}
+
+task task2 {
+    input {
+        Array[File] files
+    }
+
+    command <<<
+        cat ~{sep=" " files} >out.txt
+    >>>
+
+    output {
+        File outfile = "out.txt"
+    }
+}
+

toil/test/wdl/wdltoil_test.py

@@ -321,6 +321,28 @@ class TestWDL:
 
                 assert "StringFileCoercion.output_file" in result
 
+    @needs_docker
+    def test_gather(self, tmp_path: Path) -> None:
+        """
+        Test files with the same name from different scatter tasks.
+        """
+        with get_data("test/wdl/testfiles/gather.wdl") as wdl:
+            result_json = subprocess.check_output(
+                self.base_command
+                + [
+                    str(wdl),
+                    "-o",
+                    str(tmp_path),
+                    "--logInfo",
+                    "--retryCount=0"
+                ]
+            )
+            result = json.loads(result_json)
+
+            assert "gather.outfile" in result
+            assert isinstance(result["gather.outfile"], str)
+            assert open(result["gather.outfile"]).read() == "1\n2\n3\n"
+
     @needs_docker
     def test_wait(self, tmp_path: Path) -> None:
         """

toil/version.py

@@ -1,13 +1,13 @@
-baseVersion = '9.1.0'
+baseVersion = '9.1.2'
 cgcloudVersion = '1.6.0a1.dev393'
-version = '9.1.0-e341bb669efe78f93308e5ff1f02f7e375973511'
+version = '9.1.2-355b72ba0e425619c0367562caf2a337078dba65'
 cacheTag = 'cache-local-py3.9'
 mainCacheTag = 'cache-master-py3.9'
-distVersion = '9.1.0'
+distVersion = '9.1.2'
 exactPython = 'python3.9'
 python = 'python3.9'
-dockerTag = '9.1.0-e341bb669efe78f93308e5ff1f02f7e375973511-py3.9'
-currentCommit = 'e341bb669efe78f93308e5ff1f02f7e375973511'
+dockerTag = '9.1.2-355b72ba0e425619c0367562caf2a337078dba65-py3.9'
+currentCommit = '355b72ba0e425619c0367562caf2a337078dba65'
 dockerRegistry = 'quay.io/ucsc_cgl'
 dockerName = 'toil'
 dirty = False

toil/wdl/wdltoil.py

@@ -5048,6 +5048,9 @@ class WDLSectionJob(WDLBaseJob):
         if subscript is not None:
             # We need to include a scatter loop number.
             task_path += f".{subscript}"
+        # TODO: MyPy can't tell this dict copy will have the same type
+        child_wdl_options = cast(WDLContext, dict(self._wdl_options))
+        child_wdl_options["task_path"] = task_path
 
         if local_environment is not None:
             # Bring local environment into scope
@@ -5115,7 +5118,7 @@ class WDLSectionJob(WDLBaseJob):
                 job: WDLBaseJob = WDLWorkflowNodeJob(
                     section_graph.get(node_ids[0]),
                     rvs,
-                    wdl_options=self._wdl_options,
+                    wdl_options=child_wdl_options,
                     local=True,
                 )
             else:
@@ -5123,7 +5126,7 @@ class WDLSectionJob(WDLBaseJob):
                 job = WDLWorkflowNodeListJob(
                     [section_graph.get(node_id) for node_id in node_ids],
                     rvs,
-                    wdl_options=self._wdl_options,
+                    wdl_options=child_wdl_options,
                     local=True,
                 )
             for prev_job in prev_jobs:
@@ -5158,7 +5161,7 @@ class WDLSectionJob(WDLBaseJob):
             # And to fill in bindings from code not executed in this instantiation
             # with Null, and filter out stuff that should leave scope.
             sink = WDLCombineBindingsJob(
-                leaf_rvs, wdl_options=self._wdl_options, local=True
+                leaf_rvs, wdl_options=child_wdl_options, local=True
             )
             # It runs inside us
             self.addChild(sink)

{toil-9.1.0.dist-info → toil-9.1.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: toil
-Version: 9.1.0
+Version: 9.1.2
 Summary: Pipeline management software for clusters.
 Home-page: https://github.com/DataBiosphere/toil
 Author: Benedict Paten and the Toil community

{toil-9.1.0.dist-info → toil-9.1.2.dist-info}/RECORD

@@ -4,13 +4,13 @@ toil/common.py,sha256=SCcqojtspk3oqwStKUx-daP7fMCgWQ8TWBgh7WNcSRY,83253
 toil/deferred.py,sha256=2ShsuPqDwu0jhBQ6ocMwe4lfNegBm2lj7dOG9E7X3Zc,14152
 toil/exceptions.py,sha256=oL2d9uutwtcPkMcEUGrixKUd4y1JLSA0-N7hNJqCLSk,1883
 toil/job.py,sha256=KGcmuzoCFtWakl_y4LwBlo8Uuh1-FnGqHOXc_BwAF0A,175901
-toil/leader.py,sha256=abI47MI8zUlWZCjFce2QMwoFF9-QQ1aF8kWn7qKtrpM,85829
+toil/leader.py,sha256=jyx9pRREOfbA8X-IfVQmhNv34UvhmUJfm2Tr6-oqKwc,85659
 toil/realtimeLogger.py,sha256=2O4AUaKtvItzzCAQKORIEHZ2cDoLf_6njoKWv_uzdFc,9878
 toil/resource.py,sha256=2LmdM6zqjG_aYqQ25UixpkdKipLYnKngMGhll4I-O6M,25349
 toil/serviceManager.py,sha256=GE5IUjQmNrekYv9tclim0RlCFf4ueOQ2H4qHjMBrOcg,19978
 toil/statsAndLogging.py,sha256=JXir7TCV8OdDaAdt3VsDR2rbdNWk4ILyOwlKHVdoXT0,18696
 toil/toilState.py,sha256=DD52XQv8wRCYzF7-F6Olqa_k6u-_py-JWyjqJHRh6Z0,17989
-toil/version.py,sha256=CHyCVncfvUkQyL2LsngddgCkkBkhrtOF8yHxwkeLVYw,486
+toil/version.py,sha256=1R_b63zYENcFZU5lPdbYiY-fe6Wfugp63fYEbhmEcaA,486
 toil/worker.py,sha256=uEU3BnmFS-SQf2JW6FDvcf7t8uUXFXiInaJLTt8l9Lk,39315
 toil/batchSystems/__init__.py,sha256=T8psI5GmvE71_XmajKsfeQ3YUs6XM7rX1Fy9VNUX_-Q,1036
 toil/batchSystems/abstractBatchSystem.py,sha256=yOsKGlKOADtBhsG6OVvBY2yRJREJtPd84WisE47xBO0,33803
@@ -320,7 +320,7 @@ toil/test/wdl/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 toil/test/wdl/lint_error.wdl,sha256=vLwgk8CrPzLNRJnMLuQV22m1E1cwG-m76ZM0XgSKfLY,86
 toil/test/wdl/test.csv,sha256=1hPDzKegqv1H582Bx-4CaFBLE-jCSyZo3N6KhjhsXO8,18
 toil/test/wdl/test.tsv,sha256=O4F7lQouDB33aMklmbL2KOoLNNEDx_EUiHmANZ4pKeg,18
-toil/test/wdl/wdltoil_test.py,sha256=FyVZHRIp7qoSpospQUB4rt08mGcctI_l2OJY3KUnV1I,50554
+toil/test/wdl/wdltoil_test.py,sha256=GajkwQqFCcbUYqALEpW2KIUYhVvgnCcQH_qXwfHAzh4,51302
 toil/test/wdl/wdltoil_test_kubernetes.py,sha256=vJdmSuQJslvrfF6nadjORkdanED99xElG54EC7L2rR4,2996
 toil/test/wdl/md5sum/empty_file.json,sha256=Y4Yb2dygeV8J7wowyUuTi8qOwmRR-znb70r3jsPw688,27
 toil/test/wdl/md5sum/md5sum-gs.json,sha256=PppK5H77nRcpo_n2ttZzEPkoxrLVypGlFD6Wrwe-hfk,123
@@ -403,6 +403,7 @@ toil/test/wdl/testfiles/croo.wdl,sha256=LglQ9WjDYmMI6fsDE_CacTRMHtTRu84LX-H05IIS
 toil/test/wdl/testfiles/drop_files.wdl,sha256=6pR3s6IOIg6tvZesvF4Fh78QvzlKGz3HP3X4PUQgIWs,1929
 toil/test/wdl/testfiles/drop_files_subworkflow.wdl,sha256=Qrq7LPfQGmD8ALWUVyp3AlZ8_KcD1nqKCb5bVf9wDb4,158
 toil/test/wdl/testfiles/empty.txt,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+toil/test/wdl/testfiles/gather.wdl,sha256=kzycmVYei6LkZkcAtCM1JD6g498bMALiO-JVjBwa3hc,637
 toil/test/wdl/testfiles/not_enough_outputs.wdl,sha256=AO3v20B5nR28MZmmKYCTiqh3hRJgg6LCENAZ2nC4IYg,438
 toil/test/wdl/testfiles/random.wdl,sha256=74vORE5a-fcQs_z73uyjUzkRqpGLv22bWmlNuvE83nw,1017
 toil/test/wdl/testfiles/read_file.wdl,sha256=9UkeiNK7QQJfzARfWfApL_zQ_7O3cTAF3_IyJfBhQJs,296
@@ -442,10 +443,10 @@ toil/utils/toilStatus.py,sha256=qacxW5lcguvmhUi1b4cjm1acJik0zpsP7CsUuZyX-fo,2042
 toil/utils/toilUpdateEC2Instances.py,sha256=47972f1_oRvKOZv77EcBr3QjP2YNl8SZWhTsw6mWG-4,1318
 toil/wdl/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 toil/wdl/utils.py,sha256=U0TqbVTsap0VdeZjrUf4l71A7Zns-Hkso1B1Zdp2LRk,1316
-toil/wdl/wdltoil.py,sha256=bxg5QBKK404VW6MCOlNgTpFnrOvDlNkDLGVO7u70rSY,265800
-toil-9.1.0.dist-info/licenses/LICENSE,sha256=YVxVi652J1ZDmtC3EMP5r0EbK85-hm_pzogiULmlqUA,12862
-toil-9.1.0.dist-info/METADATA,sha256=r7k37536fUxWnVEP9c5wT-7gqeqYcl1J0gHAu942pXQ,8695
-toil-9.1.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-toil-9.1.0.dist-info/entry_points.txt,sha256=EF2yFhV2UYuJGr-OjMkUOd3RyOCgRWQbS6XJtBJ25eM,436
-toil-9.1.0.dist-info/top_level.txt,sha256=1ydj7IXvHS9tMT5OVTSSpub6ZOaQeIn3KGCgJqaikF0,5
-toil-9.1.0.dist-info/RECORD,,
+toil/wdl/wdltoil.py,sha256=DKGkqbGjjEaXt9QrnSKDzEE3JuAMNzE_jc9cxyibk7c,265992
+toil-9.1.2.dist-info/licenses/LICENSE,sha256=YVxVi652J1ZDmtC3EMP5r0EbK85-hm_pzogiULmlqUA,12862
+toil-9.1.2.dist-info/METADATA,sha256=jNXFOpp2P2OkDd2zNWN_RGSuhnsxXvWaxin8lgBE3B4,8695
+toil-9.1.2.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+toil-9.1.2.dist-info/entry_points.txt,sha256=EF2yFhV2UYuJGr-OjMkUOd3RyOCgRWQbS6XJtBJ25eM,436
+toil-9.1.2.dist-info/top_level.txt,sha256=1ydj7IXvHS9tMT5OVTSSpub6ZOaQeIn3KGCgJqaikF0,5
+toil-9.1.2.dist-info/RECORD,,