toil 9.1.1__py3-none-any.whl → 9.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (155) hide show
  1. toil/__init__.py +5 -9
  2. toil/batchSystems/abstractBatchSystem.py +23 -22
  3. toil/batchSystems/abstractGridEngineBatchSystem.py +17 -12
  4. toil/batchSystems/awsBatch.py +8 -8
  5. toil/batchSystems/cleanup_support.py +4 -4
  6. toil/batchSystems/contained_executor.py +3 -3
  7. toil/batchSystems/gridengine.py +3 -4
  8. toil/batchSystems/htcondor.py +5 -5
  9. toil/batchSystems/kubernetes.py +65 -63
  10. toil/batchSystems/local_support.py +2 -3
  11. toil/batchSystems/lsf.py +6 -7
  12. toil/batchSystems/mesos/batchSystem.py +11 -7
  13. toil/batchSystems/mesos/test/__init__.py +1 -2
  14. toil/batchSystems/options.py +9 -10
  15. toil/batchSystems/registry.py +3 -7
  16. toil/batchSystems/singleMachine.py +8 -11
  17. toil/batchSystems/slurm.py +49 -38
  18. toil/batchSystems/torque.py +3 -4
  19. toil/bus.py +36 -34
  20. toil/common.py +129 -89
  21. toil/cwl/cwltoil.py +857 -729
  22. toil/cwl/utils.py +44 -35
  23. toil/fileStores/__init__.py +3 -1
  24. toil/fileStores/abstractFileStore.py +28 -30
  25. toil/fileStores/cachingFileStore.py +8 -8
  26. toil/fileStores/nonCachingFileStore.py +10 -21
  27. toil/job.py +159 -158
  28. toil/jobStores/abstractJobStore.py +68 -69
  29. toil/jobStores/aws/jobStore.py +249 -213
  30. toil/jobStores/aws/utils.py +13 -24
  31. toil/jobStores/fileJobStore.py +28 -22
  32. toil/jobStores/googleJobStore.py +21 -17
  33. toil/jobStores/utils.py +3 -7
  34. toil/leader.py +17 -22
  35. toil/lib/accelerators.py +6 -4
  36. toil/lib/aws/__init__.py +9 -10
  37. toil/lib/aws/ami.py +33 -19
  38. toil/lib/aws/iam.py +6 -6
  39. toil/lib/aws/s3.py +259 -157
  40. toil/lib/aws/session.py +76 -76
  41. toil/lib/aws/utils.py +51 -43
  42. toil/lib/checksum.py +19 -15
  43. toil/lib/compatibility.py +3 -2
  44. toil/lib/conversions.py +45 -18
  45. toil/lib/directory.py +29 -26
  46. toil/lib/docker.py +93 -99
  47. toil/lib/dockstore.py +77 -50
  48. toil/lib/ec2.py +39 -38
  49. toil/lib/ec2nodes.py +11 -4
  50. toil/lib/exceptions.py +8 -5
  51. toil/lib/ftp_utils.py +9 -14
  52. toil/lib/generatedEC2Lists.py +161 -20
  53. toil/lib/history.py +141 -97
  54. toil/lib/history_submission.py +163 -72
  55. toil/lib/io.py +27 -17
  56. toil/lib/memoize.py +2 -1
  57. toil/lib/misc.py +15 -11
  58. toil/lib/pipes.py +40 -25
  59. toil/lib/plugins.py +12 -8
  60. toil/lib/resources.py +1 -0
  61. toil/lib/retry.py +32 -38
  62. toil/lib/threading.py +12 -12
  63. toil/lib/throttle.py +1 -2
  64. toil/lib/trs.py +113 -51
  65. toil/lib/url.py +14 -23
  66. toil/lib/web.py +7 -2
  67. toil/options/common.py +18 -15
  68. toil/options/cwl.py +2 -2
  69. toil/options/runner.py +9 -5
  70. toil/options/wdl.py +1 -3
  71. toil/provisioners/__init__.py +9 -9
  72. toil/provisioners/abstractProvisioner.py +22 -20
  73. toil/provisioners/aws/__init__.py +20 -14
  74. toil/provisioners/aws/awsProvisioner.py +10 -8
  75. toil/provisioners/clusterScaler.py +19 -18
  76. toil/provisioners/gceProvisioner.py +2 -3
  77. toil/provisioners/node.py +11 -13
  78. toil/realtimeLogger.py +4 -4
  79. toil/resource.py +5 -5
  80. toil/server/app.py +2 -2
  81. toil/server/cli/wes_cwl_runner.py +11 -11
  82. toil/server/utils.py +18 -21
  83. toil/server/wes/abstract_backend.py +9 -8
  84. toil/server/wes/amazon_wes_utils.py +3 -3
  85. toil/server/wes/tasks.py +3 -5
  86. toil/server/wes/toil_backend.py +17 -21
  87. toil/server/wsgi_app.py +3 -3
  88. toil/serviceManager.py +3 -4
  89. toil/statsAndLogging.py +12 -13
  90. toil/test/__init__.py +33 -24
  91. toil/test/batchSystems/batchSystemTest.py +12 -11
  92. toil/test/batchSystems/batch_system_plugin_test.py +3 -5
  93. toil/test/batchSystems/test_slurm.py +38 -24
  94. toil/test/cwl/conftest.py +5 -6
  95. toil/test/cwl/cwlTest.py +194 -78
  96. toil/test/cwl/download_file_uri.json +6 -0
  97. toil/test/cwl/download_file_uri_no_hostname.json +6 -0
  98. toil/test/docs/scripts/tutorial_staging.py +1 -0
  99. toil/test/jobStores/jobStoreTest.py +9 -7
  100. toil/test/lib/aws/test_iam.py +1 -3
  101. toil/test/lib/aws/test_s3.py +1 -1
  102. toil/test/lib/dockerTest.py +9 -9
  103. toil/test/lib/test_ec2.py +12 -11
  104. toil/test/lib/test_history.py +4 -4
  105. toil/test/lib/test_trs.py +16 -14
  106. toil/test/lib/test_url.py +7 -6
  107. toil/test/lib/url_plugin_test.py +12 -18
  108. toil/test/provisioners/aws/awsProvisionerTest.py +10 -8
  109. toil/test/provisioners/clusterScalerTest.py +2 -5
  110. toil/test/provisioners/clusterTest.py +1 -3
  111. toil/test/server/serverTest.py +13 -4
  112. toil/test/sort/restart_sort.py +2 -6
  113. toil/test/sort/sort.py +3 -8
  114. toil/test/src/deferredFunctionTest.py +7 -7
  115. toil/test/src/environmentTest.py +1 -2
  116. toil/test/src/fileStoreTest.py +5 -5
  117. toil/test/src/importExportFileTest.py +5 -6
  118. toil/test/src/jobServiceTest.py +22 -14
  119. toil/test/src/jobTest.py +121 -25
  120. toil/test/src/miscTests.py +5 -7
  121. toil/test/src/promisedRequirementTest.py +8 -7
  122. toil/test/src/regularLogTest.py +2 -3
  123. toil/test/src/resourceTest.py +5 -8
  124. toil/test/src/restartDAGTest.py +5 -6
  125. toil/test/src/resumabilityTest.py +2 -2
  126. toil/test/src/retainTempDirTest.py +3 -3
  127. toil/test/src/systemTest.py +3 -3
  128. toil/test/src/threadingTest.py +1 -1
  129. toil/test/src/workerTest.py +1 -2
  130. toil/test/utils/toilDebugTest.py +6 -4
  131. toil/test/utils/toilKillTest.py +1 -1
  132. toil/test/utils/utilsTest.py +15 -14
  133. toil/test/wdl/wdltoil_test.py +247 -124
  134. toil/test/wdl/wdltoil_test_kubernetes.py +2 -2
  135. toil/toilState.py +2 -3
  136. toil/utils/toilDebugFile.py +3 -8
  137. toil/utils/toilDebugJob.py +1 -2
  138. toil/utils/toilLaunchCluster.py +1 -2
  139. toil/utils/toilSshCluster.py +2 -0
  140. toil/utils/toilStats.py +19 -24
  141. toil/utils/toilStatus.py +11 -14
  142. toil/version.py +10 -10
  143. toil/wdl/wdltoil.py +313 -209
  144. toil/worker.py +18 -12
  145. {toil-9.1.1.dist-info → toil-9.2.0.dist-info}/METADATA +11 -14
  146. {toil-9.1.1.dist-info → toil-9.2.0.dist-info}/RECORD +150 -153
  147. {toil-9.1.1.dist-info → toil-9.2.0.dist-info}/WHEEL +1 -1
  148. toil/test/cwl/staging_cat.cwl +0 -27
  149. toil/test/cwl/staging_make_file.cwl +0 -25
  150. toil/test/cwl/staging_workflow.cwl +0 -43
  151. toil/test/cwl/zero_default.cwl +0 -61
  152. toil/test/utils/ABCWorkflowDebug/ABC.txt +0 -1
  153. {toil-9.1.1.dist-info → toil-9.2.0.dist-info}/entry_points.txt +0 -0
  154. {toil-9.1.1.dist-info → toil-9.2.0.dist-info}/licenses/LICENSE +0 -0
  155. {toil-9.1.1.dist-info → toil-9.2.0.dist-info}/top_level.txt +0 -0
toil/test/cwl/cwlTest.py CHANGED
@@ -12,7 +12,6 @@
12
12
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
13
  # See the License for the specific language governing permissions and
14
14
  # limitations under the License.
15
- from collections.abc import Generator
16
15
  import json
17
16
  import logging
18
17
  import os
@@ -23,10 +22,11 @@ import subprocess
23
22
  import sys
24
23
  import uuid
25
24
  import zipfile
25
+ from collections.abc import Callable, Generator
26
26
  from functools import partial
27
27
  from io import StringIO
28
28
  from pathlib import Path
29
- from typing import TYPE_CHECKING, Callable, Optional, cast
29
+ from typing import TYPE_CHECKING, cast
30
30
  from unittest.mock import Mock, call
31
31
  from urllib.request import urlretrieve
32
32
 
@@ -43,33 +43,29 @@ from schema_salad.exceptions import ValidationException
43
43
  from toil.cwl.utils import (
44
44
  DirectoryStructure,
45
45
  download_structure,
46
+ remove_redundant_mounts,
46
47
  visit_cwl_class_and_reduce,
47
48
  visit_top_cwl_class,
48
- remove_redundant_mounts
49
49
  )
50
50
  from toil.fileStores import FileID
51
51
  from toil.fileStores.abstractFileStore import AbstractFileStore
52
52
  from toil.job import WorkerImportJob
53
53
  from toil.lib.threading import cpu_count
54
- from toil.test import (
55
- get_data,
56
- )
57
- from toil.test import (
58
- pslow as slow,
59
- pneeds_docker as needs_docker,
60
- pneeds_cwl as needs_cwl,
61
- pneeds_aws_s3 as needs_aws_s3,
62
- pneeds_docker_cuda as needs_docker_cuda,
63
- pneeds_gridengine as needs_gridengine,
64
- pneeds_kubernetes as needs_kubernetes,
65
- pneeds_local_cuda as needs_local_cuda,
66
- pneeds_lsf as needs_lsf,
67
- pneeds_mesos as needs_mesos,
68
- pneeds_online as needs_online,
69
- pneeds_slurm as needs_slurm,
70
- pneeds_torque as needs_torque,
71
- pneeds_wes_server as needs_wes_server,
72
- )
54
+ from toil.test import get_data
55
+ from toil.test import pneeds_aws_s3 as needs_aws_s3
56
+ from toil.test import pneeds_cwl as needs_cwl
57
+ from toil.test import pneeds_docker as needs_docker
58
+ from toil.test import pneeds_docker_cuda as needs_docker_cuda
59
+ from toil.test import pneeds_gridengine as needs_gridengine
60
+ from toil.test import pneeds_kubernetes as needs_kubernetes
61
+ from toil.test import pneeds_local_cuda as needs_local_cuda
62
+ from toil.test import pneeds_lsf as needs_lsf
63
+ from toil.test import pneeds_mesos as needs_mesos
64
+ from toil.test import pneeds_online as needs_online
65
+ from toil.test import pneeds_slurm as needs_slurm
66
+ from toil.test import pneeds_torque as needs_torque
67
+ from toil.test import pneeds_wes_server as needs_wes_server
68
+ from toil.test import pslow as slow
73
69
 
74
70
  log = logging.getLogger(__name__)
75
71
  CONFORMANCE_TEST_TIMEOUT = 10000
@@ -78,15 +74,15 @@ CONFORMANCE_TEST_TIMEOUT = 10000
78
74
  def run_conformance_tests(
79
75
  workDir: str,
80
76
  yml: str,
81
- runner: Optional[str] = None,
77
+ runner: str | None = None,
82
78
  caching: bool = False,
83
- batchSystem: Optional[str] = None,
84
- selected_tests: Optional[str] = None,
85
- selected_tags: Optional[str] = None,
86
- skipped_tests: Optional[str] = None,
87
- extra_args: Optional[list[str]] = None,
79
+ batchSystem: str | None = None,
80
+ selected_tests: str | None = None,
81
+ selected_tags: str | None = None,
82
+ skipped_tests: str | None = None,
83
+ extra_args: list[str] | None = None,
88
84
  must_support_all_features: bool = False,
89
- junit_file: Optional[str] = None,
85
+ junit_file: str | None = None,
90
86
  ) -> None:
91
87
  """
92
88
  Run the CWL conformance tests.
@@ -225,6 +221,9 @@ def run_conformance_tests(
225
221
  log.info("Unsuccessful return code is OK")
226
222
 
227
223
 
224
+ # This is a type for a function that runs toil-cwl-runner and checks the
225
+ # result. See TestCWLWorkflow._tester and TestCWLWorkflow._debug_worker_tester
226
+ # for implementations.
228
227
  TesterFuncType = Callable[[Path, Path, "CWLObjectType", Path], None]
229
228
 
230
229
 
@@ -256,8 +255,29 @@ class TestCWLWorkflow:
256
255
  expect: "CWLObjectType",
257
256
  outdir: Path,
258
257
  out_name: str = "output",
259
- main_args: Optional[list[str]] = None,
258
+ main_args: list[str] | None = None,
260
259
  ) -> None:
260
+ """
261
+ Helper function that runs a CWL workflow and checks the result.
262
+
263
+ Implements TesterFuncType, plus a few additional parameters.
264
+
265
+ :param cwlfile: CWL workflow file path to run.
266
+ :param jobfile: Path to the input definition for the workflow run.
267
+ :param expect: Expected result of the workflow as a deserialized CWL
268
+ object. Should have one key per workflow output field. If output
269
+ files are expected from the workflow, they need to have their
270
+ absolute paths on disk under outdir already filled in.
271
+ :param outdir: Path to a directory to put the workflow's output files
272
+ in.
273
+ :param out_name: Name of the JSON key where the workflow's outputs can
274
+ be found in the output JSON from Toil.
275
+ :param main_args: Additional arguments to pass to toil-cwl-runner. This
276
+ is not part of TesterFuncType; you can use partial() to fill this
277
+ in and stamp out a TesterFuncType that runs toil-cwl-runner with
278
+ various closed-over arguments.
279
+ """
280
+
261
281
  from toil.cwl import cwltoil
262
282
 
263
283
  st = StringIO()
@@ -293,6 +313,21 @@ class TestCWLWorkflow:
293
313
  def _debug_worker_tester(
294
314
  self, cwlfile: Path, jobfile: Path, expect: "CWLObjectType", outdir: Path
295
315
  ) -> None:
316
+ """
317
+ Helper function that runs a CWL workflow with --debugWorker and checks
318
+ the result.
319
+
320
+ Implements TesterFuncType directly.
321
+
322
+ :param cwlfile: CWL workflow file path to run.
323
+ :param jobfile: Path to the input definition for the workflow run.
324
+ :param expect: Expected result of the workflow as a deserialized CWL
325
+ object. Should have one key per workflow output field. If output
326
+ files are expected from the workflow, they need to have their
327
+ absolute paths on disk under outdir already filled in.
328
+ :param outdir: Path to a directory to put the workflow's output files
329
+ in.
330
+ """
296
331
  from toil.cwl import cwltoil
297
332
 
298
333
  st = StringIO()
@@ -334,6 +369,21 @@ class TestCWLWorkflow:
334
369
  )
335
370
 
336
371
  def download(self, inputs: str, tester_fn: TesterFuncType, out_dir: Path) -> None:
372
+ """
373
+ Run a generic download test with a tester function and check the result.
374
+
375
+ Ther test is the download.cwl workflow.
376
+
377
+ The result has to match _expected_download_output on the output
378
+ directory, so it must contain an empty "output.txt" file.
379
+
380
+ :param inputs: Relative path to the inputs file within the Toil source
381
+ tree's src/toil/test/cwl directory.
382
+ :param tester_fn: The tester function to use to run the workflow and
383
+ check the result.
384
+ :param out_dir: Path to the output directory to save the workflow
385
+ output in.
386
+ """
337
387
  with get_data(f"test/cwl/{inputs}") as input_location:
338
388
  with get_data("test/cwl/download.cwl") as cwl_file:
339
389
  tester_fn(
@@ -490,8 +540,8 @@ class TestCWLWorkflow:
490
540
  main_args = [
491
541
  "--outdir",
492
542
  str(tmp_path),
493
- "#workflow/github.com/dockstore-testing/md5sum-checker:master",
494
- "https://raw.githubusercontent.com/dockstore-testing/md5sum-checker/refs/heads/master/md5sum/md5sum-input-cwl.json",
543
+ "github.com/mr-c/dockstore-tool-md5sum:master",
544
+ "https://github.com/mr-c/dockstore-tool-md5sum/raw/refs/heads/master/test.json"
495
545
  ]
496
546
  cwltoil.main(main_args, stdout=stdout)
497
547
  out = json.loads(stdout.getvalue())
@@ -596,6 +646,41 @@ class TestCWLWorkflow:
596
646
  def test_download_file(self, tmp_path: Path) -> None:
597
647
  self.download("download_file.json", self._tester, tmp_path)
598
648
 
649
+ def test_download_file_worker_import(self, tmp_path: Path) -> None:
650
+ self.download(
651
+ "download_file.json",
652
+ partial(self._tester, main_args=["--run-imports-on-workers"]),
653
+ tmp_path,
654
+ )
655
+
656
+ def test_download_file_uri(self, tmp_path: Path) -> None:
657
+ self.download("download_file_uri.json", self._tester, tmp_path)
658
+
659
+ def test_download_file_uri_worker_import(self, tmp_path: Path) -> None:
660
+ self.download(
661
+ "download_file_uri.json",
662
+ partial(self._tester, main_args=["--run-imports-on-workers"]),
663
+ tmp_path,
664
+ )
665
+
666
+ def test_download_file_uri_no_hostname(self, tmp_path: Path) -> None:
667
+ """
668
+ Test if CWL handles file: URIs without even empty hostnames.
669
+ """
670
+ # We can in fact ship an absolute file URI to an empty file if we
671
+ # assume /dev/null is available. So we can still use the helpers.
672
+ self.download("download_file_uri_no_hostname.json", self._tester, tmp_path)
673
+
674
+ def test_download_file_uri_no_hostname_worker_import(self, tmp_path: Path) -> None:
675
+ """
676
+ Test if CWL handles file: URIs without even empty hostnames, with worker import.
677
+ """
678
+ self.download(
679
+ "download_file_uri_no_hostname.json",
680
+ partial(self._tester, main_args=["--run-imports-on-workers"]),
681
+ tmp_path,
682
+ )
683
+
599
684
  @needs_aws_s3
600
685
  @pytest.mark.aws_s3
601
686
  @pytest.mark.online
@@ -827,7 +912,7 @@ class TestCWLWorkflow:
827
912
  @pytest.mark.aws_s3
828
913
  @pytest.mark.online
829
914
  def test_streamable(
830
- self, tmp_path: Path, extra_args: Optional[list[str]] = None
915
+ self, tmp_path: Path, extra_args: list[str] | None = None
831
916
  ) -> None:
832
917
  """
833
918
  Test that a file with 'streamable'=True is a named pipe.
@@ -1114,6 +1199,7 @@ def cwl_v1_0_spec(tmp_path: Path) -> Generator[Path]:
1114
1199
  finally:
1115
1200
  pass # no cleanup
1116
1201
 
1202
+
1117
1203
  @pytest.mark.integrative
1118
1204
  @pytest.mark.conformance
1119
1205
  @needs_cwl
@@ -1143,11 +1229,11 @@ class TestCWLv10Conformance:
1143
1229
  def test_run_conformance(
1144
1230
  self,
1145
1231
  cwl_v1_0_spec: Path,
1146
- batchSystem: Optional[str] = None,
1232
+ batchSystem: str | None = None,
1147
1233
  caching: bool = False,
1148
- selected_tests: Optional[str] = None,
1149
- skipped_tests: Optional[str] = None,
1150
- extra_args: Optional[list[str]] = None,
1234
+ selected_tests: str | None = None,
1235
+ skipped_tests: str | None = None,
1236
+ extra_args: list[str] | None = None,
1151
1237
  ) -> None:
1152
1238
  run_conformance_tests(
1153
1239
  workDir=str(cwl_v1_0_spec / "v1.0"),
@@ -1319,9 +1405,9 @@ class TestCWLv11Conformance:
1319
1405
  self,
1320
1406
  cwl_v1_1_spec: Path,
1321
1407
  caching: bool = False,
1322
- batchSystem: Optional[str] = None,
1323
- skipped_tests: Optional[str] = None,
1324
- extra_args: Optional[list[str]] = None,
1408
+ batchSystem: str | None = None,
1409
+ skipped_tests: str | None = None,
1410
+ extra_args: list[str] | None = None,
1325
1411
  ) -> None:
1326
1412
  run_conformance_tests(
1327
1413
  workDir=str(cwl_v1_1_spec),
@@ -1408,14 +1494,14 @@ class TestCWLv12Conformance:
1408
1494
  def test_run_conformance(
1409
1495
  self,
1410
1496
  cwl_v1_2_spec: Path,
1411
- runner: Optional[str] = None,
1497
+ runner: str | None = None,
1412
1498
  caching: bool = False,
1413
- batchSystem: Optional[str] = None,
1414
- selected_tests: Optional[str] = None,
1415
- skipped_tests: Optional[str] = None,
1416
- extra_args: Optional[list[str]] = None,
1499
+ batchSystem: str | None = None,
1500
+ selected_tests: str | None = None,
1501
+ skipped_tests: str | None = None,
1502
+ extra_args: list[str] | None = None,
1417
1503
  must_support_all_features: bool = False,
1418
- junit_file: Optional[str] = None,
1504
+ junit_file: str | None = None,
1419
1505
  ) -> None:
1420
1506
  if junit_file is None:
1421
1507
  junit_file = os.path.abspath("conformance-1.2.junit.xml")
@@ -1486,7 +1572,7 @@ class TestCWLv12Conformance:
1486
1572
  self,
1487
1573
  cwl_v1_2_spec: Path,
1488
1574
  caching: bool = False,
1489
- junit_file: Optional[str] = None,
1575
+ junit_file: str | None = None,
1490
1576
  ) -> None:
1491
1577
  if junit_file is None:
1492
1578
  junit_file = os.path.abspath("kubernetes-conformance-1.2.junit.xml")
@@ -1920,12 +2006,17 @@ def test_trim_mounts_op_nonredundant() -> None:
1920
2006
  """
1921
2007
  Make sure we don't remove all non-duplicate listings
1922
2008
  """
1923
- s: CWLObjectType = {"class": "Directory", "basename": "directory", "listing": [{"class": "File", "basename": "file", "contents": "hello world"}]}
2009
+ s: CWLObjectType = {
2010
+ "class": "Directory",
2011
+ "basename": "directory",
2012
+ "listing": [{"class": "File", "basename": "file", "contents": "hello world"}],
2013
+ }
1924
2014
  remove_redundant_mounts(s)
1925
2015
 
1926
2016
  # nothing should have been removed
1927
- assert isinstance(s['listing'], list)
1928
- assert len(s['listing']) == 1
2017
+ assert isinstance(s["listing"], list)
2018
+ assert len(s["listing"]) == 1
2019
+
1929
2020
 
1930
2021
  @needs_cwl
1931
2022
  @pytest.mark.cwl
@@ -1944,7 +2035,7 @@ def test_trim_mounts_op_redundant() -> None:
1944
2035
  "location": "file:///home/heaucques/Documents/toil/test_dir/nested_dir",
1945
2036
  "basename": "nested_dir",
1946
2037
  "listing": [],
1947
- "path": "/home/heaucques/Documents/toil/test_dir/nested_dir"
2038
+ "path": "/home/heaucques/Documents/toil/test_dir/nested_dir",
1948
2039
  },
1949
2040
  {
1950
2041
  "class": "File",
@@ -1954,16 +2045,17 @@ def test_trim_mounts_op_redundant() -> None:
1954
2045
  "nameroot": "test_file",
1955
2046
  "nameext": "",
1956
2047
  "path": "/home/heaucques/Documents/toil/test_dir/test_file",
1957
- "checksum": "sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709"
1958
- }
2048
+ "checksum": "sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709",
2049
+ },
1959
2050
  ],
1960
- "path": "/home/heaucques/Documents/toil/test_dir"
2051
+ "path": "/home/heaucques/Documents/toil/test_dir",
1961
2052
  }
1962
2053
  remove_redundant_mounts(s)
1963
2054
 
1964
2055
  # everything should have been removed
1965
- assert isinstance(s['listing'], list)
1966
- assert len(s['listing']) == 0
2056
+ assert isinstance(s["listing"], list)
2057
+ assert len(s["listing"]) == 0
2058
+
1967
2059
 
1968
2060
  @needs_cwl
1969
2061
  @pytest.mark.cwl
@@ -1982,7 +2074,7 @@ def test_trim_mounts_op_partially_redundant() -> None:
1982
2074
  "location": "file:///home/heaucques/Documents/thing",
1983
2075
  "basename": "thing2",
1984
2076
  "listing": [],
1985
- "path": "/home/heaucques/Documents/toil/thing2"
2077
+ "path": "/home/heaucques/Documents/toil/thing2",
1986
2078
  },
1987
2079
  {
1988
2080
  "class": "File",
@@ -1992,16 +2084,17 @@ def test_trim_mounts_op_partially_redundant() -> None:
1992
2084
  "nameroot": "test_file",
1993
2085
  "nameext": "",
1994
2086
  "path": "/home/heaucques/Documents/toil/test_dir/test_file",
1995
- "checksum": "sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709"
1996
- }
2087
+ "checksum": "sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709",
2088
+ },
1997
2089
  ],
1998
- "path": "/home/heaucques/Documents/toil/test_dir"
2090
+ "path": "/home/heaucques/Documents/toil/test_dir",
1999
2091
  }
2000
2092
  remove_redundant_mounts(s)
2001
2093
 
2002
2094
  # everything except the nested directory should be removed
2003
- assert isinstance(s['listing'], list)
2004
- assert len(s['listing']) == 1
2095
+ assert isinstance(s["listing"], list)
2096
+ assert len(s["listing"]) == 1
2097
+
2005
2098
 
2006
2099
  @needs_cwl
2007
2100
  @pytest.mark.cwl
@@ -2012,10 +2105,16 @@ def test_trim_mounts_op_mixed_urls_and_paths() -> None:
2012
2105
  """
2013
2106
  # Edge cases around encoding:
2014
2107
  # Ensure URL decoded file URIs match the bare path equivalent. Both of these paths should have the same shared directory
2015
- s: CWLObjectType = {"class": "Directory", "basename": "123", "location": "file:///tmp/%25/123", "listing": [{"class": "File", "path": "/tmp/%/123/456", "basename": "456"}]}
2108
+ s: CWLObjectType = {
2109
+ "class": "Directory",
2110
+ "basename": "123",
2111
+ "location": "file:///tmp/%25/123",
2112
+ "listing": [{"class": "File", "path": "/tmp/%/123/456", "basename": "456"}],
2113
+ }
2016
2114
  remove_redundant_mounts(s)
2017
- assert isinstance(s['listing'], list)
2018
- assert len(s['listing']) == 0
2115
+ assert isinstance(s["listing"], list)
2116
+ assert len(s["listing"]) == 0
2117
+
2019
2118
 
2020
2119
  @needs_cwl
2021
2120
  @pytest.mark.cwl
@@ -2023,22 +2122,39 @@ def test_trim_mounts_op_mixed_urls_and_paths() -> None:
2023
2122
  def test_trim_mounts_op_decodable_paths() -> None:
2024
2123
  """"""
2025
2124
  # Ensure path names don't get unnecessarily decoded
2026
- s: CWLObjectType = {"class": "Directory", "basename": "dir", "path": "/tmp/cat%2Ftag/dir", "listing": [{"class": "File", "path": "/tmp/cat/tag/dir/file", "basename": "file"}]}
2125
+ s: CWLObjectType = {
2126
+ "class": "Directory",
2127
+ "basename": "dir",
2128
+ "path": "/tmp/cat%2Ftag/dir",
2129
+ "listing": [
2130
+ {"class": "File", "path": "/tmp/cat/tag/dir/file", "basename": "file"}
2131
+ ],
2132
+ }
2027
2133
  remove_redundant_mounts(s)
2028
- assert isinstance(s['listing'], list)
2029
- assert len(s['listing']) == 1
2134
+ assert isinstance(s["listing"], list)
2135
+ assert len(s["listing"]) == 1
2136
+
2030
2137
 
2031
2138
  @needs_cwl
2032
2139
  @pytest.mark.cwl
2033
2140
  @pytest.mark.cwl_small
2034
2141
  def test_trim_mounts_op_multiple_encodings() -> None:
2035
2142
  # Ensure differently encoded URLs are properly decoded
2036
- s: CWLObjectType = {"class": "Directory", "basename": "dir", "location": "file:///tmp/cat%2Ftag/dir", "listing": [{"class": "File", "location": "file:///tmp/cat%2ftag/dir/file", "basename": "file"}]}
2143
+ s: CWLObjectType = {
2144
+ "class": "Directory",
2145
+ "basename": "dir",
2146
+ "location": "file:///tmp/cat%2Ftag/dir",
2147
+ "listing": [
2148
+ {
2149
+ "class": "File",
2150
+ "location": "file:///tmp/cat%2ftag/dir/file",
2151
+ "basename": "file",
2152
+ }
2153
+ ],
2154
+ }
2037
2155
  remove_redundant_mounts(s)
2038
- assert isinstance(s['listing'], list)
2039
- assert len(s['listing']) == 0
2040
-
2041
-
2156
+ assert isinstance(s["listing"], list)
2157
+ assert len(s["listing"]) == 0
2042
2158
 
2043
2159
 
2044
2160
  @needs_cwl
@@ -2163,6 +2279,7 @@ def test_import_on_workers() -> None:
2163
2279
 
2164
2280
  assert detector.detected is True
2165
2281
 
2282
+
2166
2283
  @needs_cwl
2167
2284
  @pytest.mark.cwl
2168
2285
  @pytest.mark.cwl_small
@@ -2180,13 +2297,14 @@ def test_missing_tmpdir_and_tmp_outdir(tmp_path: Path) -> None:
2180
2297
  "toil-cwl-runner",
2181
2298
  f"--jobStore=file:{tmp_path / 'jobstore'}",
2182
2299
  "--strict-memory-limit",
2183
- f'--tmpdir-prefix={tmpdir_prefix}',
2184
- f'--tmp-outdir-prefix={tmp_outdir_prefix}',
2300
+ f"--tmpdir-prefix={tmpdir_prefix}",
2301
+ f"--tmp-outdir-prefix={tmp_outdir_prefix}",
2185
2302
  str(cwl_file),
2186
2303
  ]
2187
2304
  p = subprocess.run(cmd)
2188
2305
  assert p.returncode == 0
2189
2306
 
2307
+
2190
2308
  # StreamHandler is generic, _typeshed doesn't exist at runtime, do a bit of typing trickery, see https://github.com/python/typeshed/issues/5680
2191
2309
  if TYPE_CHECKING:
2192
2310
  from _typeshed import SupportsWrite
@@ -2218,7 +2336,5 @@ class ImportWorkersMessageHandler(_stream_handler):
2218
2336
  f"Log message {record.msg} has wrong number of "
2219
2337
  f"fields in {record.args}"
2220
2338
  ) from e
2221
- if formatted.startswith(
2222
- f"Issued job '{WorkerImportJob.__name__}'"
2223
- ):
2339
+ if formatted.startswith(f"Issued job '{WorkerImportJob.__name__}'"):
2224
2340
  self.detected = True
@@ -0,0 +1,6 @@
1
+ {
2
+ "input": {
3
+ "class": "File",
4
+ "location": "file:///dev/null"
5
+ }
6
+ }
@@ -0,0 +1,6 @@
1
+ {
2
+ "input": {
3
+ "class": "File",
4
+ "location": "file:/dev/null"
5
+ }
6
+ }
@@ -4,6 +4,7 @@ from toil.common import Toil
4
4
  from toil.job import Job
5
5
  from toil.lib.io import mkdtemp
6
6
 
7
+
7
8
  class HelloWorld(Job):
8
9
  def __init__(self, id):
9
10
  Job.__init__(self)
@@ -24,7 +24,6 @@ import urllib.parse as urlparse
24
24
  import uuid
25
25
  from abc import ABCMeta, abstractmethod
26
26
  from io import BytesIO
27
- from itertools import chain, islice
28
27
  from queue import Queue
29
28
  from tempfile import mkstemp
30
29
  from threading import Thread
@@ -34,9 +33,9 @@ from urllib.request import Request, urlopen
34
33
  import pytest
35
34
  from stubserver import FTPStubServer
36
35
 
37
- from toil.common import Config, Toil
36
+ from toil.common import Config
38
37
  from toil.fileStores import FileID
39
- from toil.job import Job, JobDescription, TemporaryID
38
+ from toil.job import JobDescription, TemporaryID
40
39
  from toil.jobStores.abstractJobStore import NoSuchFileException, NoSuchJobException
41
40
  from toil.jobStores.fileJobStore import FileJobStore
42
41
  from toil.lib.io import mkdtemp
@@ -1195,12 +1194,15 @@ class AbstractEncryptedJobStoreTest:
1195
1194
  with self.jobstore_initialized.read_shared_file_stream(fileName) as f:
1196
1195
  # If the read goes through, we should fail the assert because
1197
1196
  # we read the cyphertext
1198
- assert f.read() != phrase, (
1199
- "Managed to read plaintext content with encryption off."
1200
- )
1197
+ assert (
1198
+ f.read() != phrase
1199
+ ), "Managed to read plaintext content with encryption off."
1201
1200
  except AWSBadEncryptionKeyError as e:
1202
1201
  # If the read doesn't go through, we get this.
1203
- assert "Your AWS encryption key is most likely configured incorrectly" in str(e)
1202
+ assert (
1203
+ "Your AWS encryption key is most likely configured incorrectly"
1204
+ in str(e)
1205
+ )
1204
1206
 
1205
1207
 
1206
1208
  class FileJobStoreTest(AbstractJobStoreTest.Test):
@@ -29,9 +29,7 @@ class IAMTest(ToilTest):
29
29
  """Check that given permissions and associated functions perform correctly"""
30
30
 
31
31
  def test_permissions_iam(self) -> None:
32
- granted_perms = {
33
- "*": {"Action": ["ec2:*", "iam:*", "s3:*"], "NotAction": []}
34
- }
32
+ granted_perms = {"*": {"Action": ["ec2:*", "iam:*", "s3:*"], "NotAction": []}}
35
33
  assert (
36
34
  iam.policy_permissions_allow(
37
35
  granted_perms, iam.CLUSTER_LAUNCHING_PERMISSIONS
@@ -16,7 +16,6 @@ import os
16
16
  import uuid
17
17
  from typing import TYPE_CHECKING, Optional
18
18
 
19
- from toil.jobStores.aws.jobStore import AWSJobStore
20
19
  from toil.lib.aws.session import establish_boto3_session
21
20
  from toil.lib.aws.utils import create_s3_bucket, delete_s3_bucket, get_bucket_region
22
21
  from toil.test import ToilTest, needs_aws_s3
@@ -28,6 +27,7 @@ if TYPE_CHECKING:
28
27
  from mypy_boto3_s3 import S3ServiceResource
29
28
  from mypy_boto3_s3.service_resource import Bucket
30
29
 
30
+
31
31
  @needs_aws_s3
32
32
  class S3Test(ToilTest):
33
33
  """Confirm the workarounds for us-east-1."""
@@ -13,13 +13,14 @@
13
13
  # limitations under the License.
14
14
  import logging
15
15
  import os
16
- from pathlib import Path
17
16
  import signal
18
17
  import time
19
- from typing import Optional
20
18
  import uuid
19
+ from pathlib import Path
21
20
  from threading import Thread
22
21
 
22
+ import pytest
23
+
23
24
  from docker.errors import ContainerError # type: ignore[import-not-found]
24
25
  from toil.common import Toil
25
26
  from toil.exceptions import FailedJobsException
@@ -32,9 +33,8 @@ from toil.lib.docker import (
32
33
  containerIsRunning,
33
34
  dockerKill,
34
35
  )
35
- from toil.test import pneeds_docker as needs_docker, pslow as slow
36
-
37
- import pytest
36
+ from toil.test import pneeds_docker as needs_docker
37
+ from toil.test import pslow as slow
38
38
 
39
39
  logger = logging.getLogger(__name__)
40
40
 
@@ -55,7 +55,7 @@ class TestDocker:
55
55
  caching: bool = False,
56
56
  detached: bool = True,
57
57
  rm: bool = True,
58
- deferParam: Optional[int] = None,
58
+ deferParam: int | None = None,
59
59
  ) -> None:
60
60
  """
61
61
  Run the test container that creates a file in the work dir, and sleeps
@@ -376,9 +376,9 @@ def _testDockerCleanFn(
376
376
  job: Job,
377
377
  working_dir: Path,
378
378
  detached: bool = True,
379
- rm: Optional[bool] = None,
380
- deferParam: Optional[int] = None,
381
- containerName: Optional[str] = None,
379
+ rm: bool | None = None,
380
+ deferParam: int | None = None,
381
+ containerName: str | None = None,
382
382
  ) -> None:
383
383
  """
384
384
  Test function for test docker_clean. Runs a container with given flags and