toil 8.1.0b1__py3-none-any.whl → 8.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (254) hide show
  1. toil/__init__.py +0 -35
  2. toil/batchSystems/abstractBatchSystem.py +1 -1
  3. toil/batchSystems/abstractGridEngineBatchSystem.py +1 -1
  4. toil/batchSystems/awsBatch.py +1 -1
  5. toil/batchSystems/cleanup_support.py +1 -1
  6. toil/batchSystems/kubernetes.py +53 -7
  7. toil/batchSystems/local_support.py +1 -1
  8. toil/batchSystems/mesos/batchSystem.py +13 -8
  9. toil/batchSystems/mesos/test/__init__.py +3 -2
  10. toil/batchSystems/singleMachine.py +1 -1
  11. toil/batchSystems/slurm.py +27 -26
  12. toil/bus.py +5 -3
  13. toil/common.py +39 -11
  14. toil/cwl/cwltoil.py +1 -1
  15. toil/job.py +64 -49
  16. toil/jobStores/abstractJobStore.py +24 -3
  17. toil/jobStores/fileJobStore.py +25 -1
  18. toil/jobStores/googleJobStore.py +104 -30
  19. toil/leader.py +9 -0
  20. toil/lib/accelerators.py +3 -1
  21. toil/lib/aws/utils.py.orig +504 -0
  22. toil/lib/bioio.py +1 -1
  23. toil/lib/docker.py +252 -91
  24. toil/lib/dockstore.py +11 -3
  25. toil/lib/exceptions.py +5 -3
  26. toil/lib/history.py +87 -13
  27. toil/lib/history_submission.py +23 -9
  28. toil/lib/io.py +34 -22
  29. toil/lib/misc.py +7 -1
  30. toil/lib/resources.py +2 -1
  31. toil/lib/threading.py +11 -10
  32. toil/options/common.py +8 -0
  33. toil/options/wdl.py +11 -0
  34. toil/server/api_spec/LICENSE +201 -0
  35. toil/server/api_spec/README.rst +5 -0
  36. toil/server/cli/wes_cwl_runner.py +2 -1
  37. toil/test/__init__.py +275 -115
  38. toil/test/batchSystems/batchSystemTest.py +227 -205
  39. toil/test/batchSystems/test_slurm.py +27 -0
  40. toil/test/cactus/pestis.tar.gz +0 -0
  41. toil/test/conftest.py +7 -0
  42. toil/test/cwl/2.fasta +11 -0
  43. toil/test/cwl/2.fastq +12 -0
  44. toil/test/cwl/conftest.py +1 -1
  45. toil/test/cwl/cwlTest.py +999 -867
  46. toil/test/cwl/directory/directory/file.txt +15 -0
  47. toil/test/cwl/download_directory_file.json +4 -0
  48. toil/test/cwl/download_directory_s3.json +4 -0
  49. toil/test/cwl/download_file.json +6 -0
  50. toil/test/cwl/download_http.json +6 -0
  51. toil/test/cwl/download_https.json +6 -0
  52. toil/test/cwl/download_s3.json +6 -0
  53. toil/test/cwl/download_subdirectory_file.json +5 -0
  54. toil/test/cwl/download_subdirectory_s3.json +5 -0
  55. toil/test/cwl/empty.json +1 -0
  56. toil/test/cwl/mock_mpi/fake_mpi.yml +8 -0
  57. toil/test/cwl/mock_mpi/fake_mpi_run.py +42 -0
  58. toil/test/cwl/optional-file-exists.json +6 -0
  59. toil/test/cwl/optional-file-missing.json +6 -0
  60. toil/test/cwl/preemptible_expression.json +1 -0
  61. toil/test/cwl/revsort-job-missing.json +6 -0
  62. toil/test/cwl/revsort-job.json +6 -0
  63. toil/test/cwl/s3_secondary_file.json +16 -0
  64. toil/test/cwl/seqtk_seq_job.json +6 -0
  65. toil/test/cwl/stream.json +6 -0
  66. toil/test/cwl/test_filename_conflict_resolution.ms/table.dat +0 -0
  67. toil/test/cwl/test_filename_conflict_resolution.ms/table.f0 +0 -0
  68. toil/test/cwl/test_filename_conflict_resolution.ms/table.f1 +0 -0
  69. toil/test/cwl/test_filename_conflict_resolution.ms/table.f1i +0 -0
  70. toil/test/cwl/test_filename_conflict_resolution.ms/table.f2 +0 -0
  71. toil/test/cwl/test_filename_conflict_resolution.ms/table.f2_TSM0 +0 -0
  72. toil/test/cwl/test_filename_conflict_resolution.ms/table.f3 +0 -0
  73. toil/test/cwl/test_filename_conflict_resolution.ms/table.f3_TSM0 +0 -0
  74. toil/test/cwl/test_filename_conflict_resolution.ms/table.f4 +0 -0
  75. toil/test/cwl/test_filename_conflict_resolution.ms/table.f4_TSM0 +0 -0
  76. toil/test/cwl/test_filename_conflict_resolution.ms/table.f5 +0 -0
  77. toil/test/cwl/test_filename_conflict_resolution.ms/table.info +0 -0
  78. toil/test/cwl/test_filename_conflict_resolution.ms/table.lock +0 -0
  79. toil/test/cwl/whale.txt +16 -0
  80. toil/test/docs/scripts/example_alwaysfail.py +38 -0
  81. toil/test/docs/scripts/example_alwaysfail_with_files.wdl +33 -0
  82. toil/test/docs/scripts/example_cachingbenchmark.py +117 -0
  83. toil/test/docs/scripts/stagingExampleFiles/in.txt +1 -0
  84. toil/test/docs/scripts/stagingExampleFiles/out.txt +2 -0
  85. toil/test/docs/scripts/tutorial_arguments.py +23 -0
  86. toil/test/docs/scripts/tutorial_debugging.patch +12 -0
  87. toil/test/docs/scripts/tutorial_debugging_hangs.wdl +126 -0
  88. toil/test/docs/scripts/tutorial_debugging_works.wdl +129 -0
  89. toil/test/docs/scripts/tutorial_docker.py +20 -0
  90. toil/test/docs/scripts/tutorial_dynamic.py +24 -0
  91. toil/test/docs/scripts/tutorial_encapsulation.py +28 -0
  92. toil/test/docs/scripts/tutorial_encapsulation2.py +29 -0
  93. toil/test/docs/scripts/tutorial_helloworld.py +15 -0
  94. toil/test/docs/scripts/tutorial_invokeworkflow.py +27 -0
  95. toil/test/docs/scripts/tutorial_invokeworkflow2.py +30 -0
  96. toil/test/docs/scripts/tutorial_jobfunctions.py +22 -0
  97. toil/test/docs/scripts/tutorial_managing.py +29 -0
  98. toil/test/docs/scripts/tutorial_managing2.py +56 -0
  99. toil/test/docs/scripts/tutorial_multiplejobs.py +25 -0
  100. toil/test/docs/scripts/tutorial_multiplejobs2.py +21 -0
  101. toil/test/docs/scripts/tutorial_multiplejobs3.py +22 -0
  102. toil/test/docs/scripts/tutorial_promises.py +25 -0
  103. toil/test/docs/scripts/tutorial_promises2.py +30 -0
  104. toil/test/docs/scripts/tutorial_quickstart.py +22 -0
  105. toil/test/docs/scripts/tutorial_requirements.py +44 -0
  106. toil/test/docs/scripts/tutorial_services.py +45 -0
  107. toil/test/docs/scripts/tutorial_staging.py +45 -0
  108. toil/test/docs/scripts/tutorial_stats.py +64 -0
  109. toil/test/lib/aws/test_iam.py +3 -1
  110. toil/test/lib/dockerTest.py +205 -122
  111. toil/test/lib/test_history.py +101 -77
  112. toil/test/provisioners/aws/awsProvisionerTest.py +12 -9
  113. toil/test/provisioners/clusterTest.py +4 -4
  114. toil/test/provisioners/gceProvisionerTest.py +16 -14
  115. toil/test/sort/sort.py +4 -1
  116. toil/test/src/busTest.py +17 -17
  117. toil/test/src/deferredFunctionTest.py +145 -132
  118. toil/test/src/importExportFileTest.py +71 -63
  119. toil/test/src/jobEncapsulationTest.py +27 -28
  120. toil/test/src/jobServiceTest.py +149 -133
  121. toil/test/src/jobTest.py +219 -211
  122. toil/test/src/miscTests.py +66 -60
  123. toil/test/src/promisedRequirementTest.py +163 -169
  124. toil/test/src/regularLogTest.py +24 -24
  125. toil/test/src/resourceTest.py +82 -76
  126. toil/test/src/restartDAGTest.py +51 -47
  127. toil/test/src/resumabilityTest.py +24 -19
  128. toil/test/src/retainTempDirTest.py +60 -57
  129. toil/test/src/systemTest.py +17 -13
  130. toil/test/src/threadingTest.py +29 -32
  131. toil/test/utils/ABCWorkflowDebug/B_file.txt +1 -0
  132. toil/test/utils/ABCWorkflowDebug/debugWorkflow.py +204 -0
  133. toil/test/utils/ABCWorkflowDebug/mkFile.py +16 -0
  134. toil/test/utils/ABCWorkflowDebug/sleep.cwl +12 -0
  135. toil/test/utils/ABCWorkflowDebug/sleep.yaml +1 -0
  136. toil/test/utils/toilDebugTest.py +117 -102
  137. toil/test/utils/toilKillTest.py +54 -53
  138. toil/test/utils/utilsTest.py +303 -229
  139. toil/test/wdl/lint_error.wdl +9 -0
  140. toil/test/wdl/md5sum/empty_file.json +1 -0
  141. toil/test/wdl/md5sum/md5sum-gs.json +1 -0
  142. toil/test/wdl/md5sum/md5sum.1.0.wdl +32 -0
  143. toil/test/wdl/md5sum/md5sum.input +1 -0
  144. toil/test/wdl/md5sum/md5sum.json +1 -0
  145. toil/test/wdl/md5sum/md5sum.wdl +25 -0
  146. toil/test/wdl/miniwdl_self_test/inputs-namespaced.json +1 -0
  147. toil/test/wdl/miniwdl_self_test/inputs.json +1 -0
  148. toil/test/wdl/miniwdl_self_test/self_test.wdl +40 -0
  149. toil/test/wdl/standard_library/as_map.json +16 -0
  150. toil/test/wdl/standard_library/as_map_as_input.wdl +23 -0
  151. toil/test/wdl/standard_library/as_pairs.json +7 -0
  152. toil/test/wdl/standard_library/as_pairs_as_input.wdl +23 -0
  153. toil/test/wdl/standard_library/ceil.json +3 -0
  154. toil/test/wdl/standard_library/ceil_as_command.wdl +16 -0
  155. toil/test/wdl/standard_library/ceil_as_input.wdl +16 -0
  156. toil/test/wdl/standard_library/collect_by_key.json +1 -0
  157. toil/test/wdl/standard_library/collect_by_key_as_input.wdl +23 -0
  158. toil/test/wdl/standard_library/cross.json +11 -0
  159. toil/test/wdl/standard_library/cross_as_input.wdl +19 -0
  160. toil/test/wdl/standard_library/flatten.json +7 -0
  161. toil/test/wdl/standard_library/flatten_as_input.wdl +18 -0
  162. toil/test/wdl/standard_library/floor.json +3 -0
  163. toil/test/wdl/standard_library/floor_as_command.wdl +16 -0
  164. toil/test/wdl/standard_library/floor_as_input.wdl +16 -0
  165. toil/test/wdl/standard_library/keys.json +8 -0
  166. toil/test/wdl/standard_library/keys_as_input.wdl +24 -0
  167. toil/test/wdl/standard_library/length.json +7 -0
  168. toil/test/wdl/standard_library/length_as_input.wdl +16 -0
  169. toil/test/wdl/standard_library/length_as_input_with_map.json +7 -0
  170. toil/test/wdl/standard_library/length_as_input_with_map.wdl +17 -0
  171. toil/test/wdl/standard_library/length_invalid.json +3 -0
  172. toil/test/wdl/standard_library/range.json +3 -0
  173. toil/test/wdl/standard_library/range_0.json +3 -0
  174. toil/test/wdl/standard_library/range_as_input.wdl +17 -0
  175. toil/test/wdl/standard_library/range_invalid.json +3 -0
  176. toil/test/wdl/standard_library/read_boolean.json +3 -0
  177. toil/test/wdl/standard_library/read_boolean_as_command.wdl +17 -0
  178. toil/test/wdl/standard_library/read_float.json +3 -0
  179. toil/test/wdl/standard_library/read_float_as_command.wdl +17 -0
  180. toil/test/wdl/standard_library/read_int.json +3 -0
  181. toil/test/wdl/standard_library/read_int_as_command.wdl +17 -0
  182. toil/test/wdl/standard_library/read_json.json +3 -0
  183. toil/test/wdl/standard_library/read_json_as_output.wdl +31 -0
  184. toil/test/wdl/standard_library/read_lines.json +3 -0
  185. toil/test/wdl/standard_library/read_lines_as_output.wdl +31 -0
  186. toil/test/wdl/standard_library/read_map.json +3 -0
  187. toil/test/wdl/standard_library/read_map_as_output.wdl +31 -0
  188. toil/test/wdl/standard_library/read_string.json +3 -0
  189. toil/test/wdl/standard_library/read_string_as_command.wdl +17 -0
  190. toil/test/wdl/standard_library/read_tsv.json +3 -0
  191. toil/test/wdl/standard_library/read_tsv_as_output.wdl +31 -0
  192. toil/test/wdl/standard_library/round.json +3 -0
  193. toil/test/wdl/standard_library/round_as_command.wdl +16 -0
  194. toil/test/wdl/standard_library/round_as_input.wdl +16 -0
  195. toil/test/wdl/standard_library/size.json +3 -0
  196. toil/test/wdl/standard_library/size_as_command.wdl +17 -0
  197. toil/test/wdl/standard_library/size_as_output.wdl +36 -0
  198. toil/test/wdl/standard_library/stderr.json +3 -0
  199. toil/test/wdl/standard_library/stderr_as_output.wdl +30 -0
  200. toil/test/wdl/standard_library/stdout.json +3 -0
  201. toil/test/wdl/standard_library/stdout_as_output.wdl +30 -0
  202. toil/test/wdl/standard_library/sub.json +3 -0
  203. toil/test/wdl/standard_library/sub_as_input.wdl +17 -0
  204. toil/test/wdl/standard_library/sub_as_input_with_file.wdl +17 -0
  205. toil/test/wdl/standard_library/transpose.json +6 -0
  206. toil/test/wdl/standard_library/transpose_as_input.wdl +18 -0
  207. toil/test/wdl/standard_library/write_json.json +6 -0
  208. toil/test/wdl/standard_library/write_json_as_command.wdl +17 -0
  209. toil/test/wdl/standard_library/write_lines.json +7 -0
  210. toil/test/wdl/standard_library/write_lines_as_command.wdl +17 -0
  211. toil/test/wdl/standard_library/write_map.json +6 -0
  212. toil/test/wdl/standard_library/write_map_as_command.wdl +17 -0
  213. toil/test/wdl/standard_library/write_tsv.json +6 -0
  214. toil/test/wdl/standard_library/write_tsv_as_command.wdl +17 -0
  215. toil/test/wdl/standard_library/zip.json +12 -0
  216. toil/test/wdl/standard_library/zip_as_input.wdl +19 -0
  217. toil/test/wdl/test.csv +3 -0
  218. toil/test/wdl/test.tsv +3 -0
  219. toil/test/wdl/testfiles/croo.wdl +38 -0
  220. toil/test/wdl/testfiles/drop_files.wdl +62 -0
  221. toil/test/wdl/testfiles/drop_files_subworkflow.wdl +13 -0
  222. toil/test/wdl/testfiles/empty.txt +0 -0
  223. toil/test/wdl/testfiles/not_enough_outputs.wdl +33 -0
  224. toil/test/wdl/testfiles/random.wdl +66 -0
  225. toil/test/wdl/testfiles/string_file_coercion.json +1 -0
  226. toil/test/wdl/testfiles/string_file_coercion.wdl +35 -0
  227. toil/test/wdl/testfiles/test.json +4 -0
  228. toil/test/wdl/testfiles/test_boolean.txt +1 -0
  229. toil/test/wdl/testfiles/test_float.txt +1 -0
  230. toil/test/wdl/testfiles/test_int.txt +1 -0
  231. toil/test/wdl/testfiles/test_lines.txt +5 -0
  232. toil/test/wdl/testfiles/test_map.txt +2 -0
  233. toil/test/wdl/testfiles/test_string.txt +1 -0
  234. toil/test/wdl/testfiles/url_to_file.wdl +13 -0
  235. toil/test/wdl/testfiles/url_to_optional_file.wdl +13 -0
  236. toil/test/wdl/testfiles/vocab.json +1 -0
  237. toil/test/wdl/testfiles/vocab.wdl +66 -0
  238. toil/test/wdl/testfiles/wait.wdl +34 -0
  239. toil/test/wdl/wdl_specification/type_pair.json +23 -0
  240. toil/test/wdl/wdl_specification/type_pair_basic.wdl +36 -0
  241. toil/test/wdl/wdl_specification/type_pair_with_files.wdl +36 -0
  242. toil/test/wdl/wdl_specification/v1_spec.json +1 -0
  243. toil/test/wdl/wdl_specification/v1_spec_declaration.wdl +39 -0
  244. toil/test/wdl/wdltoil_test.py +680 -407
  245. toil/test/wdl/wdltoil_test_kubernetes.py +2 -2
  246. toil/version.py +9 -9
  247. toil/wdl/wdltoil.py +336 -123
  248. {toil-8.1.0b1.dist-info → toil-8.2.0.dist-info}/METADATA +5 -4
  249. toil-8.2.0.dist-info/RECORD +439 -0
  250. {toil-8.1.0b1.dist-info → toil-8.2.0.dist-info}/WHEEL +1 -1
  251. toil-8.1.0b1.dist-info/RECORD +0 -259
  252. {toil-8.1.0b1.dist-info → toil-8.2.0.dist-info}/entry_points.txt +0 -0
  253. {toil-8.1.0b1.dist-info → toil-8.2.0.dist-info/licenses}/LICENSE +0 -0
  254. {toil-8.1.0b1.dist-info → toil-8.2.0.dist-info}/top_level.txt +0 -0
@@ -14,39 +14,42 @@
14
14
  import inspect
15
15
  import logging
16
16
  import os
17
+ import re
18
+ from pathlib import Path
17
19
  import random
18
20
  import sys
21
+ from types import FrameType
19
22
  from uuid import uuid4
23
+ from typing import cast, Union
20
24
 
21
25
  from toil.common import getNodeID
22
26
  from toil.lib.exceptions import panic, raise_
23
27
  from toil.lib.io import AtomicFileCreate, atomic_install, atomic_tmp_file, mkdtemp
24
- from toil.lib.misc import CalledProcessErrorStderr, call_command
25
- from toil.test import ToilTest, slow
28
+ from toil.lib.misc import CalledProcessErrorStderr, StrPath, call_command
29
+ from toil.test import pslow as slow
30
+
31
+ import pytest
26
32
 
27
33
  log = logging.getLogger(__name__)
28
34
  logging.basicConfig()
29
35
 
30
36
 
31
- class MiscTests(ToilTest):
37
+ class TestMisc:
32
38
  """
33
39
  This class contains miscellaneous tests that don't have enough content to be their own test
34
40
  file, and that don't logically fit in with any of the other test suites.
35
41
  """
36
42
 
37
- def setUp(self):
38
- super().setUp()
39
- self.testDir = self._createTempDir()
40
-
41
- def testIDStability(self):
43
+ def testIDStability(self) -> None:
42
44
  prevNodeID = None
43
45
  for i in range(10, 1):
44
46
  nodeID = getNodeID()
45
- self.assertEqual(nodeID, prevNodeID)
47
+ assert nodeID == prevNodeID
46
48
  prevNodeID = nodeID
47
49
 
48
50
  @slow
49
- def testGetSizeOfDirectoryWorks(self):
51
+ @pytest.mark.slow
52
+ def testGetSizeOfDirectoryWorks(self, tmp_path: Path) -> None:
50
53
  """A test to make sure toil.common.getDirSizeRecursively does not
51
54
  underestimate the amount of disk space needed.
52
55
 
@@ -58,9 +61,9 @@ class MiscTests(ToilTest):
58
61
  from toil.common import getDirSizeRecursively
59
62
 
60
63
  # a list of the directories used in the test
61
- directories = [self.testDir]
62
- # A dict of {FILENAME: FILESIZE} for all files used in the test
63
- files = {}
64
+ directories: list[StrPath] = [tmp_path]
65
+ # A dict of {FILENAME: FILESIZE or FILELINK} for all files used in the test
66
+ files: dict[Path, Union[int, str]] = {}
64
67
  # Create a random directory structure
65
68
  for i in range(0, 10):
66
69
  directories.append(mkdtemp(dir=random.choice(directories), prefix="test"))
@@ -68,11 +71,11 @@ class MiscTests(ToilTest):
68
71
  # these are fresh files of size [1, 10] MB and 25% of the time they are hard links to old
69
72
  # files.
70
73
  while len(files) <= 50:
71
- fileName = os.path.join(random.choice(directories), self._getRandomName())
74
+ fileName = Path(random.choice(directories)) / self._getRandomName()
72
75
  if random.randint(0, 100) < 75:
73
76
  # Create a fresh file in the range of 1-10 MB
74
77
  fileSize = int(round(random.random(), 2) * 10 * 1024 * 1024)
75
- with open(fileName, "wb") as fileHandle:
78
+ with fileName.open("wb") as fileHandle:
76
79
  fileHandle.write(os.urandom(fileSize))
77
80
  files[fileName] = fileSize
78
81
  else:
@@ -83,94 +86,96 @@ class MiscTests(ToilTest):
83
86
  os.link(linkSrc, fileName)
84
87
  files[fileName] = "Link to %s" % linkSrc
85
88
 
86
- computedDirectorySize = getDirSizeRecursively(self.testDir)
89
+ computedDirectorySize = getDirSizeRecursively(tmp_path)
87
90
  totalExpectedSize = sum(x for x in list(files.values()) if isinstance(x, int))
88
- self.assertGreaterEqual(computedDirectorySize, totalExpectedSize)
91
+ assert computedDirectorySize >= totalExpectedSize
89
92
 
90
93
  @staticmethod
91
- def _getRandomName():
92
- return uuid4().hex
94
+ def _getRandomName() -> str:
95
+ return str(uuid4().hex)
93
96
 
94
- def _get_test_out_file(self, tail):
95
- outf = os.path.join(self.testDir, self.id() + "." + tail)
96
- if os.path.exists(outf):
97
- os.unlink(outf)
97
+ def _get_test_out_file(self, testDir: Path, tail: str) -> Path:
98
+ outf = testDir / f"test.{tail}"
99
+ if outf.exists():
100
+ outf.unlink()
98
101
  return outf
99
102
 
100
- def _write_test_file(self, outf_tmp):
103
+ def _write_test_file(self, outf_tmp: str) -> None:
101
104
  with open(outf_tmp, "w") as fh:
102
- fh.write(self.id() + "\n")
105
+ fh.write(Path(outf_tmp).as_uri() + "\n")
103
106
 
104
- def test_atomic_install(self):
105
- outf = self._get_test_out_file(".foo.gz")
107
+ def test_atomic_install(self, tmp_path: Path) -> None:
108
+ outf = self._get_test_out_file(tmp_path, ".foo.gz")
106
109
  outf_tmp = atomic_tmp_file(outf)
107
110
  self._write_test_file(outf_tmp)
108
111
  atomic_install(outf_tmp, outf)
109
- self.assertTrue(os.path.exists(outf))
112
+ assert outf.exists()
110
113
 
111
- def test_atomic_install_dev(self):
114
+ def test_atomic_install_dev(self) -> None:
112
115
  devn = "/dev/null"
113
116
  tmp = atomic_tmp_file(devn)
114
- self.assertEqual(tmp, devn)
117
+ assert tmp == devn
115
118
  atomic_install(tmp, devn)
116
119
 
117
- def test_atomic_context_ok(self):
118
- outf = self._get_test_out_file(".tar")
120
+ def test_atomic_context_ok(self, tmp_path: Path) -> None:
121
+ outf = self._get_test_out_file(tmp_path, ".tar")
119
122
  with AtomicFileCreate(outf) as outf_tmp:
120
123
  self._write_test_file(outf_tmp)
121
- self.assertTrue(os.path.exists(outf))
124
+ assert outf.exists()
122
125
 
123
- def test_atomic_context_error(self):
124
- outf = self._get_test_out_file(".tar")
126
+ def test_atomic_context_error(self, tmp_path: Path) -> None:
127
+ outf = self._get_test_out_file(tmp_path, ".tar")
125
128
  try:
126
129
  with AtomicFileCreate(outf) as outf_tmp:
127
130
  self._write_test_file(outf_tmp)
128
131
  raise Exception("stop!")
129
132
  except Exception as ex:
130
- self.assertEqual(str(ex), "stop!")
131
- self.assertFalse(os.path.exists(outf))
133
+ assert str(ex) == "stop!"
134
+ assert not os.path.exists(outf)
132
135
 
133
- def test_call_command_ok(self):
136
+ def test_call_command_ok(self) -> None:
134
137
  o = call_command(["echo", "Fred"])
135
- self.assertEqual("Fred\n", o)
136
- self.assertTrue(isinstance(o, str), str(type(o)))
138
+ assert "Fred\n" == o
139
+ assert isinstance(o, str), str(type(o))
137
140
 
138
- def test_call_command_err(self):
139
- with self.assertRaisesRegex(
141
+ def test_call_command_err(self) -> None:
142
+ with pytest.raises(
140
143
  CalledProcessErrorStderr,
141
- "^Command '\\['cat', '/dev/Frankenheimer']' exit status 1: cat: /dev/Frankenheimer: No such file or directory\n$",
144
+ match=re.escape(
145
+ "Command '['cat', '/dev/Frankenheimer']' exit status 1: cat: /dev/Frankenheimer: No such file or directory"
146
+ ),
142
147
  ):
143
148
  call_command(["cat", "/dev/Frankenheimer"])
144
149
 
145
150
 
146
- class TestPanic(ToilTest):
147
- def test_panic_by_hand(self):
151
+ class TestPanic:
152
+ def test_panic_by_hand(self) -> None:
148
153
  try:
149
154
  self.try_and_panic_by_hand()
150
155
  except:
151
156
  self.__assert_raised_exception_is_primary()
152
157
 
153
- def test_panic(self):
158
+ def test_panic(self) -> None:
154
159
  try:
155
160
  self.try_and_panic()
156
161
  except:
157
162
  self.__assert_raised_exception_is_primary()
158
163
 
159
- def test_panic_with_secondary(self):
164
+ def test_panic_with_secondary(self) -> None:
160
165
  try:
161
166
  self.try_and_panic_with_secondary()
162
167
  except:
163
168
  self.__assert_raised_exception_is_primary()
164
169
 
165
- def test_nested_panic(self):
170
+ def test_nested_panic(self) -> None:
166
171
  try:
167
172
  self.try_and_nested_panic_with_secondary()
168
173
  except:
169
174
  self.__assert_raised_exception_is_primary()
170
175
 
171
- def try_and_panic_by_hand(self):
176
+ def try_and_panic_by_hand(self) -> None:
172
177
  try:
173
- self.line_of_primary_exc = inspect.currentframe().f_lineno + 1
178
+ self.line_of_primary_exc = (inspect.currentframe()).f_lineno + 1 # type: ignore[union-attr]
174
179
  raise ValueError("primary")
175
180
  except Exception:
176
181
  exc_type, exc_value, traceback = sys.exc_info()
@@ -180,35 +185,36 @@ class TestPanic(ToilTest):
180
185
  pass
181
186
  raise_(exc_type, exc_value, traceback)
182
187
 
183
- def try_and_panic(self):
188
+ def try_and_panic(self) -> None:
184
189
  try:
185
- self.line_of_primary_exc = inspect.currentframe().f_lineno + 1
190
+ self.line_of_primary_exc = (inspect.currentframe()).f_lineno + 1 # type: ignore[union-attr]
186
191
  raise ValueError("primary")
187
192
  except:
188
193
  with panic(log):
189
194
  pass
190
195
 
191
- def try_and_panic_with_secondary(self):
196
+ def try_and_panic_with_secondary(self) -> None:
192
197
  try:
193
- self.line_of_primary_exc = inspect.currentframe().f_lineno + 1
198
+ self.line_of_primary_exc = (inspect.currentframe()).f_lineno + 1 # type: ignore[union-attr]
194
199
  raise ValueError("primary")
195
200
  except:
196
201
  with panic(log):
197
202
  raise RuntimeError("secondary")
198
203
 
199
- def try_and_nested_panic_with_secondary(self):
204
+ def try_and_nested_panic_with_secondary(self) -> None:
200
205
  try:
201
- self.line_of_primary_exc = inspect.currentframe().f_lineno + 1
206
+ self.line_of_primary_exc = (inspect.currentframe()).f_lineno + 1 # type: ignore[union-attr]
202
207
  raise ValueError("primary")
203
208
  except:
204
209
  with panic(log):
205
210
  with panic(log):
206
211
  raise RuntimeError("secondary")
207
212
 
208
- def __assert_raised_exception_is_primary(self):
213
+ def __assert_raised_exception_is_primary(self) -> None:
209
214
  exc_type, exc_value, exc_traceback = sys.exc_info()
210
- self.assertEqual(exc_type, ValueError)
211
- self.assertEqual(str(exc_value), "primary")
215
+ assert exc_type == ValueError
216
+ assert str(exc_value) == "primary"
217
+ assert exc_traceback is not None
212
218
  while exc_traceback.tb_next is not None:
213
219
  exc_traceback = exc_traceback.tb_next
214
- self.assertEqual(exc_traceback.tb_lineno, self.line_of_primary_exc)
220
+ assert exc_traceback.tb_lineno == self.line_of_primary_exc
@@ -12,172 +12,162 @@
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
14
 
15
+ from collections.abc import Generator
16
+ import argparse
15
17
  import logging
16
18
  import os
19
+ from pathlib import Path
17
20
  import time
21
+ from typing import Any
18
22
 
19
- import toil.test.batchSystems.batchSystemTest as batchSystemTest
23
+ from toil.test.batchSystems import batchSystemTest
20
24
  from toil.batchSystems.mesos.test import MesosTestSupport
21
- from toil.job import Job, PromisedRequirement
25
+ from toil.fileStores import FileID
26
+ from toil.job import Job, PromisedRequirement, JobFunctionWrappingJob, Promise
22
27
  from toil.lib.retry import retry_flaky_test
23
- from toil.test import needs_mesos, slow
28
+ from toil.test import pneeds_mesos as needs_mesos, pslow as slow
24
29
 
25
- log = logging.getLogger(__name__)
26
-
27
-
28
- class hidden:
29
- """
30
- Hide abstract base class from unittest's test case loader.
31
-
32
- http://stackoverflow.com/questions/1323455/python-unit-test-with-base-and-sub-class#answer-25695512
33
- """
30
+ import pytest
34
31
 
35
- class AbstractPromisedRequirementsTest(
36
- batchSystemTest.hidden.AbstractBatchSystemJobTest
37
- ):
38
- """An abstract base class for testing Toil workflows with promised requirements."""
39
-
40
- @slow
41
- def testConcurrencyDynamic(self):
42
- """
43
- Asserts that promised core resources are allocated properly using a dynamic Toil workflow
44
- """
45
- for coresPerJob in self.allocatedCores:
46
- log.debug(
47
- "Testing %d cores per job with CPU count %d",
48
- coresPerJob,
49
- self.cpuCount,
50
- )
51
- tempDir = self._createTempDir("testFiles")
52
- counterPath = self.getCounterPath(tempDir)
53
-
54
- root = Job.wrapJobFn(
55
- maxConcurrency,
56
- self.cpuCount,
57
- counterPath,
58
- coresPerJob,
59
- cores=1,
60
- memory="1M",
61
- disk="1M",
62
- )
63
- values = Job.Runner.startToil(root, self.getOptions(tempDir))
64
- maxValue = max(values)
65
- self.assertLessEqual(maxValue, self.cpuCount // coresPerJob)
66
-
67
- @slow
68
- @retry_flaky_test(
69
- prepare=[
70
- batchSystemTest.hidden.AbstractBatchSystemJobTest.tearDown,
71
- batchSystemTest.hidden.AbstractBatchSystemJobTest.setUp,
72
- ]
73
- )
74
- def testConcurrencyStatic(self):
75
- """
76
- Asserts that promised core resources are allocated properly using a static DAG
77
- """
78
- for coresPerJob in self.allocatedCores:
79
- log.debug(
80
- "Testing %d cores per job with CPU count %d",
81
- coresPerJob,
82
- self.cpuCount,
83
- )
84
- tempDir = self._createTempDir("testFiles")
85
- counterPath = self.getCounterPath(tempDir)
32
+ log = logging.getLogger(__name__)
86
33
 
87
- root = Job()
88
- one = Job.wrapFn(getOne, cores=0.1, memory="32M", disk="1M")
89
- thirtyTwoMb = Job.wrapFn(
90
- getThirtyTwoMb, cores=0.1, memory="32M", disk="1M"
91
- )
92
- root.addChild(one)
93
- root.addChild(thirtyTwoMb)
94
- for _ in range(self.cpuCount):
95
- root.addFollowOn(
96
- Job.wrapFn(
97
- batchSystemTest.measureConcurrency,
98
- counterPath,
99
- cores=PromisedRequirement(
100
- lambda x: x * coresPerJob, one.rv()
101
- ),
102
- memory=PromisedRequirement(thirtyTwoMb.rv()),
103
- disk="1M",
104
- )
105
- )
106
- Job.Runner.startToil(root, self.getOptions(tempDir))
107
- _, maxValue = batchSystemTest.getCounters(counterPath)
108
- self.assertLessEqual(maxValue, self.cpuCount // coresPerJob)
109
-
110
- def getOptions(self, tempDir, caching=True):
111
- options = super().getOptions(tempDir)
112
- # defaultCores defaults to 1 - this is coincidentally the core requirement relied upon by this
113
- # test, so we change defaultCores to 2 to make the test more strict
114
- options.defaultCores = 2
115
- options.caching = caching
116
- return options
117
-
118
- def getCounterPath(self, tempDir):
119
- """
120
- Returns path to a counter file
121
- :param str tempDir: path to test directory
122
- :return: path to counter file
123
- """
124
- counterPath = os.path.join(tempDir, "counter")
125
- batchSystemTest.resetCounters(counterPath)
126
- minValue, maxValue = batchSystemTest.getCounters(counterPath)
127
- assert (minValue, maxValue) == (0, 0)
128
- return counterPath
129
-
130
- def testPromisesWithJobStoreFileObjects(self, caching=True):
131
- """
132
- Check whether FileID objects are being pickled properly when used as return
133
- values of functions. Then ensure that lambdas of promised FileID objects can be
134
- used to describe the requirements of a subsequent job. This type of operation will be
135
- used commonly in Toil scripts.
136
- :return: None
137
- """
138
- file1 = 1024
139
- file2 = 512
140
- F1 = Job.wrapJobFn(_writer, file1)
141
- F2 = Job.wrapJobFn(_writer, file2)
142
- G = Job.wrapJobFn(
143
- _follower,
144
- file1 + file2,
145
- disk=PromisedRequirement(
146
- lambda x, y: x.size + y.size, F1.rv(), F2.rv()
147
- ),
148
- )
149
- F1.addChild(F2)
150
- F2.addChild(G)
151
34
 
152
- Job.Runner.startToil(
153
- F1, self.getOptions(self._createTempDir("testFiles"), caching=caching)
35
+ class AbstractPromisedRequirementsTest(batchSystemTest.AbstractBatchSystemJobTest):
36
+ """An abstract base class for testing Toil workflows with promised requirements."""
37
+
38
+ @slow
39
+ @pytest.mark.slow
40
+ def testConcurrencyDynamic(self, tmp_path: Path) -> None:
41
+ """
42
+ Asserts that promised core resources are allocated properly using a dynamic Toil workflow
43
+ """
44
+ for coresPerJob in self.allocatedCores:
45
+ log.debug(
46
+ "Testing %d cores per job with CPU count %d",
47
+ coresPerJob,
48
+ self.cpuCount,
154
49
  )
155
-
156
- def testPromisesWithNonCachingFileStore(self):
157
- self.testPromisesWithJobStoreFileObjects(caching=False)
158
-
159
- @slow
160
- def testPromiseRequirementRaceStatic(self):
161
- """
162
- Checks for a race condition when using promised requirements and child job functions.
163
- """
164
- A = Job.wrapJobFn(
165
- logDiskUsage, "A", sleep=5, disk=PromisedRequirement(1024)
50
+ tempDir = tmp_path / f"testFiles_{coresPerJob}"
51
+ tempDir.mkdir()
52
+ counterPath = self.getCounterPath(tempDir)
53
+
54
+ root = Job.wrapJobFn(
55
+ maxConcurrency,
56
+ self.cpuCount,
57
+ counterPath,
58
+ coresPerJob,
59
+ cores=1,
60
+ memory="1M",
61
+ disk="1M",
166
62
  )
167
- B = Job.wrapJobFn(
168
- logDiskUsage, "B", disk=PromisedRequirement(lambda x: x + 1024, A.rv())
63
+ values = Job.Runner.startToil(root, self.getOptions(tempDir))
64
+ maxValue = max(values)
65
+ assert maxValue <= (self.cpuCount // coresPerJob)
66
+
67
+ @slow
68
+ @pytest.mark.slow
69
+ @retry_flaky_test(prepare=[])
70
+ def testConcurrencyStatic(self, tmp_path: Path) -> None:
71
+ """
72
+ Asserts that promised core resources are allocated properly using a static DAG
73
+ """
74
+ for coresPerJob in self.allocatedCores:
75
+ log.debug(
76
+ "Testing %d cores per job with CPU count %d",
77
+ coresPerJob,
78
+ self.cpuCount,
169
79
  )
170
- A.addChild(B)
171
- Job.Runner.startToil(A, self.getOptions(self._createTempDir("testFiles")))
80
+ tempDir = tmp_path / f"testFiles_{coresPerJob}"
81
+ tempDir.mkdir()
82
+ counterPath = self.getCounterPath(tempDir)
83
+
84
+ root = Job()
85
+ one = Job.wrapFn(getOne, cores=0.1, memory="32M", disk="1M")
86
+ thirtyTwoMb = Job.wrapFn(getThirtyTwoMb, cores=0.1, memory="32M", disk="1M")
87
+ root.addChild(one)
88
+ root.addChild(thirtyTwoMb)
89
+ for _ in range(self.cpuCount):
90
+ root.addFollowOn(
91
+ Job.wrapFn(
92
+ batchSystemTest.measureConcurrency,
93
+ counterPath,
94
+ cores=PromisedRequirement(lambda x: x * coresPerJob, one.rv()),
95
+ memory=PromisedRequirement(thirtyTwoMb.rv()),
96
+ disk="1M",
97
+ )
98
+ )
99
+ Job.Runner.startToil(root, self.getOptions(tempDir))
100
+ _, maxValue = batchSystemTest.getCounters(counterPath)
101
+ assert maxValue <= (self.cpuCount // coresPerJob)
102
+
103
+ def getOptions(self, tempDir: Path, caching: bool = True) -> argparse.Namespace:
104
+ options = super().getOptions(tempDir)
105
+ # defaultCores defaults to 1 - this is coincidentally the core requirement relied upon by this
106
+ # test, so we change defaultCores to 2 to make the test more strict
107
+ options.defaultCores = 2
108
+ options.caching = caching
109
+ return options
110
+
111
+ def getCounterPath(self, tempDir: Path) -> Path:
112
+ """
113
+ Returns path to a counter file
114
+ :param tempDir: path to test directory
115
+ :return: path to counter file
116
+ """
117
+ counterPath = tempDir / "counter"
118
+ batchSystemTest.resetCounters(counterPath)
119
+ minValue, maxValue = batchSystemTest.getCounters(counterPath)
120
+ assert (minValue, maxValue) == (0, 0)
121
+ return counterPath
122
+
123
+ def testPromisesWithJobStoreFileObjects(
124
+ self, tmp_path: Path, caching: bool = True
125
+ ) -> None:
126
+ """
127
+ Check whether FileID objects are being pickled properly when used as return
128
+ values of functions. Then ensure that lambdas of promised FileID objects can be
129
+ used to describe the requirements of a subsequent job. This type of operation will be
130
+ used commonly in Toil scripts.
131
+ :return: None
132
+ """
133
+ file1 = 1024
134
+ file2 = 512
135
+ F1 = Job.wrapJobFn(_writer, file1)
136
+ F2 = Job.wrapJobFn(_writer, file2)
137
+ G = Job.wrapJobFn(
138
+ _follower,
139
+ file1 + file2,
140
+ disk=PromisedRequirement(lambda x, y: x.size + y.size, F1.rv(), F2.rv()),
141
+ )
142
+ F1.addChild(F2)
143
+ F2.addChild(G)
144
+
145
+ Job.Runner.startToil(F1, self.getOptions(tmp_path, caching=caching))
146
+
147
+ def testPromisesWithNonCachingFileStore(self, tmp_path: Path) -> None:
148
+ self.testPromisesWithJobStoreFileObjects(tmp_path, caching=False)
149
+
150
+ @slow
151
+ @pytest.mark.slow
152
+ def testPromiseRequirementRaceStatic(self, tmp_path: Path) -> None:
153
+ """
154
+ Checks for a race condition when using promised requirements and child job functions.
155
+ """
156
+ A = Job.wrapJobFn(logDiskUsage, "A", sleep=5, disk=PromisedRequirement(1024))
157
+ B = Job.wrapJobFn(
158
+ logDiskUsage, "B", disk=PromisedRequirement(lambda x: x + 1024, A.rv())
159
+ )
160
+ A.addChild(B)
161
+ Job.Runner.startToil(A, self.getOptions(tmp_path))
172
162
 
173
163
 
174
- def _writer(job, fileSize):
164
+ def _writer(job: JobFunctionWrappingJob, fileSize: int) -> FileID:
175
165
  """
176
166
  Write a local file and return the FileID obtained from running writeGlobalFile on
177
167
  it.
178
168
 
179
- :param job job: job
180
- :param int fileSize: Size of the file in bytes
169
+ :param job: job
170
+ :param fileSize: Size of the file in bytes
181
171
  :returns: the result of writeGlobalFile on a locally created file
182
172
  :rtype: job.FileID
183
173
  """
@@ -186,26 +176,28 @@ def _writer(job, fileSize):
186
176
  return job.fileStore.writeGlobalFile(fH.name)
187
177
 
188
178
 
189
- def _follower(job, expectedDisk):
179
+ def _follower(job: Job, expectedDisk: Any) -> None:
190
180
  """
191
181
  This job follows the _writer jobs and ensures the expected disk is used.
192
182
 
193
- :param job job: job
183
+ :param job: job
194
184
  :param expectedDisk: Expect disk to be used by this job
195
185
  :return: None
196
186
  """
197
187
  assert job.disk == expectedDisk
198
188
 
199
189
 
200
- def maxConcurrency(job, cpuCount, filename, coresPerJob):
190
+ def maxConcurrency(
191
+ job: Job, cpuCount: int, filename: Path, coresPerJob: int
192
+ ) -> list[Promise]:
201
193
  """
202
194
  Returns the max number of concurrent tasks when using a PromisedRequirement instance
203
195
  to allocate the number of cores per job.
204
196
 
205
- :param int cpuCount: number of available cpus
206
- :param str filename: path to counter file
207
- :param int coresPerJob: number of cores assigned to each job
208
- :return int max concurrency value:
197
+ :param cpuCount: number of available cpus
198
+ :param filename: path to counter file
199
+ :param coresPerJob: number of cores assigned to each job
200
+ :return: max concurrency value per CPU
209
201
  """
210
202
  one = job.addChildFn(getOne, cores=0.1, memory="32M", disk="1M")
211
203
  thirtyTwoMb = job.addChildFn(getThirtyTwoMb, cores=0.1, memory="32M", disk="1M")
@@ -223,15 +215,15 @@ def maxConcurrency(job, cpuCount, filename, coresPerJob):
223
215
  return values
224
216
 
225
217
 
226
- def getOne():
218
+ def getOne() -> int:
227
219
  return 1
228
220
 
229
221
 
230
- def getThirtyTwoMb():
222
+ def getThirtyTwoMb() -> str:
231
223
  return "32M"
232
224
 
233
225
 
234
- def logDiskUsage(job, funcName, sleep=0):
226
+ def logDiskUsage(job: JobFunctionWrappingJob, funcName: str, sleep: int = 0) -> int:
235
227
  """
236
228
  Logs the job's disk usage to master and sleeps for specified amount of time.
237
229
 
@@ -243,34 +235,36 @@ def logDiskUsage(job, funcName, sleep=0):
243
235
  return diskUsage
244
236
 
245
237
 
246
- class SingleMachinePromisedRequirementsTest(hidden.AbstractPromisedRequirementsTest):
238
+ class TestSingleMachinePromisedRequirements(AbstractPromisedRequirementsTest):
247
239
  """
248
240
  Tests against the SingleMachine batch system
249
241
  """
250
242
 
251
- def getBatchSystemName(self):
243
+ def getBatchSystemName(self) -> str:
252
244
  return "single_machine"
253
245
 
254
- def tearDown(self):
246
+ def tearDown(self) -> None:
255
247
  pass
256
248
 
257
249
 
258
250
  @needs_mesos
259
- class MesosPromisedRequirementsTest(
260
- hidden.AbstractPromisedRequirementsTest, MesosTestSupport
261
- ):
251
+ class TestMesosPromisedRequirements(AbstractPromisedRequirementsTest, MesosTestSupport):
262
252
  """
263
253
  Tests against the Mesos batch system
264
254
  """
265
255
 
266
- def getOptions(self, tempDir, caching=True):
256
+ @pytest.fixture(autouse=True)
257
+ def mesos_support(self) -> Generator[None]:
258
+ try:
259
+ self._startMesos(self.cpuCount)
260
+ yield
261
+ finally:
262
+ self._stopMesos()
263
+
264
+ def getOptions(self, tempDir: Path, caching: bool = True) -> argparse.Namespace:
267
265
  options = super().getOptions(tempDir, caching=caching)
268
266
  options.mesos_endpoint = "localhost:5050"
269
267
  return options
270
268
 
271
- def getBatchSystemName(self):
272
- self._startMesos(self.cpuCount)
269
+ def getBatchSystemName(self) -> str:
273
270
  return "mesos"
274
-
275
- def tearDown(self):
276
- self._stopMesos()