toil 5.12.0__py3-none-any.whl → 6.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (164) hide show
  1. toil/__init__.py +18 -13
  2. toil/batchSystems/abstractBatchSystem.py +39 -13
  3. toil/batchSystems/abstractGridEngineBatchSystem.py +24 -24
  4. toil/batchSystems/awsBatch.py +14 -14
  5. toil/batchSystems/cleanup_support.py +7 -3
  6. toil/batchSystems/contained_executor.py +3 -3
  7. toil/batchSystems/htcondor.py +0 -1
  8. toil/batchSystems/kubernetes.py +34 -31
  9. toil/batchSystems/local_support.py +3 -1
  10. toil/batchSystems/lsf.py +7 -7
  11. toil/batchSystems/mesos/batchSystem.py +7 -7
  12. toil/batchSystems/options.py +32 -83
  13. toil/batchSystems/registry.py +104 -23
  14. toil/batchSystems/singleMachine.py +16 -13
  15. toil/batchSystems/slurm.py +87 -16
  16. toil/batchSystems/torque.py +0 -1
  17. toil/bus.py +44 -8
  18. toil/common.py +544 -753
  19. toil/cwl/__init__.py +28 -32
  20. toil/cwl/cwltoil.py +595 -574
  21. toil/cwl/utils.py +55 -10
  22. toil/exceptions.py +1 -1
  23. toil/fileStores/__init__.py +2 -2
  24. toil/fileStores/abstractFileStore.py +88 -14
  25. toil/fileStores/cachingFileStore.py +610 -549
  26. toil/fileStores/nonCachingFileStore.py +46 -22
  27. toil/job.py +182 -101
  28. toil/jobStores/abstractJobStore.py +161 -95
  29. toil/jobStores/aws/jobStore.py +23 -9
  30. toil/jobStores/aws/utils.py +6 -6
  31. toil/jobStores/fileJobStore.py +116 -18
  32. toil/jobStores/googleJobStore.py +16 -7
  33. toil/jobStores/utils.py +5 -6
  34. toil/leader.py +87 -56
  35. toil/lib/accelerators.py +10 -5
  36. toil/lib/aws/__init__.py +3 -14
  37. toil/lib/aws/ami.py +22 -9
  38. toil/lib/aws/iam.py +21 -13
  39. toil/lib/aws/session.py +2 -16
  40. toil/lib/aws/utils.py +4 -5
  41. toil/lib/compatibility.py +1 -1
  42. toil/lib/conversions.py +26 -3
  43. toil/lib/docker.py +22 -23
  44. toil/lib/ec2.py +10 -6
  45. toil/lib/ec2nodes.py +106 -100
  46. toil/lib/encryption/_nacl.py +2 -1
  47. toil/lib/generatedEC2Lists.py +325 -18
  48. toil/lib/io.py +49 -2
  49. toil/lib/misc.py +1 -1
  50. toil/lib/resources.py +9 -2
  51. toil/lib/threading.py +101 -38
  52. toil/options/common.py +736 -0
  53. toil/options/cwl.py +336 -0
  54. toil/options/wdl.py +37 -0
  55. toil/provisioners/abstractProvisioner.py +9 -4
  56. toil/provisioners/aws/__init__.py +3 -6
  57. toil/provisioners/aws/awsProvisioner.py +6 -0
  58. toil/provisioners/clusterScaler.py +3 -2
  59. toil/provisioners/gceProvisioner.py +2 -2
  60. toil/realtimeLogger.py +2 -1
  61. toil/resource.py +24 -18
  62. toil/server/app.py +2 -3
  63. toil/server/cli/wes_cwl_runner.py +4 -4
  64. toil/server/utils.py +1 -1
  65. toil/server/wes/abstract_backend.py +3 -2
  66. toil/server/wes/amazon_wes_utils.py +5 -4
  67. toil/server/wes/tasks.py +2 -3
  68. toil/server/wes/toil_backend.py +2 -10
  69. toil/server/wsgi_app.py +2 -0
  70. toil/serviceManager.py +12 -10
  71. toil/statsAndLogging.py +41 -9
  72. toil/test/__init__.py +29 -54
  73. toil/test/batchSystems/batchSystemTest.py +11 -111
  74. toil/test/batchSystems/test_slurm.py +24 -8
  75. toil/test/cactus/__init__.py +0 -0
  76. toil/test/cactus/test_cactus_integration.py +58 -0
  77. toil/test/cwl/cwlTest.py +438 -223
  78. toil/test/cwl/glob_dir.cwl +15 -0
  79. toil/test/cwl/preemptible.cwl +21 -0
  80. toil/test/cwl/preemptible_expression.cwl +28 -0
  81. toil/test/cwl/revsort.cwl +1 -1
  82. toil/test/cwl/revsort2.cwl +1 -1
  83. toil/test/docs/scriptsTest.py +2 -3
  84. toil/test/jobStores/jobStoreTest.py +34 -21
  85. toil/test/lib/aws/test_iam.py +4 -14
  86. toil/test/lib/aws/test_utils.py +0 -3
  87. toil/test/lib/dockerTest.py +4 -4
  88. toil/test/lib/test_ec2.py +12 -17
  89. toil/test/mesos/helloWorld.py +4 -5
  90. toil/test/mesos/stress.py +1 -1
  91. toil/test/{wdl/conftest.py → options/__init__.py} +0 -10
  92. toil/test/options/options.py +37 -0
  93. toil/test/provisioners/aws/awsProvisionerTest.py +9 -5
  94. toil/test/provisioners/clusterScalerTest.py +6 -4
  95. toil/test/provisioners/clusterTest.py +23 -11
  96. toil/test/provisioners/gceProvisionerTest.py +0 -6
  97. toil/test/provisioners/restartScript.py +3 -2
  98. toil/test/server/serverTest.py +1 -1
  99. toil/test/sort/restart_sort.py +2 -1
  100. toil/test/sort/sort.py +2 -1
  101. toil/test/sort/sortTest.py +2 -13
  102. toil/test/src/autoDeploymentTest.py +45 -45
  103. toil/test/src/busTest.py +5 -5
  104. toil/test/src/checkpointTest.py +2 -2
  105. toil/test/src/deferredFunctionTest.py +1 -1
  106. toil/test/src/fileStoreTest.py +32 -16
  107. toil/test/src/helloWorldTest.py +1 -1
  108. toil/test/src/importExportFileTest.py +1 -1
  109. toil/test/src/jobDescriptionTest.py +2 -1
  110. toil/test/src/jobServiceTest.py +1 -1
  111. toil/test/src/jobTest.py +18 -18
  112. toil/test/src/miscTests.py +5 -3
  113. toil/test/src/promisedRequirementTest.py +3 -3
  114. toil/test/src/realtimeLoggerTest.py +1 -1
  115. toil/test/src/resourceTest.py +2 -2
  116. toil/test/src/restartDAGTest.py +1 -1
  117. toil/test/src/resumabilityTest.py +36 -2
  118. toil/test/src/retainTempDirTest.py +1 -1
  119. toil/test/src/systemTest.py +2 -2
  120. toil/test/src/toilContextManagerTest.py +2 -2
  121. toil/test/src/userDefinedJobArgTypeTest.py +1 -1
  122. toil/test/utils/toilDebugTest.py +98 -32
  123. toil/test/utils/toilKillTest.py +2 -2
  124. toil/test/utils/utilsTest.py +23 -3
  125. toil/test/wdl/wdltoil_test.py +223 -45
  126. toil/toilState.py +7 -6
  127. toil/utils/toilClean.py +1 -1
  128. toil/utils/toilConfig.py +36 -0
  129. toil/utils/toilDebugFile.py +60 -33
  130. toil/utils/toilDebugJob.py +39 -12
  131. toil/utils/toilDestroyCluster.py +1 -1
  132. toil/utils/toilKill.py +1 -1
  133. toil/utils/toilLaunchCluster.py +13 -2
  134. toil/utils/toilMain.py +3 -2
  135. toil/utils/toilRsyncCluster.py +1 -1
  136. toil/utils/toilSshCluster.py +1 -1
  137. toil/utils/toilStats.py +445 -305
  138. toil/utils/toilStatus.py +2 -5
  139. toil/version.py +10 -10
  140. toil/wdl/utils.py +2 -122
  141. toil/wdl/wdltoil.py +1257 -492
  142. toil/worker.py +55 -46
  143. toil-6.1.0.dist-info/METADATA +124 -0
  144. toil-6.1.0.dist-info/RECORD +241 -0
  145. {toil-5.12.0.dist-info → toil-6.1.0.dist-info}/WHEEL +1 -1
  146. {toil-5.12.0.dist-info → toil-6.1.0.dist-info}/entry_points.txt +0 -1
  147. toil/batchSystems/parasol.py +0 -379
  148. toil/batchSystems/tes.py +0 -459
  149. toil/test/batchSystems/parasolTestSupport.py +0 -117
  150. toil/test/wdl/builtinTest.py +0 -506
  151. toil/test/wdl/toilwdlTest.py +0 -522
  152. toil/wdl/toilwdl.py +0 -141
  153. toil/wdl/versions/dev.py +0 -107
  154. toil/wdl/versions/draft2.py +0 -980
  155. toil/wdl/versions/v1.py +0 -794
  156. toil/wdl/wdl_analysis.py +0 -116
  157. toil/wdl/wdl_functions.py +0 -997
  158. toil/wdl/wdl_synthesis.py +0 -1011
  159. toil/wdl/wdl_types.py +0 -243
  160. toil-5.12.0.dist-info/METADATA +0 -118
  161. toil-5.12.0.dist-info/RECORD +0 -244
  162. /toil/{wdl/versions → options}/__init__.py +0 -0
  163. {toil-5.12.0.dist-info → toil-6.1.0.dist-info}/LICENSE +0 -0
  164. {toil-5.12.0.dist-info → toil-6.1.0.dist-info}/top_level.txt +0 -0
toil/utils/toilStatus.py CHANGED
@@ -12,12 +12,9 @@
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
14
  """Tool for reporting on job status."""
15
- from collections import defaultdict
16
15
  import logging
17
16
  import os
18
17
  import sys
19
- from functools import reduce
20
- import json
21
18
  from typing import Any, Dict, List, Optional, Set
22
19
 
23
20
  from toil.bus import replay_message_bus
@@ -85,7 +82,7 @@ class ToilStatus:
85
82
  with job.getLogFileHandle(self.jobStore) as fH:
86
83
  # TODO: This looks intended to be machine-readable, but the format is
87
84
  # unspecified and no escaping is done. But keep these tags around.
88
- print(StatsAndLogging.formatLogStream(fH, job_name=f"LOG_FILE_OF_JOB:{job} LOG:"))
85
+ print(StatsAndLogging.formatLogStream(fH, stream_name=f"LOG_FILE_OF_JOB:{job} LOG:"))
89
86
  else:
90
87
  print(f"LOG_FILE_OF_JOB: {job} LOG: Job has no log file")
91
88
 
@@ -320,7 +317,7 @@ class ToilStatus:
320
317
 
321
318
  def main() -> None:
322
319
  """Reports the state of a Toil workflow."""
323
- parser = parser_with_common_options()
320
+ parser = parser_with_common_options(prog="toil status")
324
321
  parser.add_argument("--failIfNotComplete", action="store_true",
325
322
  help="Return exit value of 1 if toil jobs not all completed. default=%(default)s",
326
323
  default=False)
toil/version.py CHANGED
@@ -1,14 +1,14 @@
1
- baseVersion = '5.12.0'
1
+ baseVersion = '6.1.0'
2
2
  cgcloudVersion = '1.6.0a1.dev393'
3
- version = '5.12.0-6d5a5b83b649cd8adf34a5cfe89e7690c95189d3'
4
- cacheTag = 'cache-local-py3.9'
5
- mainCacheTag = 'cache-master-py3.9'
6
- distVersion = '5.12.0'
7
- exactPython = 'python3.9'
8
- python = 'python3.9'
9
- dockerTag = '5.12.0-6d5a5b83b649cd8adf34a5cfe89e7690c95189d3-py3.9'
10
- currentCommit = '6d5a5b83b649cd8adf34a5cfe89e7690c95189d3'
3
+ version = '6.1.0-3f9cba3766e52866ea80d0934498f8c8f3129c3f'
4
+ cacheTag = 'cache-local-py3.10'
5
+ mainCacheTag = 'cache-master-py3.10'
6
+ distVersion = '6.1.0'
7
+ exactPython = 'python3.10'
8
+ python = 'python3.10'
9
+ dockerTag = '6.1.0-3f9cba3766e52866ea80d0934498f8c8f3129c3f-py3.10'
10
+ currentCommit = '3f9cba3766e52866ea80d0934498f8c8f3129c3f'
11
11
  dockerRegistry = 'quay.io/ucsc_cgl'
12
12
  dockerName = 'toil'
13
13
  dirty = False
14
- cwltool_version = '3.1.20230601100705'
14
+ cwltool_version = '3.1.20240112164112'
toil/wdl/utils.py CHANGED
@@ -11,12 +11,10 @@
11
11
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
- import json
14
+ from typing import Iterable
15
15
 
16
- from toil.wdl.wdl_analysis import AnalyzeWDL
17
16
 
18
-
19
- def get_version(iterable) -> str:
17
+ def get_version(iterable: Iterable[str]) -> str:
20
18
  """
21
19
  Get the version of the WDL document.
22
20
 
@@ -35,121 +33,3 @@ def get_version(iterable) -> str:
35
33
  break
36
34
  # only draft-2 doesn't contain the version declaration
37
35
  return 'draft-2'
38
-
39
-
40
- def get_analyzer(wdl_file: str) -> AnalyzeWDL:
41
- """
42
- Creates an instance of an AnalyzeWDL implementation based on the version.
43
-
44
- :param wdl_file: The path to the WDL file.
45
- """
46
- with open(wdl_file) as f:
47
- version = get_version(f)
48
-
49
- if version == 'draft-2':
50
- from toil.wdl.versions.draft2 import AnalyzeDraft2WDL
51
- return AnalyzeDraft2WDL(wdl_file)
52
- elif version == '1.0':
53
- from toil.wdl.versions.v1 import AnalyzeV1WDL
54
- return AnalyzeV1WDL(wdl_file)
55
- elif version == 'development':
56
- from toil.wdl.versions.dev import AnalyzeDevelopmentWDL
57
- return AnalyzeDevelopmentWDL(wdl_file)
58
- else:
59
- raise RuntimeError(f"Unsupported WDL version: '{version}'.")
60
-
61
-
62
- def dict_from_JSON(JSON_file: str) -> dict:
63
- """
64
- Takes a WDL-mapped json file and creates a dict containing the bindings.
65
-
66
- :param JSON_file: A required JSON file containing WDL variable bindings.
67
- """
68
- json_dict = {}
69
-
70
- # TODO: Add context support for variables within multiple wdl files
71
-
72
- with open(JSON_file) as data_file:
73
- data = json.load(data_file)
74
- for d in data:
75
- if isinstance(data[d], str):
76
- json_dict[d] = f'"{data[d]}"'
77
- else:
78
- json_dict[d] = data[d]
79
- return json_dict
80
-
81
-
82
- def write_mappings(parser: AnalyzeWDL, filename: str = 'mappings.out') -> None:
83
- """
84
- Takes an AnalyzeWDL instance and writes the final task dict and workflow
85
- dict to the given file.
86
-
87
- :param parser: An AnalyzeWDL instance.
88
- :param filename: The name of a file to write to.
89
- """
90
- from collections import OrderedDict
91
-
92
- class Formatter:
93
- def __init__(self):
94
- self.types = {}
95
- self.htchar = '\t'
96
- self.lfchar = '\n'
97
- self.indent = 0
98
- self.set_formater(object, self.__class__.format_object)
99
- self.set_formater(dict, self.__class__.format_dict)
100
- self.set_formater(list, self.__class__.format_list)
101
- self.set_formater(tuple, self.__class__.format_tuple)
102
-
103
- def set_formater(self, obj, callback):
104
- self.types[obj] = callback
105
-
106
- def __call__(self, value, **args):
107
- for key in args:
108
- setattr(self, key, args[key])
109
- formater = self.types[type(value) if type(value) in self.types else object]
110
- return formater(self, value, self.indent)
111
-
112
- def format_object(self, value, indent):
113
- return repr(value)
114
-
115
- def format_dict(self, value, indent):
116
- items = [
117
- self.lfchar + self.htchar * (indent + 1) + repr(key) + ': ' +
118
- (self.types[type(value[key]) if type(value[key]) in self.types else object])(self, value[key],
119
- indent + 1)
120
- for key in value]
121
- return '{%s}' % (','.join(items) + self.lfchar + self.htchar * indent)
122
-
123
- def format_list(self, value, indent):
124
- items = [
125
- self.lfchar + self.htchar * (indent + 1) + (
126
- self.types[type(item) if type(item) in self.types else object])(self, item, indent + 1)
127
- for item in value]
128
- return '[%s]' % (','.join(items) + self.lfchar + self.htchar * indent)
129
-
130
- def format_tuple(self, value, indent):
131
- items = [
132
- self.lfchar + self.htchar * (indent + 1) + (
133
- self.types[type(item) if type(item) in self.types else object])(self, item, indent + 1)
134
- for item in value]
135
- return '(%s)' % (','.join(items) + self.lfchar + self.htchar * indent)
136
-
137
- pretty = Formatter()
138
-
139
- def format_ordereddict(self, value, indent):
140
- items = [
141
- self.lfchar + self.htchar * (indent + 1) +
142
- "(" + repr(key) + ', ' + (self.types[
143
- type(value[key]) if type(value[key]) in self.types else object
144
- ])(self, value[key], indent + 1) + ")"
145
- for key in value
146
- ]
147
- return 'OrderedDict([%s])' % (','.join(items) +
148
- self.lfchar + self.htchar * indent)
149
-
150
- pretty.set_formater(OrderedDict, format_ordereddict)
151
-
152
- with open(filename, 'w') as f:
153
- f.write(pretty(parser.tasks_dictionary))
154
- f.write('\n\n\n\n\n\n')
155
- f.write(pretty(parser.workflows_dictionary))