toil 5.12.0__py3-none-any.whl → 6.1.0a1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (157) hide show
  1. toil/__init__.py +18 -13
  2. toil/batchSystems/abstractBatchSystem.py +21 -10
  3. toil/batchSystems/abstractGridEngineBatchSystem.py +2 -2
  4. toil/batchSystems/awsBatch.py +14 -14
  5. toil/batchSystems/contained_executor.py +3 -3
  6. toil/batchSystems/htcondor.py +0 -1
  7. toil/batchSystems/kubernetes.py +34 -31
  8. toil/batchSystems/local_support.py +3 -1
  9. toil/batchSystems/mesos/batchSystem.py +7 -7
  10. toil/batchSystems/options.py +32 -83
  11. toil/batchSystems/registry.py +104 -23
  12. toil/batchSystems/singleMachine.py +16 -13
  13. toil/batchSystems/slurm.py +3 -3
  14. toil/batchSystems/torque.py +0 -1
  15. toil/bus.py +6 -8
  16. toil/common.py +532 -743
  17. toil/cwl/__init__.py +28 -32
  18. toil/cwl/cwltoil.py +523 -520
  19. toil/cwl/utils.py +55 -10
  20. toil/fileStores/__init__.py +2 -2
  21. toil/fileStores/abstractFileStore.py +36 -11
  22. toil/fileStores/cachingFileStore.py +607 -530
  23. toil/fileStores/nonCachingFileStore.py +43 -10
  24. toil/job.py +140 -75
  25. toil/jobStores/abstractJobStore.py +147 -79
  26. toil/jobStores/aws/jobStore.py +23 -9
  27. toil/jobStores/aws/utils.py +1 -2
  28. toil/jobStores/fileJobStore.py +117 -19
  29. toil/jobStores/googleJobStore.py +16 -7
  30. toil/jobStores/utils.py +5 -6
  31. toil/leader.py +71 -43
  32. toil/lib/accelerators.py +10 -5
  33. toil/lib/aws/__init__.py +3 -14
  34. toil/lib/aws/ami.py +22 -9
  35. toil/lib/aws/iam.py +21 -13
  36. toil/lib/aws/session.py +2 -16
  37. toil/lib/aws/utils.py +4 -5
  38. toil/lib/compatibility.py +1 -1
  39. toil/lib/conversions.py +7 -3
  40. toil/lib/docker.py +22 -23
  41. toil/lib/ec2.py +10 -6
  42. toil/lib/ec2nodes.py +106 -100
  43. toil/lib/encryption/_nacl.py +2 -1
  44. toil/lib/generatedEC2Lists.py +325 -18
  45. toil/lib/io.py +21 -0
  46. toil/lib/misc.py +1 -1
  47. toil/lib/resources.py +1 -1
  48. toil/lib/threading.py +74 -26
  49. toil/options/common.py +738 -0
  50. toil/options/cwl.py +336 -0
  51. toil/options/wdl.py +32 -0
  52. toil/provisioners/abstractProvisioner.py +1 -4
  53. toil/provisioners/aws/__init__.py +3 -6
  54. toil/provisioners/aws/awsProvisioner.py +6 -0
  55. toil/provisioners/clusterScaler.py +3 -2
  56. toil/provisioners/gceProvisioner.py +2 -2
  57. toil/realtimeLogger.py +2 -1
  58. toil/resource.py +24 -18
  59. toil/server/app.py +2 -3
  60. toil/server/cli/wes_cwl_runner.py +4 -4
  61. toil/server/utils.py +1 -1
  62. toil/server/wes/abstract_backend.py +3 -2
  63. toil/server/wes/amazon_wes_utils.py +5 -4
  64. toil/server/wes/tasks.py +2 -3
  65. toil/server/wes/toil_backend.py +2 -10
  66. toil/server/wsgi_app.py +2 -0
  67. toil/serviceManager.py +12 -10
  68. toil/statsAndLogging.py +5 -1
  69. toil/test/__init__.py +29 -54
  70. toil/test/batchSystems/batchSystemTest.py +11 -111
  71. toil/test/batchSystems/test_slurm.py +3 -2
  72. toil/test/cwl/cwlTest.py +213 -90
  73. toil/test/cwl/glob_dir.cwl +15 -0
  74. toil/test/cwl/preemptible.cwl +21 -0
  75. toil/test/cwl/preemptible_expression.cwl +28 -0
  76. toil/test/cwl/revsort.cwl +1 -1
  77. toil/test/cwl/revsort2.cwl +1 -1
  78. toil/test/docs/scriptsTest.py +0 -1
  79. toil/test/jobStores/jobStoreTest.py +27 -16
  80. toil/test/lib/aws/test_iam.py +4 -14
  81. toil/test/lib/aws/test_utils.py +0 -3
  82. toil/test/lib/dockerTest.py +4 -4
  83. toil/test/lib/test_ec2.py +11 -16
  84. toil/test/mesos/helloWorld.py +4 -5
  85. toil/test/mesos/stress.py +1 -1
  86. toil/test/provisioners/aws/awsProvisionerTest.py +9 -5
  87. toil/test/provisioners/clusterScalerTest.py +6 -4
  88. toil/test/provisioners/clusterTest.py +14 -3
  89. toil/test/provisioners/gceProvisionerTest.py +0 -6
  90. toil/test/provisioners/restartScript.py +3 -2
  91. toil/test/server/serverTest.py +1 -1
  92. toil/test/sort/restart_sort.py +2 -1
  93. toil/test/sort/sort.py +2 -1
  94. toil/test/sort/sortTest.py +2 -13
  95. toil/test/src/autoDeploymentTest.py +45 -45
  96. toil/test/src/busTest.py +5 -5
  97. toil/test/src/checkpointTest.py +2 -2
  98. toil/test/src/deferredFunctionTest.py +1 -1
  99. toil/test/src/fileStoreTest.py +32 -16
  100. toil/test/src/helloWorldTest.py +1 -1
  101. toil/test/src/importExportFileTest.py +1 -1
  102. toil/test/src/jobDescriptionTest.py +2 -1
  103. toil/test/src/jobServiceTest.py +1 -1
  104. toil/test/src/jobTest.py +18 -18
  105. toil/test/src/miscTests.py +5 -3
  106. toil/test/src/promisedRequirementTest.py +3 -3
  107. toil/test/src/realtimeLoggerTest.py +1 -1
  108. toil/test/src/resourceTest.py +2 -2
  109. toil/test/src/restartDAGTest.py +1 -1
  110. toil/test/src/resumabilityTest.py +36 -2
  111. toil/test/src/retainTempDirTest.py +1 -1
  112. toil/test/src/systemTest.py +2 -2
  113. toil/test/src/toilContextManagerTest.py +2 -2
  114. toil/test/src/userDefinedJobArgTypeTest.py +1 -1
  115. toil/test/utils/toilDebugTest.py +98 -32
  116. toil/test/utils/toilKillTest.py +2 -2
  117. toil/test/utils/utilsTest.py +20 -0
  118. toil/test/wdl/wdltoil_test.py +148 -45
  119. toil/toilState.py +7 -6
  120. toil/utils/toilClean.py +1 -1
  121. toil/utils/toilConfig.py +36 -0
  122. toil/utils/toilDebugFile.py +60 -33
  123. toil/utils/toilDebugJob.py +39 -12
  124. toil/utils/toilDestroyCluster.py +1 -1
  125. toil/utils/toilKill.py +1 -1
  126. toil/utils/toilLaunchCluster.py +13 -2
  127. toil/utils/toilMain.py +3 -2
  128. toil/utils/toilRsyncCluster.py +1 -1
  129. toil/utils/toilSshCluster.py +1 -1
  130. toil/utils/toilStats.py +240 -143
  131. toil/utils/toilStatus.py +1 -4
  132. toil/version.py +11 -11
  133. toil/wdl/utils.py +2 -122
  134. toil/wdl/wdltoil.py +999 -386
  135. toil/worker.py +25 -31
  136. {toil-5.12.0.dist-info → toil-6.1.0a1.dist-info}/METADATA +60 -53
  137. toil-6.1.0a1.dist-info/RECORD +237 -0
  138. {toil-5.12.0.dist-info → toil-6.1.0a1.dist-info}/WHEEL +1 -1
  139. {toil-5.12.0.dist-info → toil-6.1.0a1.dist-info}/entry_points.txt +0 -1
  140. toil/batchSystems/parasol.py +0 -379
  141. toil/batchSystems/tes.py +0 -459
  142. toil/test/batchSystems/parasolTestSupport.py +0 -117
  143. toil/test/wdl/builtinTest.py +0 -506
  144. toil/test/wdl/conftest.py +0 -23
  145. toil/test/wdl/toilwdlTest.py +0 -522
  146. toil/wdl/toilwdl.py +0 -141
  147. toil/wdl/versions/dev.py +0 -107
  148. toil/wdl/versions/draft2.py +0 -980
  149. toil/wdl/versions/v1.py +0 -794
  150. toil/wdl/wdl_analysis.py +0 -116
  151. toil/wdl/wdl_functions.py +0 -997
  152. toil/wdl/wdl_synthesis.py +0 -1011
  153. toil/wdl/wdl_types.py +0 -243
  154. toil-5.12.0.dist-info/RECORD +0 -244
  155. /toil/{wdl/versions → options}/__init__.py +0 -0
  156. {toil-5.12.0.dist-info → toil-6.1.0a1.dist-info}/LICENSE +0 -0
  157. {toil-5.12.0.dist-info → toil-6.1.0a1.dist-info}/top_level.txt +0 -0
toil/utils/toilStatus.py CHANGED
@@ -12,12 +12,9 @@
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
14
  """Tool for reporting on job status."""
15
- from collections import defaultdict
16
15
  import logging
17
16
  import os
18
17
  import sys
19
- from functools import reduce
20
- import json
21
18
  from typing import Any, Dict, List, Optional, Set
22
19
 
23
20
  from toil.bus import replay_message_bus
@@ -320,7 +317,7 @@ class ToilStatus:
320
317
 
321
318
  def main() -> None:
322
319
  """Reports the state of a Toil workflow."""
323
- parser = parser_with_common_options()
320
+ parser = parser_with_common_options(prog="toil status")
324
321
  parser.add_argument("--failIfNotComplete", action="store_true",
325
322
  help="Return exit value of 1 if toil jobs not all completed. default=%(default)s",
326
323
  default=False)
toil/version.py CHANGED
@@ -1,14 +1,14 @@
1
- baseVersion = '5.12.0'
1
+ baseVersion = '6.1.0a1'
2
2
  cgcloudVersion = '1.6.0a1.dev393'
3
- version = '5.12.0-6d5a5b83b649cd8adf34a5cfe89e7690c95189d3'
4
- cacheTag = 'cache-local-py3.9'
5
- mainCacheTag = 'cache-master-py3.9'
6
- distVersion = '5.12.0'
7
- exactPython = 'python3.9'
8
- python = 'python3.9'
9
- dockerTag = '5.12.0-6d5a5b83b649cd8adf34a5cfe89e7690c95189d3-py3.9'
10
- currentCommit = '6d5a5b83b649cd8adf34a5cfe89e7690c95189d3'
3
+ version = '6.1.0a1-04b966f2417ebf2752e6f216e7fbda3ce20b4a37-dirty'
4
+ cacheTag = 'cache-local-py3.11'
5
+ mainCacheTag = 'cache-master-py3.11'
6
+ distVersion = '6.1.0a1'
7
+ exactPython = 'python3.11'
8
+ python = 'python3.11'
9
+ dockerTag = '6.1.0a1-04b966f2417ebf2752e6f216e7fbda3ce20b4a37-dirty-py3.11'
10
+ currentCommit = '04b966f2417ebf2752e6f216e7fbda3ce20b4a37'
11
11
  dockerRegistry = 'quay.io/ucsc_cgl'
12
12
  dockerName = 'toil'
13
- dirty = False
14
- cwltool_version = '3.1.20230601100705'
13
+ dirty = True
14
+ cwltool_version = '3.1.20240112164112'
toil/wdl/utils.py CHANGED
@@ -11,12 +11,10 @@
11
11
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
- import json
14
+ from typing import Iterable
15
15
 
16
- from toil.wdl.wdl_analysis import AnalyzeWDL
17
16
 
18
-
19
- def get_version(iterable) -> str:
17
+ def get_version(iterable: Iterable[str]) -> str:
20
18
  """
21
19
  Get the version of the WDL document.
22
20
 
@@ -35,121 +33,3 @@ def get_version(iterable) -> str:
35
33
  break
36
34
  # only draft-2 doesn't contain the version declaration
37
35
  return 'draft-2'
38
-
39
-
40
- def get_analyzer(wdl_file: str) -> AnalyzeWDL:
41
- """
42
- Creates an instance of an AnalyzeWDL implementation based on the version.
43
-
44
- :param wdl_file: The path to the WDL file.
45
- """
46
- with open(wdl_file) as f:
47
- version = get_version(f)
48
-
49
- if version == 'draft-2':
50
- from toil.wdl.versions.draft2 import AnalyzeDraft2WDL
51
- return AnalyzeDraft2WDL(wdl_file)
52
- elif version == '1.0':
53
- from toil.wdl.versions.v1 import AnalyzeV1WDL
54
- return AnalyzeV1WDL(wdl_file)
55
- elif version == 'development':
56
- from toil.wdl.versions.dev import AnalyzeDevelopmentWDL
57
- return AnalyzeDevelopmentWDL(wdl_file)
58
- else:
59
- raise RuntimeError(f"Unsupported WDL version: '{version}'.")
60
-
61
-
62
- def dict_from_JSON(JSON_file: str) -> dict:
63
- """
64
- Takes a WDL-mapped json file and creates a dict containing the bindings.
65
-
66
- :param JSON_file: A required JSON file containing WDL variable bindings.
67
- """
68
- json_dict = {}
69
-
70
- # TODO: Add context support for variables within multiple wdl files
71
-
72
- with open(JSON_file) as data_file:
73
- data = json.load(data_file)
74
- for d in data:
75
- if isinstance(data[d], str):
76
- json_dict[d] = f'"{data[d]}"'
77
- else:
78
- json_dict[d] = data[d]
79
- return json_dict
80
-
81
-
82
- def write_mappings(parser: AnalyzeWDL, filename: str = 'mappings.out') -> None:
83
- """
84
- Takes an AnalyzeWDL instance and writes the final task dict and workflow
85
- dict to the given file.
86
-
87
- :param parser: An AnalyzeWDL instance.
88
- :param filename: The name of a file to write to.
89
- """
90
- from collections import OrderedDict
91
-
92
- class Formatter:
93
- def __init__(self):
94
- self.types = {}
95
- self.htchar = '\t'
96
- self.lfchar = '\n'
97
- self.indent = 0
98
- self.set_formater(object, self.__class__.format_object)
99
- self.set_formater(dict, self.__class__.format_dict)
100
- self.set_formater(list, self.__class__.format_list)
101
- self.set_formater(tuple, self.__class__.format_tuple)
102
-
103
- def set_formater(self, obj, callback):
104
- self.types[obj] = callback
105
-
106
- def __call__(self, value, **args):
107
- for key in args:
108
- setattr(self, key, args[key])
109
- formater = self.types[type(value) if type(value) in self.types else object]
110
- return formater(self, value, self.indent)
111
-
112
- def format_object(self, value, indent):
113
- return repr(value)
114
-
115
- def format_dict(self, value, indent):
116
- items = [
117
- self.lfchar + self.htchar * (indent + 1) + repr(key) + ': ' +
118
- (self.types[type(value[key]) if type(value[key]) in self.types else object])(self, value[key],
119
- indent + 1)
120
- for key in value]
121
- return '{%s}' % (','.join(items) + self.lfchar + self.htchar * indent)
122
-
123
- def format_list(self, value, indent):
124
- items = [
125
- self.lfchar + self.htchar * (indent + 1) + (
126
- self.types[type(item) if type(item) in self.types else object])(self, item, indent + 1)
127
- for item in value]
128
- return '[%s]' % (','.join(items) + self.lfchar + self.htchar * indent)
129
-
130
- def format_tuple(self, value, indent):
131
- items = [
132
- self.lfchar + self.htchar * (indent + 1) + (
133
- self.types[type(item) if type(item) in self.types else object])(self, item, indent + 1)
134
- for item in value]
135
- return '(%s)' % (','.join(items) + self.lfchar + self.htchar * indent)
136
-
137
- pretty = Formatter()
138
-
139
- def format_ordereddict(self, value, indent):
140
- items = [
141
- self.lfchar + self.htchar * (indent + 1) +
142
- "(" + repr(key) + ', ' + (self.types[
143
- type(value[key]) if type(value[key]) in self.types else object
144
- ])(self, value[key], indent + 1) + ")"
145
- for key in value
146
- ]
147
- return 'OrderedDict([%s])' % (','.join(items) +
148
- self.lfchar + self.htchar * indent)
149
-
150
- pretty.set_formater(OrderedDict, format_ordereddict)
151
-
152
- with open(filename, 'w') as f:
153
- f.write(pretty(parser.tasks_dictionary))
154
- f.write('\n\n\n\n\n\n')
155
- f.write(pretty(parser.workflows_dictionary))