toil 5.12.0__py3-none-any.whl → 6.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (164) hide show
  1. toil/__init__.py +18 -13
  2. toil/batchSystems/abstractBatchSystem.py +39 -13
  3. toil/batchSystems/abstractGridEngineBatchSystem.py +24 -24
  4. toil/batchSystems/awsBatch.py +14 -14
  5. toil/batchSystems/cleanup_support.py +7 -3
  6. toil/batchSystems/contained_executor.py +3 -3
  7. toil/batchSystems/htcondor.py +0 -1
  8. toil/batchSystems/kubernetes.py +34 -31
  9. toil/batchSystems/local_support.py +3 -1
  10. toil/batchSystems/lsf.py +7 -7
  11. toil/batchSystems/mesos/batchSystem.py +7 -7
  12. toil/batchSystems/options.py +32 -83
  13. toil/batchSystems/registry.py +104 -23
  14. toil/batchSystems/singleMachine.py +16 -13
  15. toil/batchSystems/slurm.py +87 -16
  16. toil/batchSystems/torque.py +0 -1
  17. toil/bus.py +44 -8
  18. toil/common.py +544 -753
  19. toil/cwl/__init__.py +28 -32
  20. toil/cwl/cwltoil.py +595 -574
  21. toil/cwl/utils.py +55 -10
  22. toil/exceptions.py +1 -1
  23. toil/fileStores/__init__.py +2 -2
  24. toil/fileStores/abstractFileStore.py +88 -14
  25. toil/fileStores/cachingFileStore.py +610 -549
  26. toil/fileStores/nonCachingFileStore.py +46 -22
  27. toil/job.py +182 -101
  28. toil/jobStores/abstractJobStore.py +161 -95
  29. toil/jobStores/aws/jobStore.py +23 -9
  30. toil/jobStores/aws/utils.py +6 -6
  31. toil/jobStores/fileJobStore.py +116 -18
  32. toil/jobStores/googleJobStore.py +16 -7
  33. toil/jobStores/utils.py +5 -6
  34. toil/leader.py +87 -56
  35. toil/lib/accelerators.py +10 -5
  36. toil/lib/aws/__init__.py +3 -14
  37. toil/lib/aws/ami.py +22 -9
  38. toil/lib/aws/iam.py +21 -13
  39. toil/lib/aws/session.py +2 -16
  40. toil/lib/aws/utils.py +4 -5
  41. toil/lib/compatibility.py +1 -1
  42. toil/lib/conversions.py +26 -3
  43. toil/lib/docker.py +22 -23
  44. toil/lib/ec2.py +10 -6
  45. toil/lib/ec2nodes.py +106 -100
  46. toil/lib/encryption/_nacl.py +2 -1
  47. toil/lib/generatedEC2Lists.py +325 -18
  48. toil/lib/io.py +49 -2
  49. toil/lib/misc.py +1 -1
  50. toil/lib/resources.py +9 -2
  51. toil/lib/threading.py +101 -38
  52. toil/options/common.py +736 -0
  53. toil/options/cwl.py +336 -0
  54. toil/options/wdl.py +37 -0
  55. toil/provisioners/abstractProvisioner.py +9 -4
  56. toil/provisioners/aws/__init__.py +3 -6
  57. toil/provisioners/aws/awsProvisioner.py +6 -0
  58. toil/provisioners/clusterScaler.py +3 -2
  59. toil/provisioners/gceProvisioner.py +2 -2
  60. toil/realtimeLogger.py +2 -1
  61. toil/resource.py +24 -18
  62. toil/server/app.py +2 -3
  63. toil/server/cli/wes_cwl_runner.py +4 -4
  64. toil/server/utils.py +1 -1
  65. toil/server/wes/abstract_backend.py +3 -2
  66. toil/server/wes/amazon_wes_utils.py +5 -4
  67. toil/server/wes/tasks.py +2 -3
  68. toil/server/wes/toil_backend.py +2 -10
  69. toil/server/wsgi_app.py +2 -0
  70. toil/serviceManager.py +12 -10
  71. toil/statsAndLogging.py +41 -9
  72. toil/test/__init__.py +29 -54
  73. toil/test/batchSystems/batchSystemTest.py +11 -111
  74. toil/test/batchSystems/test_slurm.py +24 -8
  75. toil/test/cactus/__init__.py +0 -0
  76. toil/test/cactus/test_cactus_integration.py +58 -0
  77. toil/test/cwl/cwlTest.py +438 -223
  78. toil/test/cwl/glob_dir.cwl +15 -0
  79. toil/test/cwl/preemptible.cwl +21 -0
  80. toil/test/cwl/preemptible_expression.cwl +28 -0
  81. toil/test/cwl/revsort.cwl +1 -1
  82. toil/test/cwl/revsort2.cwl +1 -1
  83. toil/test/docs/scriptsTest.py +2 -3
  84. toil/test/jobStores/jobStoreTest.py +34 -21
  85. toil/test/lib/aws/test_iam.py +4 -14
  86. toil/test/lib/aws/test_utils.py +0 -3
  87. toil/test/lib/dockerTest.py +4 -4
  88. toil/test/lib/test_ec2.py +12 -17
  89. toil/test/mesos/helloWorld.py +4 -5
  90. toil/test/mesos/stress.py +1 -1
  91. toil/test/{wdl/conftest.py → options/__init__.py} +0 -10
  92. toil/test/options/options.py +37 -0
  93. toil/test/provisioners/aws/awsProvisionerTest.py +9 -5
  94. toil/test/provisioners/clusterScalerTest.py +6 -4
  95. toil/test/provisioners/clusterTest.py +23 -11
  96. toil/test/provisioners/gceProvisionerTest.py +0 -6
  97. toil/test/provisioners/restartScript.py +3 -2
  98. toil/test/server/serverTest.py +1 -1
  99. toil/test/sort/restart_sort.py +2 -1
  100. toil/test/sort/sort.py +2 -1
  101. toil/test/sort/sortTest.py +2 -13
  102. toil/test/src/autoDeploymentTest.py +45 -45
  103. toil/test/src/busTest.py +5 -5
  104. toil/test/src/checkpointTest.py +2 -2
  105. toil/test/src/deferredFunctionTest.py +1 -1
  106. toil/test/src/fileStoreTest.py +32 -16
  107. toil/test/src/helloWorldTest.py +1 -1
  108. toil/test/src/importExportFileTest.py +1 -1
  109. toil/test/src/jobDescriptionTest.py +2 -1
  110. toil/test/src/jobServiceTest.py +1 -1
  111. toil/test/src/jobTest.py +18 -18
  112. toil/test/src/miscTests.py +5 -3
  113. toil/test/src/promisedRequirementTest.py +3 -3
  114. toil/test/src/realtimeLoggerTest.py +1 -1
  115. toil/test/src/resourceTest.py +2 -2
  116. toil/test/src/restartDAGTest.py +1 -1
  117. toil/test/src/resumabilityTest.py +36 -2
  118. toil/test/src/retainTempDirTest.py +1 -1
  119. toil/test/src/systemTest.py +2 -2
  120. toil/test/src/toilContextManagerTest.py +2 -2
  121. toil/test/src/userDefinedJobArgTypeTest.py +1 -1
  122. toil/test/utils/toilDebugTest.py +98 -32
  123. toil/test/utils/toilKillTest.py +2 -2
  124. toil/test/utils/utilsTest.py +23 -3
  125. toil/test/wdl/wdltoil_test.py +223 -45
  126. toil/toilState.py +7 -6
  127. toil/utils/toilClean.py +1 -1
  128. toil/utils/toilConfig.py +36 -0
  129. toil/utils/toilDebugFile.py +60 -33
  130. toil/utils/toilDebugJob.py +39 -12
  131. toil/utils/toilDestroyCluster.py +1 -1
  132. toil/utils/toilKill.py +1 -1
  133. toil/utils/toilLaunchCluster.py +13 -2
  134. toil/utils/toilMain.py +3 -2
  135. toil/utils/toilRsyncCluster.py +1 -1
  136. toil/utils/toilSshCluster.py +1 -1
  137. toil/utils/toilStats.py +445 -305
  138. toil/utils/toilStatus.py +2 -5
  139. toil/version.py +10 -10
  140. toil/wdl/utils.py +2 -122
  141. toil/wdl/wdltoil.py +1257 -492
  142. toil/worker.py +55 -46
  143. toil-6.1.0.dist-info/METADATA +124 -0
  144. toil-6.1.0.dist-info/RECORD +241 -0
  145. {toil-5.12.0.dist-info → toil-6.1.0.dist-info}/WHEEL +1 -1
  146. {toil-5.12.0.dist-info → toil-6.1.0.dist-info}/entry_points.txt +0 -1
  147. toil/batchSystems/parasol.py +0 -379
  148. toil/batchSystems/tes.py +0 -459
  149. toil/test/batchSystems/parasolTestSupport.py +0 -117
  150. toil/test/wdl/builtinTest.py +0 -506
  151. toil/test/wdl/toilwdlTest.py +0 -522
  152. toil/wdl/toilwdl.py +0 -141
  153. toil/wdl/versions/dev.py +0 -107
  154. toil/wdl/versions/draft2.py +0 -980
  155. toil/wdl/versions/v1.py +0 -794
  156. toil/wdl/wdl_analysis.py +0 -116
  157. toil/wdl/wdl_functions.py +0 -997
  158. toil/wdl/wdl_synthesis.py +0 -1011
  159. toil/wdl/wdl_types.py +0 -243
  160. toil-5.12.0.dist-info/METADATA +0 -118
  161. toil-5.12.0.dist-info/RECORD +0 -244
  162. /toil/{wdl/versions → options}/__init__.py +0 -0
  163. {toil-5.12.0.dist-info → toil-6.1.0.dist-info}/LICENSE +0 -0
  164. {toil-5.12.0.dist-info → toil-6.1.0.dist-info}/top_level.txt +0 -0
toil/wdl/versions/v1.py DELETED
@@ -1,794 +0,0 @@
1
- # Copyright (C) 2020-2021 Regents of the University of California
2
- #
3
- # Licensed under the Apache License, Version 2.0 (the "License");
4
- # you may not use this file except in compliance with the License.
5
- # You may obtain a copy of the License at
6
- #
7
- # http://www.apache.org/licenses/LICENSE-2.0
8
- #
9
- # Unless required by applicable law or agreed to in writing, software
10
- # distributed under the License is distributed on an "AS IS" BASIS,
11
- # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
- # See the License for the specific language governing permissions and
13
- # limitations under the License.
14
- import logging
15
- from collections import OrderedDict
16
- from typing import Union
17
-
18
- from wdlparse.v1.WdlV1Lexer import FileStream, WdlV1Lexer
19
- from wdlparse.v1.WdlV1Parser import CommonTokenStream, WdlV1Parser
20
-
21
- from toil.wdl.wdl_analysis import AnalyzeWDL
22
-
23
- logger = logging.getLogger(__name__)
24
-
25
-
26
- def is_context(ctx, classname: Union[str, tuple]) -> bool:
27
- """
28
- Returns whether an ANTLR4 context object is of the precise type `classname`.
29
-
30
- :param ctx: An ANTLR4 context object.
31
- :param classname: The class name(s) as a string or a tuple of strings. If a
32
- tuple is provided, this returns True if the context object
33
- matches one of the class names.
34
- """
35
- # we check for `ctx.__class__.__name__` so that it's portable across multiple similar auto-generated parsers.
36
- if isinstance(classname, str):
37
- return ctx.__class__.__name__ == classname
38
- return ctx.__class__.__name__ in classname
39
-
40
-
41
- class AnalyzeV1WDL(AnalyzeWDL):
42
- """
43
- AnalyzeWDL implementation for the 1.0 version using ANTLR4.
44
-
45
- See: https://github.com/openwdl/wdl/blob/main/versions/1.0/SPEC.md
46
- https://github.com/openwdl/wdl/blob/main/versions/1.0/parsers/antlr4/WdlV1Parser.g4
47
- """
48
-
49
- @property
50
- def version(self) -> str:
51
- return '1.0'
52
-
53
- def analyze(self):
54
- """
55
- Analyzes the WDL file passed into the constructor and generates the two
56
- intermediate data structures: `self.workflows_dictionary` and
57
- `self.tasks_dictionary`.
58
- """
59
- lexer = WdlV1Lexer(FileStream(self.wdl_file))
60
- parser = WdlV1Parser(input=CommonTokenStream(lexer))
61
- tree = parser.document()
62
- self.visit_document(tree)
63
-
64
- def visit_document(self, ctx):
65
- """
66
- Root of tree. Contains `version` followed by an optional workflow and
67
- any number of `document_element`s.
68
- """
69
- wf = ctx.workflow()
70
- if wf:
71
- self.visit_workflow(wf)
72
-
73
- for element in ctx.document_element():
74
- self.visit_document_element(element)
75
-
76
- def visit_document_element(self, ctx):
77
- """
78
- Contains one of the following: 'import_doc', 'struct', or 'task'.
79
- """
80
- element = ctx.children[0]
81
- # task
82
- if is_context(element, 'TaskContext'):
83
- return self.visit_task(element)
84
- # struct
85
- elif is_context(element, 'StructContext'):
86
- # TODO: add support for structs.
87
- raise NotImplementedError('Struct is not supported.')
88
- # import_doc
89
- elif is_context(element, 'Import_docContext'):
90
- # TODO: add support for imports.
91
- raise NotImplementedError('Import is not supported.')
92
- else:
93
- raise RuntimeError(f'Unrecognized document element in visitDocument(): {type(element)}')
94
-
95
- # Workflow section
96
-
97
- def visit_workflow(self, ctx):
98
- """
99
- Contains an 'identifier' and an array of `workflow_element`s.
100
- """
101
- identifier = ctx.Identifier().getText()
102
- wf = self.workflows_dictionary.setdefault(identifier, OrderedDict())
103
-
104
- for element in ctx.workflow_element():
105
- section = element.children[0]
106
- # input
107
- if is_context(section, 'Workflow_inputContext'):
108
- # loop through all inputs and add to the workflow dictionary.
109
- # treating this the same as workflow declarations for now
110
- for wf_input in self.visit_workflow_input(section):
111
- wf[f'declaration{self.declaration_number}'] = wf_input
112
- self.declaration_number += 1
113
- # output
114
- elif is_context(section, 'Workflow_outputContext'):
115
- # TODO: add support for workflow level outputs in wdl_synthesis
116
- wf['wf_outputs'] = self.visit_workflow_output(section)
117
- # inner_element
118
- # i.e.: non-input declarations, scatters, calls, and conditionals
119
- elif is_context(section, 'Inner_workflow_elementContext'):
120
- wf_key, contents = self.visit_inner_workflow_element(section)
121
- wf[wf_key] = contents
122
- # parameter_meta and meta
123
- elif is_context(section, ('Parameter_meta_elementContext', 'Meta_elementContext')):
124
- # ignore additional metadata information for now.
125
- pass
126
- else:
127
- raise RuntimeError(f'Unrecognized workflow element in visitWorkflow(): {type(section)}')
128
-
129
- def visit_workflow_input(self, ctx):
130
- """
131
- Contains an array of 'any_decls', which can be unbound or bound declarations.
132
- Example:
133
- input {
134
- String in_str = "twenty"
135
- Int in_int
136
- }
137
-
138
- Returns a list of tuple=(name, type, expr).
139
- """
140
- return [self.visit_any_decls(decl) for decl in ctx.any_decls()]
141
-
142
- def visit_workflow_output(self, ctx):
143
- """
144
- Contains an array of 'bound_decls' (unbound_decls not allowed).
145
- Example:
146
- output {
147
- String out_str = "output"
148
- }
149
-
150
- Returns a list of tuple=(name, type, expr).
151
- """
152
- return [self.visit_bound_decls(decl) for decl in ctx.bound_decls()]
153
-
154
- def visit_inner_workflow_element(self, ctx):
155
- """
156
- Returns a tuple=(unique_key, dict), where dict contains the contents of
157
- the given inner workflow element.
158
- """
159
- element = ctx.children[0]
160
-
161
- # bound_decls
162
- # i.e.: declarations declared outside of input section
163
- if is_context(element, 'Bound_declsContext'):
164
- key = f'declaration{self.declaration_number}'
165
- self.declaration_number += 1
166
- return key, self.visit_bound_decls(element)
167
- # call
168
- elif is_context(element, 'CallContext'):
169
- key = f'call{self.call_number}'
170
- self.call_number += 1
171
- return key, self.visit_call(element)
172
- # scatter
173
- elif is_context(element, 'ScatterContext'):
174
- key = f'scatter{self.scatter_number}'
175
- self.scatter_number += 1
176
- return key, self.visit_scatter(element)
177
- # conditional
178
- elif is_context(element, 'ConditionalContext'):
179
- key = f'if{self.if_number}'
180
- self.if_number += 1
181
- return key, self.visit_conditional(element)
182
- else:
183
- raise RuntimeError(f'Unrecognized workflow element in visitInner_workflow_element(): {type(element)}')
184
-
185
- def visit_call(self, ctx):
186
- """
187
- Pattern: CALL call_name call_alias? call_body?
188
- Example WDL syntax: call task_1 {input: arr=arr}
189
-
190
- Returns a dict={task, alias, io}.
191
- """
192
- name = '.'.join(identifier.getText() for identifier in ctx.call_name().Identifier())
193
- alias = ctx.call_alias().Identifier().getText() if ctx.call_alias() else name
194
-
195
- body = OrderedDict()
196
- # make sure that '{}' and '{input: ...}' are provided
197
- if ctx.call_body() and ctx.call_body().call_inputs():
198
- for input_ in ctx.call_body().call_inputs().call_input():
199
- body[input_.Identifier().getText()] = self.visit_expr(input_.expr())
200
-
201
- return {
202
- 'task': name,
203
- 'alias': alias,
204
- 'io': body
205
- }
206
-
207
- def visit_scatter(self, ctx):
208
- """
209
- Pattern: SCATTER LPAREN Identifier In expr RPAREN LBRACE inner_workflow_element* RBRACE
210
- Example WDL syntax: scatter ( i in items) { ... }
211
-
212
- Returns a dict={item, collection, body}.
213
- """
214
- item = ctx.Identifier().getText()
215
- expr = self.visit_expr(ctx.expr())
216
- body = OrderedDict()
217
- for element in ctx.inner_workflow_element():
218
- body_key, contents = self.visit_inner_workflow_element(element)
219
- body[body_key] = contents
220
- return {
221
- 'item': item,
222
- 'collection': expr,
223
- 'body': body
224
- }
225
-
226
- def visit_conditional(self, ctx):
227
- """
228
- Pattern: IF LPAREN expr RPAREN LBRACE inner_workflow_element* RBRACE
229
- Example WDL syntax: if (condition) { ... }
230
-
231
- Returns a dict={expression, body}.
232
- """
233
- # see https://github.com/openwdl/wdl/blob/main/versions/1.0/SPEC.md#conditionals
234
- expr = self.visit_expr(ctx.expr())
235
-
236
- body = OrderedDict()
237
- for element in ctx.inner_workflow_element():
238
- body_key, contents = self.visit_inner_workflow_element(element)
239
- body[body_key] = contents
240
-
241
- return {
242
- 'expression': expr,
243
- 'body': body
244
- }
245
-
246
- # Task section
247
-
248
- def visit_task(self, ctx):
249
- """
250
- Root of a task definition. Contains an `identifier` and an array of
251
- `task_element`s.
252
- """
253
- identifier = ctx.Identifier().getText()
254
- task = self.tasks_dictionary.setdefault(identifier, OrderedDict())
255
- # print(f'Visiting task: {identifier}')
256
-
257
- for element in ctx.task_element():
258
- section = element.children[0]
259
- # input
260
- if is_context(section, 'Task_inputContext'):
261
- task.setdefault('inputs', []).extend(self.visit_task_input(section))
262
- # output
263
- elif is_context(section, 'Task_outputContext'):
264
- task['outputs'] = self.visit_task_output(section)
265
- # command
266
- elif is_context(section, 'Task_commandContext'):
267
- task['raw_commandline'] = self.visit_task_command(section)
268
- # runtime
269
- elif is_context(section, 'Task_runtimeContext'):
270
- task['runtime'] = self.visit_task_runtime(section)
271
- # bound_decls
272
- elif is_context(section, 'Bound_declsContext'):
273
- # treating this the same as inputs for now
274
- decl = self.visit_bound_decls(section)
275
- task.setdefault('inputs', []).append(decl)
276
- # parameter_meta, and meta
277
- elif is_context(section, ('Parameter_meta_elementContext', 'Meta_elementContext')):
278
- pass
279
- else:
280
- raise RuntimeError(f'Unrecognized task element in visitTask(): {type(section)}')
281
-
282
- def visit_task_input(self, ctx):
283
- """
284
- Contains an array of 'any_decls', which can be unbound or bound declarations.
285
- Example:
286
- input {
287
- String in_str = "twenty"
288
- Int in_int
289
- }
290
-
291
- Returns a list of tuple=(name, type, expr)
292
- """
293
- return [self.visit_any_decls(decl) for decl in ctx.any_decls()]
294
-
295
- def visit_task_output(self, ctx):
296
- """
297
- Contains an array of 'bound_decls' (unbound_decls not allowed).
298
- Example:
299
- output {
300
- String out_str = read_string(stdout())
301
- }
302
-
303
- Returns a list of tuple=(name, type, expr)
304
- """
305
- return [self.visit_bound_decls(decl) for decl in ctx.bound_decls()]
306
-
307
- def visit_task_command(self, ctx):
308
- """
309
- Parses the command section of the WDL task.
310
-
311
- Contains a `string_part` plus any number of `expr_with_string`s.
312
- The following example command:
313
- 'echo ${var1} ${var2} > output_file.txt'
314
- Has 3 parts:
315
- 1. string_part: 'echo '
316
- 2. expr_with_string, which has two parts:
317
- - expr_part: 'var1'
318
- - string_part: ' '
319
- 1. expr_with_string, which has two parts:
320
- - expr_part: 'var2'
321
- - string_part: ' > output_file.txt'
322
-
323
- Returns a list=[] of strings representing the parts of the command.
324
- e.g. [string_part, expr_part, string_part, ...]
325
- """
326
- parts = []
327
-
328
- # add the first part
329
- str_part = self.visit_task_command_string_part(ctx.task_command_string_part())
330
- if str_part:
331
- parts.append(f"r'''{str_part}'''")
332
-
333
- # add the rest
334
- for group in ctx.task_command_expr_with_string():
335
- expr_part, str_part = self.visit_task_command_expr_with_string(group)
336
- parts.append(expr_part)
337
- if str_part:
338
- parts.append(f"r'''{str_part}'''")
339
-
340
- return parts
341
-
342
- def visit_task_command_string_part(self, ctx):
343
- """
344
- Returns a string representing the string_part.
345
- """
346
- # join here because a string that contains '$', '{', or '}' is split
347
- return ''.join(part.getText() for part in ctx.CommandStringPart())
348
-
349
- def visit_task_command_expr_with_string(self, ctx):
350
- """
351
- Returns a tuple=(expr_part, string_part).
352
- """
353
- return (self.visit_task_command_expr_part(ctx.task_command_expr_part()),
354
- self.visit_task_command_string_part(ctx.task_command_string_part()))
355
-
356
- def visit_task_command_expr_part(self, ctx):
357
- """
358
- Contains the expression inside ${expr}. Same function as `self.visit_string_expr_part()`.
359
-
360
- Returns the expression.
361
- """
362
- return self.visit_string_expr_part(ctx)
363
-
364
- def visit_task_runtime(self, ctx):
365
- """
366
- Contains an array of `task_runtime_kv`s.
367
-
368
- Returns a dict={key: value} where key can be 'docker', 'cpu', 'memory',
369
- 'cores', or 'disks'.
370
- """
371
- return OrderedDict((kv.children[0].getText(), # runtime key
372
- self.visit_expr(kv.expr())) # runtime value
373
- for kv in ctx.task_runtime_kv())
374
-
375
- # Shared
376
-
377
- def visit_any_decls(self, ctx):
378
- """
379
- Contains a bound or unbound declaration.
380
- """
381
- if ctx.bound_decls():
382
- return self.visit_bound_decls(ctx.bound_decls())
383
- elif ctx.unbound_decls():
384
- return self.visit_unbound_decls(ctx.unbound_decls())
385
- else:
386
- raise RuntimeError(f'Unrecognized declaration: {type(ctx)}')
387
-
388
- def visit_unbound_decls(self, ctx):
389
- """
390
- Contains an unbound declaration. E.g.: `String in_str`.
391
-
392
- Returns a tuple=(name, type, expr), where `expr` is None.
393
- """
394
- name = ctx.Identifier().getText()
395
- type_ = self.visit_wdl_type(ctx.wdl_type())
396
- return name, type_, None
397
-
398
- def visit_bound_decls(self, ctx):
399
- """
400
- Contains a bound declaration. E.g.: `String in_str = "some string"`.
401
-
402
- Returns a tuple=(name, type, expr).
403
- """
404
- name = ctx.Identifier().getText()
405
- type_ = self.visit_wdl_type(ctx.wdl_type())
406
- expr = self.visit_expr(ctx.expr())
407
-
408
- return name, type_, expr
409
-
410
- def visit_wdl_type(self, ctx):
411
- """
412
- Returns a WDLType instance.
413
- """
414
- identifier = ctx.type_base().children[0]
415
- optional = ctx.OPTIONAL() is not None
416
-
417
- # primitives
418
- if is_context(identifier, 'TerminalNodeImpl'):
419
-
420
- # TODO: implement Object type
421
- return self.create_wdl_primitive_type(key=identifier.getText(), optional=optional)
422
- # compound types
423
- else:
424
- name = identifier.children[0].getText() # the first child is the name of the type.
425
- type_ = identifier.wdl_type()
426
- if isinstance(type_, list):
427
- elements = [self.visit_wdl_type(element) for element in type_]
428
- else:
429
- elements = [self.visit_wdl_type(type_)]
430
- return self.create_wdl_compound_type(key=name, elements=elements, optional=optional)
431
-
432
- def visit_primitive_literal(self, ctx):
433
- """
434
- Returns the primitive literal as a string.
435
- """
436
- is_bool = ctx.BoolLiteral()
437
- if is_bool:
438
- val = is_bool.getText()
439
- if val not in ('true', 'false'):
440
- raise TypeError(f'Parsed boolean ({val}) must be expressed as "true" or "false".')
441
- return val.capitalize()
442
- elif is_context(ctx.children[0], 'StringContext'):
443
- return self.visit_string(ctx.children[0])
444
- elif is_context(ctx.children[0], ('TerminalNodeImpl', # this also includes variables
445
- 'NumberContext')):
446
- return ctx.children[0].getText()
447
- else:
448
- raise RuntimeError(f'Primitive literal has unknown child: {type(ctx.children[0])}.')
449
-
450
- def visit_number(self, ctx):
451
- """
452
- Contains an `IntLiteral` or a `FloatLiteral`.
453
- """
454
- return ctx.children[0].getText()
455
-
456
- def visit_string(self, ctx):
457
- """
458
- Contains a `string_part` followed by an array of `string_expr_with_string_part`s.
459
- """
460
- string = self.visit_string_part(ctx.string_part())
461
-
462
- for part in ctx.string_expr_with_string_part():
463
- string += f' + {self.visit_string_expr_with_string_part(part)}'
464
-
465
- return string
466
-
467
- def visit_string_expr_with_string_part(self, ctx):
468
- """
469
- Contains a `string_expr_part` and a `string_part`.
470
- """
471
- expr = self.visit_string_expr_part(ctx.string_expr_part())
472
- part = self.visit_string_part(ctx.string_part())
473
-
474
- if not part:
475
- return expr
476
-
477
- return f'{expr} + {part}'
478
-
479
- def visit_string_expr_part(self, ctx):
480
- """
481
- Contains an array of `expression_placeholder_option`s and an `expr`.
482
- """
483
- # See https://github.com/openwdl/wdl/blob/main/versions/1.0/parsers/antlr4/WdlV1Parser.g4#L56
484
-
485
- options = {}
486
-
487
- for opt in ctx.expression_placeholder_option():
488
- key, val = self.visit_expression_placeholder_option(opt)
489
- options[key] = val
490
-
491
- expr = self.visit_expr(ctx.expr())
492
-
493
- if len(options) == 0:
494
- return expr
495
- elif 'sep' in options:
496
- sep = options['sep']
497
- return f'{sep}.join(str(x) for x in {expr})'
498
- elif 'default' in options:
499
- default = options['default']
500
- return f'({expr} if {expr} else {default})'
501
- else:
502
- raise NotImplementedError(options)
503
-
504
- def visit_string_part(self, ctx):
505
- """
506
- Returns a string representing the string_part.
507
- """
508
- # join here because a string that contains '$', '{', or '}' is split
509
- part = ''.join(part.getText() for part in ctx.StringPart())
510
-
511
- if part:
512
- return f"'{part}'"
513
- return None
514
-
515
- def visit_expression_placeholder_option(self, ctx):
516
- """
517
- Expression placeholder options.
518
-
519
- Can match one of the following:
520
- BoolLiteral EQUAL (string | number)
521
- DEFAULT EQUAL (string | number)
522
- SEP EQUAL (string | number)
523
-
524
- See https://github.com/openwdl/wdl/blob/main/versions/1.0/SPEC.md#expression-placeholder-options
525
-
526
- e.g.: ${sep=", " array_value}
527
- e.g.: ${true="--yes" false="--no" boolean_value}
528
- e.g.: ${default="foo" optional_value}
529
-
530
- Returns a tuple=(key, value)
531
- """
532
- assert len(ctx.children) == 3
533
-
534
- param = ctx.children[0].getText()
535
- str_or_num = ctx.children[2]
536
- val = self.visit_string(str_or_num) \
537
- if is_context(str_or_num, 'StringContext') else self.visit_number(str_or_num)
538
-
539
- return param, val
540
-
541
- def visit_expr(self, ctx):
542
- """
543
- Expression root.
544
- """
545
- return self.visit_infix0(ctx.expr_infix())
546
-
547
- def visit_infix0(self, ctx):
548
- """
549
- Expression infix0 (LOR).
550
- """
551
- infix = ctx.expr_infix0()
552
- if is_context(infix, 'LorContext'):
553
- return self.visit_lor(infix)
554
- return self.visit_infix1(infix)
555
-
556
- def visit_lor(self, ctx):
557
- """
558
- Logical OR expression.
559
- """
560
- lhs = self.visit_infix0(ctx)
561
- rhs = self.visit_infix1(ctx)
562
- return f'{lhs} or {rhs}'
563
-
564
- def visit_infix1(self, ctx):
565
- """
566
- Expression infix1 (LAND).
567
- """
568
- infix = ctx.expr_infix1()
569
- if is_context(infix, 'LandContext'):
570
- return self.visit_land(infix)
571
- return self.visit_infix2(infix)
572
-
573
- def visit_land(self, ctx):
574
- """
575
- Logical AND expresion.
576
- """
577
- lhs = self.visit_infix1(ctx)
578
- rhs = self.visit_infix2(ctx)
579
- return f'{lhs} and {rhs}'
580
-
581
- def visit_infix2(self, ctx):
582
- """
583
- Expression infix2 (comparisons).
584
- """
585
- infix = ctx.expr_infix2()
586
- if is_context(infix, 'EqeqContext'):
587
- return self._visit_infix2(infix, '==')
588
- elif is_context(infix, 'NeqContext'):
589
- return self._visit_infix2(infix, '!=')
590
- elif is_context(infix, 'LteContext'):
591
- return self._visit_infix2(infix, '<=')
592
- elif is_context(infix, 'GteContext'):
593
- return self._visit_infix2(infix, '>=')
594
- elif is_context(infix, 'LtContext'):
595
- return self._visit_infix2(infix, '<')
596
- elif is_context(infix, 'GtContext'):
597
- return self._visit_infix2(infix, '>')
598
- # continue down our path
599
- return self.visit_infix3(infix)
600
-
601
- def _visit_infix2(self, ctx, operation: str):
602
- """
603
- :param operation: Operation as a string.
604
- """
605
- lhs = self.visit_infix2(ctx)
606
- rhs = self.visit_infix3(ctx)
607
- return f'{lhs} {operation} {rhs}'
608
-
609
- def visit_infix3(self, ctx):
610
- """
611
- Expression infix3 (add/subtract).
612
- """
613
- infix = ctx.expr_infix3()
614
- if is_context(infix, 'AddContext'):
615
- return self._visit_infix3(infix, '+')
616
- elif is_context(infix, 'SubContext'):
617
- return self._visit_infix3(infix, '-')
618
- # continue down our path
619
- return self.visit_infix4(infix)
620
-
621
- def _visit_infix3(self, ctx, operation: str):
622
- """
623
- :param operation: Operation as a string.
624
- """
625
- lhs = self.visit_infix3(ctx)
626
- rhs = self.visit_infix4(ctx)
627
- return f'{lhs} {operation} {rhs}'
628
-
629
- def visit_infix4(self, ctx):
630
- """
631
- Expression infix4 (multiply/divide/modulo).
632
- """
633
- infix = ctx.expr_infix4()
634
- if is_context(infix, 'MulContext'):
635
- return self._visit_infix4(infix, '*')
636
- elif is_context(infix, 'DivideContext'):
637
- return self._visit_infix4(infix, '/')
638
- elif is_context(infix, 'ModContext'):
639
- return self._visit_infix4(infix, '%')
640
- # continue down our path
641
- return self.visit_infix5(infix)
642
-
643
- def _visit_infix4(self, ctx, operation: str):
644
- """
645
- :param operation: Operation as a string.
646
- """
647
- lhs = self.visit_infix4(ctx)
648
- rhs = self.visit_infix5(ctx)
649
- return f'{lhs} {operation} {rhs}'
650
-
651
- def visit_infix5(self, ctx):
652
- """
653
- Expression infix5.
654
- """
655
- return self.visit_expr_core(ctx.expr_infix5().expr_core())
656
-
657
- def visit_expr_core(self, expr):
658
- """
659
- Expression core.
660
- """
661
- # TODO: implement map_literal, object_literal, and left_name
662
-
663
- if is_context(expr, 'ApplyContext'):
664
- return self.visit_apply(expr)
665
- elif is_context(expr, 'Array_literalContext'):
666
- return self.visit_array_literal(expr)
667
- elif is_context(expr, 'Pair_literalContext'):
668
- return self.visit_pair_literal(expr)
669
- elif is_context(expr, 'IfthenelseContext'):
670
- return self.visit_ifthenelse(expr)
671
- elif is_context(expr, 'Expression_groupContext'):
672
- return self.visit_expression_group(expr)
673
- elif is_context(expr, 'AtContext'):
674
- return self.visit_at(expr)
675
- elif is_context(expr, 'Get_nameContext'):
676
- return self.visit_get_name(expr)
677
- elif is_context(expr, 'NegateContext'):
678
- return self.visit_negate(expr)
679
- elif is_context(expr, 'UnarysignedContext'):
680
- return self.visit_unarysigned(expr)
681
- elif is_context(expr, 'PrimitivesContext'):
682
- return self.visit_primitives(expr)
683
-
684
- raise NotImplementedError(f"Expression context '{type(expr)}' is not supported.")
685
-
686
- # expr_core
687
-
688
- def visit_apply(self, ctx):
689
- """
690
- A function call.
691
- Pattern: Identifier LPAREN (expr (COMMA expr)*)? RPAREN
692
- """
693
- fn = ctx.Identifier().getText()
694
- params = ', '.join(self.visit_expr(expr) for expr in ctx.expr())
695
-
696
- if fn == 'stdout':
697
- return '_toil_wdl_internal__stdout_file'
698
- elif fn == 'stderr':
699
- return '_toil_wdl_internal__stderr_file'
700
- elif fn in ('range', 'zip'):
701
- # replace python built-in functions
702
- return f'wdl_{fn}'
703
-
704
- call = f'{fn}({params}'
705
-
706
- # append necessary params for i/o functions
707
- if fn == 'glob':
708
- return call + ', tempDir)'
709
- elif fn == 'size':
710
- return call + (params + ', ' if params else '') + 'fileStore=fileStore)'
711
- elif fn in ('write_lines', 'write_tsv', 'write_json', 'write_map'):
712
- return call + ', temp_dir=tempDir, file_store=fileStore)'
713
- else:
714
- return call + ')'
715
-
716
- def visit_array_literal(self, ctx):
717
- """
718
- Pattern: LBRACK (expr (COMMA expr)*)* RBRACK
719
- """
720
- return f"[{', '.join(self.visit_expr(expr) for expr in ctx.expr())}]"
721
-
722
- def visit_pair_literal(self, ctx):
723
- """
724
- Pattern: LPAREN expr COMMA expr RPAREN
725
- """
726
- return f"({self.visit_expr(ctx.expr(0))}, {self.visit_expr(ctx.expr(1))})"
727
-
728
- def visit_ifthenelse(self, ctx):
729
- """
730
- Ternary expression.
731
- Pattern: IF expr THEN expr ELSE expr
732
- """
733
- if_true = self.visit_expr(ctx.expr(0))
734
- condition = self.visit_expr(ctx.expr(1))
735
- if_false = self.visit_expr(ctx.expr(2))
736
-
737
- return f'({condition} if {if_true} else {if_false})'
738
-
739
- def visit_expression_group(self, ctx):
740
- """
741
- Pattern: LPAREN expr RPAREN
742
- """
743
- return f'({self.visit_expr(ctx.expr())})'
744
-
745
- def visit_at(self, ctx):
746
- """
747
- Array or map lookup.
748
- Pattern: expr_core LBRACK expr RBRACK
749
- """
750
- expr_core = self.visit_expr_core(ctx.expr_core())
751
- expr = self.visit_expr(ctx.expr())
752
-
753
- # parenthesis must be removed because 'i[0]' works, but '(i)[0]' does not
754
- if expr_core[0] == '(' and expr_core[-1] == ')':
755
- expr_core = expr_core[1:-1]
756
-
757
- return f'{expr_core}[{expr}]'
758
-
759
- def visit_get_name(self, ctx):
760
- """
761
- Member access.
762
- Pattern: expr_core DOT Identifier
763
- """
764
- expr_core = self.visit_expr_core(ctx.expr_core())
765
- identifier = ctx.Identifier().getText()
766
-
767
- if identifier in ('left', 'right'):
768
- # hack-y way to make sure pair.left and pair.right are parsed correctly.
769
- return f'({expr_core}.{identifier})'
770
-
771
- return f'({expr_core}_{identifier})'
772
-
773
- def visit_negate(self, ctx):
774
- """
775
- Pattern: NOT expr
776
- """
777
- return f'(not {self.visit_expr(ctx.expr())})'
778
-
779
- def visit_unarysigned(self, ctx):
780
- """
781
- Pattern: (PLUS | MINUS) expr
782
- """
783
- plus: bool = ctx.PLUS() is not None
784
- expr = self.visit_expr(ctx.expr())
785
-
786
- if plus:
787
- return f'(+{expr})'
788
- return f'(-{expr})'
789
-
790
- def visit_primitives(self, ctx):
791
- """
792
- Expression alias for primitive literal.
793
- """
794
- return self.visit_primitive_literal(ctx.primitive_literal())