toil 5.12.0__py3-none-any.whl → 6.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- toil/__init__.py +18 -13
- toil/batchSystems/abstractBatchSystem.py +39 -13
- toil/batchSystems/abstractGridEngineBatchSystem.py +24 -24
- toil/batchSystems/awsBatch.py +14 -14
- toil/batchSystems/cleanup_support.py +7 -3
- toil/batchSystems/contained_executor.py +3 -3
- toil/batchSystems/htcondor.py +0 -1
- toil/batchSystems/kubernetes.py +34 -31
- toil/batchSystems/local_support.py +3 -1
- toil/batchSystems/lsf.py +7 -7
- toil/batchSystems/mesos/batchSystem.py +7 -7
- toil/batchSystems/options.py +32 -83
- toil/batchSystems/registry.py +104 -23
- toil/batchSystems/singleMachine.py +16 -13
- toil/batchSystems/slurm.py +87 -16
- toil/batchSystems/torque.py +0 -1
- toil/bus.py +44 -8
- toil/common.py +544 -753
- toil/cwl/__init__.py +28 -32
- toil/cwl/cwltoil.py +595 -574
- toil/cwl/utils.py +55 -10
- toil/exceptions.py +1 -1
- toil/fileStores/__init__.py +2 -2
- toil/fileStores/abstractFileStore.py +88 -14
- toil/fileStores/cachingFileStore.py +610 -549
- toil/fileStores/nonCachingFileStore.py +46 -22
- toil/job.py +182 -101
- toil/jobStores/abstractJobStore.py +161 -95
- toil/jobStores/aws/jobStore.py +23 -9
- toil/jobStores/aws/utils.py +6 -6
- toil/jobStores/fileJobStore.py +116 -18
- toil/jobStores/googleJobStore.py +16 -7
- toil/jobStores/utils.py +5 -6
- toil/leader.py +87 -56
- toil/lib/accelerators.py +10 -5
- toil/lib/aws/__init__.py +3 -14
- toil/lib/aws/ami.py +22 -9
- toil/lib/aws/iam.py +21 -13
- toil/lib/aws/session.py +2 -16
- toil/lib/aws/utils.py +4 -5
- toil/lib/compatibility.py +1 -1
- toil/lib/conversions.py +26 -3
- toil/lib/docker.py +22 -23
- toil/lib/ec2.py +10 -6
- toil/lib/ec2nodes.py +106 -100
- toil/lib/encryption/_nacl.py +2 -1
- toil/lib/generatedEC2Lists.py +325 -18
- toil/lib/io.py +49 -2
- toil/lib/misc.py +1 -1
- toil/lib/resources.py +9 -2
- toil/lib/threading.py +101 -38
- toil/options/common.py +736 -0
- toil/options/cwl.py +336 -0
- toil/options/wdl.py +37 -0
- toil/provisioners/abstractProvisioner.py +9 -4
- toil/provisioners/aws/__init__.py +3 -6
- toil/provisioners/aws/awsProvisioner.py +6 -0
- toil/provisioners/clusterScaler.py +3 -2
- toil/provisioners/gceProvisioner.py +2 -2
- toil/realtimeLogger.py +2 -1
- toil/resource.py +24 -18
- toil/server/app.py +2 -3
- toil/server/cli/wes_cwl_runner.py +4 -4
- toil/server/utils.py +1 -1
- toil/server/wes/abstract_backend.py +3 -2
- toil/server/wes/amazon_wes_utils.py +5 -4
- toil/server/wes/tasks.py +2 -3
- toil/server/wes/toil_backend.py +2 -10
- toil/server/wsgi_app.py +2 -0
- toil/serviceManager.py +12 -10
- toil/statsAndLogging.py +41 -9
- toil/test/__init__.py +29 -54
- toil/test/batchSystems/batchSystemTest.py +11 -111
- toil/test/batchSystems/test_slurm.py +24 -8
- toil/test/cactus/__init__.py +0 -0
- toil/test/cactus/test_cactus_integration.py +58 -0
- toil/test/cwl/cwlTest.py +438 -223
- toil/test/cwl/glob_dir.cwl +15 -0
- toil/test/cwl/preemptible.cwl +21 -0
- toil/test/cwl/preemptible_expression.cwl +28 -0
- toil/test/cwl/revsort.cwl +1 -1
- toil/test/cwl/revsort2.cwl +1 -1
- toil/test/docs/scriptsTest.py +2 -3
- toil/test/jobStores/jobStoreTest.py +34 -21
- toil/test/lib/aws/test_iam.py +4 -14
- toil/test/lib/aws/test_utils.py +0 -3
- toil/test/lib/dockerTest.py +4 -4
- toil/test/lib/test_ec2.py +12 -17
- toil/test/mesos/helloWorld.py +4 -5
- toil/test/mesos/stress.py +1 -1
- toil/test/{wdl/conftest.py → options/__init__.py} +0 -10
- toil/test/options/options.py +37 -0
- toil/test/provisioners/aws/awsProvisionerTest.py +9 -5
- toil/test/provisioners/clusterScalerTest.py +6 -4
- toil/test/provisioners/clusterTest.py +23 -11
- toil/test/provisioners/gceProvisionerTest.py +0 -6
- toil/test/provisioners/restartScript.py +3 -2
- toil/test/server/serverTest.py +1 -1
- toil/test/sort/restart_sort.py +2 -1
- toil/test/sort/sort.py +2 -1
- toil/test/sort/sortTest.py +2 -13
- toil/test/src/autoDeploymentTest.py +45 -45
- toil/test/src/busTest.py +5 -5
- toil/test/src/checkpointTest.py +2 -2
- toil/test/src/deferredFunctionTest.py +1 -1
- toil/test/src/fileStoreTest.py +32 -16
- toil/test/src/helloWorldTest.py +1 -1
- toil/test/src/importExportFileTest.py +1 -1
- toil/test/src/jobDescriptionTest.py +2 -1
- toil/test/src/jobServiceTest.py +1 -1
- toil/test/src/jobTest.py +18 -18
- toil/test/src/miscTests.py +5 -3
- toil/test/src/promisedRequirementTest.py +3 -3
- toil/test/src/realtimeLoggerTest.py +1 -1
- toil/test/src/resourceTest.py +2 -2
- toil/test/src/restartDAGTest.py +1 -1
- toil/test/src/resumabilityTest.py +36 -2
- toil/test/src/retainTempDirTest.py +1 -1
- toil/test/src/systemTest.py +2 -2
- toil/test/src/toilContextManagerTest.py +2 -2
- toil/test/src/userDefinedJobArgTypeTest.py +1 -1
- toil/test/utils/toilDebugTest.py +98 -32
- toil/test/utils/toilKillTest.py +2 -2
- toil/test/utils/utilsTest.py +23 -3
- toil/test/wdl/wdltoil_test.py +223 -45
- toil/toilState.py +7 -6
- toil/utils/toilClean.py +1 -1
- toil/utils/toilConfig.py +36 -0
- toil/utils/toilDebugFile.py +60 -33
- toil/utils/toilDebugJob.py +39 -12
- toil/utils/toilDestroyCluster.py +1 -1
- toil/utils/toilKill.py +1 -1
- toil/utils/toilLaunchCluster.py +13 -2
- toil/utils/toilMain.py +3 -2
- toil/utils/toilRsyncCluster.py +1 -1
- toil/utils/toilSshCluster.py +1 -1
- toil/utils/toilStats.py +445 -305
- toil/utils/toilStatus.py +2 -5
- toil/version.py +10 -10
- toil/wdl/utils.py +2 -122
- toil/wdl/wdltoil.py +1257 -492
- toil/worker.py +55 -46
- toil-6.1.0.dist-info/METADATA +124 -0
- toil-6.1.0.dist-info/RECORD +241 -0
- {toil-5.12.0.dist-info → toil-6.1.0.dist-info}/WHEEL +1 -1
- {toil-5.12.0.dist-info → toil-6.1.0.dist-info}/entry_points.txt +0 -1
- toil/batchSystems/parasol.py +0 -379
- toil/batchSystems/tes.py +0 -459
- toil/test/batchSystems/parasolTestSupport.py +0 -117
- toil/test/wdl/builtinTest.py +0 -506
- toil/test/wdl/toilwdlTest.py +0 -522
- toil/wdl/toilwdl.py +0 -141
- toil/wdl/versions/dev.py +0 -107
- toil/wdl/versions/draft2.py +0 -980
- toil/wdl/versions/v1.py +0 -794
- toil/wdl/wdl_analysis.py +0 -116
- toil/wdl/wdl_functions.py +0 -997
- toil/wdl/wdl_synthesis.py +0 -1011
- toil/wdl/wdl_types.py +0 -243
- toil-5.12.0.dist-info/METADATA +0 -118
- toil-5.12.0.dist-info/RECORD +0 -244
- /toil/{wdl/versions → options}/__init__.py +0 -0
- {toil-5.12.0.dist-info → toil-6.1.0.dist-info}/LICENSE +0 -0
- {toil-5.12.0.dist-info → toil-6.1.0.dist-info}/top_level.txt +0 -0
toil/wdl/versions/dev.py
DELETED
|
@@ -1,107 +0,0 @@
|
|
|
1
|
-
# Copyright (C) 2020-2021 Regents of the University of California
|
|
2
|
-
#
|
|
3
|
-
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
-
# you may not use this file except in compliance with the License.
|
|
5
|
-
# You may obtain a copy of the License at
|
|
6
|
-
#
|
|
7
|
-
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
-
#
|
|
9
|
-
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
-
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
-
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
-
# See the License for the specific language governing permissions and
|
|
13
|
-
# limitations under the License.
|
|
14
|
-
import logging
|
|
15
|
-
|
|
16
|
-
from wdlparse.dev.WdlLexer import FileStream, WdlLexer
|
|
17
|
-
from wdlparse.dev.WdlParser import CommonTokenStream, WdlParser
|
|
18
|
-
|
|
19
|
-
from toil.wdl.versions.v1 import AnalyzeV1WDL, is_context
|
|
20
|
-
from toil.wdl.wdl_types import WDLType
|
|
21
|
-
|
|
22
|
-
logger = logging.getLogger(__name__)
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
class AnalyzeDevelopmentWDL(AnalyzeV1WDL): # extend from 1.0
|
|
26
|
-
"""
|
|
27
|
-
AnalyzeWDL implementation for the development version using ANTLR4.
|
|
28
|
-
|
|
29
|
-
See: https://github.com/openwdl/wdl/blob/main/versions/development/SPEC.md
|
|
30
|
-
https://github.com/openwdl/wdl/blob/main/versions/development/parsers/antlr4/WdlParser.g4
|
|
31
|
-
"""
|
|
32
|
-
|
|
33
|
-
@property
|
|
34
|
-
def version(self) -> str:
|
|
35
|
-
return 'development'
|
|
36
|
-
|
|
37
|
-
def analyze(self):
|
|
38
|
-
"""
|
|
39
|
-
Analyzes the WDL file passed into the constructor and generates the two
|
|
40
|
-
intermediate data structures: `self.workflows_dictionary` and
|
|
41
|
-
`self.tasks_dictionary`.
|
|
42
|
-
"""
|
|
43
|
-
lexer = WdlLexer(FileStream(self.wdl_file))
|
|
44
|
-
parser = WdlParser(input=CommonTokenStream(lexer))
|
|
45
|
-
tree = parser.document()
|
|
46
|
-
self.visit_document(tree)
|
|
47
|
-
|
|
48
|
-
def visit_document(self, ctx: WdlParser.DocumentContext) -> None:
|
|
49
|
-
"""
|
|
50
|
-
Similar to version 1.0, except the 'workflow' element is included in
|
|
51
|
-
`ctx.document_element()`.
|
|
52
|
-
"""
|
|
53
|
-
for element in ctx.document_element():
|
|
54
|
-
self.visit_document_element(element)
|
|
55
|
-
|
|
56
|
-
def visit_document_element(self, ctx: WdlParser.Document_elementContext) -> None:
|
|
57
|
-
"""
|
|
58
|
-
Similar to version 1.0, except this also contains 'workflow'.
|
|
59
|
-
"""
|
|
60
|
-
element = ctx.children[0]
|
|
61
|
-
if is_context(element, 'WorkflowContext'):
|
|
62
|
-
self.visit_workflow(element)
|
|
63
|
-
else:
|
|
64
|
-
# let super take care of the rest.
|
|
65
|
-
super().visit_document_element(ctx)
|
|
66
|
-
|
|
67
|
-
def visit_call(self, ctx: WdlParser.CallContext) -> dict:
|
|
68
|
-
"""
|
|
69
|
-
Similar to version 1.0, except `ctx.call_afters()` is added.
|
|
70
|
-
"""
|
|
71
|
-
# TODO: implement call_afters
|
|
72
|
-
# See: https://github.com/openwdl/wdl/blob/main/versions/development/SPEC.md#call-statement
|
|
73
|
-
return super().visit_call(ctx)
|
|
74
|
-
|
|
75
|
-
def visit_string_expr_part(self, ctx: WdlParser.String_expr_partContext) -> str:
|
|
76
|
-
"""
|
|
77
|
-
Similar to version 1.0, except `ctx.expression_placeholder_option()`
|
|
78
|
-
is removed.
|
|
79
|
-
"""
|
|
80
|
-
# expression placeholder options are removed in development
|
|
81
|
-
# See: https://github.com/openwdl/wdl/blob/main/versions/development/parsers/antlr4/WdlParser.g4#L55
|
|
82
|
-
|
|
83
|
-
return self.visit_expr(ctx.expr())
|
|
84
|
-
|
|
85
|
-
def visit_wdl_type(self, ctx: WdlParser.Wdl_typeContext) -> WDLType:
|
|
86
|
-
"""
|
|
87
|
-
Similar to version 1.0, except Directory type is added.
|
|
88
|
-
"""
|
|
89
|
-
identifier = ctx.type_base().children[0]
|
|
90
|
-
|
|
91
|
-
if identifier == 'Directory':
|
|
92
|
-
# TODO: implement Directory type
|
|
93
|
-
raise NotImplementedError('Directory type is not implemented.')
|
|
94
|
-
else:
|
|
95
|
-
# let super take care of the rest.
|
|
96
|
-
return super().visit_wdl_type(ctx)
|
|
97
|
-
|
|
98
|
-
def visit_expr_core(self, expr: WdlParser.Expr_coreContext) -> str:
|
|
99
|
-
"""
|
|
100
|
-
Similar to version 1.0, except struct literal is added.
|
|
101
|
-
"""
|
|
102
|
-
if is_context(expr, 'Struct_literalContext'):
|
|
103
|
-
# TODO: implement struct literal
|
|
104
|
-
raise NotImplementedError(f'WDL struct is not implemented.')
|
|
105
|
-
else:
|
|
106
|
-
# let super take care of the rest.
|
|
107
|
-
return super().visit_expr_core(expr)
|