toil 5.12.0__py3-none-any.whl → 6.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- toil/__init__.py +18 -13
- toil/batchSystems/abstractBatchSystem.py +39 -13
- toil/batchSystems/abstractGridEngineBatchSystem.py +24 -24
- toil/batchSystems/awsBatch.py +14 -14
- toil/batchSystems/cleanup_support.py +7 -3
- toil/batchSystems/contained_executor.py +3 -3
- toil/batchSystems/htcondor.py +0 -1
- toil/batchSystems/kubernetes.py +34 -31
- toil/batchSystems/local_support.py +3 -1
- toil/batchSystems/lsf.py +7 -7
- toil/batchSystems/mesos/batchSystem.py +7 -7
- toil/batchSystems/options.py +32 -83
- toil/batchSystems/registry.py +104 -23
- toil/batchSystems/singleMachine.py +16 -13
- toil/batchSystems/slurm.py +87 -16
- toil/batchSystems/torque.py +0 -1
- toil/bus.py +44 -8
- toil/common.py +544 -753
- toil/cwl/__init__.py +28 -32
- toil/cwl/cwltoil.py +595 -574
- toil/cwl/utils.py +55 -10
- toil/exceptions.py +1 -1
- toil/fileStores/__init__.py +2 -2
- toil/fileStores/abstractFileStore.py +88 -14
- toil/fileStores/cachingFileStore.py +610 -549
- toil/fileStores/nonCachingFileStore.py +46 -22
- toil/job.py +182 -101
- toil/jobStores/abstractJobStore.py +161 -95
- toil/jobStores/aws/jobStore.py +23 -9
- toil/jobStores/aws/utils.py +6 -6
- toil/jobStores/fileJobStore.py +116 -18
- toil/jobStores/googleJobStore.py +16 -7
- toil/jobStores/utils.py +5 -6
- toil/leader.py +87 -56
- toil/lib/accelerators.py +10 -5
- toil/lib/aws/__init__.py +3 -14
- toil/lib/aws/ami.py +22 -9
- toil/lib/aws/iam.py +21 -13
- toil/lib/aws/session.py +2 -16
- toil/lib/aws/utils.py +4 -5
- toil/lib/compatibility.py +1 -1
- toil/lib/conversions.py +26 -3
- toil/lib/docker.py +22 -23
- toil/lib/ec2.py +10 -6
- toil/lib/ec2nodes.py +106 -100
- toil/lib/encryption/_nacl.py +2 -1
- toil/lib/generatedEC2Lists.py +325 -18
- toil/lib/io.py +49 -2
- toil/lib/misc.py +1 -1
- toil/lib/resources.py +9 -2
- toil/lib/threading.py +101 -38
- toil/options/common.py +736 -0
- toil/options/cwl.py +336 -0
- toil/options/wdl.py +37 -0
- toil/provisioners/abstractProvisioner.py +9 -4
- toil/provisioners/aws/__init__.py +3 -6
- toil/provisioners/aws/awsProvisioner.py +6 -0
- toil/provisioners/clusterScaler.py +3 -2
- toil/provisioners/gceProvisioner.py +2 -2
- toil/realtimeLogger.py +2 -1
- toil/resource.py +24 -18
- toil/server/app.py +2 -3
- toil/server/cli/wes_cwl_runner.py +4 -4
- toil/server/utils.py +1 -1
- toil/server/wes/abstract_backend.py +3 -2
- toil/server/wes/amazon_wes_utils.py +5 -4
- toil/server/wes/tasks.py +2 -3
- toil/server/wes/toil_backend.py +2 -10
- toil/server/wsgi_app.py +2 -0
- toil/serviceManager.py +12 -10
- toil/statsAndLogging.py +41 -9
- toil/test/__init__.py +29 -54
- toil/test/batchSystems/batchSystemTest.py +11 -111
- toil/test/batchSystems/test_slurm.py +24 -8
- toil/test/cactus/__init__.py +0 -0
- toil/test/cactus/test_cactus_integration.py +58 -0
- toil/test/cwl/cwlTest.py +438 -223
- toil/test/cwl/glob_dir.cwl +15 -0
- toil/test/cwl/preemptible.cwl +21 -0
- toil/test/cwl/preemptible_expression.cwl +28 -0
- toil/test/cwl/revsort.cwl +1 -1
- toil/test/cwl/revsort2.cwl +1 -1
- toil/test/docs/scriptsTest.py +2 -3
- toil/test/jobStores/jobStoreTest.py +34 -21
- toil/test/lib/aws/test_iam.py +4 -14
- toil/test/lib/aws/test_utils.py +0 -3
- toil/test/lib/dockerTest.py +4 -4
- toil/test/lib/test_ec2.py +12 -17
- toil/test/mesos/helloWorld.py +4 -5
- toil/test/mesos/stress.py +1 -1
- toil/test/{wdl/conftest.py → options/__init__.py} +0 -10
- toil/test/options/options.py +37 -0
- toil/test/provisioners/aws/awsProvisionerTest.py +9 -5
- toil/test/provisioners/clusterScalerTest.py +6 -4
- toil/test/provisioners/clusterTest.py +23 -11
- toil/test/provisioners/gceProvisionerTest.py +0 -6
- toil/test/provisioners/restartScript.py +3 -2
- toil/test/server/serverTest.py +1 -1
- toil/test/sort/restart_sort.py +2 -1
- toil/test/sort/sort.py +2 -1
- toil/test/sort/sortTest.py +2 -13
- toil/test/src/autoDeploymentTest.py +45 -45
- toil/test/src/busTest.py +5 -5
- toil/test/src/checkpointTest.py +2 -2
- toil/test/src/deferredFunctionTest.py +1 -1
- toil/test/src/fileStoreTest.py +32 -16
- toil/test/src/helloWorldTest.py +1 -1
- toil/test/src/importExportFileTest.py +1 -1
- toil/test/src/jobDescriptionTest.py +2 -1
- toil/test/src/jobServiceTest.py +1 -1
- toil/test/src/jobTest.py +18 -18
- toil/test/src/miscTests.py +5 -3
- toil/test/src/promisedRequirementTest.py +3 -3
- toil/test/src/realtimeLoggerTest.py +1 -1
- toil/test/src/resourceTest.py +2 -2
- toil/test/src/restartDAGTest.py +1 -1
- toil/test/src/resumabilityTest.py +36 -2
- toil/test/src/retainTempDirTest.py +1 -1
- toil/test/src/systemTest.py +2 -2
- toil/test/src/toilContextManagerTest.py +2 -2
- toil/test/src/userDefinedJobArgTypeTest.py +1 -1
- toil/test/utils/toilDebugTest.py +98 -32
- toil/test/utils/toilKillTest.py +2 -2
- toil/test/utils/utilsTest.py +23 -3
- toil/test/wdl/wdltoil_test.py +223 -45
- toil/toilState.py +7 -6
- toil/utils/toilClean.py +1 -1
- toil/utils/toilConfig.py +36 -0
- toil/utils/toilDebugFile.py +60 -33
- toil/utils/toilDebugJob.py +39 -12
- toil/utils/toilDestroyCluster.py +1 -1
- toil/utils/toilKill.py +1 -1
- toil/utils/toilLaunchCluster.py +13 -2
- toil/utils/toilMain.py +3 -2
- toil/utils/toilRsyncCluster.py +1 -1
- toil/utils/toilSshCluster.py +1 -1
- toil/utils/toilStats.py +445 -305
- toil/utils/toilStatus.py +2 -5
- toil/version.py +10 -10
- toil/wdl/utils.py +2 -122
- toil/wdl/wdltoil.py +1257 -492
- toil/worker.py +55 -46
- toil-6.1.0.dist-info/METADATA +124 -0
- toil-6.1.0.dist-info/RECORD +241 -0
- {toil-5.12.0.dist-info → toil-6.1.0.dist-info}/WHEEL +1 -1
- {toil-5.12.0.dist-info → toil-6.1.0.dist-info}/entry_points.txt +0 -1
- toil/batchSystems/parasol.py +0 -379
- toil/batchSystems/tes.py +0 -459
- toil/test/batchSystems/parasolTestSupport.py +0 -117
- toil/test/wdl/builtinTest.py +0 -506
- toil/test/wdl/toilwdlTest.py +0 -522
- toil/wdl/toilwdl.py +0 -141
- toil/wdl/versions/dev.py +0 -107
- toil/wdl/versions/draft2.py +0 -980
- toil/wdl/versions/v1.py +0 -794
- toil/wdl/wdl_analysis.py +0 -116
- toil/wdl/wdl_functions.py +0 -997
- toil/wdl/wdl_synthesis.py +0 -1011
- toil/wdl/wdl_types.py +0 -243
- toil-5.12.0.dist-info/METADATA +0 -118
- toil-5.12.0.dist-info/RECORD +0 -244
- /toil/{wdl/versions → options}/__init__.py +0 -0
- {toil-5.12.0.dist-info → toil-6.1.0.dist-info}/LICENSE +0 -0
- {toil-5.12.0.dist-info → toil-6.1.0.dist-info}/top_level.txt +0 -0
toil/wdl/wdl_analysis.py
DELETED
|
@@ -1,116 +0,0 @@
|
|
|
1
|
-
# Copyright (C) 2018-2021 UCSC Computational Genomics Lab
|
|
2
|
-
#
|
|
3
|
-
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
-
# you may not use this file except in compliance with the License.
|
|
5
|
-
# You may obtain a copy of the License at
|
|
6
|
-
#
|
|
7
|
-
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
-
#
|
|
9
|
-
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
-
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
-
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
-
# See the License for the specific language governing permissions and
|
|
13
|
-
# limitations under the License.
|
|
14
|
-
import logging
|
|
15
|
-
from collections import OrderedDict
|
|
16
|
-
|
|
17
|
-
from toil.wdl.wdl_types import (WDLArrayType,
|
|
18
|
-
WDLBooleanType,
|
|
19
|
-
WDLFileType,
|
|
20
|
-
WDLFloatType,
|
|
21
|
-
WDLIntType,
|
|
22
|
-
WDLMapType,
|
|
23
|
-
WDLPairType,
|
|
24
|
-
WDLStringType)
|
|
25
|
-
|
|
26
|
-
logger = logging.getLogger(__name__)
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
class AnalyzeWDL:
|
|
30
|
-
"""
|
|
31
|
-
An interface to analyze a WDL file. Each version corresponds to a subclass that
|
|
32
|
-
restructures the WDL document into 2 intermediate data structures (python
|
|
33
|
-
dictionaries):
|
|
34
|
-
"workflows_dictionary": containing the parsed workflow information.
|
|
35
|
-
"tasks_dictionary": containing the parsed task information.
|
|
36
|
-
|
|
37
|
-
These are then fed into wdl_synthesis.py which uses them to write a native python
|
|
38
|
-
script for use with Toil.
|
|
39
|
-
|
|
40
|
-
Requires a WDL file. The WDL file contains ordered commands.
|
|
41
|
-
"""
|
|
42
|
-
def __init__(self, wdl_file: str):
|
|
43
|
-
self.wdl_file = wdl_file
|
|
44
|
-
|
|
45
|
-
# holds task skeletons from WDL task objects
|
|
46
|
-
self.tasks_dictionary = OrderedDict()
|
|
47
|
-
|
|
48
|
-
# holds workflow structure from WDL workflow objects
|
|
49
|
-
self.workflows_dictionary = OrderedDict()
|
|
50
|
-
|
|
51
|
-
# unique iterator to add to declaration names
|
|
52
|
-
self.declaration_number = 0
|
|
53
|
-
|
|
54
|
-
# unique iterator to add to call names
|
|
55
|
-
self.call_number = 0
|
|
56
|
-
|
|
57
|
-
# unique iterator to add to scatter names
|
|
58
|
-
self.scatter_number = 0
|
|
59
|
-
|
|
60
|
-
# unique iterator to add to if names
|
|
61
|
-
self.if_number = 0
|
|
62
|
-
|
|
63
|
-
@property
|
|
64
|
-
def version(self) -> str:
|
|
65
|
-
"""
|
|
66
|
-
Returns the version of the WDL document as a string.
|
|
67
|
-
"""
|
|
68
|
-
raise NotImplementedError
|
|
69
|
-
|
|
70
|
-
def analyze(self):
|
|
71
|
-
"""
|
|
72
|
-
Analyzes the WDL file passed into the constructor and generates the two
|
|
73
|
-
intermediate data structures: `self.workflows_dictionary` and
|
|
74
|
-
`self.tasks_dictionary`.
|
|
75
|
-
|
|
76
|
-
:return: Returns nothing.
|
|
77
|
-
"""
|
|
78
|
-
|
|
79
|
-
def write_AST(self, out_dir):
|
|
80
|
-
"""
|
|
81
|
-
Writes a file with the AST for a wdl file in the out_dir.
|
|
82
|
-
"""
|
|
83
|
-
|
|
84
|
-
primitive_types = {
|
|
85
|
-
'String': WDLStringType,
|
|
86
|
-
'Int': WDLIntType,
|
|
87
|
-
'Float': WDLFloatType,
|
|
88
|
-
'Boolean': WDLBooleanType,
|
|
89
|
-
'File': WDLFileType
|
|
90
|
-
}
|
|
91
|
-
|
|
92
|
-
compound_types = {
|
|
93
|
-
'Array': WDLArrayType,
|
|
94
|
-
'Pair': WDLPairType,
|
|
95
|
-
'Map': WDLMapType
|
|
96
|
-
}
|
|
97
|
-
|
|
98
|
-
def create_wdl_primitive_type(self, key: str, optional: bool = False):
|
|
99
|
-
"""
|
|
100
|
-
Returns an instance of WDLType.
|
|
101
|
-
"""
|
|
102
|
-
type_ = self.primitive_types.get(key)
|
|
103
|
-
if type_:
|
|
104
|
-
return type_(optional=optional)
|
|
105
|
-
else:
|
|
106
|
-
raise RuntimeError(f'Unsupported primitive type: {key}')
|
|
107
|
-
|
|
108
|
-
def create_wdl_compound_type(self, key: str, elements: list, optional: bool = False):
|
|
109
|
-
"""
|
|
110
|
-
Returns an instance of WDLCompoundType.
|
|
111
|
-
"""
|
|
112
|
-
type_ = self.compound_types.get(key)
|
|
113
|
-
if type_:
|
|
114
|
-
return type_(*elements, optional=optional)
|
|
115
|
-
else:
|
|
116
|
-
raise RuntimeError(f'Unsupported compound type: {key}')
|