toil 5.12.0__py3-none-any.whl → 6.1.0a1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- toil/__init__.py +18 -13
- toil/batchSystems/abstractBatchSystem.py +21 -10
- toil/batchSystems/abstractGridEngineBatchSystem.py +2 -2
- toil/batchSystems/awsBatch.py +14 -14
- toil/batchSystems/contained_executor.py +3 -3
- toil/batchSystems/htcondor.py +0 -1
- toil/batchSystems/kubernetes.py +34 -31
- toil/batchSystems/local_support.py +3 -1
- toil/batchSystems/mesos/batchSystem.py +7 -7
- toil/batchSystems/options.py +32 -83
- toil/batchSystems/registry.py +104 -23
- toil/batchSystems/singleMachine.py +16 -13
- toil/batchSystems/slurm.py +3 -3
- toil/batchSystems/torque.py +0 -1
- toil/bus.py +6 -8
- toil/common.py +532 -743
- toil/cwl/__init__.py +28 -32
- toil/cwl/cwltoil.py +523 -520
- toil/cwl/utils.py +55 -10
- toil/fileStores/__init__.py +2 -2
- toil/fileStores/abstractFileStore.py +36 -11
- toil/fileStores/cachingFileStore.py +607 -530
- toil/fileStores/nonCachingFileStore.py +43 -10
- toil/job.py +140 -75
- toil/jobStores/abstractJobStore.py +147 -79
- toil/jobStores/aws/jobStore.py +23 -9
- toil/jobStores/aws/utils.py +1 -2
- toil/jobStores/fileJobStore.py +117 -19
- toil/jobStores/googleJobStore.py +16 -7
- toil/jobStores/utils.py +5 -6
- toil/leader.py +71 -43
- toil/lib/accelerators.py +10 -5
- toil/lib/aws/__init__.py +3 -14
- toil/lib/aws/ami.py +22 -9
- toil/lib/aws/iam.py +21 -13
- toil/lib/aws/session.py +2 -16
- toil/lib/aws/utils.py +4 -5
- toil/lib/compatibility.py +1 -1
- toil/lib/conversions.py +7 -3
- toil/lib/docker.py +22 -23
- toil/lib/ec2.py +10 -6
- toil/lib/ec2nodes.py +106 -100
- toil/lib/encryption/_nacl.py +2 -1
- toil/lib/generatedEC2Lists.py +325 -18
- toil/lib/io.py +21 -0
- toil/lib/misc.py +1 -1
- toil/lib/resources.py +1 -1
- toil/lib/threading.py +74 -26
- toil/options/common.py +738 -0
- toil/options/cwl.py +336 -0
- toil/options/wdl.py +32 -0
- toil/provisioners/abstractProvisioner.py +1 -4
- toil/provisioners/aws/__init__.py +3 -6
- toil/provisioners/aws/awsProvisioner.py +6 -0
- toil/provisioners/clusterScaler.py +3 -2
- toil/provisioners/gceProvisioner.py +2 -2
- toil/realtimeLogger.py +2 -1
- toil/resource.py +24 -18
- toil/server/app.py +2 -3
- toil/server/cli/wes_cwl_runner.py +4 -4
- toil/server/utils.py +1 -1
- toil/server/wes/abstract_backend.py +3 -2
- toil/server/wes/amazon_wes_utils.py +5 -4
- toil/server/wes/tasks.py +2 -3
- toil/server/wes/toil_backend.py +2 -10
- toil/server/wsgi_app.py +2 -0
- toil/serviceManager.py +12 -10
- toil/statsAndLogging.py +5 -1
- toil/test/__init__.py +29 -54
- toil/test/batchSystems/batchSystemTest.py +11 -111
- toil/test/batchSystems/test_slurm.py +3 -2
- toil/test/cwl/cwlTest.py +213 -90
- toil/test/cwl/glob_dir.cwl +15 -0
- toil/test/cwl/preemptible.cwl +21 -0
- toil/test/cwl/preemptible_expression.cwl +28 -0
- toil/test/cwl/revsort.cwl +1 -1
- toil/test/cwl/revsort2.cwl +1 -1
- toil/test/docs/scriptsTest.py +0 -1
- toil/test/jobStores/jobStoreTest.py +27 -16
- toil/test/lib/aws/test_iam.py +4 -14
- toil/test/lib/aws/test_utils.py +0 -3
- toil/test/lib/dockerTest.py +4 -4
- toil/test/lib/test_ec2.py +11 -16
- toil/test/mesos/helloWorld.py +4 -5
- toil/test/mesos/stress.py +1 -1
- toil/test/provisioners/aws/awsProvisionerTest.py +9 -5
- toil/test/provisioners/clusterScalerTest.py +6 -4
- toil/test/provisioners/clusterTest.py +14 -3
- toil/test/provisioners/gceProvisionerTest.py +0 -6
- toil/test/provisioners/restartScript.py +3 -2
- toil/test/server/serverTest.py +1 -1
- toil/test/sort/restart_sort.py +2 -1
- toil/test/sort/sort.py +2 -1
- toil/test/sort/sortTest.py +2 -13
- toil/test/src/autoDeploymentTest.py +45 -45
- toil/test/src/busTest.py +5 -5
- toil/test/src/checkpointTest.py +2 -2
- toil/test/src/deferredFunctionTest.py +1 -1
- toil/test/src/fileStoreTest.py +32 -16
- toil/test/src/helloWorldTest.py +1 -1
- toil/test/src/importExportFileTest.py +1 -1
- toil/test/src/jobDescriptionTest.py +2 -1
- toil/test/src/jobServiceTest.py +1 -1
- toil/test/src/jobTest.py +18 -18
- toil/test/src/miscTests.py +5 -3
- toil/test/src/promisedRequirementTest.py +3 -3
- toil/test/src/realtimeLoggerTest.py +1 -1
- toil/test/src/resourceTest.py +2 -2
- toil/test/src/restartDAGTest.py +1 -1
- toil/test/src/resumabilityTest.py +36 -2
- toil/test/src/retainTempDirTest.py +1 -1
- toil/test/src/systemTest.py +2 -2
- toil/test/src/toilContextManagerTest.py +2 -2
- toil/test/src/userDefinedJobArgTypeTest.py +1 -1
- toil/test/utils/toilDebugTest.py +98 -32
- toil/test/utils/toilKillTest.py +2 -2
- toil/test/utils/utilsTest.py +20 -0
- toil/test/wdl/wdltoil_test.py +148 -45
- toil/toilState.py +7 -6
- toil/utils/toilClean.py +1 -1
- toil/utils/toilConfig.py +36 -0
- toil/utils/toilDebugFile.py +60 -33
- toil/utils/toilDebugJob.py +39 -12
- toil/utils/toilDestroyCluster.py +1 -1
- toil/utils/toilKill.py +1 -1
- toil/utils/toilLaunchCluster.py +13 -2
- toil/utils/toilMain.py +3 -2
- toil/utils/toilRsyncCluster.py +1 -1
- toil/utils/toilSshCluster.py +1 -1
- toil/utils/toilStats.py +240 -143
- toil/utils/toilStatus.py +1 -4
- toil/version.py +11 -11
- toil/wdl/utils.py +2 -122
- toil/wdl/wdltoil.py +999 -386
- toil/worker.py +25 -31
- {toil-5.12.0.dist-info → toil-6.1.0a1.dist-info}/METADATA +60 -53
- toil-6.1.0a1.dist-info/RECORD +237 -0
- {toil-5.12.0.dist-info → toil-6.1.0a1.dist-info}/WHEEL +1 -1
- {toil-5.12.0.dist-info → toil-6.1.0a1.dist-info}/entry_points.txt +0 -1
- toil/batchSystems/parasol.py +0 -379
- toil/batchSystems/tes.py +0 -459
- toil/test/batchSystems/parasolTestSupport.py +0 -117
- toil/test/wdl/builtinTest.py +0 -506
- toil/test/wdl/conftest.py +0 -23
- toil/test/wdl/toilwdlTest.py +0 -522
- toil/wdl/toilwdl.py +0 -141
- toil/wdl/versions/dev.py +0 -107
- toil/wdl/versions/draft2.py +0 -980
- toil/wdl/versions/v1.py +0 -794
- toil/wdl/wdl_analysis.py +0 -116
- toil/wdl/wdl_functions.py +0 -997
- toil/wdl/wdl_synthesis.py +0 -1011
- toil/wdl/wdl_types.py +0 -243
- toil-5.12.0.dist-info/RECORD +0 -244
- /toil/{wdl/versions → options}/__init__.py +0 -0
- {toil-5.12.0.dist-info → toil-6.1.0a1.dist-info}/LICENSE +0 -0
- {toil-5.12.0.dist-info → toil-6.1.0a1.dist-info}/top_level.txt +0 -0
toil/utils/toilStatus.py
CHANGED
|
@@ -12,12 +12,9 @@
|
|
|
12
12
|
# See the License for the specific language governing permissions and
|
|
13
13
|
# limitations under the License.
|
|
14
14
|
"""Tool for reporting on job status."""
|
|
15
|
-
from collections import defaultdict
|
|
16
15
|
import logging
|
|
17
16
|
import os
|
|
18
17
|
import sys
|
|
19
|
-
from functools import reduce
|
|
20
|
-
import json
|
|
21
18
|
from typing import Any, Dict, List, Optional, Set
|
|
22
19
|
|
|
23
20
|
from toil.bus import replay_message_bus
|
|
@@ -320,7 +317,7 @@ class ToilStatus:
|
|
|
320
317
|
|
|
321
318
|
def main() -> None:
|
|
322
319
|
"""Reports the state of a Toil workflow."""
|
|
323
|
-
parser = parser_with_common_options()
|
|
320
|
+
parser = parser_with_common_options(prog="toil status")
|
|
324
321
|
parser.add_argument("--failIfNotComplete", action="store_true",
|
|
325
322
|
help="Return exit value of 1 if toil jobs not all completed. default=%(default)s",
|
|
326
323
|
default=False)
|
toil/version.py
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
|
-
baseVersion = '
|
|
1
|
+
baseVersion = '6.1.0a1'
|
|
2
2
|
cgcloudVersion = '1.6.0a1.dev393'
|
|
3
|
-
version = '
|
|
4
|
-
cacheTag = 'cache-local-py3.
|
|
5
|
-
mainCacheTag = 'cache-master-py3.
|
|
6
|
-
distVersion = '
|
|
7
|
-
exactPython = 'python3.
|
|
8
|
-
python = 'python3.
|
|
9
|
-
dockerTag = '
|
|
10
|
-
currentCommit = '
|
|
3
|
+
version = '6.1.0a1-04b966f2417ebf2752e6f216e7fbda3ce20b4a37-dirty'
|
|
4
|
+
cacheTag = 'cache-local-py3.11'
|
|
5
|
+
mainCacheTag = 'cache-master-py3.11'
|
|
6
|
+
distVersion = '6.1.0a1'
|
|
7
|
+
exactPython = 'python3.11'
|
|
8
|
+
python = 'python3.11'
|
|
9
|
+
dockerTag = '6.1.0a1-04b966f2417ebf2752e6f216e7fbda3ce20b4a37-dirty-py3.11'
|
|
10
|
+
currentCommit = '04b966f2417ebf2752e6f216e7fbda3ce20b4a37'
|
|
11
11
|
dockerRegistry = 'quay.io/ucsc_cgl'
|
|
12
12
|
dockerName = 'toil'
|
|
13
|
-
dirty =
|
|
14
|
-
cwltool_version = '3.1.
|
|
13
|
+
dirty = True
|
|
14
|
+
cwltool_version = '3.1.20240112164112'
|
toil/wdl/utils.py
CHANGED
|
@@ -11,12 +11,10 @@
|
|
|
11
11
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
12
|
# See the License for the specific language governing permissions and
|
|
13
13
|
# limitations under the License.
|
|
14
|
-
import
|
|
14
|
+
from typing import Iterable
|
|
15
15
|
|
|
16
|
-
from toil.wdl.wdl_analysis import AnalyzeWDL
|
|
17
16
|
|
|
18
|
-
|
|
19
|
-
def get_version(iterable) -> str:
|
|
17
|
+
def get_version(iterable: Iterable[str]) -> str:
|
|
20
18
|
"""
|
|
21
19
|
Get the version of the WDL document.
|
|
22
20
|
|
|
@@ -35,121 +33,3 @@ def get_version(iterable) -> str:
|
|
|
35
33
|
break
|
|
36
34
|
# only draft-2 doesn't contain the version declaration
|
|
37
35
|
return 'draft-2'
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
def get_analyzer(wdl_file: str) -> AnalyzeWDL:
|
|
41
|
-
"""
|
|
42
|
-
Creates an instance of an AnalyzeWDL implementation based on the version.
|
|
43
|
-
|
|
44
|
-
:param wdl_file: The path to the WDL file.
|
|
45
|
-
"""
|
|
46
|
-
with open(wdl_file) as f:
|
|
47
|
-
version = get_version(f)
|
|
48
|
-
|
|
49
|
-
if version == 'draft-2':
|
|
50
|
-
from toil.wdl.versions.draft2 import AnalyzeDraft2WDL
|
|
51
|
-
return AnalyzeDraft2WDL(wdl_file)
|
|
52
|
-
elif version == '1.0':
|
|
53
|
-
from toil.wdl.versions.v1 import AnalyzeV1WDL
|
|
54
|
-
return AnalyzeV1WDL(wdl_file)
|
|
55
|
-
elif version == 'development':
|
|
56
|
-
from toil.wdl.versions.dev import AnalyzeDevelopmentWDL
|
|
57
|
-
return AnalyzeDevelopmentWDL(wdl_file)
|
|
58
|
-
else:
|
|
59
|
-
raise RuntimeError(f"Unsupported WDL version: '{version}'.")
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
def dict_from_JSON(JSON_file: str) -> dict:
|
|
63
|
-
"""
|
|
64
|
-
Takes a WDL-mapped json file and creates a dict containing the bindings.
|
|
65
|
-
|
|
66
|
-
:param JSON_file: A required JSON file containing WDL variable bindings.
|
|
67
|
-
"""
|
|
68
|
-
json_dict = {}
|
|
69
|
-
|
|
70
|
-
# TODO: Add context support for variables within multiple wdl files
|
|
71
|
-
|
|
72
|
-
with open(JSON_file) as data_file:
|
|
73
|
-
data = json.load(data_file)
|
|
74
|
-
for d in data:
|
|
75
|
-
if isinstance(data[d], str):
|
|
76
|
-
json_dict[d] = f'"{data[d]}"'
|
|
77
|
-
else:
|
|
78
|
-
json_dict[d] = data[d]
|
|
79
|
-
return json_dict
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
def write_mappings(parser: AnalyzeWDL, filename: str = 'mappings.out') -> None:
|
|
83
|
-
"""
|
|
84
|
-
Takes an AnalyzeWDL instance and writes the final task dict and workflow
|
|
85
|
-
dict to the given file.
|
|
86
|
-
|
|
87
|
-
:param parser: An AnalyzeWDL instance.
|
|
88
|
-
:param filename: The name of a file to write to.
|
|
89
|
-
"""
|
|
90
|
-
from collections import OrderedDict
|
|
91
|
-
|
|
92
|
-
class Formatter:
|
|
93
|
-
def __init__(self):
|
|
94
|
-
self.types = {}
|
|
95
|
-
self.htchar = '\t'
|
|
96
|
-
self.lfchar = '\n'
|
|
97
|
-
self.indent = 0
|
|
98
|
-
self.set_formater(object, self.__class__.format_object)
|
|
99
|
-
self.set_formater(dict, self.__class__.format_dict)
|
|
100
|
-
self.set_formater(list, self.__class__.format_list)
|
|
101
|
-
self.set_formater(tuple, self.__class__.format_tuple)
|
|
102
|
-
|
|
103
|
-
def set_formater(self, obj, callback):
|
|
104
|
-
self.types[obj] = callback
|
|
105
|
-
|
|
106
|
-
def __call__(self, value, **args):
|
|
107
|
-
for key in args:
|
|
108
|
-
setattr(self, key, args[key])
|
|
109
|
-
formater = self.types[type(value) if type(value) in self.types else object]
|
|
110
|
-
return formater(self, value, self.indent)
|
|
111
|
-
|
|
112
|
-
def format_object(self, value, indent):
|
|
113
|
-
return repr(value)
|
|
114
|
-
|
|
115
|
-
def format_dict(self, value, indent):
|
|
116
|
-
items = [
|
|
117
|
-
self.lfchar + self.htchar * (indent + 1) + repr(key) + ': ' +
|
|
118
|
-
(self.types[type(value[key]) if type(value[key]) in self.types else object])(self, value[key],
|
|
119
|
-
indent + 1)
|
|
120
|
-
for key in value]
|
|
121
|
-
return '{%s}' % (','.join(items) + self.lfchar + self.htchar * indent)
|
|
122
|
-
|
|
123
|
-
def format_list(self, value, indent):
|
|
124
|
-
items = [
|
|
125
|
-
self.lfchar + self.htchar * (indent + 1) + (
|
|
126
|
-
self.types[type(item) if type(item) in self.types else object])(self, item, indent + 1)
|
|
127
|
-
for item in value]
|
|
128
|
-
return '[%s]' % (','.join(items) + self.lfchar + self.htchar * indent)
|
|
129
|
-
|
|
130
|
-
def format_tuple(self, value, indent):
|
|
131
|
-
items = [
|
|
132
|
-
self.lfchar + self.htchar * (indent + 1) + (
|
|
133
|
-
self.types[type(item) if type(item) in self.types else object])(self, item, indent + 1)
|
|
134
|
-
for item in value]
|
|
135
|
-
return '(%s)' % (','.join(items) + self.lfchar + self.htchar * indent)
|
|
136
|
-
|
|
137
|
-
pretty = Formatter()
|
|
138
|
-
|
|
139
|
-
def format_ordereddict(self, value, indent):
|
|
140
|
-
items = [
|
|
141
|
-
self.lfchar + self.htchar * (indent + 1) +
|
|
142
|
-
"(" + repr(key) + ', ' + (self.types[
|
|
143
|
-
type(value[key]) if type(value[key]) in self.types else object
|
|
144
|
-
])(self, value[key], indent + 1) + ")"
|
|
145
|
-
for key in value
|
|
146
|
-
]
|
|
147
|
-
return 'OrderedDict([%s])' % (','.join(items) +
|
|
148
|
-
self.lfchar + self.htchar * indent)
|
|
149
|
-
|
|
150
|
-
pretty.set_formater(OrderedDict, format_ordereddict)
|
|
151
|
-
|
|
152
|
-
with open(filename, 'w') as f:
|
|
153
|
-
f.write(pretty(parser.tasks_dictionary))
|
|
154
|
-
f.write('\n\n\n\n\n\n')
|
|
155
|
-
f.write(pretty(parser.workflows_dictionary))
|