toil 5.12.0__py3-none-any.whl → 6.1.0a1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- toil/__init__.py +18 -13
- toil/batchSystems/abstractBatchSystem.py +21 -10
- toil/batchSystems/abstractGridEngineBatchSystem.py +2 -2
- toil/batchSystems/awsBatch.py +14 -14
- toil/batchSystems/contained_executor.py +3 -3
- toil/batchSystems/htcondor.py +0 -1
- toil/batchSystems/kubernetes.py +34 -31
- toil/batchSystems/local_support.py +3 -1
- toil/batchSystems/mesos/batchSystem.py +7 -7
- toil/batchSystems/options.py +32 -83
- toil/batchSystems/registry.py +104 -23
- toil/batchSystems/singleMachine.py +16 -13
- toil/batchSystems/slurm.py +3 -3
- toil/batchSystems/torque.py +0 -1
- toil/bus.py +6 -8
- toil/common.py +532 -743
- toil/cwl/__init__.py +28 -32
- toil/cwl/cwltoil.py +523 -520
- toil/cwl/utils.py +55 -10
- toil/fileStores/__init__.py +2 -2
- toil/fileStores/abstractFileStore.py +36 -11
- toil/fileStores/cachingFileStore.py +607 -530
- toil/fileStores/nonCachingFileStore.py +43 -10
- toil/job.py +140 -75
- toil/jobStores/abstractJobStore.py +147 -79
- toil/jobStores/aws/jobStore.py +23 -9
- toil/jobStores/aws/utils.py +1 -2
- toil/jobStores/fileJobStore.py +117 -19
- toil/jobStores/googleJobStore.py +16 -7
- toil/jobStores/utils.py +5 -6
- toil/leader.py +71 -43
- toil/lib/accelerators.py +10 -5
- toil/lib/aws/__init__.py +3 -14
- toil/lib/aws/ami.py +22 -9
- toil/lib/aws/iam.py +21 -13
- toil/lib/aws/session.py +2 -16
- toil/lib/aws/utils.py +4 -5
- toil/lib/compatibility.py +1 -1
- toil/lib/conversions.py +7 -3
- toil/lib/docker.py +22 -23
- toil/lib/ec2.py +10 -6
- toil/lib/ec2nodes.py +106 -100
- toil/lib/encryption/_nacl.py +2 -1
- toil/lib/generatedEC2Lists.py +325 -18
- toil/lib/io.py +21 -0
- toil/lib/misc.py +1 -1
- toil/lib/resources.py +1 -1
- toil/lib/threading.py +74 -26
- toil/options/common.py +738 -0
- toil/options/cwl.py +336 -0
- toil/options/wdl.py +32 -0
- toil/provisioners/abstractProvisioner.py +1 -4
- toil/provisioners/aws/__init__.py +3 -6
- toil/provisioners/aws/awsProvisioner.py +6 -0
- toil/provisioners/clusterScaler.py +3 -2
- toil/provisioners/gceProvisioner.py +2 -2
- toil/realtimeLogger.py +2 -1
- toil/resource.py +24 -18
- toil/server/app.py +2 -3
- toil/server/cli/wes_cwl_runner.py +4 -4
- toil/server/utils.py +1 -1
- toil/server/wes/abstract_backend.py +3 -2
- toil/server/wes/amazon_wes_utils.py +5 -4
- toil/server/wes/tasks.py +2 -3
- toil/server/wes/toil_backend.py +2 -10
- toil/server/wsgi_app.py +2 -0
- toil/serviceManager.py +12 -10
- toil/statsAndLogging.py +5 -1
- toil/test/__init__.py +29 -54
- toil/test/batchSystems/batchSystemTest.py +11 -111
- toil/test/batchSystems/test_slurm.py +3 -2
- toil/test/cwl/cwlTest.py +213 -90
- toil/test/cwl/glob_dir.cwl +15 -0
- toil/test/cwl/preemptible.cwl +21 -0
- toil/test/cwl/preemptible_expression.cwl +28 -0
- toil/test/cwl/revsort.cwl +1 -1
- toil/test/cwl/revsort2.cwl +1 -1
- toil/test/docs/scriptsTest.py +0 -1
- toil/test/jobStores/jobStoreTest.py +27 -16
- toil/test/lib/aws/test_iam.py +4 -14
- toil/test/lib/aws/test_utils.py +0 -3
- toil/test/lib/dockerTest.py +4 -4
- toil/test/lib/test_ec2.py +11 -16
- toil/test/mesos/helloWorld.py +4 -5
- toil/test/mesos/stress.py +1 -1
- toil/test/provisioners/aws/awsProvisionerTest.py +9 -5
- toil/test/provisioners/clusterScalerTest.py +6 -4
- toil/test/provisioners/clusterTest.py +14 -3
- toil/test/provisioners/gceProvisionerTest.py +0 -6
- toil/test/provisioners/restartScript.py +3 -2
- toil/test/server/serverTest.py +1 -1
- toil/test/sort/restart_sort.py +2 -1
- toil/test/sort/sort.py +2 -1
- toil/test/sort/sortTest.py +2 -13
- toil/test/src/autoDeploymentTest.py +45 -45
- toil/test/src/busTest.py +5 -5
- toil/test/src/checkpointTest.py +2 -2
- toil/test/src/deferredFunctionTest.py +1 -1
- toil/test/src/fileStoreTest.py +32 -16
- toil/test/src/helloWorldTest.py +1 -1
- toil/test/src/importExportFileTest.py +1 -1
- toil/test/src/jobDescriptionTest.py +2 -1
- toil/test/src/jobServiceTest.py +1 -1
- toil/test/src/jobTest.py +18 -18
- toil/test/src/miscTests.py +5 -3
- toil/test/src/promisedRequirementTest.py +3 -3
- toil/test/src/realtimeLoggerTest.py +1 -1
- toil/test/src/resourceTest.py +2 -2
- toil/test/src/restartDAGTest.py +1 -1
- toil/test/src/resumabilityTest.py +36 -2
- toil/test/src/retainTempDirTest.py +1 -1
- toil/test/src/systemTest.py +2 -2
- toil/test/src/toilContextManagerTest.py +2 -2
- toil/test/src/userDefinedJobArgTypeTest.py +1 -1
- toil/test/utils/toilDebugTest.py +98 -32
- toil/test/utils/toilKillTest.py +2 -2
- toil/test/utils/utilsTest.py +20 -0
- toil/test/wdl/wdltoil_test.py +148 -45
- toil/toilState.py +7 -6
- toil/utils/toilClean.py +1 -1
- toil/utils/toilConfig.py +36 -0
- toil/utils/toilDebugFile.py +60 -33
- toil/utils/toilDebugJob.py +39 -12
- toil/utils/toilDestroyCluster.py +1 -1
- toil/utils/toilKill.py +1 -1
- toil/utils/toilLaunchCluster.py +13 -2
- toil/utils/toilMain.py +3 -2
- toil/utils/toilRsyncCluster.py +1 -1
- toil/utils/toilSshCluster.py +1 -1
- toil/utils/toilStats.py +240 -143
- toil/utils/toilStatus.py +1 -4
- toil/version.py +11 -11
- toil/wdl/utils.py +2 -122
- toil/wdl/wdltoil.py +999 -386
- toil/worker.py +25 -31
- {toil-5.12.0.dist-info → toil-6.1.0a1.dist-info}/METADATA +60 -53
- toil-6.1.0a1.dist-info/RECORD +237 -0
- {toil-5.12.0.dist-info → toil-6.1.0a1.dist-info}/WHEEL +1 -1
- {toil-5.12.0.dist-info → toil-6.1.0a1.dist-info}/entry_points.txt +0 -1
- toil/batchSystems/parasol.py +0 -379
- toil/batchSystems/tes.py +0 -459
- toil/test/batchSystems/parasolTestSupport.py +0 -117
- toil/test/wdl/builtinTest.py +0 -506
- toil/test/wdl/conftest.py +0 -23
- toil/test/wdl/toilwdlTest.py +0 -522
- toil/wdl/toilwdl.py +0 -141
- toil/wdl/versions/dev.py +0 -107
- toil/wdl/versions/draft2.py +0 -980
- toil/wdl/versions/v1.py +0 -794
- toil/wdl/wdl_analysis.py +0 -116
- toil/wdl/wdl_functions.py +0 -997
- toil/wdl/wdl_synthesis.py +0 -1011
- toil/wdl/wdl_types.py +0 -243
- toil-5.12.0.dist-info/RECORD +0 -244
- /toil/{wdl/versions → options}/__init__.py +0 -0
- {toil-5.12.0.dist-info → toil-6.1.0a1.dist-info}/LICENSE +0 -0
- {toil-5.12.0.dist-info → toil-6.1.0a1.dist-info}/top_level.txt +0 -0
toil/options/cwl.py
ADDED
|
@@ -0,0 +1,336 @@
|
|
|
1
|
+
import os
|
|
2
|
+
from argparse import ArgumentParser
|
|
3
|
+
|
|
4
|
+
from configargparse import SUPPRESS
|
|
5
|
+
|
|
6
|
+
from toil.version import baseVersion
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def add_cwl_options(parser: ArgumentParser, suppress: bool = True) -> None:
|
|
10
|
+
"""
|
|
11
|
+
Add CWL options to the parser. This only adds nonpositional CWL arguments.
|
|
12
|
+
|
|
13
|
+
:param parser: Parser to add options to
|
|
14
|
+
:param suppress: Suppress help output
|
|
15
|
+
:return: None
|
|
16
|
+
"""
|
|
17
|
+
suppress_help = SUPPRESS if suppress else None
|
|
18
|
+
parser.add_argument("--not-strict", action="store_true", help=suppress_help)
|
|
19
|
+
parser.add_argument(
|
|
20
|
+
"--enable-dev",
|
|
21
|
+
action="store_true",
|
|
22
|
+
help=suppress_help or suppress_help or "Enable loading and running development versions of CWL",
|
|
23
|
+
)
|
|
24
|
+
parser.add_argument(
|
|
25
|
+
"--enable-ext",
|
|
26
|
+
action="store_true",
|
|
27
|
+
help=suppress_help or "Enable loading and running 'cwltool:' extensions to the CWL standards.",
|
|
28
|
+
default=False,
|
|
29
|
+
)
|
|
30
|
+
parser.add_argument("--quiet", dest="quiet", action="store_true", default=False, help=suppress_help)
|
|
31
|
+
parser.add_argument("--basedir", type=str, help=suppress_help) # TODO: Might be hard-coded?
|
|
32
|
+
parser.add_argument("--outdir", type=str, default=None, help=suppress_help)
|
|
33
|
+
parser.add_argument("--version", action="version", version=baseVersion,
|
|
34
|
+
help=suppress_help or "show program's version number and exit")
|
|
35
|
+
parser.add_argument(
|
|
36
|
+
"--log-dir",
|
|
37
|
+
type=str,
|
|
38
|
+
default="",
|
|
39
|
+
help=suppress_help or "Log your tools stdout/stderr to this location outside of container",
|
|
40
|
+
)
|
|
41
|
+
# this is as a result of suppressed help statements not working well with mutually_exclusive_groups, which will
|
|
42
|
+
# cause an assertion error
|
|
43
|
+
# https://github.com/python/cpython/issues/62090
|
|
44
|
+
dockergroup = parser.add_mutually_exclusive_group() if not suppress_help else parser.add_argument_group()
|
|
45
|
+
dockergroup.add_argument(
|
|
46
|
+
"--user-space-docker-cmd",
|
|
47
|
+
help=suppress_help or "(Linux/OS X only) Specify a user space docker command (like "
|
|
48
|
+
"udocker or dx-docker) that will be used to call 'pull' and 'run'",
|
|
49
|
+
)
|
|
50
|
+
dockergroup.add_argument(
|
|
51
|
+
"--singularity",
|
|
52
|
+
action="store_true",
|
|
53
|
+
default=False,
|
|
54
|
+
help=suppress_help or "Use Singularity runtime for running containers. "
|
|
55
|
+
"Requires Singularity v2.6.1+ and Linux with kernel version v3.18+ or "
|
|
56
|
+
"with overlayfs support backported.",
|
|
57
|
+
)
|
|
58
|
+
dockergroup.add_argument(
|
|
59
|
+
"--podman",
|
|
60
|
+
action="store_true",
|
|
61
|
+
default=False,
|
|
62
|
+
help=suppress_help or "Use Podman runtime for running containers. ",
|
|
63
|
+
)
|
|
64
|
+
dockergroup.add_argument(
|
|
65
|
+
"--no-container",
|
|
66
|
+
action="store_true",
|
|
67
|
+
help=suppress_help or "Do not execute jobs in a "
|
|
68
|
+
"Docker container, even when `DockerRequirement` "
|
|
69
|
+
"is specified under `hints`.",
|
|
70
|
+
)
|
|
71
|
+
dockergroup.add_argument(
|
|
72
|
+
"--leave-container",
|
|
73
|
+
action="store_false",
|
|
74
|
+
default=True,
|
|
75
|
+
help=suppress_help or "Do not delete Docker container used by jobs after they exit",
|
|
76
|
+
dest="rm_container",
|
|
77
|
+
)
|
|
78
|
+
extra_dockergroup = parser.add_argument_group()
|
|
79
|
+
extra_dockergroup.add_argument(
|
|
80
|
+
"--custom-net",
|
|
81
|
+
help=suppress_help or "Specify docker network name to pass to docker run command",
|
|
82
|
+
)
|
|
83
|
+
cidgroup = parser.add_argument_group(
|
|
84
|
+
"Options for recording the Docker container identifier into a file."
|
|
85
|
+
)
|
|
86
|
+
cidgroup.add_argument(
|
|
87
|
+
# Disabled as containerid is now saved by default
|
|
88
|
+
"--record-container-id",
|
|
89
|
+
action="store_true",
|
|
90
|
+
default=False,
|
|
91
|
+
help=suppress_help or SUPPRESS,
|
|
92
|
+
dest="record_container_id",
|
|
93
|
+
)
|
|
94
|
+
|
|
95
|
+
cidgroup.add_argument(
|
|
96
|
+
"--cidfile-dir",
|
|
97
|
+
type=str,
|
|
98
|
+
help=suppress_help or "Store the Docker container ID into a file in the specified directory.",
|
|
99
|
+
default=None,
|
|
100
|
+
dest="cidfile_dir",
|
|
101
|
+
)
|
|
102
|
+
|
|
103
|
+
cidgroup.add_argument(
|
|
104
|
+
"--cidfile-prefix",
|
|
105
|
+
type=str,
|
|
106
|
+
help=suppress_help or "Specify a prefix to the container ID filename. "
|
|
107
|
+
"Final file name will be followed by a timestamp. "
|
|
108
|
+
"The default is no prefix.",
|
|
109
|
+
default=None,
|
|
110
|
+
dest="cidfile_prefix",
|
|
111
|
+
)
|
|
112
|
+
|
|
113
|
+
parser.add_argument(
|
|
114
|
+
"--preserve-environment",
|
|
115
|
+
type=str,
|
|
116
|
+
nargs="+",
|
|
117
|
+
help=suppress_help or "Preserve specified environment variables when running"
|
|
118
|
+
" CommandLineTools",
|
|
119
|
+
metavar=("VAR1 VAR2"),
|
|
120
|
+
default=("PATH",),
|
|
121
|
+
dest="preserve_environment",
|
|
122
|
+
)
|
|
123
|
+
parser.add_argument(
|
|
124
|
+
"--preserve-entire-environment",
|
|
125
|
+
action="store_true",
|
|
126
|
+
help=suppress_help or "Preserve all environment variable when running CommandLineTools.",
|
|
127
|
+
default=False,
|
|
128
|
+
dest="preserve_entire_environment",
|
|
129
|
+
)
|
|
130
|
+
parser.add_argument(
|
|
131
|
+
"--destBucket",
|
|
132
|
+
type=str,
|
|
133
|
+
help=suppress_help or "Specify a cloud bucket endpoint for output files.",
|
|
134
|
+
)
|
|
135
|
+
parser.add_argument("--beta-dependency-resolvers-configuration", default=None, help=suppress_help)
|
|
136
|
+
parser.add_argument("--beta-dependencies-directory", default=None, help=suppress_help)
|
|
137
|
+
parser.add_argument("--beta-use-biocontainers", default=None, action="store_true", help=suppress_help)
|
|
138
|
+
parser.add_argument("--beta-conda-dependencies", default=None, action="store_true", help=suppress_help)
|
|
139
|
+
parser.add_argument(
|
|
140
|
+
"--tmpdir-prefix",
|
|
141
|
+
type=str,
|
|
142
|
+
help=suppress_help or "Path prefix for temporary directories",
|
|
143
|
+
default=None,
|
|
144
|
+
)
|
|
145
|
+
parser.add_argument(
|
|
146
|
+
"--tmp-outdir-prefix",
|
|
147
|
+
type=str,
|
|
148
|
+
help=suppress_help or "Path prefix for intermediate output directories",
|
|
149
|
+
default=None,
|
|
150
|
+
)
|
|
151
|
+
parser.add_argument(
|
|
152
|
+
"--force-docker-pull",
|
|
153
|
+
action="store_true",
|
|
154
|
+
default=False,
|
|
155
|
+
dest="force_docker_pull",
|
|
156
|
+
help=suppress_help or "Pull latest docker image even if it is locally present",
|
|
157
|
+
)
|
|
158
|
+
parser.add_argument(
|
|
159
|
+
"--no-match-user",
|
|
160
|
+
action="store_true",
|
|
161
|
+
default=False,
|
|
162
|
+
help=suppress_help or "Disable passing the current uid to `docker run --user`",
|
|
163
|
+
)
|
|
164
|
+
parser.add_argument(
|
|
165
|
+
"--no-read-only",
|
|
166
|
+
action="store_true",
|
|
167
|
+
default=False,
|
|
168
|
+
help=suppress_help or "Do not set root directory in the container as read-only",
|
|
169
|
+
)
|
|
170
|
+
parser.add_argument(
|
|
171
|
+
"--strict-memory-limit",
|
|
172
|
+
action="store_true",
|
|
173
|
+
help=suppress_help or "When running with "
|
|
174
|
+
"software containers and the Docker engine, pass either the "
|
|
175
|
+
"calculated memory allocation from ResourceRequirements or the "
|
|
176
|
+
"default of 1 gigabyte to Docker's --memory option.",
|
|
177
|
+
)
|
|
178
|
+
parser.add_argument(
|
|
179
|
+
"--strict-cpu-limit",
|
|
180
|
+
action="store_true",
|
|
181
|
+
help=suppress_help or "When running with "
|
|
182
|
+
"software containers and the Docker engine, pass either the "
|
|
183
|
+
"calculated cpu allocation from ResourceRequirements or the "
|
|
184
|
+
"default of 1 core to Docker's --cpu option. "
|
|
185
|
+
"Requires docker version >= v1.13.",
|
|
186
|
+
)
|
|
187
|
+
parser.add_argument(
|
|
188
|
+
"--relax-path-checks",
|
|
189
|
+
action="store_true",
|
|
190
|
+
default=False,
|
|
191
|
+
help=suppress_help or "Relax requirements on path names to permit "
|
|
192
|
+
"spaces and hash characters.",
|
|
193
|
+
dest="relax_path_checks",
|
|
194
|
+
)
|
|
195
|
+
parser.add_argument(
|
|
196
|
+
"--default-container",
|
|
197
|
+
help=suppress_help or "Specify a default docker container that will be "
|
|
198
|
+
"used if the workflow fails to specify one.",
|
|
199
|
+
)
|
|
200
|
+
parser.add_argument(
|
|
201
|
+
"--disable-validate",
|
|
202
|
+
dest="do_validate",
|
|
203
|
+
action="store_false",
|
|
204
|
+
default=True,
|
|
205
|
+
help=suppress_help or SUPPRESS,
|
|
206
|
+
)
|
|
207
|
+
parser.add_argument(
|
|
208
|
+
"--fast-parser",
|
|
209
|
+
dest="fast_parser",
|
|
210
|
+
action="store_true",
|
|
211
|
+
default=False,
|
|
212
|
+
help=suppress_help or SUPPRESS,
|
|
213
|
+
)
|
|
214
|
+
# same workaround as dockergroup
|
|
215
|
+
checkgroup = parser.add_mutually_exclusive_group() if not suppress_help else parser.add_argument_group()
|
|
216
|
+
checkgroup.add_argument(
|
|
217
|
+
"--compute-checksum",
|
|
218
|
+
action="store_true",
|
|
219
|
+
default=True,
|
|
220
|
+
help=suppress_help or "Compute checksum of contents while collecting outputs",
|
|
221
|
+
dest="compute_checksum",
|
|
222
|
+
)
|
|
223
|
+
checkgroup.add_argument(
|
|
224
|
+
"--no-compute-checksum",
|
|
225
|
+
action="store_false",
|
|
226
|
+
help=suppress_help or "Do not compute checksum of contents while collecting outputs",
|
|
227
|
+
dest="compute_checksum",
|
|
228
|
+
)
|
|
229
|
+
|
|
230
|
+
parser.add_argument(
|
|
231
|
+
"--eval-timeout",
|
|
232
|
+
help=suppress_help or "Time to wait for a Javascript expression to evaluate before giving "
|
|
233
|
+
"an error, default 20s.",
|
|
234
|
+
type=float,
|
|
235
|
+
default=20,
|
|
236
|
+
)
|
|
237
|
+
parser.add_argument(
|
|
238
|
+
"--overrides",
|
|
239
|
+
type=str,
|
|
240
|
+
default=None,
|
|
241
|
+
help=suppress_help or "Read process requirement overrides from file.",
|
|
242
|
+
)
|
|
243
|
+
|
|
244
|
+
parser.add_argument(
|
|
245
|
+
"--mpi-config-file",
|
|
246
|
+
type=str,
|
|
247
|
+
default=None,
|
|
248
|
+
help=suppress_help or "Platform specific configuration for MPI (parallel "
|
|
249
|
+
"launcher, its flag etc). See the cwltool README "
|
|
250
|
+
"section 'Running MPI-based tools' for details of the format: "
|
|
251
|
+
"https://github.com/common-workflow-language/cwltool#running-mpi-based-tools-that-need-to-be-launched",
|
|
252
|
+
)
|
|
253
|
+
parser.add_argument(
|
|
254
|
+
"--bypass-file-store",
|
|
255
|
+
action="store_true",
|
|
256
|
+
default=False,
|
|
257
|
+
help=suppress_help or "Do not use Toil's file store and assume all "
|
|
258
|
+
"paths are accessible in place from all nodes.",
|
|
259
|
+
dest="bypass_file_store",
|
|
260
|
+
)
|
|
261
|
+
parser.add_argument(
|
|
262
|
+
"--reference-inputs",
|
|
263
|
+
action="store_true",
|
|
264
|
+
default=False,
|
|
265
|
+
help=suppress_help or "Do not copy remote inputs into Toil's file "
|
|
266
|
+
"store and assume they are accessible in place from "
|
|
267
|
+
"all nodes.",
|
|
268
|
+
dest="reference_inputs",
|
|
269
|
+
)
|
|
270
|
+
parser.add_argument(
|
|
271
|
+
"--disable-streaming",
|
|
272
|
+
action="store_true",
|
|
273
|
+
default=False,
|
|
274
|
+
help=suppress_help or "Disable file streaming for files that have 'streamable' flag True",
|
|
275
|
+
dest="disable_streaming",
|
|
276
|
+
)
|
|
277
|
+
|
|
278
|
+
provgroup = parser.add_argument_group(
|
|
279
|
+
"Options for recording provenance information of the execution"
|
|
280
|
+
)
|
|
281
|
+
provgroup.add_argument(
|
|
282
|
+
"--provenance",
|
|
283
|
+
help=suppress_help or "Save provenance to specified folder as a "
|
|
284
|
+
"Research Object that captures and aggregates "
|
|
285
|
+
"workflow execution and data products.",
|
|
286
|
+
type=str,
|
|
287
|
+
)
|
|
288
|
+
|
|
289
|
+
provgroup.add_argument(
|
|
290
|
+
"--enable-user-provenance",
|
|
291
|
+
default=False,
|
|
292
|
+
action="store_true",
|
|
293
|
+
help=suppress_help or "Record user account info as part of provenance.",
|
|
294
|
+
dest="user_provenance",
|
|
295
|
+
)
|
|
296
|
+
provgroup.add_argument(
|
|
297
|
+
"--disable-user-provenance",
|
|
298
|
+
default=False,
|
|
299
|
+
action="store_false",
|
|
300
|
+
help=suppress_help or "Do not record user account info in provenance.",
|
|
301
|
+
dest="user_provenance",
|
|
302
|
+
)
|
|
303
|
+
provgroup.add_argument(
|
|
304
|
+
"--enable-host-provenance",
|
|
305
|
+
default=False,
|
|
306
|
+
action="store_true",
|
|
307
|
+
help=suppress_help or "Record host info as part of provenance.",
|
|
308
|
+
dest="host_provenance",
|
|
309
|
+
)
|
|
310
|
+
provgroup.add_argument(
|
|
311
|
+
"--disable-host-provenance",
|
|
312
|
+
default=False,
|
|
313
|
+
action="store_false",
|
|
314
|
+
help=suppress_help or "Do not record host info in provenance.",
|
|
315
|
+
dest="host_provenance",
|
|
316
|
+
)
|
|
317
|
+
provgroup.add_argument(
|
|
318
|
+
"--orcid",
|
|
319
|
+
help=suppress_help or "Record user ORCID identifier as part of "
|
|
320
|
+
"provenance, e.g. https://orcid.org/0000-0002-1825-0097 "
|
|
321
|
+
"or 0000-0002-1825-0097. Alternatively the environment variable "
|
|
322
|
+
"ORCID may be set.",
|
|
323
|
+
dest="orcid",
|
|
324
|
+
default=os.environ.get("ORCID", ""),
|
|
325
|
+
type=str,
|
|
326
|
+
)
|
|
327
|
+
provgroup.add_argument(
|
|
328
|
+
"--full-name",
|
|
329
|
+
help=suppress_help or "Record full name of user as part of provenance, "
|
|
330
|
+
"e.g. Josiah Carberry. You may need to use shell quotes to preserve "
|
|
331
|
+
"spaces. Alternatively the environment variable CWL_FULL_NAME may "
|
|
332
|
+
"be set.",
|
|
333
|
+
dest="cwl_full_name",
|
|
334
|
+
default=os.environ.get("CWL_FULL_NAME", ""),
|
|
335
|
+
type=str,
|
|
336
|
+
)
|
toil/options/wdl.py
ADDED
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
from argparse import ArgumentParser
|
|
2
|
+
|
|
3
|
+
from configargparse import SUPPRESS
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def add_wdl_options(parser: ArgumentParser, suppress: bool = True) -> None:
|
|
7
|
+
"""
|
|
8
|
+
Add WDL options to a parser. This only adds nonpositional WDL arguments
|
|
9
|
+
:param parser: Parser to add options to
|
|
10
|
+
:param suppress: Suppress help output
|
|
11
|
+
:return: None
|
|
12
|
+
"""
|
|
13
|
+
suppress_help = SUPPRESS if suppress else None
|
|
14
|
+
# include arg names without a wdl specifier if suppress is False
|
|
15
|
+
# this is to avoid possible duplicate options in custom toil scripts, ex outputFile can be a common argument name
|
|
16
|
+
output_dialect_arguments = ["--wdlOutputDialect"] + (["--outputDialect"] if not suppress else [])
|
|
17
|
+
parser.add_argument(*output_dialect_arguments, dest="output_dialect", type=str, default='cromwell',
|
|
18
|
+
choices=['cromwell', 'miniwdl'],
|
|
19
|
+
help=suppress_help or ("JSON output format dialect. 'cromwell' just returns the workflow's "
|
|
20
|
+
"output values as JSON, while 'miniwdl' nests that under an 'outputs' "
|
|
21
|
+
"key, and includes a 'dir' key where files are written."))
|
|
22
|
+
output_directory_arguments = ["--wdlOutputDirectory"] + (["--outputDirectory", "-o"] if not suppress else [])
|
|
23
|
+
parser.add_argument(*output_directory_arguments, dest="output_directory", type=str, default=None,
|
|
24
|
+
help=suppress_help or (
|
|
25
|
+
"Directory or URI prefix to save output files at. By default a new directory is created "
|
|
26
|
+
"in the current directory."))
|
|
27
|
+
output_file_arguments = ["--wdlOutputFile"] + (["--outputFile", "-m"] if not suppress else [])
|
|
28
|
+
parser.add_argument(*output_file_arguments, dest="output_file", type=str, default=None,
|
|
29
|
+
help=suppress_help or "File or URI to save output JSON to.")
|
|
30
|
+
reference_inputs_arguments = ["--wdlReferenceInputs"] + (["--referenceInputs"] if not suppress else [])
|
|
31
|
+
parser.add_argument(*reference_inputs_arguments, dest="reference_inputs", type=bool, default=False,
|
|
32
|
+
help=suppress_help or "Pass input files by URL")
|
|
@@ -15,7 +15,6 @@ import configparser
|
|
|
15
15
|
import json
|
|
16
16
|
import logging
|
|
17
17
|
import os.path
|
|
18
|
-
import platform
|
|
19
18
|
import subprocess
|
|
20
19
|
import tempfile
|
|
21
20
|
import textwrap
|
|
@@ -477,15 +476,13 @@ class AbstractProvisioner(ABC):
|
|
|
477
476
|
raise NotImplementedError
|
|
478
477
|
|
|
479
478
|
@abstractmethod
|
|
480
|
-
def getNodeShape(self, instance_type: str, preemptible=False):
|
|
479
|
+
def getNodeShape(self, instance_type: str, preemptible=False) -> Shape:
|
|
481
480
|
"""
|
|
482
481
|
The shape of a preemptible or non-preemptible node managed by this provisioner. The node
|
|
483
482
|
shape defines key properties of a machine, such as its number of cores or the time
|
|
484
483
|
between billing intervals.
|
|
485
484
|
|
|
486
485
|
:param str instance_type: Instance type name to return the shape of.
|
|
487
|
-
|
|
488
|
-
:rtype: Shape
|
|
489
486
|
"""
|
|
490
487
|
raise NotImplementedError
|
|
491
488
|
|
|
@@ -13,18 +13,15 @@
|
|
|
13
13
|
# limitations under the License.
|
|
14
14
|
import datetime
|
|
15
15
|
import logging
|
|
16
|
-
import os
|
|
17
16
|
from collections import namedtuple
|
|
18
17
|
from operator import attrgetter
|
|
19
18
|
from statistics import mean, stdev
|
|
20
|
-
from typing import
|
|
19
|
+
from typing import List, Optional
|
|
21
20
|
|
|
22
21
|
from toil.lib.aws import (get_aws_zone_from_boto,
|
|
23
22
|
get_aws_zone_from_environment,
|
|
24
23
|
get_aws_zone_from_environment_region,
|
|
25
|
-
get_aws_zone_from_metadata
|
|
26
|
-
running_on_ec2,
|
|
27
|
-
zone_to_region)
|
|
24
|
+
get_aws_zone_from_metadata)
|
|
28
25
|
|
|
29
26
|
logger = logging.getLogger(__name__)
|
|
30
27
|
|
|
@@ -146,7 +143,7 @@ def optimize_spot_bid(boto2_ec2, instance_type, spot_bid, zone_options: List[str
|
|
|
146
143
|
the instance in a sensible zone.
|
|
147
144
|
|
|
148
145
|
:param zone_options: The collection of allowed zones to consider, within
|
|
149
|
-
|
|
146
|
+
the region associated with the Boto2 connection.
|
|
150
147
|
"""
|
|
151
148
|
spot_history = _get_spot_history(boto2_ec2, instance_type)
|
|
152
149
|
if spot_history:
|
|
@@ -98,6 +98,12 @@ _S3_BUCKET_MAX_NAME_LEN = 63
|
|
|
98
98
|
# The suffix of the S3 bucket associated with the cluster
|
|
99
99
|
_S3_BUCKET_INTERNAL_SUFFIX = '--internal'
|
|
100
100
|
|
|
101
|
+
# prevent removal of these imports
|
|
102
|
+
str(boto.ec2)
|
|
103
|
+
str(boto.iam)
|
|
104
|
+
str(boto.vpc)
|
|
105
|
+
|
|
106
|
+
|
|
101
107
|
|
|
102
108
|
def awsRetryPredicate(e):
|
|
103
109
|
if isinstance(e, socket.gaierror):
|
|
@@ -32,12 +32,13 @@ from toil.batchSystems.abstractBatchSystem import (AbstractBatchSystem,
|
|
|
32
32
|
AbstractScalableBatchSystem,
|
|
33
33
|
NodeInfo)
|
|
34
34
|
from toil.bus import ClusterDesiredSizeMessage, ClusterSizeMessage
|
|
35
|
-
from toil.common import Config
|
|
35
|
+
from toil.common import Config
|
|
36
|
+
from toil.options.common import defaultTargetTime
|
|
36
37
|
from toil.job import JobDescription, ServiceJobDescription
|
|
37
38
|
from toil.lib.conversions import bytes2human, human2bytes
|
|
38
39
|
from toil.lib.retry import old_retry
|
|
39
40
|
from toil.lib.threading import ExceptionalThread
|
|
40
|
-
from toil.lib.throttle import
|
|
41
|
+
from toil.lib.throttle import throttle
|
|
41
42
|
from toil.provisioners.abstractProvisioner import AbstractProvisioner, Shape
|
|
42
43
|
|
|
43
44
|
if TYPE_CHECKING:
|
|
@@ -26,7 +26,7 @@ from libcloud.compute.types import Provider
|
|
|
26
26
|
|
|
27
27
|
from toil.jobStores.googleJobStore import GoogleJobStore
|
|
28
28
|
from toil.lib.conversions import human2bytes
|
|
29
|
-
from toil.lib.
|
|
29
|
+
from toil.lib.compatibility import compat_bytes_recursive
|
|
30
30
|
from toil.provisioners import NoSuchClusterException
|
|
31
31
|
from toil.provisioners.abstractProvisioner import AbstractProvisioner, Shape
|
|
32
32
|
from toil.provisioners.node import Node
|
|
@@ -197,7 +197,7 @@ class GCEProvisioner(AbstractProvisioner):
|
|
|
197
197
|
|
|
198
198
|
logger.debug('Launched leader')
|
|
199
199
|
|
|
200
|
-
def getNodeShape(self, instance_type: str, preemptible=False):
|
|
200
|
+
def getNodeShape(self, instance_type: str, preemptible=False) -> Shape:
|
|
201
201
|
# TODO: read this value only once
|
|
202
202
|
sizes = self._gceDriver.list_sizes(location=self._zone)
|
|
203
203
|
sizes = [x for x in sizes if x.name == instance_type]
|
toil/realtimeLogger.py
CHANGED
|
@@ -168,7 +168,8 @@ class RealtimeLogger(metaclass=RealtimeLoggerMetaclass):
|
|
|
168
168
|
def _stopLeader(cls) -> None:
|
|
169
169
|
"""Stop the server on the leader."""
|
|
170
170
|
with cls.lock:
|
|
171
|
-
|
|
171
|
+
if cls.initialized == 0:
|
|
172
|
+
raise RuntimeError("Can't stop the server on the leader as the leader was never initialized.")
|
|
172
173
|
cls.initialized -= 1
|
|
173
174
|
if cls.initialized == 0:
|
|
174
175
|
if cls.loggingServer:
|
toil/resource.py
CHANGED
|
@@ -23,27 +23,25 @@ from collections import namedtuple
|
|
|
23
23
|
from contextlib import closing
|
|
24
24
|
from io import BytesIO
|
|
25
25
|
from pydoc import locate
|
|
26
|
-
from
|
|
26
|
+
from types import ModuleType
|
|
27
|
+
from typing import (IO,
|
|
28
|
+
TYPE_CHECKING,
|
|
29
|
+
BinaryIO,
|
|
30
|
+
Callable,
|
|
31
|
+
Optional,
|
|
32
|
+
Sequence,
|
|
33
|
+
Type)
|
|
27
34
|
from urllib.error import HTTPError
|
|
28
35
|
from urllib.request import urlopen
|
|
29
36
|
from zipfile import ZipFile
|
|
30
37
|
|
|
31
|
-
from typing import (TYPE_CHECKING,
|
|
32
|
-
Optional,
|
|
33
|
-
Callable,
|
|
34
|
-
IO,
|
|
35
|
-
Type,
|
|
36
|
-
Sequence,
|
|
37
|
-
BinaryIO)
|
|
38
|
-
|
|
39
38
|
from toil import inVirtualEnv
|
|
39
|
+
from toil.lib.io import mkdtemp
|
|
40
40
|
from toil.lib.iterables import concat
|
|
41
41
|
from toil.lib.memoize import strict_bool
|
|
42
42
|
from toil.lib.retry import ErrorCondition, retry
|
|
43
43
|
from toil.version import exactPython
|
|
44
44
|
|
|
45
|
-
from types import ModuleType
|
|
46
|
-
|
|
47
45
|
if TYPE_CHECKING:
|
|
48
46
|
from toil.jobStores.abstractJobStore import AbstractJobStore
|
|
49
47
|
|
|
@@ -117,7 +115,8 @@ class Resource(namedtuple('Resource', ('name', 'pathHash', 'url', 'contentHash')
|
|
|
117
115
|
# .. and register its location in an environment variable such that child processes
|
|
118
116
|
# can find it.
|
|
119
117
|
os.environ[cls.rootDirPathEnvName] = resourceRootDirPath
|
|
120
|
-
|
|
118
|
+
if not os.path.isdir(resourceRootDirPath):
|
|
119
|
+
raise RuntimeError("Resource root path is not a directory.")
|
|
121
120
|
|
|
122
121
|
@classmethod
|
|
123
122
|
def cleanSystem(cls) -> None:
|
|
@@ -152,7 +151,9 @@ class Resource(namedtuple('Resource', ('name', 'pathHash', 'url', 'contentHash')
|
|
|
152
151
|
return None
|
|
153
152
|
else:
|
|
154
153
|
self = cls.unpickle(s)
|
|
155
|
-
|
|
154
|
+
if self.pathHash != pathHash:
|
|
155
|
+
raise RuntimeError("The Resource's path is incorrect.")
|
|
156
|
+
|
|
156
157
|
return self
|
|
157
158
|
|
|
158
159
|
def download(self, callback: Optional[Callable[[str], None]] = None) -> None:
|
|
@@ -243,7 +244,8 @@ class Resource(namedtuple('Resource', ('name', 'pathHash', 'url', 'contentHash')
|
|
|
243
244
|
with closing(urlopen(self.url)) as content:
|
|
244
245
|
buf = content.read()
|
|
245
246
|
contentHash = hashlib.md5(buf)
|
|
246
|
-
|
|
247
|
+
if contentHash.hexdigest() != self.contentHash:
|
|
248
|
+
raise RuntimeError("The Resource's contents is incorrect.")
|
|
247
249
|
dstFile.write(buf)
|
|
248
250
|
|
|
249
251
|
|
|
@@ -323,7 +325,8 @@ class VirtualEnvResource(DirectoryResource):
|
|
|
323
325
|
|
|
324
326
|
@classmethod
|
|
325
327
|
def _load(cls, path: str) -> BytesIO:
|
|
326
|
-
|
|
328
|
+
if os.path.basename(path) != 'site-packages':
|
|
329
|
+
raise RuntimeError("An incorrect path was passed through.")
|
|
327
330
|
bytesIO = BytesIO()
|
|
328
331
|
with ZipFile(file=bytesIO, mode='w') as zipFile:
|
|
329
332
|
for dirName, _, fileList in os.walk(path):
|
|
@@ -404,7 +407,8 @@ class ModuleDescriptor(namedtuple('ModuleDescriptor', ('dirPath', 'name', 'fromV
|
|
|
404
407
|
nameList = [filePath.pop()]
|
|
405
408
|
for package in reversed(module.__package__.split('.')):
|
|
406
409
|
dirPathTail = filePath.pop()
|
|
407
|
-
|
|
410
|
+
if dirPathTail != package:
|
|
411
|
+
raise RuntimeError("Incorrect path to package.")
|
|
408
412
|
nameList.append(dirPathTail)
|
|
409
413
|
name = '.'.join(reversed(nameList))
|
|
410
414
|
dirPath = os.path.sep.join(filePath)
|
|
@@ -421,7 +425,8 @@ class ModuleDescriptor(namedtuple('ModuleDescriptor', ('dirPath', 'name', 'fromV
|
|
|
421
425
|
|
|
422
426
|
for package in reversed(name.split('.')):
|
|
423
427
|
dirPathTail = filePath.pop()
|
|
424
|
-
|
|
428
|
+
if dirPathTail != package:
|
|
429
|
+
raise RuntimeError("Incorrect path to package.")
|
|
425
430
|
dirPath = os.path.abspath(os.path.sep.join(filePath))
|
|
426
431
|
absPrefix = os.path.abspath(sys.prefix)
|
|
427
432
|
inVenv = inVirtualEnv()
|
|
@@ -591,7 +596,8 @@ class ModuleDescriptor(namedtuple('ModuleDescriptor', ('dirPath', 'name', 'fromV
|
|
|
591
596
|
|
|
592
597
|
@classmethod
|
|
593
598
|
def fromCommand(cls, command: Sequence[str]) -> "ModuleDescriptor":
|
|
594
|
-
|
|
599
|
+
if len(command) != 3:
|
|
600
|
+
raise RuntimeError("Incorrect number of arguments (Expected 3).")
|
|
595
601
|
return cls(dirPath=command[0], name=command[1], fromVirtualEnv=strict_bool(command[2]))
|
|
596
602
|
|
|
597
603
|
def makeLoadable(self) -> "ModuleDescriptor":
|
toil/server/app.py
CHANGED
|
@@ -14,12 +14,11 @@
|
|
|
14
14
|
import argparse
|
|
15
15
|
import logging
|
|
16
16
|
import os
|
|
17
|
-
from typing import Type
|
|
18
17
|
|
|
19
18
|
import connexion # type: ignore
|
|
19
|
+
from configargparse import ArgumentParser
|
|
20
20
|
|
|
21
21
|
from toil.lib.aws import get_current_aws_region, running_on_ec2, running_on_ecs
|
|
22
|
-
from toil.lib.misc import get_public_ip
|
|
23
22
|
from toil.server.wes.toil_backend import ToilBackend
|
|
24
23
|
from toil.server.wsgi_app import run_app
|
|
25
24
|
from toil.version import version
|
|
@@ -28,7 +27,7 @@ logger = logging.getLogger(__name__)
|
|
|
28
27
|
|
|
29
28
|
|
|
30
29
|
def parser_with_server_options() -> argparse.ArgumentParser:
|
|
31
|
-
parser =
|
|
30
|
+
parser = ArgumentParser(description="Toil server mode.")
|
|
32
31
|
|
|
33
32
|
parser.add_argument("--debug", action="store_true", default=False,
|
|
34
33
|
help="Enable debug mode.")
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import argparse
|
|
2
1
|
import json
|
|
3
2
|
import logging
|
|
4
3
|
import os
|
|
@@ -13,6 +12,7 @@ from urllib.parse import urldefrag, urljoin, urlparse
|
|
|
13
12
|
import requests
|
|
14
13
|
import ruamel.yaml
|
|
15
14
|
import schema_salad
|
|
15
|
+
from configargparse import ArgumentParser
|
|
16
16
|
from wes_client.util import WESClient # type: ignore
|
|
17
17
|
from wes_client.util import wes_reponse as wes_response
|
|
18
18
|
|
|
@@ -147,7 +147,7 @@ class WESClientWithWorkflowEngineParameters(WESClient): # type: ignore
|
|
|
147
147
|
|
|
148
148
|
:param workflow_params_file: The URL or path to the CWL input file.
|
|
149
149
|
"""
|
|
150
|
-
loader = schema_salad.ref_resolver.Loader(
|
|
150
|
+
loader = schema_salad.ref_resolver.Loader( # type:ignore
|
|
151
151
|
{"location": {"@type": "@id"}, "path": {"@type": "@id"}}
|
|
152
152
|
)
|
|
153
153
|
|
|
@@ -396,7 +396,7 @@ def submit_run(client: WESClientWithWorkflowEngineParameters,
|
|
|
396
396
|
input_file,
|
|
397
397
|
attachments=attachments,
|
|
398
398
|
workflow_engine_parameters=engine_options)
|
|
399
|
-
return run_result
|
|
399
|
+
return str(run_result["run_id"])
|
|
400
400
|
|
|
401
401
|
|
|
402
402
|
def poll_run(client: WESClientWithWorkflowEngineParameters, run_id: str) -> bool:
|
|
@@ -425,7 +425,7 @@ def print_logs_and_exit(client: WESClientWithWorkflowEngineParameters, run_id: s
|
|
|
425
425
|
|
|
426
426
|
|
|
427
427
|
def main() -> None:
|
|
428
|
-
parser =
|
|
428
|
+
parser = ArgumentParser(description="A CWL runner that runs workflows through WES.")
|
|
429
429
|
|
|
430
430
|
# the first two positional arguments are the CWL file and its input file
|
|
431
431
|
parser.add_argument("cwl_file", type=str)
|
toil/server/utils.py
CHANGED
|
@@ -193,7 +193,7 @@ class AbstractStateStore:
|
|
|
193
193
|
This is a key-value store, with keys namespaced by workflow ID. Concurrent
|
|
194
194
|
access from multiple threads or processes is safe and globally consistent.
|
|
195
195
|
|
|
196
|
-
Keys and workflow IDs are restricted to [-a-zA-Z0-9_]
|
|
196
|
+
Keys and workflow IDs are restricted to ``[-a-zA-Z0-9_]``, because backends may
|
|
197
197
|
use them as path or URL components.
|
|
198
198
|
|
|
199
199
|
Key values are either a string, or None if the key is not set.
|