toil 5.12.0__py3-none-any.whl → 6.1.0a1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (157) hide show
  1. toil/__init__.py +18 -13
  2. toil/batchSystems/abstractBatchSystem.py +21 -10
  3. toil/batchSystems/abstractGridEngineBatchSystem.py +2 -2
  4. toil/batchSystems/awsBatch.py +14 -14
  5. toil/batchSystems/contained_executor.py +3 -3
  6. toil/batchSystems/htcondor.py +0 -1
  7. toil/batchSystems/kubernetes.py +34 -31
  8. toil/batchSystems/local_support.py +3 -1
  9. toil/batchSystems/mesos/batchSystem.py +7 -7
  10. toil/batchSystems/options.py +32 -83
  11. toil/batchSystems/registry.py +104 -23
  12. toil/batchSystems/singleMachine.py +16 -13
  13. toil/batchSystems/slurm.py +3 -3
  14. toil/batchSystems/torque.py +0 -1
  15. toil/bus.py +6 -8
  16. toil/common.py +532 -743
  17. toil/cwl/__init__.py +28 -32
  18. toil/cwl/cwltoil.py +523 -520
  19. toil/cwl/utils.py +55 -10
  20. toil/fileStores/__init__.py +2 -2
  21. toil/fileStores/abstractFileStore.py +36 -11
  22. toil/fileStores/cachingFileStore.py +607 -530
  23. toil/fileStores/nonCachingFileStore.py +43 -10
  24. toil/job.py +140 -75
  25. toil/jobStores/abstractJobStore.py +147 -79
  26. toil/jobStores/aws/jobStore.py +23 -9
  27. toil/jobStores/aws/utils.py +1 -2
  28. toil/jobStores/fileJobStore.py +117 -19
  29. toil/jobStores/googleJobStore.py +16 -7
  30. toil/jobStores/utils.py +5 -6
  31. toil/leader.py +71 -43
  32. toil/lib/accelerators.py +10 -5
  33. toil/lib/aws/__init__.py +3 -14
  34. toil/lib/aws/ami.py +22 -9
  35. toil/lib/aws/iam.py +21 -13
  36. toil/lib/aws/session.py +2 -16
  37. toil/lib/aws/utils.py +4 -5
  38. toil/lib/compatibility.py +1 -1
  39. toil/lib/conversions.py +7 -3
  40. toil/lib/docker.py +22 -23
  41. toil/lib/ec2.py +10 -6
  42. toil/lib/ec2nodes.py +106 -100
  43. toil/lib/encryption/_nacl.py +2 -1
  44. toil/lib/generatedEC2Lists.py +325 -18
  45. toil/lib/io.py +21 -0
  46. toil/lib/misc.py +1 -1
  47. toil/lib/resources.py +1 -1
  48. toil/lib/threading.py +74 -26
  49. toil/options/common.py +738 -0
  50. toil/options/cwl.py +336 -0
  51. toil/options/wdl.py +32 -0
  52. toil/provisioners/abstractProvisioner.py +1 -4
  53. toil/provisioners/aws/__init__.py +3 -6
  54. toil/provisioners/aws/awsProvisioner.py +6 -0
  55. toil/provisioners/clusterScaler.py +3 -2
  56. toil/provisioners/gceProvisioner.py +2 -2
  57. toil/realtimeLogger.py +2 -1
  58. toil/resource.py +24 -18
  59. toil/server/app.py +2 -3
  60. toil/server/cli/wes_cwl_runner.py +4 -4
  61. toil/server/utils.py +1 -1
  62. toil/server/wes/abstract_backend.py +3 -2
  63. toil/server/wes/amazon_wes_utils.py +5 -4
  64. toil/server/wes/tasks.py +2 -3
  65. toil/server/wes/toil_backend.py +2 -10
  66. toil/server/wsgi_app.py +2 -0
  67. toil/serviceManager.py +12 -10
  68. toil/statsAndLogging.py +5 -1
  69. toil/test/__init__.py +29 -54
  70. toil/test/batchSystems/batchSystemTest.py +11 -111
  71. toil/test/batchSystems/test_slurm.py +3 -2
  72. toil/test/cwl/cwlTest.py +213 -90
  73. toil/test/cwl/glob_dir.cwl +15 -0
  74. toil/test/cwl/preemptible.cwl +21 -0
  75. toil/test/cwl/preemptible_expression.cwl +28 -0
  76. toil/test/cwl/revsort.cwl +1 -1
  77. toil/test/cwl/revsort2.cwl +1 -1
  78. toil/test/docs/scriptsTest.py +0 -1
  79. toil/test/jobStores/jobStoreTest.py +27 -16
  80. toil/test/lib/aws/test_iam.py +4 -14
  81. toil/test/lib/aws/test_utils.py +0 -3
  82. toil/test/lib/dockerTest.py +4 -4
  83. toil/test/lib/test_ec2.py +11 -16
  84. toil/test/mesos/helloWorld.py +4 -5
  85. toil/test/mesos/stress.py +1 -1
  86. toil/test/provisioners/aws/awsProvisionerTest.py +9 -5
  87. toil/test/provisioners/clusterScalerTest.py +6 -4
  88. toil/test/provisioners/clusterTest.py +14 -3
  89. toil/test/provisioners/gceProvisionerTest.py +0 -6
  90. toil/test/provisioners/restartScript.py +3 -2
  91. toil/test/server/serverTest.py +1 -1
  92. toil/test/sort/restart_sort.py +2 -1
  93. toil/test/sort/sort.py +2 -1
  94. toil/test/sort/sortTest.py +2 -13
  95. toil/test/src/autoDeploymentTest.py +45 -45
  96. toil/test/src/busTest.py +5 -5
  97. toil/test/src/checkpointTest.py +2 -2
  98. toil/test/src/deferredFunctionTest.py +1 -1
  99. toil/test/src/fileStoreTest.py +32 -16
  100. toil/test/src/helloWorldTest.py +1 -1
  101. toil/test/src/importExportFileTest.py +1 -1
  102. toil/test/src/jobDescriptionTest.py +2 -1
  103. toil/test/src/jobServiceTest.py +1 -1
  104. toil/test/src/jobTest.py +18 -18
  105. toil/test/src/miscTests.py +5 -3
  106. toil/test/src/promisedRequirementTest.py +3 -3
  107. toil/test/src/realtimeLoggerTest.py +1 -1
  108. toil/test/src/resourceTest.py +2 -2
  109. toil/test/src/restartDAGTest.py +1 -1
  110. toil/test/src/resumabilityTest.py +36 -2
  111. toil/test/src/retainTempDirTest.py +1 -1
  112. toil/test/src/systemTest.py +2 -2
  113. toil/test/src/toilContextManagerTest.py +2 -2
  114. toil/test/src/userDefinedJobArgTypeTest.py +1 -1
  115. toil/test/utils/toilDebugTest.py +98 -32
  116. toil/test/utils/toilKillTest.py +2 -2
  117. toil/test/utils/utilsTest.py +20 -0
  118. toil/test/wdl/wdltoil_test.py +148 -45
  119. toil/toilState.py +7 -6
  120. toil/utils/toilClean.py +1 -1
  121. toil/utils/toilConfig.py +36 -0
  122. toil/utils/toilDebugFile.py +60 -33
  123. toil/utils/toilDebugJob.py +39 -12
  124. toil/utils/toilDestroyCluster.py +1 -1
  125. toil/utils/toilKill.py +1 -1
  126. toil/utils/toilLaunchCluster.py +13 -2
  127. toil/utils/toilMain.py +3 -2
  128. toil/utils/toilRsyncCluster.py +1 -1
  129. toil/utils/toilSshCluster.py +1 -1
  130. toil/utils/toilStats.py +240 -143
  131. toil/utils/toilStatus.py +1 -4
  132. toil/version.py +11 -11
  133. toil/wdl/utils.py +2 -122
  134. toil/wdl/wdltoil.py +999 -386
  135. toil/worker.py +25 -31
  136. {toil-5.12.0.dist-info → toil-6.1.0a1.dist-info}/METADATA +60 -53
  137. toil-6.1.0a1.dist-info/RECORD +237 -0
  138. {toil-5.12.0.dist-info → toil-6.1.0a1.dist-info}/WHEEL +1 -1
  139. {toil-5.12.0.dist-info → toil-6.1.0a1.dist-info}/entry_points.txt +0 -1
  140. toil/batchSystems/parasol.py +0 -379
  141. toil/batchSystems/tes.py +0 -459
  142. toil/test/batchSystems/parasolTestSupport.py +0 -117
  143. toil/test/wdl/builtinTest.py +0 -506
  144. toil/test/wdl/conftest.py +0 -23
  145. toil/test/wdl/toilwdlTest.py +0 -522
  146. toil/wdl/toilwdl.py +0 -141
  147. toil/wdl/versions/dev.py +0 -107
  148. toil/wdl/versions/draft2.py +0 -980
  149. toil/wdl/versions/v1.py +0 -794
  150. toil/wdl/wdl_analysis.py +0 -116
  151. toil/wdl/wdl_functions.py +0 -997
  152. toil/wdl/wdl_synthesis.py +0 -1011
  153. toil/wdl/wdl_types.py +0 -243
  154. toil-5.12.0.dist-info/RECORD +0 -244
  155. /toil/{wdl/versions → options}/__init__.py +0 -0
  156. {toil-5.12.0.dist-info → toil-6.1.0a1.dist-info}/LICENSE +0 -0
  157. {toil-5.12.0.dist-info → toil-6.1.0a1.dist-info}/top_level.txt +0 -0
toil/options/cwl.py ADDED
@@ -0,0 +1,336 @@
1
+ import os
2
+ from argparse import ArgumentParser
3
+
4
+ from configargparse import SUPPRESS
5
+
6
+ from toil.version import baseVersion
7
+
8
+
9
+ def add_cwl_options(parser: ArgumentParser, suppress: bool = True) -> None:
10
+ """
11
+ Add CWL options to the parser. This only adds nonpositional CWL arguments.
12
+
13
+ :param parser: Parser to add options to
14
+ :param suppress: Suppress help output
15
+ :return: None
16
+ """
17
+ suppress_help = SUPPRESS if suppress else None
18
+ parser.add_argument("--not-strict", action="store_true", help=suppress_help)
19
+ parser.add_argument(
20
+ "--enable-dev",
21
+ action="store_true",
22
+ help=suppress_help or suppress_help or "Enable loading and running development versions of CWL",
23
+ )
24
+ parser.add_argument(
25
+ "--enable-ext",
26
+ action="store_true",
27
+ help=suppress_help or "Enable loading and running 'cwltool:' extensions to the CWL standards.",
28
+ default=False,
29
+ )
30
+ parser.add_argument("--quiet", dest="quiet", action="store_true", default=False, help=suppress_help)
31
+ parser.add_argument("--basedir", type=str, help=suppress_help) # TODO: Might be hard-coded?
32
+ parser.add_argument("--outdir", type=str, default=None, help=suppress_help)
33
+ parser.add_argument("--version", action="version", version=baseVersion,
34
+ help=suppress_help or "show program's version number and exit")
35
+ parser.add_argument(
36
+ "--log-dir",
37
+ type=str,
38
+ default="",
39
+ help=suppress_help or "Log your tools stdout/stderr to this location outside of container",
40
+ )
41
+ # this is as a result of suppressed help statements not working well with mutually_exclusive_groups, which will
42
+ # cause an assertion error
43
+ # https://github.com/python/cpython/issues/62090
44
+ dockergroup = parser.add_mutually_exclusive_group() if not suppress_help else parser.add_argument_group()
45
+ dockergroup.add_argument(
46
+ "--user-space-docker-cmd",
47
+ help=suppress_help or "(Linux/OS X only) Specify a user space docker command (like "
48
+ "udocker or dx-docker) that will be used to call 'pull' and 'run'",
49
+ )
50
+ dockergroup.add_argument(
51
+ "--singularity",
52
+ action="store_true",
53
+ default=False,
54
+ help=suppress_help or "Use Singularity runtime for running containers. "
55
+ "Requires Singularity v2.6.1+ and Linux with kernel version v3.18+ or "
56
+ "with overlayfs support backported.",
57
+ )
58
+ dockergroup.add_argument(
59
+ "--podman",
60
+ action="store_true",
61
+ default=False,
62
+ help=suppress_help or "Use Podman runtime for running containers. ",
63
+ )
64
+ dockergroup.add_argument(
65
+ "--no-container",
66
+ action="store_true",
67
+ help=suppress_help or "Do not execute jobs in a "
68
+ "Docker container, even when `DockerRequirement` "
69
+ "is specified under `hints`.",
70
+ )
71
+ dockergroup.add_argument(
72
+ "--leave-container",
73
+ action="store_false",
74
+ default=True,
75
+ help=suppress_help or "Do not delete Docker container used by jobs after they exit",
76
+ dest="rm_container",
77
+ )
78
+ extra_dockergroup = parser.add_argument_group()
79
+ extra_dockergroup.add_argument(
80
+ "--custom-net",
81
+ help=suppress_help or "Specify docker network name to pass to docker run command",
82
+ )
83
+ cidgroup = parser.add_argument_group(
84
+ "Options for recording the Docker container identifier into a file."
85
+ )
86
+ cidgroup.add_argument(
87
+ # Disabled as containerid is now saved by default
88
+ "--record-container-id",
89
+ action="store_true",
90
+ default=False,
91
+ help=suppress_help or SUPPRESS,
92
+ dest="record_container_id",
93
+ )
94
+
95
+ cidgroup.add_argument(
96
+ "--cidfile-dir",
97
+ type=str,
98
+ help=suppress_help or "Store the Docker container ID into a file in the specified directory.",
99
+ default=None,
100
+ dest="cidfile_dir",
101
+ )
102
+
103
+ cidgroup.add_argument(
104
+ "--cidfile-prefix",
105
+ type=str,
106
+ help=suppress_help or "Specify a prefix to the container ID filename. "
107
+ "Final file name will be followed by a timestamp. "
108
+ "The default is no prefix.",
109
+ default=None,
110
+ dest="cidfile_prefix",
111
+ )
112
+
113
+ parser.add_argument(
114
+ "--preserve-environment",
115
+ type=str,
116
+ nargs="+",
117
+ help=suppress_help or "Preserve specified environment variables when running"
118
+ " CommandLineTools",
119
+ metavar=("VAR1 VAR2"),
120
+ default=("PATH",),
121
+ dest="preserve_environment",
122
+ )
123
+ parser.add_argument(
124
+ "--preserve-entire-environment",
125
+ action="store_true",
126
+ help=suppress_help or "Preserve all environment variable when running CommandLineTools.",
127
+ default=False,
128
+ dest="preserve_entire_environment",
129
+ )
130
+ parser.add_argument(
131
+ "--destBucket",
132
+ type=str,
133
+ help=suppress_help or "Specify a cloud bucket endpoint for output files.",
134
+ )
135
+ parser.add_argument("--beta-dependency-resolvers-configuration", default=None, help=suppress_help)
136
+ parser.add_argument("--beta-dependencies-directory", default=None, help=suppress_help)
137
+ parser.add_argument("--beta-use-biocontainers", default=None, action="store_true", help=suppress_help)
138
+ parser.add_argument("--beta-conda-dependencies", default=None, action="store_true", help=suppress_help)
139
+ parser.add_argument(
140
+ "--tmpdir-prefix",
141
+ type=str,
142
+ help=suppress_help or "Path prefix for temporary directories",
143
+ default=None,
144
+ )
145
+ parser.add_argument(
146
+ "--tmp-outdir-prefix",
147
+ type=str,
148
+ help=suppress_help or "Path prefix for intermediate output directories",
149
+ default=None,
150
+ )
151
+ parser.add_argument(
152
+ "--force-docker-pull",
153
+ action="store_true",
154
+ default=False,
155
+ dest="force_docker_pull",
156
+ help=suppress_help or "Pull latest docker image even if it is locally present",
157
+ )
158
+ parser.add_argument(
159
+ "--no-match-user",
160
+ action="store_true",
161
+ default=False,
162
+ help=suppress_help or "Disable passing the current uid to `docker run --user`",
163
+ )
164
+ parser.add_argument(
165
+ "--no-read-only",
166
+ action="store_true",
167
+ default=False,
168
+ help=suppress_help or "Do not set root directory in the container as read-only",
169
+ )
170
+ parser.add_argument(
171
+ "--strict-memory-limit",
172
+ action="store_true",
173
+ help=suppress_help or "When running with "
174
+ "software containers and the Docker engine, pass either the "
175
+ "calculated memory allocation from ResourceRequirements or the "
176
+ "default of 1 gigabyte to Docker's --memory option.",
177
+ )
178
+ parser.add_argument(
179
+ "--strict-cpu-limit",
180
+ action="store_true",
181
+ help=suppress_help or "When running with "
182
+ "software containers and the Docker engine, pass either the "
183
+ "calculated cpu allocation from ResourceRequirements or the "
184
+ "default of 1 core to Docker's --cpu option. "
185
+ "Requires docker version >= v1.13.",
186
+ )
187
+ parser.add_argument(
188
+ "--relax-path-checks",
189
+ action="store_true",
190
+ default=False,
191
+ help=suppress_help or "Relax requirements on path names to permit "
192
+ "spaces and hash characters.",
193
+ dest="relax_path_checks",
194
+ )
195
+ parser.add_argument(
196
+ "--default-container",
197
+ help=suppress_help or "Specify a default docker container that will be "
198
+ "used if the workflow fails to specify one.",
199
+ )
200
+ parser.add_argument(
201
+ "--disable-validate",
202
+ dest="do_validate",
203
+ action="store_false",
204
+ default=True,
205
+ help=suppress_help or SUPPRESS,
206
+ )
207
+ parser.add_argument(
208
+ "--fast-parser",
209
+ dest="fast_parser",
210
+ action="store_true",
211
+ default=False,
212
+ help=suppress_help or SUPPRESS,
213
+ )
214
+ # same workaround as dockergroup
215
+ checkgroup = parser.add_mutually_exclusive_group() if not suppress_help else parser.add_argument_group()
216
+ checkgroup.add_argument(
217
+ "--compute-checksum",
218
+ action="store_true",
219
+ default=True,
220
+ help=suppress_help or "Compute checksum of contents while collecting outputs",
221
+ dest="compute_checksum",
222
+ )
223
+ checkgroup.add_argument(
224
+ "--no-compute-checksum",
225
+ action="store_false",
226
+ help=suppress_help or "Do not compute checksum of contents while collecting outputs",
227
+ dest="compute_checksum",
228
+ )
229
+
230
+ parser.add_argument(
231
+ "--eval-timeout",
232
+ help=suppress_help or "Time to wait for a Javascript expression to evaluate before giving "
233
+ "an error, default 20s.",
234
+ type=float,
235
+ default=20,
236
+ )
237
+ parser.add_argument(
238
+ "--overrides",
239
+ type=str,
240
+ default=None,
241
+ help=suppress_help or "Read process requirement overrides from file.",
242
+ )
243
+
244
+ parser.add_argument(
245
+ "--mpi-config-file",
246
+ type=str,
247
+ default=None,
248
+ help=suppress_help or "Platform specific configuration for MPI (parallel "
249
+ "launcher, its flag etc). See the cwltool README "
250
+ "section 'Running MPI-based tools' for details of the format: "
251
+ "https://github.com/common-workflow-language/cwltool#running-mpi-based-tools-that-need-to-be-launched",
252
+ )
253
+ parser.add_argument(
254
+ "--bypass-file-store",
255
+ action="store_true",
256
+ default=False,
257
+ help=suppress_help or "Do not use Toil's file store and assume all "
258
+ "paths are accessible in place from all nodes.",
259
+ dest="bypass_file_store",
260
+ )
261
+ parser.add_argument(
262
+ "--reference-inputs",
263
+ action="store_true",
264
+ default=False,
265
+ help=suppress_help or "Do not copy remote inputs into Toil's file "
266
+ "store and assume they are accessible in place from "
267
+ "all nodes.",
268
+ dest="reference_inputs",
269
+ )
270
+ parser.add_argument(
271
+ "--disable-streaming",
272
+ action="store_true",
273
+ default=False,
274
+ help=suppress_help or "Disable file streaming for files that have 'streamable' flag True",
275
+ dest="disable_streaming",
276
+ )
277
+
278
+ provgroup = parser.add_argument_group(
279
+ "Options for recording provenance information of the execution"
280
+ )
281
+ provgroup.add_argument(
282
+ "--provenance",
283
+ help=suppress_help or "Save provenance to specified folder as a "
284
+ "Research Object that captures and aggregates "
285
+ "workflow execution and data products.",
286
+ type=str,
287
+ )
288
+
289
+ provgroup.add_argument(
290
+ "--enable-user-provenance",
291
+ default=False,
292
+ action="store_true",
293
+ help=suppress_help or "Record user account info as part of provenance.",
294
+ dest="user_provenance",
295
+ )
296
+ provgroup.add_argument(
297
+ "--disable-user-provenance",
298
+ default=False,
299
+ action="store_false",
300
+ help=suppress_help or "Do not record user account info in provenance.",
301
+ dest="user_provenance",
302
+ )
303
+ provgroup.add_argument(
304
+ "--enable-host-provenance",
305
+ default=False,
306
+ action="store_true",
307
+ help=suppress_help or "Record host info as part of provenance.",
308
+ dest="host_provenance",
309
+ )
310
+ provgroup.add_argument(
311
+ "--disable-host-provenance",
312
+ default=False,
313
+ action="store_false",
314
+ help=suppress_help or "Do not record host info in provenance.",
315
+ dest="host_provenance",
316
+ )
317
+ provgroup.add_argument(
318
+ "--orcid",
319
+ help=suppress_help or "Record user ORCID identifier as part of "
320
+ "provenance, e.g. https://orcid.org/0000-0002-1825-0097 "
321
+ "or 0000-0002-1825-0097. Alternatively the environment variable "
322
+ "ORCID may be set.",
323
+ dest="orcid",
324
+ default=os.environ.get("ORCID", ""),
325
+ type=str,
326
+ )
327
+ provgroup.add_argument(
328
+ "--full-name",
329
+ help=suppress_help or "Record full name of user as part of provenance, "
330
+ "e.g. Josiah Carberry. You may need to use shell quotes to preserve "
331
+ "spaces. Alternatively the environment variable CWL_FULL_NAME may "
332
+ "be set.",
333
+ dest="cwl_full_name",
334
+ default=os.environ.get("CWL_FULL_NAME", ""),
335
+ type=str,
336
+ )
toil/options/wdl.py ADDED
@@ -0,0 +1,32 @@
1
+ from argparse import ArgumentParser
2
+
3
+ from configargparse import SUPPRESS
4
+
5
+
6
+ def add_wdl_options(parser: ArgumentParser, suppress: bool = True) -> None:
7
+ """
8
+ Add WDL options to a parser. This only adds nonpositional WDL arguments
9
+ :param parser: Parser to add options to
10
+ :param suppress: Suppress help output
11
+ :return: None
12
+ """
13
+ suppress_help = SUPPRESS if suppress else None
14
+ # include arg names without a wdl specifier if suppress is False
15
+ # this is to avoid possible duplicate options in custom toil scripts, ex outputFile can be a common argument name
16
+ output_dialect_arguments = ["--wdlOutputDialect"] + (["--outputDialect"] if not suppress else [])
17
+ parser.add_argument(*output_dialect_arguments, dest="output_dialect", type=str, default='cromwell',
18
+ choices=['cromwell', 'miniwdl'],
19
+ help=suppress_help or ("JSON output format dialect. 'cromwell' just returns the workflow's "
20
+ "output values as JSON, while 'miniwdl' nests that under an 'outputs' "
21
+ "key, and includes a 'dir' key where files are written."))
22
+ output_directory_arguments = ["--wdlOutputDirectory"] + (["--outputDirectory", "-o"] if not suppress else [])
23
+ parser.add_argument(*output_directory_arguments, dest="output_directory", type=str, default=None,
24
+ help=suppress_help or (
25
+ "Directory or URI prefix to save output files at. By default a new directory is created "
26
+ "in the current directory."))
27
+ output_file_arguments = ["--wdlOutputFile"] + (["--outputFile", "-m"] if not suppress else [])
28
+ parser.add_argument(*output_file_arguments, dest="output_file", type=str, default=None,
29
+ help=suppress_help or "File or URI to save output JSON to.")
30
+ reference_inputs_arguments = ["--wdlReferenceInputs"] + (["--referenceInputs"] if not suppress else [])
31
+ parser.add_argument(*reference_inputs_arguments, dest="reference_inputs", type=bool, default=False,
32
+ help=suppress_help or "Pass input files by URL")
@@ -15,7 +15,6 @@ import configparser
15
15
  import json
16
16
  import logging
17
17
  import os.path
18
- import platform
19
18
  import subprocess
20
19
  import tempfile
21
20
  import textwrap
@@ -477,15 +476,13 @@ class AbstractProvisioner(ABC):
477
476
  raise NotImplementedError
478
477
 
479
478
  @abstractmethod
480
- def getNodeShape(self, instance_type: str, preemptible=False):
479
+ def getNodeShape(self, instance_type: str, preemptible=False) -> Shape:
481
480
  """
482
481
  The shape of a preemptible or non-preemptible node managed by this provisioner. The node
483
482
  shape defines key properties of a machine, such as its number of cores or the time
484
483
  between billing intervals.
485
484
 
486
485
  :param str instance_type: Instance type name to return the shape of.
487
-
488
- :rtype: Shape
489
486
  """
490
487
  raise NotImplementedError
491
488
 
@@ -13,18 +13,15 @@
13
13
  # limitations under the License.
14
14
  import datetime
15
15
  import logging
16
- import os
17
16
  from collections import namedtuple
18
17
  from operator import attrgetter
19
18
  from statistics import mean, stdev
20
- from typing import Any, List, Optional
19
+ from typing import List, Optional
21
20
 
22
21
  from toil.lib.aws import (get_aws_zone_from_boto,
23
22
  get_aws_zone_from_environment,
24
23
  get_aws_zone_from_environment_region,
25
- get_aws_zone_from_metadata,
26
- running_on_ec2,
27
- zone_to_region)
24
+ get_aws_zone_from_metadata)
28
25
 
29
26
  logger = logging.getLogger(__name__)
30
27
 
@@ -146,7 +143,7 @@ def optimize_spot_bid(boto2_ec2, instance_type, spot_bid, zone_options: List[str
146
143
  the instance in a sensible zone.
147
144
 
148
145
  :param zone_options: The collection of allowed zones to consider, within
149
- the region associated with the Boto2 connection.
146
+ the region associated with the Boto2 connection.
150
147
  """
151
148
  spot_history = _get_spot_history(boto2_ec2, instance_type)
152
149
  if spot_history:
@@ -98,6 +98,12 @@ _S3_BUCKET_MAX_NAME_LEN = 63
98
98
  # The suffix of the S3 bucket associated with the cluster
99
99
  _S3_BUCKET_INTERNAL_SUFFIX = '--internal'
100
100
 
101
+ # prevent removal of these imports
102
+ str(boto.ec2)
103
+ str(boto.iam)
104
+ str(boto.vpc)
105
+
106
+
101
107
 
102
108
  def awsRetryPredicate(e):
103
109
  if isinstance(e, socket.gaierror):
@@ -32,12 +32,13 @@ from toil.batchSystems.abstractBatchSystem import (AbstractBatchSystem,
32
32
  AbstractScalableBatchSystem,
33
33
  NodeInfo)
34
34
  from toil.bus import ClusterDesiredSizeMessage, ClusterSizeMessage
35
- from toil.common import Config, defaultTargetTime
35
+ from toil.common import Config
36
+ from toil.options.common import defaultTargetTime
36
37
  from toil.job import JobDescription, ServiceJobDescription
37
38
  from toil.lib.conversions import bytes2human, human2bytes
38
39
  from toil.lib.retry import old_retry
39
40
  from toil.lib.threading import ExceptionalThread
40
- from toil.lib.throttle import LocalThrottle, throttle
41
+ from toil.lib.throttle import throttle
41
42
  from toil.provisioners.abstractProvisioner import AbstractProvisioner, Shape
42
43
 
43
44
  if TYPE_CHECKING:
@@ -26,7 +26,7 @@ from libcloud.compute.types import Provider
26
26
 
27
27
  from toil.jobStores.googleJobStore import GoogleJobStore
28
28
  from toil.lib.conversions import human2bytes
29
- from toil.lib.util.compatibility import compat_bytes_recursive
29
+ from toil.lib.compatibility import compat_bytes_recursive
30
30
  from toil.provisioners import NoSuchClusterException
31
31
  from toil.provisioners.abstractProvisioner import AbstractProvisioner, Shape
32
32
  from toil.provisioners.node import Node
@@ -197,7 +197,7 @@ class GCEProvisioner(AbstractProvisioner):
197
197
 
198
198
  logger.debug('Launched leader')
199
199
 
200
- def getNodeShape(self, instance_type: str, preemptible=False):
200
+ def getNodeShape(self, instance_type: str, preemptible=False) -> Shape:
201
201
  # TODO: read this value only once
202
202
  sizes = self._gceDriver.list_sizes(location=self._zone)
203
203
  sizes = [x for x in sizes if x.name == instance_type]
toil/realtimeLogger.py CHANGED
@@ -168,7 +168,8 @@ class RealtimeLogger(metaclass=RealtimeLoggerMetaclass):
168
168
  def _stopLeader(cls) -> None:
169
169
  """Stop the server on the leader."""
170
170
  with cls.lock:
171
- assert cls.initialized > 0
171
+ if cls.initialized == 0:
172
+ raise RuntimeError("Can't stop the server on the leader as the leader was never initialized.")
172
173
  cls.initialized -= 1
173
174
  if cls.initialized == 0:
174
175
  if cls.loggingServer:
toil/resource.py CHANGED
@@ -23,27 +23,25 @@ from collections import namedtuple
23
23
  from contextlib import closing
24
24
  from io import BytesIO
25
25
  from pydoc import locate
26
- from tempfile import mkdtemp
26
+ from types import ModuleType
27
+ from typing import (IO,
28
+ TYPE_CHECKING,
29
+ BinaryIO,
30
+ Callable,
31
+ Optional,
32
+ Sequence,
33
+ Type)
27
34
  from urllib.error import HTTPError
28
35
  from urllib.request import urlopen
29
36
  from zipfile import ZipFile
30
37
 
31
- from typing import (TYPE_CHECKING,
32
- Optional,
33
- Callable,
34
- IO,
35
- Type,
36
- Sequence,
37
- BinaryIO)
38
-
39
38
  from toil import inVirtualEnv
39
+ from toil.lib.io import mkdtemp
40
40
  from toil.lib.iterables import concat
41
41
  from toil.lib.memoize import strict_bool
42
42
  from toil.lib.retry import ErrorCondition, retry
43
43
  from toil.version import exactPython
44
44
 
45
- from types import ModuleType
46
-
47
45
  if TYPE_CHECKING:
48
46
  from toil.jobStores.abstractJobStore import AbstractJobStore
49
47
 
@@ -117,7 +115,8 @@ class Resource(namedtuple('Resource', ('name', 'pathHash', 'url', 'contentHash')
117
115
  # .. and register its location in an environment variable such that child processes
118
116
  # can find it.
119
117
  os.environ[cls.rootDirPathEnvName] = resourceRootDirPath
120
- assert os.path.isdir(resourceRootDirPath)
118
+ if not os.path.isdir(resourceRootDirPath):
119
+ raise RuntimeError("Resource root path is not a directory.")
121
120
 
122
121
  @classmethod
123
122
  def cleanSystem(cls) -> None:
@@ -152,7 +151,9 @@ class Resource(namedtuple('Resource', ('name', 'pathHash', 'url', 'contentHash')
152
151
  return None
153
152
  else:
154
153
  self = cls.unpickle(s)
155
- assert self.pathHash == pathHash
154
+ if self.pathHash != pathHash:
155
+ raise RuntimeError("The Resource's path is incorrect.")
156
+
156
157
  return self
157
158
 
158
159
  def download(self, callback: Optional[Callable[[str], None]] = None) -> None:
@@ -243,7 +244,8 @@ class Resource(namedtuple('Resource', ('name', 'pathHash', 'url', 'contentHash')
243
244
  with closing(urlopen(self.url)) as content:
244
245
  buf = content.read()
245
246
  contentHash = hashlib.md5(buf)
246
- assert contentHash.hexdigest() == self.contentHash
247
+ if contentHash.hexdigest() != self.contentHash:
248
+ raise RuntimeError("The Resource's contents is incorrect.")
247
249
  dstFile.write(buf)
248
250
 
249
251
 
@@ -323,7 +325,8 @@ class VirtualEnvResource(DirectoryResource):
323
325
 
324
326
  @classmethod
325
327
  def _load(cls, path: str) -> BytesIO:
326
- assert os.path.basename(path) == 'site-packages'
328
+ if os.path.basename(path) != 'site-packages':
329
+ raise RuntimeError("An incorrect path was passed through.")
327
330
  bytesIO = BytesIO()
328
331
  with ZipFile(file=bytesIO, mode='w') as zipFile:
329
332
  for dirName, _, fileList in os.walk(path):
@@ -404,7 +407,8 @@ class ModuleDescriptor(namedtuple('ModuleDescriptor', ('dirPath', 'name', 'fromV
404
407
  nameList = [filePath.pop()]
405
408
  for package in reversed(module.__package__.split('.')):
406
409
  dirPathTail = filePath.pop()
407
- assert dirPathTail == package
410
+ if dirPathTail != package:
411
+ raise RuntimeError("Incorrect path to package.")
408
412
  nameList.append(dirPathTail)
409
413
  name = '.'.join(reversed(nameList))
410
414
  dirPath = os.path.sep.join(filePath)
@@ -421,7 +425,8 @@ class ModuleDescriptor(namedtuple('ModuleDescriptor', ('dirPath', 'name', 'fromV
421
425
 
422
426
  for package in reversed(name.split('.')):
423
427
  dirPathTail = filePath.pop()
424
- assert dirPathTail == package
428
+ if dirPathTail != package:
429
+ raise RuntimeError("Incorrect path to package.")
425
430
  dirPath = os.path.abspath(os.path.sep.join(filePath))
426
431
  absPrefix = os.path.abspath(sys.prefix)
427
432
  inVenv = inVirtualEnv()
@@ -591,7 +596,8 @@ class ModuleDescriptor(namedtuple('ModuleDescriptor', ('dirPath', 'name', 'fromV
591
596
 
592
597
  @classmethod
593
598
  def fromCommand(cls, command: Sequence[str]) -> "ModuleDescriptor":
594
- assert len(command) == 3
599
+ if len(command) != 3:
600
+ raise RuntimeError("Incorrect number of arguments (Expected 3).")
595
601
  return cls(dirPath=command[0], name=command[1], fromVirtualEnv=strict_bool(command[2]))
596
602
 
597
603
  def makeLoadable(self) -> "ModuleDescriptor":
toil/server/app.py CHANGED
@@ -14,12 +14,11 @@
14
14
  import argparse
15
15
  import logging
16
16
  import os
17
- from typing import Type
18
17
 
19
18
  import connexion # type: ignore
19
+ from configargparse import ArgumentParser
20
20
 
21
21
  from toil.lib.aws import get_current_aws_region, running_on_ec2, running_on_ecs
22
- from toil.lib.misc import get_public_ip
23
22
  from toil.server.wes.toil_backend import ToilBackend
24
23
  from toil.server.wsgi_app import run_app
25
24
  from toil.version import version
@@ -28,7 +27,7 @@ logger = logging.getLogger(__name__)
28
27
 
29
28
 
30
29
  def parser_with_server_options() -> argparse.ArgumentParser:
31
- parser = argparse.ArgumentParser(description="Toil server mode.")
30
+ parser = ArgumentParser(description="Toil server mode.")
32
31
 
33
32
  parser.add_argument("--debug", action="store_true", default=False,
34
33
  help="Enable debug mode.")
@@ -1,4 +1,3 @@
1
- import argparse
2
1
  import json
3
2
  import logging
4
3
  import os
@@ -13,6 +12,7 @@ from urllib.parse import urldefrag, urljoin, urlparse
13
12
  import requests
14
13
  import ruamel.yaml
15
14
  import schema_salad
15
+ from configargparse import ArgumentParser
16
16
  from wes_client.util import WESClient # type: ignore
17
17
  from wes_client.util import wes_reponse as wes_response
18
18
 
@@ -147,7 +147,7 @@ class WESClientWithWorkflowEngineParameters(WESClient): # type: ignore
147
147
 
148
148
  :param workflow_params_file: The URL or path to the CWL input file.
149
149
  """
150
- loader = schema_salad.ref_resolver.Loader(
150
+ loader = schema_salad.ref_resolver.Loader( # type:ignore
151
151
  {"location": {"@type": "@id"}, "path": {"@type": "@id"}}
152
152
  )
153
153
 
@@ -396,7 +396,7 @@ def submit_run(client: WESClientWithWorkflowEngineParameters,
396
396
  input_file,
397
397
  attachments=attachments,
398
398
  workflow_engine_parameters=engine_options)
399
- return run_result.get("run_id", None)
399
+ return str(run_result["run_id"])
400
400
 
401
401
 
402
402
  def poll_run(client: WESClientWithWorkflowEngineParameters, run_id: str) -> bool:
@@ -425,7 +425,7 @@ def print_logs_and_exit(client: WESClientWithWorkflowEngineParameters, run_id: s
425
425
 
426
426
 
427
427
  def main() -> None:
428
- parser = argparse.ArgumentParser(description="A CWL runner that runs workflows through WES.")
428
+ parser = ArgumentParser(description="A CWL runner that runs workflows through WES.")
429
429
 
430
430
  # the first two positional arguments are the CWL file and its input file
431
431
  parser.add_argument("cwl_file", type=str)
toil/server/utils.py CHANGED
@@ -193,7 +193,7 @@ class AbstractStateStore:
193
193
  This is a key-value store, with keys namespaced by workflow ID. Concurrent
194
194
  access from multiple threads or processes is safe and globally consistent.
195
195
 
196
- Keys and workflow IDs are restricted to [-a-zA-Z0-9_], because backends may
196
+ Keys and workflow IDs are restricted to ``[-a-zA-Z0-9_]``, because backends may
197
197
  use them as path or URL components.
198
198
 
199
199
  Key values are either a string, or None if the key is not set.