toil 9.1.1__py3-none-any.whl → 9.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- toil/__init__.py +5 -9
- toil/batchSystems/abstractBatchSystem.py +23 -22
- toil/batchSystems/abstractGridEngineBatchSystem.py +17 -12
- toil/batchSystems/awsBatch.py +8 -8
- toil/batchSystems/cleanup_support.py +4 -4
- toil/batchSystems/contained_executor.py +3 -3
- toil/batchSystems/gridengine.py +3 -4
- toil/batchSystems/htcondor.py +5 -5
- toil/batchSystems/kubernetes.py +65 -63
- toil/batchSystems/local_support.py +2 -3
- toil/batchSystems/lsf.py +6 -7
- toil/batchSystems/mesos/batchSystem.py +11 -7
- toil/batchSystems/mesos/test/__init__.py +1 -2
- toil/batchSystems/options.py +9 -10
- toil/batchSystems/registry.py +3 -7
- toil/batchSystems/singleMachine.py +8 -11
- toil/batchSystems/slurm.py +49 -38
- toil/batchSystems/torque.py +3 -4
- toil/bus.py +36 -34
- toil/common.py +129 -89
- toil/cwl/cwltoil.py +857 -729
- toil/cwl/utils.py +44 -35
- toil/fileStores/__init__.py +3 -1
- toil/fileStores/abstractFileStore.py +28 -30
- toil/fileStores/cachingFileStore.py +8 -8
- toil/fileStores/nonCachingFileStore.py +10 -21
- toil/job.py +159 -158
- toil/jobStores/abstractJobStore.py +68 -69
- toil/jobStores/aws/jobStore.py +249 -213
- toil/jobStores/aws/utils.py +13 -24
- toil/jobStores/fileJobStore.py +28 -22
- toil/jobStores/googleJobStore.py +21 -17
- toil/jobStores/utils.py +3 -7
- toil/leader.py +17 -22
- toil/lib/accelerators.py +6 -4
- toil/lib/aws/__init__.py +9 -10
- toil/lib/aws/ami.py +33 -19
- toil/lib/aws/iam.py +6 -6
- toil/lib/aws/s3.py +259 -157
- toil/lib/aws/session.py +76 -76
- toil/lib/aws/utils.py +51 -43
- toil/lib/checksum.py +19 -15
- toil/lib/compatibility.py +3 -2
- toil/lib/conversions.py +45 -18
- toil/lib/directory.py +29 -26
- toil/lib/docker.py +93 -99
- toil/lib/dockstore.py +77 -50
- toil/lib/ec2.py +39 -38
- toil/lib/ec2nodes.py +11 -4
- toil/lib/exceptions.py +8 -5
- toil/lib/ftp_utils.py +9 -14
- toil/lib/generatedEC2Lists.py +161 -20
- toil/lib/history.py +141 -97
- toil/lib/history_submission.py +163 -72
- toil/lib/io.py +27 -17
- toil/lib/memoize.py +2 -1
- toil/lib/misc.py +15 -11
- toil/lib/pipes.py +40 -25
- toil/lib/plugins.py +12 -8
- toil/lib/resources.py +1 -0
- toil/lib/retry.py +32 -38
- toil/lib/threading.py +12 -12
- toil/lib/throttle.py +1 -2
- toil/lib/trs.py +113 -51
- toil/lib/url.py +14 -23
- toil/lib/web.py +7 -2
- toil/options/common.py +18 -15
- toil/options/cwl.py +2 -2
- toil/options/runner.py +9 -5
- toil/options/wdl.py +1 -3
- toil/provisioners/__init__.py +9 -9
- toil/provisioners/abstractProvisioner.py +22 -20
- toil/provisioners/aws/__init__.py +20 -14
- toil/provisioners/aws/awsProvisioner.py +10 -8
- toil/provisioners/clusterScaler.py +19 -18
- toil/provisioners/gceProvisioner.py +2 -3
- toil/provisioners/node.py +11 -13
- toil/realtimeLogger.py +4 -4
- toil/resource.py +5 -5
- toil/server/app.py +2 -2
- toil/server/cli/wes_cwl_runner.py +11 -11
- toil/server/utils.py +18 -21
- toil/server/wes/abstract_backend.py +9 -8
- toil/server/wes/amazon_wes_utils.py +3 -3
- toil/server/wes/tasks.py +3 -5
- toil/server/wes/toil_backend.py +17 -21
- toil/server/wsgi_app.py +3 -3
- toil/serviceManager.py +3 -4
- toil/statsAndLogging.py +12 -13
- toil/test/__init__.py +33 -24
- toil/test/batchSystems/batchSystemTest.py +12 -11
- toil/test/batchSystems/batch_system_plugin_test.py +3 -5
- toil/test/batchSystems/test_slurm.py +38 -24
- toil/test/cwl/conftest.py +5 -6
- toil/test/cwl/cwlTest.py +194 -78
- toil/test/cwl/download_file_uri.json +6 -0
- toil/test/cwl/download_file_uri_no_hostname.json +6 -0
- toil/test/docs/scripts/tutorial_staging.py +1 -0
- toil/test/jobStores/jobStoreTest.py +9 -7
- toil/test/lib/aws/test_iam.py +1 -3
- toil/test/lib/aws/test_s3.py +1 -1
- toil/test/lib/dockerTest.py +9 -9
- toil/test/lib/test_ec2.py +12 -11
- toil/test/lib/test_history.py +4 -4
- toil/test/lib/test_trs.py +16 -14
- toil/test/lib/test_url.py +7 -6
- toil/test/lib/url_plugin_test.py +12 -18
- toil/test/provisioners/aws/awsProvisionerTest.py +10 -8
- toil/test/provisioners/clusterScalerTest.py +2 -5
- toil/test/provisioners/clusterTest.py +1 -3
- toil/test/server/serverTest.py +13 -4
- toil/test/sort/restart_sort.py +2 -6
- toil/test/sort/sort.py +3 -8
- toil/test/src/deferredFunctionTest.py +7 -7
- toil/test/src/environmentTest.py +1 -2
- toil/test/src/fileStoreTest.py +5 -5
- toil/test/src/importExportFileTest.py +5 -6
- toil/test/src/jobServiceTest.py +22 -14
- toil/test/src/jobTest.py +121 -25
- toil/test/src/miscTests.py +5 -7
- toil/test/src/promisedRequirementTest.py +8 -7
- toil/test/src/regularLogTest.py +2 -3
- toil/test/src/resourceTest.py +5 -8
- toil/test/src/restartDAGTest.py +5 -6
- toil/test/src/resumabilityTest.py +2 -2
- toil/test/src/retainTempDirTest.py +3 -3
- toil/test/src/systemTest.py +3 -3
- toil/test/src/threadingTest.py +1 -1
- toil/test/src/workerTest.py +1 -2
- toil/test/utils/toilDebugTest.py +6 -4
- toil/test/utils/toilKillTest.py +1 -1
- toil/test/utils/utilsTest.py +15 -14
- toil/test/wdl/wdltoil_test.py +247 -124
- toil/test/wdl/wdltoil_test_kubernetes.py +2 -2
- toil/toilState.py +2 -3
- toil/utils/toilDebugFile.py +3 -8
- toil/utils/toilDebugJob.py +1 -2
- toil/utils/toilLaunchCluster.py +1 -2
- toil/utils/toilSshCluster.py +2 -0
- toil/utils/toilStats.py +19 -24
- toil/utils/toilStatus.py +11 -14
- toil/version.py +10 -10
- toil/wdl/wdltoil.py +313 -209
- toil/worker.py +18 -12
- {toil-9.1.1.dist-info → toil-9.2.0.dist-info}/METADATA +11 -14
- {toil-9.1.1.dist-info → toil-9.2.0.dist-info}/RECORD +150 -153
- {toil-9.1.1.dist-info → toil-9.2.0.dist-info}/WHEEL +1 -1
- toil/test/cwl/staging_cat.cwl +0 -27
- toil/test/cwl/staging_make_file.cwl +0 -25
- toil/test/cwl/staging_workflow.cwl +0 -43
- toil/test/cwl/zero_default.cwl +0 -61
- toil/test/utils/ABCWorkflowDebug/ABC.txt +0 -1
- {toil-9.1.1.dist-info → toil-9.2.0.dist-info}/entry_points.txt +0 -0
- {toil-9.1.1.dist-info → toil-9.2.0.dist-info}/licenses/LICENSE +0 -0
- {toil-9.1.1.dist-info → toil-9.2.0.dist-info}/top_level.txt +0 -0
toil/worker.py
CHANGED
|
@@ -28,9 +28,9 @@ import sys
|
|
|
28
28
|
import threading
|
|
29
29
|
import time
|
|
30
30
|
import traceback
|
|
31
|
-
from collections.abc import Iterator
|
|
31
|
+
from collections.abc import Callable, Iterator
|
|
32
32
|
from contextlib import contextmanager
|
|
33
|
-
from typing import Any
|
|
33
|
+
from typing import Any
|
|
34
34
|
|
|
35
35
|
from configargparse import ArgParser
|
|
36
36
|
|
|
@@ -62,9 +62,10 @@ class StatsDict(MagicExpando):
|
|
|
62
62
|
|
|
63
63
|
jobs: list[MagicExpando]
|
|
64
64
|
|
|
65
|
+
|
|
65
66
|
def nextChainable(
|
|
66
67
|
predecessor: JobDescription, job_store: AbstractJobStore, config: Config
|
|
67
|
-
) ->
|
|
68
|
+
) -> JobDescription | None:
|
|
68
69
|
"""
|
|
69
70
|
Returns the next chainable job's JobDescription after the given predecessor
|
|
70
71
|
JobDescription, if one exists, or None if the chain must terminate.
|
|
@@ -167,6 +168,7 @@ def nextChainable(
|
|
|
167
168
|
# Made it through! This job is chainable.
|
|
168
169
|
return successor
|
|
169
170
|
|
|
171
|
+
|
|
170
172
|
def unstick_worker(interval: float = 120, timeout: float = 120) -> None:
|
|
171
173
|
"""
|
|
172
174
|
Thread function that tries to prevent the process from getting stuck.
|
|
@@ -189,7 +191,7 @@ def unstick_worker(interval: float = 120, timeout: float = 120) -> None:
|
|
|
189
191
|
# Figure out our process ID
|
|
190
192
|
pid = os.getpid()
|
|
191
193
|
|
|
192
|
-
child:
|
|
194
|
+
child: subprocess.Popen[bytes] | None = None
|
|
193
195
|
|
|
194
196
|
def clean_up_child() -> None:
|
|
195
197
|
"""
|
|
@@ -233,7 +235,9 @@ def unstick_worker(interval: float = 120, timeout: float = 120) -> None:
|
|
|
233
235
|
)
|
|
234
236
|
except FileNotFoundError:
|
|
235
237
|
# If there isn't an lsof, don't try and use it.
|
|
236
|
-
logger.info(
|
|
238
|
+
logger.info(
|
|
239
|
+
"lsof is not available. We will not be able to use it to unstick a stuck Toil leader."
|
|
240
|
+
)
|
|
237
241
|
return
|
|
238
242
|
try:
|
|
239
243
|
child.wait(timeout=timeout)
|
|
@@ -247,19 +251,23 @@ def unstick_worker(interval: float = 120, timeout: float = 120) -> None:
|
|
|
247
251
|
if child.returncode != 0:
|
|
248
252
|
# Something went wrong, which is suspicious. Either it failed or it
|
|
249
253
|
# timed out and could not be killed promptly.
|
|
250
|
-
logger.warning(
|
|
254
|
+
logger.warning(
|
|
255
|
+
"Could not list open files on ourselves. Return code: %s",
|
|
256
|
+
child.returncode,
|
|
257
|
+
)
|
|
251
258
|
|
|
252
259
|
# Wait the interval.
|
|
253
260
|
time.sleep(interval)
|
|
254
261
|
|
|
262
|
+
|
|
255
263
|
def workerScript(
|
|
256
264
|
job_store: AbstractJobStore,
|
|
257
265
|
config: Config,
|
|
258
266
|
job_name: str,
|
|
259
267
|
job_store_id: str,
|
|
260
268
|
redirect_output_to_log_file: bool = True,
|
|
261
|
-
local_worker_temp_dir:
|
|
262
|
-
debug_flags:
|
|
269
|
+
local_worker_temp_dir: str | None = None,
|
|
270
|
+
debug_flags: set[str] | None = None,
|
|
263
271
|
) -> int:
|
|
264
272
|
"""
|
|
265
273
|
Worker process script, runs a job.
|
|
@@ -575,9 +583,7 @@ def workerScript(
|
|
|
575
583
|
caching=config.caching,
|
|
576
584
|
)
|
|
577
585
|
try:
|
|
578
|
-
with job._executor(
|
|
579
|
-
stats=statsDict, fileStore=fileStore
|
|
580
|
-
):
|
|
586
|
+
with job._executor(stats=statsDict, fileStore=fileStore):
|
|
581
587
|
with deferredFunctionManager.open() as defer:
|
|
582
588
|
with fileStore.open(job):
|
|
583
589
|
# Get the next block function to wait on committing this job
|
|
@@ -978,7 +984,7 @@ def in_contexts(contexts: list[str]) -> Iterator[None]:
|
|
|
978
984
|
yield
|
|
979
985
|
|
|
980
986
|
|
|
981
|
-
def main(argv:
|
|
987
|
+
def main(argv: list[str] | None = None) -> None:
|
|
982
988
|
if argv is None:
|
|
983
989
|
argv = sys.argv
|
|
984
990
|
# Parse our command line
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: toil
|
|
3
|
-
Version: 9.
|
|
3
|
+
Version: 9.2.0
|
|
4
4
|
Summary: Pipeline management software for clusters.
|
|
5
5
|
Home-page: https://github.com/DataBiosphere/toil
|
|
6
6
|
Author: Benedict Paten and the Toil community
|
|
@@ -16,11 +16,11 @@ Classifier: Natural Language :: English
|
|
|
16
16
|
Classifier: Operating System :: MacOS :: MacOS X
|
|
17
17
|
Classifier: Operating System :: POSIX
|
|
18
18
|
Classifier: Operating System :: POSIX :: Linux
|
|
19
|
-
Classifier: Programming Language :: Python :: 3.9
|
|
20
19
|
Classifier: Programming Language :: Python :: 3.10
|
|
21
20
|
Classifier: Programming Language :: Python :: 3.11
|
|
22
21
|
Classifier: Programming Language :: Python :: 3.12
|
|
23
22
|
Classifier: Programming Language :: Python :: 3.13
|
|
23
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
24
24
|
Classifier: Topic :: Scientific/Engineering
|
|
25
25
|
Classifier: Topic :: Scientific/Engineering :: Bio-Informatics
|
|
26
26
|
Classifier: Topic :: Scientific/Engineering :: Astronomy
|
|
@@ -29,7 +29,7 @@ Classifier: Topic :: Scientific/Engineering :: Information Analysis
|
|
|
29
29
|
Classifier: Topic :: Scientific/Engineering :: Medical Science Apps.
|
|
30
30
|
Classifier: Topic :: System :: Distributed Computing
|
|
31
31
|
Classifier: Topic :: Utilities
|
|
32
|
-
Requires-Python: >=3.
|
|
32
|
+
Requires-Python: >=3.10
|
|
33
33
|
Description-Content-Type: text/x-rst
|
|
34
34
|
License-File: LICENSE
|
|
35
35
|
Requires-Dist: dill<0.4,>=0.3.2
|
|
@@ -40,7 +40,6 @@ Requires-Dist: python-dateutil
|
|
|
40
40
|
Requires-Dist: psutil<8,>=6.1.0
|
|
41
41
|
Requires-Dist: PyPubSub<5,>=4.0.3
|
|
42
42
|
Requires-Dist: addict<2.5,>=2.2.1
|
|
43
|
-
Requires-Dist: backports.zoneinfo[tzdata]; python_version < "3.9"
|
|
44
43
|
Requires-Dist: enlighten<2,>=1.5.2
|
|
45
44
|
Requires-Dist: configargparse<2,>=1.7
|
|
46
45
|
Requires-Dist: ruamel.yaml>=0.15
|
|
@@ -55,11 +54,11 @@ Requires-Dist: mypy-boto3-s3<2,>=1.28.3.post2; extra == "aws"
|
|
|
55
54
|
Requires-Dist: moto<6,>=5.0.3; extra == "aws"
|
|
56
55
|
Requires-Dist: ec2_metadata<3,>=2.14.0; extra == "aws"
|
|
57
56
|
Provides-Extra: cwl
|
|
58
|
-
Requires-Dist: cwltool==3.1.
|
|
57
|
+
Requires-Dist: cwltool==3.1.20260108082145; extra == "cwl"
|
|
59
58
|
Requires-Dist: schema-salad<9,>=8.4.20230128170514; extra == "cwl"
|
|
60
59
|
Requires-Dist: galaxy-tool-util<26; extra == "cwl"
|
|
61
60
|
Requires-Dist: galaxy-util<26; extra == "cwl"
|
|
62
|
-
Requires-Dist: ruamel.yaml<=0.19,>=0.15; extra == "cwl"
|
|
61
|
+
Requires-Dist: ruamel.yaml<=0.19.1,>=0.15; extra == "cwl"
|
|
63
62
|
Requires-Dist: ruamel.yaml.clib>=0.2.6; extra == "cwl"
|
|
64
63
|
Requires-Dist: networkx!=2.8.1,<4; extra == "cwl"
|
|
65
64
|
Requires-Dist: CacheControl[filecache]; extra == "cwl"
|
|
@@ -77,15 +76,14 @@ Requires-Dist: types-urllib3; extra == "kubernetes"
|
|
|
77
76
|
Requires-Dist: types-PyYAML; extra == "kubernetes"
|
|
78
77
|
Requires-Dist: idna>=2; extra == "kubernetes"
|
|
79
78
|
Provides-Extra: wdl
|
|
80
|
-
Requires-Dist: miniwdl==1.13.
|
|
79
|
+
Requires-Dist: miniwdl==1.13.1; extra == "wdl"
|
|
81
80
|
Requires-Dist: wdlparse==0.1.0; extra == "wdl"
|
|
82
|
-
Requires-Dist: graphlib-backport==1.0; python_version < "3.9" and extra == "wdl"
|
|
83
81
|
Provides-Extra: server
|
|
84
82
|
Requires-Dist: connexion[swagger-ui]<4,>=3; extra == "server"
|
|
85
83
|
Requires-Dist: flask<4,>=3; extra == "server"
|
|
86
84
|
Requires-Dist: werkzeug<4,>=3; extra == "server"
|
|
87
85
|
Requires-Dist: flask-cors<7,>=5.0.1; extra == "server"
|
|
88
|
-
Requires-Dist: gunicorn<
|
|
86
|
+
Requires-Dist: gunicorn<26,>=23.0.0; extra == "server"
|
|
89
87
|
Requires-Dist: celery<6,>=5.4.0; extra == "server"
|
|
90
88
|
Requires-Dist: wes-service<6,>=5.0.0; extra == "server"
|
|
91
89
|
Requires-Dist: ruamel.yaml<1,>=0.18.7; extra == "server"
|
|
@@ -99,11 +97,11 @@ Requires-Dist: mypy-boto3-iam<2,>=1.28.3.post2; extra == "all"
|
|
|
99
97
|
Requires-Dist: mypy-boto3-s3<2,>=1.28.3.post2; extra == "all"
|
|
100
98
|
Requires-Dist: moto<6,>=5.0.3; extra == "all"
|
|
101
99
|
Requires-Dist: ec2_metadata<3,>=2.14.0; extra == "all"
|
|
102
|
-
Requires-Dist: cwltool==3.1.
|
|
100
|
+
Requires-Dist: cwltool==3.1.20260108082145; extra == "all"
|
|
103
101
|
Requires-Dist: schema-salad<9,>=8.4.20230128170514; extra == "all"
|
|
104
102
|
Requires-Dist: galaxy-tool-util<26; extra == "all"
|
|
105
103
|
Requires-Dist: galaxy-util<26; extra == "all"
|
|
106
|
-
Requires-Dist: ruamel.yaml<=0.19,>=0.15; extra == "all"
|
|
104
|
+
Requires-Dist: ruamel.yaml<=0.19.1,>=0.15; extra == "all"
|
|
107
105
|
Requires-Dist: ruamel.yaml.clib>=0.2.6; extra == "all"
|
|
108
106
|
Requires-Dist: networkx!=2.8.1,<4; extra == "all"
|
|
109
107
|
Requires-Dist: CacheControl[filecache]; extra == "all"
|
|
@@ -117,14 +115,13 @@ Requires-Dist: kubernetes-stubs==v22.6.0post1; extra == "all"
|
|
|
117
115
|
Requires-Dist: types-urllib3; extra == "all"
|
|
118
116
|
Requires-Dist: types-PyYAML; extra == "all"
|
|
119
117
|
Requires-Dist: idna>=2; extra == "all"
|
|
120
|
-
Requires-Dist: miniwdl==1.13.
|
|
118
|
+
Requires-Dist: miniwdl==1.13.1; extra == "all"
|
|
121
119
|
Requires-Dist: wdlparse==0.1.0; extra == "all"
|
|
122
|
-
Requires-Dist: graphlib-backport==1.0; python_version < "3.9" and extra == "all"
|
|
123
120
|
Requires-Dist: connexion[swagger-ui]<4,>=3; extra == "all"
|
|
124
121
|
Requires-Dist: flask<4,>=3; extra == "all"
|
|
125
122
|
Requires-Dist: werkzeug<4,>=3; extra == "all"
|
|
126
123
|
Requires-Dist: flask-cors<7,>=5.0.1; extra == "all"
|
|
127
|
-
Requires-Dist: gunicorn<
|
|
124
|
+
Requires-Dist: gunicorn<26,>=23.0.0; extra == "all"
|
|
128
125
|
Requires-Dist: celery<6,>=5.4.0; extra == "all"
|
|
129
126
|
Requires-Dist: wes-service<6,>=5.0.0; extra == "all"
|
|
130
127
|
Requires-Dist: ruamel.yaml<1,>=0.18.7; extra == "all"
|