toil 9.1.1__py3-none-any.whl → 9.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (155) hide show
  1. toil/__init__.py +5 -9
  2. toil/batchSystems/abstractBatchSystem.py +23 -22
  3. toil/batchSystems/abstractGridEngineBatchSystem.py +17 -12
  4. toil/batchSystems/awsBatch.py +8 -8
  5. toil/batchSystems/cleanup_support.py +4 -4
  6. toil/batchSystems/contained_executor.py +3 -3
  7. toil/batchSystems/gridengine.py +3 -4
  8. toil/batchSystems/htcondor.py +5 -5
  9. toil/batchSystems/kubernetes.py +65 -63
  10. toil/batchSystems/local_support.py +2 -3
  11. toil/batchSystems/lsf.py +6 -7
  12. toil/batchSystems/mesos/batchSystem.py +11 -7
  13. toil/batchSystems/mesos/test/__init__.py +1 -2
  14. toil/batchSystems/options.py +9 -10
  15. toil/batchSystems/registry.py +3 -7
  16. toil/batchSystems/singleMachine.py +8 -11
  17. toil/batchSystems/slurm.py +49 -38
  18. toil/batchSystems/torque.py +3 -4
  19. toil/bus.py +36 -34
  20. toil/common.py +129 -89
  21. toil/cwl/cwltoil.py +857 -729
  22. toil/cwl/utils.py +44 -35
  23. toil/fileStores/__init__.py +3 -1
  24. toil/fileStores/abstractFileStore.py +28 -30
  25. toil/fileStores/cachingFileStore.py +8 -8
  26. toil/fileStores/nonCachingFileStore.py +10 -21
  27. toil/job.py +159 -158
  28. toil/jobStores/abstractJobStore.py +68 -69
  29. toil/jobStores/aws/jobStore.py +249 -213
  30. toil/jobStores/aws/utils.py +13 -24
  31. toil/jobStores/fileJobStore.py +28 -22
  32. toil/jobStores/googleJobStore.py +21 -17
  33. toil/jobStores/utils.py +3 -7
  34. toil/leader.py +17 -22
  35. toil/lib/accelerators.py +6 -4
  36. toil/lib/aws/__init__.py +9 -10
  37. toil/lib/aws/ami.py +33 -19
  38. toil/lib/aws/iam.py +6 -6
  39. toil/lib/aws/s3.py +259 -157
  40. toil/lib/aws/session.py +76 -76
  41. toil/lib/aws/utils.py +51 -43
  42. toil/lib/checksum.py +19 -15
  43. toil/lib/compatibility.py +3 -2
  44. toil/lib/conversions.py +45 -18
  45. toil/lib/directory.py +29 -26
  46. toil/lib/docker.py +93 -99
  47. toil/lib/dockstore.py +77 -50
  48. toil/lib/ec2.py +39 -38
  49. toil/lib/ec2nodes.py +11 -4
  50. toil/lib/exceptions.py +8 -5
  51. toil/lib/ftp_utils.py +9 -14
  52. toil/lib/generatedEC2Lists.py +161 -20
  53. toil/lib/history.py +141 -97
  54. toil/lib/history_submission.py +163 -72
  55. toil/lib/io.py +27 -17
  56. toil/lib/memoize.py +2 -1
  57. toil/lib/misc.py +15 -11
  58. toil/lib/pipes.py +40 -25
  59. toil/lib/plugins.py +12 -8
  60. toil/lib/resources.py +1 -0
  61. toil/lib/retry.py +32 -38
  62. toil/lib/threading.py +12 -12
  63. toil/lib/throttle.py +1 -2
  64. toil/lib/trs.py +113 -51
  65. toil/lib/url.py +14 -23
  66. toil/lib/web.py +7 -2
  67. toil/options/common.py +18 -15
  68. toil/options/cwl.py +2 -2
  69. toil/options/runner.py +9 -5
  70. toil/options/wdl.py +1 -3
  71. toil/provisioners/__init__.py +9 -9
  72. toil/provisioners/abstractProvisioner.py +22 -20
  73. toil/provisioners/aws/__init__.py +20 -14
  74. toil/provisioners/aws/awsProvisioner.py +10 -8
  75. toil/provisioners/clusterScaler.py +19 -18
  76. toil/provisioners/gceProvisioner.py +2 -3
  77. toil/provisioners/node.py +11 -13
  78. toil/realtimeLogger.py +4 -4
  79. toil/resource.py +5 -5
  80. toil/server/app.py +2 -2
  81. toil/server/cli/wes_cwl_runner.py +11 -11
  82. toil/server/utils.py +18 -21
  83. toil/server/wes/abstract_backend.py +9 -8
  84. toil/server/wes/amazon_wes_utils.py +3 -3
  85. toil/server/wes/tasks.py +3 -5
  86. toil/server/wes/toil_backend.py +17 -21
  87. toil/server/wsgi_app.py +3 -3
  88. toil/serviceManager.py +3 -4
  89. toil/statsAndLogging.py +12 -13
  90. toil/test/__init__.py +33 -24
  91. toil/test/batchSystems/batchSystemTest.py +12 -11
  92. toil/test/batchSystems/batch_system_plugin_test.py +3 -5
  93. toil/test/batchSystems/test_slurm.py +38 -24
  94. toil/test/cwl/conftest.py +5 -6
  95. toil/test/cwl/cwlTest.py +194 -78
  96. toil/test/cwl/download_file_uri.json +6 -0
  97. toil/test/cwl/download_file_uri_no_hostname.json +6 -0
  98. toil/test/docs/scripts/tutorial_staging.py +1 -0
  99. toil/test/jobStores/jobStoreTest.py +9 -7
  100. toil/test/lib/aws/test_iam.py +1 -3
  101. toil/test/lib/aws/test_s3.py +1 -1
  102. toil/test/lib/dockerTest.py +9 -9
  103. toil/test/lib/test_ec2.py +12 -11
  104. toil/test/lib/test_history.py +4 -4
  105. toil/test/lib/test_trs.py +16 -14
  106. toil/test/lib/test_url.py +7 -6
  107. toil/test/lib/url_plugin_test.py +12 -18
  108. toil/test/provisioners/aws/awsProvisionerTest.py +10 -8
  109. toil/test/provisioners/clusterScalerTest.py +2 -5
  110. toil/test/provisioners/clusterTest.py +1 -3
  111. toil/test/server/serverTest.py +13 -4
  112. toil/test/sort/restart_sort.py +2 -6
  113. toil/test/sort/sort.py +3 -8
  114. toil/test/src/deferredFunctionTest.py +7 -7
  115. toil/test/src/environmentTest.py +1 -2
  116. toil/test/src/fileStoreTest.py +5 -5
  117. toil/test/src/importExportFileTest.py +5 -6
  118. toil/test/src/jobServiceTest.py +22 -14
  119. toil/test/src/jobTest.py +121 -25
  120. toil/test/src/miscTests.py +5 -7
  121. toil/test/src/promisedRequirementTest.py +8 -7
  122. toil/test/src/regularLogTest.py +2 -3
  123. toil/test/src/resourceTest.py +5 -8
  124. toil/test/src/restartDAGTest.py +5 -6
  125. toil/test/src/resumabilityTest.py +2 -2
  126. toil/test/src/retainTempDirTest.py +3 -3
  127. toil/test/src/systemTest.py +3 -3
  128. toil/test/src/threadingTest.py +1 -1
  129. toil/test/src/workerTest.py +1 -2
  130. toil/test/utils/toilDebugTest.py +6 -4
  131. toil/test/utils/toilKillTest.py +1 -1
  132. toil/test/utils/utilsTest.py +15 -14
  133. toil/test/wdl/wdltoil_test.py +247 -124
  134. toil/test/wdl/wdltoil_test_kubernetes.py +2 -2
  135. toil/toilState.py +2 -3
  136. toil/utils/toilDebugFile.py +3 -8
  137. toil/utils/toilDebugJob.py +1 -2
  138. toil/utils/toilLaunchCluster.py +1 -2
  139. toil/utils/toilSshCluster.py +2 -0
  140. toil/utils/toilStats.py +19 -24
  141. toil/utils/toilStatus.py +11 -14
  142. toil/version.py +10 -10
  143. toil/wdl/wdltoil.py +313 -209
  144. toil/worker.py +18 -12
  145. {toil-9.1.1.dist-info → toil-9.2.0.dist-info}/METADATA +11 -14
  146. {toil-9.1.1.dist-info → toil-9.2.0.dist-info}/RECORD +150 -153
  147. {toil-9.1.1.dist-info → toil-9.2.0.dist-info}/WHEEL +1 -1
  148. toil/test/cwl/staging_cat.cwl +0 -27
  149. toil/test/cwl/staging_make_file.cwl +0 -25
  150. toil/test/cwl/staging_workflow.cwl +0 -43
  151. toil/test/cwl/zero_default.cwl +0 -61
  152. toil/test/utils/ABCWorkflowDebug/ABC.txt +0 -1
  153. {toil-9.1.1.dist-info → toil-9.2.0.dist-info}/entry_points.txt +0 -0
  154. {toil-9.1.1.dist-info → toil-9.2.0.dist-info}/licenses/LICENSE +0 -0
  155. {toil-9.1.1.dist-info → toil-9.2.0.dist-info}/top_level.txt +0 -0
toil/worker.py CHANGED
@@ -28,9 +28,9 @@ import sys
28
28
  import threading
29
29
  import time
30
30
  import traceback
31
- from collections.abc import Iterator
31
+ from collections.abc import Callable, Iterator
32
32
  from contextlib import contextmanager
33
- from typing import Any, Callable, Optional
33
+ from typing import Any
34
34
 
35
35
  from configargparse import ArgParser
36
36
 
@@ -62,9 +62,10 @@ class StatsDict(MagicExpando):
62
62
 
63
63
  jobs: list[MagicExpando]
64
64
 
65
+
65
66
  def nextChainable(
66
67
  predecessor: JobDescription, job_store: AbstractJobStore, config: Config
67
- ) -> Optional[JobDescription]:
68
+ ) -> JobDescription | None:
68
69
  """
69
70
  Returns the next chainable job's JobDescription after the given predecessor
70
71
  JobDescription, if one exists, or None if the chain must terminate.
@@ -167,6 +168,7 @@ def nextChainable(
167
168
  # Made it through! This job is chainable.
168
169
  return successor
169
170
 
171
+
170
172
  def unstick_worker(interval: float = 120, timeout: float = 120) -> None:
171
173
  """
172
174
  Thread function that tries to prevent the process from getting stuck.
@@ -189,7 +191,7 @@ def unstick_worker(interval: float = 120, timeout: float = 120) -> None:
189
191
  # Figure out our process ID
190
192
  pid = os.getpid()
191
193
 
192
- child: Optional[subprocess.Popen[bytes]] = None
194
+ child: subprocess.Popen[bytes] | None = None
193
195
 
194
196
  def clean_up_child() -> None:
195
197
  """
@@ -233,7 +235,9 @@ def unstick_worker(interval: float = 120, timeout: float = 120) -> None:
233
235
  )
234
236
  except FileNotFoundError:
235
237
  # If there isn't an lsof, don't try and use it.
236
- logger.info("lsof is not available. We will not be able to use it to unstick a stuck Toil leader.")
238
+ logger.info(
239
+ "lsof is not available. We will not be able to use it to unstick a stuck Toil leader."
240
+ )
237
241
  return
238
242
  try:
239
243
  child.wait(timeout=timeout)
@@ -247,19 +251,23 @@ def unstick_worker(interval: float = 120, timeout: float = 120) -> None:
247
251
  if child.returncode != 0:
248
252
  # Something went wrong, which is suspicious. Either it failed or it
249
253
  # timed out and could not be killed promptly.
250
- logger.warning("Could not list open files on ourselves. Return code: %s", child.returncode)
254
+ logger.warning(
255
+ "Could not list open files on ourselves. Return code: %s",
256
+ child.returncode,
257
+ )
251
258
 
252
259
  # Wait the interval.
253
260
  time.sleep(interval)
254
261
 
262
+
255
263
  def workerScript(
256
264
  job_store: AbstractJobStore,
257
265
  config: Config,
258
266
  job_name: str,
259
267
  job_store_id: str,
260
268
  redirect_output_to_log_file: bool = True,
261
- local_worker_temp_dir: Optional[str] = None,
262
- debug_flags: Optional[set[str]] = None,
269
+ local_worker_temp_dir: str | None = None,
270
+ debug_flags: set[str] | None = None,
263
271
  ) -> int:
264
272
  """
265
273
  Worker process script, runs a job.
@@ -575,9 +583,7 @@ def workerScript(
575
583
  caching=config.caching,
576
584
  )
577
585
  try:
578
- with job._executor(
579
- stats=statsDict, fileStore=fileStore
580
- ):
586
+ with job._executor(stats=statsDict, fileStore=fileStore):
581
587
  with deferredFunctionManager.open() as defer:
582
588
  with fileStore.open(job):
583
589
  # Get the next block function to wait on committing this job
@@ -978,7 +984,7 @@ def in_contexts(contexts: list[str]) -> Iterator[None]:
978
984
  yield
979
985
 
980
986
 
981
- def main(argv: Optional[list[str]] = None) -> None:
987
+ def main(argv: list[str] | None = None) -> None:
982
988
  if argv is None:
983
989
  argv = sys.argv
984
990
  # Parse our command line
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: toil
3
- Version: 9.1.1
3
+ Version: 9.2.0
4
4
  Summary: Pipeline management software for clusters.
5
5
  Home-page: https://github.com/DataBiosphere/toil
6
6
  Author: Benedict Paten and the Toil community
@@ -16,11 +16,11 @@ Classifier: Natural Language :: English
16
16
  Classifier: Operating System :: MacOS :: MacOS X
17
17
  Classifier: Operating System :: POSIX
18
18
  Classifier: Operating System :: POSIX :: Linux
19
- Classifier: Programming Language :: Python :: 3.9
20
19
  Classifier: Programming Language :: Python :: 3.10
21
20
  Classifier: Programming Language :: Python :: 3.11
22
21
  Classifier: Programming Language :: Python :: 3.12
23
22
  Classifier: Programming Language :: Python :: 3.13
23
+ Classifier: Programming Language :: Python :: 3.14
24
24
  Classifier: Topic :: Scientific/Engineering
25
25
  Classifier: Topic :: Scientific/Engineering :: Bio-Informatics
26
26
  Classifier: Topic :: Scientific/Engineering :: Astronomy
@@ -29,7 +29,7 @@ Classifier: Topic :: Scientific/Engineering :: Information Analysis
29
29
  Classifier: Topic :: Scientific/Engineering :: Medical Science Apps.
30
30
  Classifier: Topic :: System :: Distributed Computing
31
31
  Classifier: Topic :: Utilities
32
- Requires-Python: >=3.9
32
+ Requires-Python: >=3.10
33
33
  Description-Content-Type: text/x-rst
34
34
  License-File: LICENSE
35
35
  Requires-Dist: dill<0.4,>=0.3.2
@@ -40,7 +40,6 @@ Requires-Dist: python-dateutil
40
40
  Requires-Dist: psutil<8,>=6.1.0
41
41
  Requires-Dist: PyPubSub<5,>=4.0.3
42
42
  Requires-Dist: addict<2.5,>=2.2.1
43
- Requires-Dist: backports.zoneinfo[tzdata]; python_version < "3.9"
44
43
  Requires-Dist: enlighten<2,>=1.5.2
45
44
  Requires-Dist: configargparse<2,>=1.7
46
45
  Requires-Dist: ruamel.yaml>=0.15
@@ -55,11 +54,11 @@ Requires-Dist: mypy-boto3-s3<2,>=1.28.3.post2; extra == "aws"
55
54
  Requires-Dist: moto<6,>=5.0.3; extra == "aws"
56
55
  Requires-Dist: ec2_metadata<3,>=2.14.0; extra == "aws"
57
56
  Provides-Extra: cwl
58
- Requires-Dist: cwltool==3.1.20250715140722; extra == "cwl"
57
+ Requires-Dist: cwltool==3.1.20260108082145; extra == "cwl"
59
58
  Requires-Dist: schema-salad<9,>=8.4.20230128170514; extra == "cwl"
60
59
  Requires-Dist: galaxy-tool-util<26; extra == "cwl"
61
60
  Requires-Dist: galaxy-util<26; extra == "cwl"
62
- Requires-Dist: ruamel.yaml<=0.19,>=0.15; extra == "cwl"
61
+ Requires-Dist: ruamel.yaml<=0.19.1,>=0.15; extra == "cwl"
63
62
  Requires-Dist: ruamel.yaml.clib>=0.2.6; extra == "cwl"
64
63
  Requires-Dist: networkx!=2.8.1,<4; extra == "cwl"
65
64
  Requires-Dist: CacheControl[filecache]; extra == "cwl"
@@ -77,15 +76,14 @@ Requires-Dist: types-urllib3; extra == "kubernetes"
77
76
  Requires-Dist: types-PyYAML; extra == "kubernetes"
78
77
  Requires-Dist: idna>=2; extra == "kubernetes"
79
78
  Provides-Extra: wdl
80
- Requires-Dist: miniwdl==1.13.0; extra == "wdl"
79
+ Requires-Dist: miniwdl==1.13.1; extra == "wdl"
81
80
  Requires-Dist: wdlparse==0.1.0; extra == "wdl"
82
- Requires-Dist: graphlib-backport==1.0; python_version < "3.9" and extra == "wdl"
83
81
  Provides-Extra: server
84
82
  Requires-Dist: connexion[swagger-ui]<4,>=3; extra == "server"
85
83
  Requires-Dist: flask<4,>=3; extra == "server"
86
84
  Requires-Dist: werkzeug<4,>=3; extra == "server"
87
85
  Requires-Dist: flask-cors<7,>=5.0.1; extra == "server"
88
- Requires-Dist: gunicorn<24,>=23.0.0; extra == "server"
86
+ Requires-Dist: gunicorn<26,>=23.0.0; extra == "server"
89
87
  Requires-Dist: celery<6,>=5.4.0; extra == "server"
90
88
  Requires-Dist: wes-service<6,>=5.0.0; extra == "server"
91
89
  Requires-Dist: ruamel.yaml<1,>=0.18.7; extra == "server"
@@ -99,11 +97,11 @@ Requires-Dist: mypy-boto3-iam<2,>=1.28.3.post2; extra == "all"
99
97
  Requires-Dist: mypy-boto3-s3<2,>=1.28.3.post2; extra == "all"
100
98
  Requires-Dist: moto<6,>=5.0.3; extra == "all"
101
99
  Requires-Dist: ec2_metadata<3,>=2.14.0; extra == "all"
102
- Requires-Dist: cwltool==3.1.20250715140722; extra == "all"
100
+ Requires-Dist: cwltool==3.1.20260108082145; extra == "all"
103
101
  Requires-Dist: schema-salad<9,>=8.4.20230128170514; extra == "all"
104
102
  Requires-Dist: galaxy-tool-util<26; extra == "all"
105
103
  Requires-Dist: galaxy-util<26; extra == "all"
106
- Requires-Dist: ruamel.yaml<=0.19,>=0.15; extra == "all"
104
+ Requires-Dist: ruamel.yaml<=0.19.1,>=0.15; extra == "all"
107
105
  Requires-Dist: ruamel.yaml.clib>=0.2.6; extra == "all"
108
106
  Requires-Dist: networkx!=2.8.1,<4; extra == "all"
109
107
  Requires-Dist: CacheControl[filecache]; extra == "all"
@@ -117,14 +115,13 @@ Requires-Dist: kubernetes-stubs==v22.6.0post1; extra == "all"
117
115
  Requires-Dist: types-urllib3; extra == "all"
118
116
  Requires-Dist: types-PyYAML; extra == "all"
119
117
  Requires-Dist: idna>=2; extra == "all"
120
- Requires-Dist: miniwdl==1.13.0; extra == "all"
118
+ Requires-Dist: miniwdl==1.13.1; extra == "all"
121
119
  Requires-Dist: wdlparse==0.1.0; extra == "all"
122
- Requires-Dist: graphlib-backport==1.0; python_version < "3.9" and extra == "all"
123
120
  Requires-Dist: connexion[swagger-ui]<4,>=3; extra == "all"
124
121
  Requires-Dist: flask<4,>=3; extra == "all"
125
122
  Requires-Dist: werkzeug<4,>=3; extra == "all"
126
123
  Requires-Dist: flask-cors<7,>=5.0.1; extra == "all"
127
- Requires-Dist: gunicorn<24,>=23.0.0; extra == "all"
124
+ Requires-Dist: gunicorn<26,>=23.0.0; extra == "all"
128
125
  Requires-Dist: celery<6,>=5.4.0; extra == "all"
129
126
  Requires-Dist: wes-service<6,>=5.0.0; extra == "all"
130
127
  Requires-Dist: ruamel.yaml<1,>=0.18.7; extra == "all"