toil 9.1.1__py3-none-any.whl → 9.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (155) hide show
  1. toil/__init__.py +5 -9
  2. toil/batchSystems/abstractBatchSystem.py +23 -22
  3. toil/batchSystems/abstractGridEngineBatchSystem.py +17 -12
  4. toil/batchSystems/awsBatch.py +8 -8
  5. toil/batchSystems/cleanup_support.py +4 -4
  6. toil/batchSystems/contained_executor.py +3 -3
  7. toil/batchSystems/gridengine.py +3 -4
  8. toil/batchSystems/htcondor.py +5 -5
  9. toil/batchSystems/kubernetes.py +65 -63
  10. toil/batchSystems/local_support.py +2 -3
  11. toil/batchSystems/lsf.py +6 -7
  12. toil/batchSystems/mesos/batchSystem.py +11 -7
  13. toil/batchSystems/mesos/test/__init__.py +1 -2
  14. toil/batchSystems/options.py +9 -10
  15. toil/batchSystems/registry.py +3 -7
  16. toil/batchSystems/singleMachine.py +8 -11
  17. toil/batchSystems/slurm.py +49 -38
  18. toil/batchSystems/torque.py +3 -4
  19. toil/bus.py +36 -34
  20. toil/common.py +129 -89
  21. toil/cwl/cwltoil.py +857 -729
  22. toil/cwl/utils.py +44 -35
  23. toil/fileStores/__init__.py +3 -1
  24. toil/fileStores/abstractFileStore.py +28 -30
  25. toil/fileStores/cachingFileStore.py +8 -8
  26. toil/fileStores/nonCachingFileStore.py +10 -21
  27. toil/job.py +159 -158
  28. toil/jobStores/abstractJobStore.py +68 -69
  29. toil/jobStores/aws/jobStore.py +249 -213
  30. toil/jobStores/aws/utils.py +13 -24
  31. toil/jobStores/fileJobStore.py +28 -22
  32. toil/jobStores/googleJobStore.py +21 -17
  33. toil/jobStores/utils.py +3 -7
  34. toil/leader.py +17 -22
  35. toil/lib/accelerators.py +6 -4
  36. toil/lib/aws/__init__.py +9 -10
  37. toil/lib/aws/ami.py +33 -19
  38. toil/lib/aws/iam.py +6 -6
  39. toil/lib/aws/s3.py +259 -157
  40. toil/lib/aws/session.py +76 -76
  41. toil/lib/aws/utils.py +51 -43
  42. toil/lib/checksum.py +19 -15
  43. toil/lib/compatibility.py +3 -2
  44. toil/lib/conversions.py +45 -18
  45. toil/lib/directory.py +29 -26
  46. toil/lib/docker.py +93 -99
  47. toil/lib/dockstore.py +77 -50
  48. toil/lib/ec2.py +39 -38
  49. toil/lib/ec2nodes.py +11 -4
  50. toil/lib/exceptions.py +8 -5
  51. toil/lib/ftp_utils.py +9 -14
  52. toil/lib/generatedEC2Lists.py +161 -20
  53. toil/lib/history.py +141 -97
  54. toil/lib/history_submission.py +163 -72
  55. toil/lib/io.py +27 -17
  56. toil/lib/memoize.py +2 -1
  57. toil/lib/misc.py +15 -11
  58. toil/lib/pipes.py +40 -25
  59. toil/lib/plugins.py +12 -8
  60. toil/lib/resources.py +1 -0
  61. toil/lib/retry.py +32 -38
  62. toil/lib/threading.py +12 -12
  63. toil/lib/throttle.py +1 -2
  64. toil/lib/trs.py +113 -51
  65. toil/lib/url.py +14 -23
  66. toil/lib/web.py +7 -2
  67. toil/options/common.py +18 -15
  68. toil/options/cwl.py +2 -2
  69. toil/options/runner.py +9 -5
  70. toil/options/wdl.py +1 -3
  71. toil/provisioners/__init__.py +9 -9
  72. toil/provisioners/abstractProvisioner.py +22 -20
  73. toil/provisioners/aws/__init__.py +20 -14
  74. toil/provisioners/aws/awsProvisioner.py +10 -8
  75. toil/provisioners/clusterScaler.py +19 -18
  76. toil/provisioners/gceProvisioner.py +2 -3
  77. toil/provisioners/node.py +11 -13
  78. toil/realtimeLogger.py +4 -4
  79. toil/resource.py +5 -5
  80. toil/server/app.py +2 -2
  81. toil/server/cli/wes_cwl_runner.py +11 -11
  82. toil/server/utils.py +18 -21
  83. toil/server/wes/abstract_backend.py +9 -8
  84. toil/server/wes/amazon_wes_utils.py +3 -3
  85. toil/server/wes/tasks.py +3 -5
  86. toil/server/wes/toil_backend.py +17 -21
  87. toil/server/wsgi_app.py +3 -3
  88. toil/serviceManager.py +3 -4
  89. toil/statsAndLogging.py +12 -13
  90. toil/test/__init__.py +33 -24
  91. toil/test/batchSystems/batchSystemTest.py +12 -11
  92. toil/test/batchSystems/batch_system_plugin_test.py +3 -5
  93. toil/test/batchSystems/test_slurm.py +38 -24
  94. toil/test/cwl/conftest.py +5 -6
  95. toil/test/cwl/cwlTest.py +194 -78
  96. toil/test/cwl/download_file_uri.json +6 -0
  97. toil/test/cwl/download_file_uri_no_hostname.json +6 -0
  98. toil/test/docs/scripts/tutorial_staging.py +1 -0
  99. toil/test/jobStores/jobStoreTest.py +9 -7
  100. toil/test/lib/aws/test_iam.py +1 -3
  101. toil/test/lib/aws/test_s3.py +1 -1
  102. toil/test/lib/dockerTest.py +9 -9
  103. toil/test/lib/test_ec2.py +12 -11
  104. toil/test/lib/test_history.py +4 -4
  105. toil/test/lib/test_trs.py +16 -14
  106. toil/test/lib/test_url.py +7 -6
  107. toil/test/lib/url_plugin_test.py +12 -18
  108. toil/test/provisioners/aws/awsProvisionerTest.py +10 -8
  109. toil/test/provisioners/clusterScalerTest.py +2 -5
  110. toil/test/provisioners/clusterTest.py +1 -3
  111. toil/test/server/serverTest.py +13 -4
  112. toil/test/sort/restart_sort.py +2 -6
  113. toil/test/sort/sort.py +3 -8
  114. toil/test/src/deferredFunctionTest.py +7 -7
  115. toil/test/src/environmentTest.py +1 -2
  116. toil/test/src/fileStoreTest.py +5 -5
  117. toil/test/src/importExportFileTest.py +5 -6
  118. toil/test/src/jobServiceTest.py +22 -14
  119. toil/test/src/jobTest.py +121 -25
  120. toil/test/src/miscTests.py +5 -7
  121. toil/test/src/promisedRequirementTest.py +8 -7
  122. toil/test/src/regularLogTest.py +2 -3
  123. toil/test/src/resourceTest.py +5 -8
  124. toil/test/src/restartDAGTest.py +5 -6
  125. toil/test/src/resumabilityTest.py +2 -2
  126. toil/test/src/retainTempDirTest.py +3 -3
  127. toil/test/src/systemTest.py +3 -3
  128. toil/test/src/threadingTest.py +1 -1
  129. toil/test/src/workerTest.py +1 -2
  130. toil/test/utils/toilDebugTest.py +6 -4
  131. toil/test/utils/toilKillTest.py +1 -1
  132. toil/test/utils/utilsTest.py +15 -14
  133. toil/test/wdl/wdltoil_test.py +247 -124
  134. toil/test/wdl/wdltoil_test_kubernetes.py +2 -2
  135. toil/toilState.py +2 -3
  136. toil/utils/toilDebugFile.py +3 -8
  137. toil/utils/toilDebugJob.py +1 -2
  138. toil/utils/toilLaunchCluster.py +1 -2
  139. toil/utils/toilSshCluster.py +2 -0
  140. toil/utils/toilStats.py +19 -24
  141. toil/utils/toilStatus.py +11 -14
  142. toil/version.py +10 -10
  143. toil/wdl/wdltoil.py +313 -209
  144. toil/worker.py +18 -12
  145. {toil-9.1.1.dist-info → toil-9.2.0.dist-info}/METADATA +11 -14
  146. {toil-9.1.1.dist-info → toil-9.2.0.dist-info}/RECORD +150 -153
  147. {toil-9.1.1.dist-info → toil-9.2.0.dist-info}/WHEEL +1 -1
  148. toil/test/cwl/staging_cat.cwl +0 -27
  149. toil/test/cwl/staging_make_file.cwl +0 -25
  150. toil/test/cwl/staging_workflow.cwl +0 -43
  151. toil/test/cwl/zero_default.cwl +0 -61
  152. toil/test/utils/ABCWorkflowDebug/ABC.txt +0 -1
  153. {toil-9.1.1.dist-info → toil-9.2.0.dist-info}/entry_points.txt +0 -0
  154. {toil-9.1.1.dist-info → toil-9.2.0.dist-info}/licenses/LICENSE +0 -0
  155. {toil-9.1.1.dist-info → toil-9.2.0.dist-info}/top_level.txt +0 -0
toil/test/lib/test_ec2.py CHANGED
@@ -13,15 +13,14 @@
13
13
  # limitations under the License.
14
14
  import logging
15
15
  import os
16
-
17
16
  from unittest import mock
18
17
 
19
18
  from toil.lib.aws.ami import (
19
+ ReleaseFeedUnavailableError,
20
20
  aws_marketplace_flatcar_ami_search,
21
21
  feed_flatcar_ami_release,
22
22
  flatcar_release_feed_ami,
23
23
  get_flatcar_ami,
24
- ReleaseFeedUnavailableError
25
24
  )
26
25
  from toil.test import ToilTest, needs_aws_ec2, needs_online
27
26
 
@@ -33,27 +32,29 @@ logging.basicConfig(level=logging.DEBUG)
33
32
  class FlatcarFeedTest(ToilTest):
34
33
  """Test accessing the Flatcar AMI release feed, independent of the AWS API"""
35
34
 
36
- # Note that we need to support getting an empty list back, because
37
- # sometimes the Flatcar feeds are just down, and we can't fail CI at those
38
- # times.
35
+ # Note that we need to support getting no AMI back, because sometimes the
36
+ # Flatcar feeds are just down, and we can't fail CI at those times.
39
37
 
40
38
  def test_parse_archive_feed(self):
41
39
  """Make sure we can get a Flatcar release from the Internet Archive."""
42
40
  ami = flatcar_release_feed_ami("us-west-2", "amd64", "archive")
43
- self.assertEqual(len(ami), len("ami-02b46c73fed689d1c"))
44
- self.assertTrue(ami.startswith("ami-"))
41
+ if ami is not None:
42
+ self.assertEqual(len(ami), len("ami-02b46c73fed689d1c"))
43
+ self.assertTrue(ami.startswith("ami-"))
45
44
 
46
45
  def test_parse_beta_feed(self):
47
46
  """Make sure we can get a Flatcar release from the beta channel."""
48
47
  ami = flatcar_release_feed_ami("us-west-2", "amd64", "beta")
49
- self.assertEqual(len(ami), len("ami-02b46c73fed689d1c"))
50
- self.assertTrue(ami.startswith("ami-"))
48
+ if ami is not None:
49
+ self.assertEqual(len(ami), len("ami-02b46c73fed689d1c"))
50
+ self.assertTrue(ami.startswith("ami-"))
51
51
 
52
52
  def test_parse_stable_feed(self):
53
53
  """Make sure we can get a Flatcar release from the stable channel."""
54
54
  ami = flatcar_release_feed_ami("us-west-2", "amd64", "stable")
55
- self.assertEqual(len(ami), len("ami-02b46c73fed689d1c"))
56
- self.assertTrue(ami.startswith("ami-"))
55
+ if ami is not None:
56
+ self.assertEqual(len(ami), len("ami-02b46c73fed689d1c"))
57
+ self.assertTrue(ami.startswith("ami-"))
57
58
 
58
59
 
59
60
  @needs_aws_ec2
@@ -12,15 +12,15 @@
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
14
 
15
- from collections.abc import Generator
16
- from pathlib import Path
17
15
  import logging
18
16
  import time
19
-
20
- from toil.lib.history import HistoryManager
17
+ from collections.abc import Generator
18
+ from pathlib import Path
21
19
 
22
20
  import pytest
23
21
 
22
+ from toil.lib.history import HistoryManager
23
+
24
24
  logger = logging.getLogger(__name__)
25
25
  logging.basicConfig(level=logging.DEBUG)
26
26
 
toil/test/lib/test_trs.py CHANGED
@@ -12,20 +12,21 @@
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
14
 
15
- import io
16
15
  import logging
17
- import pytest
18
- from typing import IO
19
16
  import urllib.request
17
+ from typing import IO
20
18
  from urllib.error import URLError
21
19
 
20
+ import pytest
21
+
22
22
  from toil.lib.retry import retry
23
- from toil.lib.trs import find_workflow, fetch_workflow
23
+ from toil.lib.trs import fetch_workflow, find_workflow
24
24
  from toil.test import ToilTest, needs_online
25
25
 
26
26
  logger = logging.getLogger(__name__)
27
27
  logging.basicConfig(level=logging.DEBUG)
28
28
 
29
+
29
30
  @pytest.mark.integrative
30
31
  @needs_online
31
32
  class DockstoreLookupTest(ToilTest):
@@ -57,7 +58,9 @@ class DockstoreLookupTest(ToilTest):
57
58
  PAGE_URL = "https://dockstore.org/workflows/github.com/dockstore/bcc2020-training/HelloWorld:master?tab=info"
58
59
  trs_id, trs_version, language = find_workflow(PAGE_URL)
59
60
 
60
- self.assertEqual(trs_id, "#workflow/github.com/dockstore/bcc2020-training/HelloWorld")
61
+ self.assertEqual(
62
+ trs_id, "#workflow/github.com/dockstore/bcc2020-training/HelloWorld"
63
+ )
61
64
  self.assertEqual(trs_version, "master")
62
65
  self.assertEqual(language, "WDL")
63
66
 
@@ -65,7 +68,7 @@ class DockstoreLookupTest(ToilTest):
65
68
  TRS_ID = "#workflow/github.com/dockstore-testing/md5sum-checker"
66
69
  TRS_VERSION = "master"
67
70
  trs_id, trs_version, language = find_workflow(f"{TRS_ID}:{TRS_VERSION}")
68
-
71
+
69
72
  self.assertEqual(trs_id, TRS_ID)
70
73
  self.assertEqual(trs_version, TRS_VERSION)
71
74
  self.assertEqual(language, "CWL")
@@ -90,7 +93,7 @@ class DockstoreLookupTest(ToilTest):
90
93
  # checker workflow itself.
91
94
  WORKFLOW_URL = "https://raw.githubusercontent.com/dockstore-testing/md5sum-checker/master/md5sum/md5sum-workflow.cwl"
92
95
  looked_up = fetch_workflow(TRS_ID, TRS_VERSION, LANGUAGE)
93
-
96
+
94
97
  data_from_lookup = self.read_result(looked_up).read()
95
98
  data_from_source = self.read_result(WORKFLOW_URL).read()
96
99
  self.assertEqual(data_from_lookup, data_from_source)
@@ -104,7 +107,7 @@ class DockstoreLookupTest(ToilTest):
104
107
  fetch_workflow(TRS_ID, TRS_VERSION, LANGUAGE)
105
108
  # This lookup is definitely cached
106
109
  looked_up = fetch_workflow(TRS_ID, TRS_VERSION, LANGUAGE)
107
-
110
+
108
111
  data_from_lookup = self.read_result(looked_up).read()
109
112
  data_from_source = self.read_result(WORKFLOW_URL).read()
110
113
  self.assertEqual(data_from_lookup, data_from_source)
@@ -112,7 +115,7 @@ class DockstoreLookupTest(ToilTest):
112
115
  def test_lookup_from_trs_with_version(self) -> None:
113
116
  TRS_VERSIONED_ID = "#workflow/github.com/dockstore-testing/md5sum-checker:workflowWithHTTPImport"
114
117
  trs_id, trs_version, language = find_workflow(TRS_VERSIONED_ID)
115
-
118
+
116
119
  parts = TRS_VERSIONED_ID.split(":")
117
120
 
118
121
  self.assertEqual(trs_id, parts[0])
@@ -130,10 +133,12 @@ class DockstoreLookupTest(ToilTest):
130
133
  looked_up = find_workflow(TRS_VERSIONED_ID)
131
134
 
132
135
  def test_lookup_from_trs_nonexistent_version(self) -> None:
133
- TRS_VERSIONED_ID = "#workflow/github.com/dockstore-testing/md5sum-checker:notARealVersion"
136
+ TRS_VERSIONED_ID = (
137
+ "#workflow/github.com/dockstore-testing/md5sum-checker:notARealVersion"
138
+ )
134
139
  with self.assertRaises(FileNotFoundError):
135
140
  looked_up = find_workflow(TRS_VERSIONED_ID)
136
-
141
+
137
142
  def test_get_nonexistent_workflow(self) -> None:
138
143
  TRS_ID = "#workflow/github.com/adamnovak/veryfakerepo"
139
144
  TRS_VERSION = "notARealVersion"
@@ -156,6 +161,3 @@ class DockstoreLookupTest(ToilTest):
156
161
  LANGUAGE = "CWL"
157
162
  with self.assertRaises(FileNotFoundError):
158
163
  looked_up = fetch_workflow(TRS_ID, TRS_VERSION, LANGUAGE)
159
-
160
-
161
-
toil/test/lib/test_url.py CHANGED
@@ -11,20 +11,18 @@
11
11
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
- import getpass
15
14
  import logging
16
15
 
17
16
  from pytest_httpserver import HTTPServer
18
17
 
19
- from toil.lib.misc import get_user_name
20
18
  from toil.lib.url import URLAccess
21
- from toil.test import needs_aws_s3, needs_online
22
-
19
+ from toil.test import needs_aws_s3
23
20
 
24
21
  logger = logging.getLogger(__name__)
25
22
  logging.basicConfig(level=logging.DEBUG)
26
23
 
27
- class TestURLAccess():
24
+
25
+ class TestURLAccess:
28
26
  """
29
27
  Test URLAccess class handling read, list,
30
28
  and checking the size/existence of resources at given URL
@@ -54,8 +52,11 @@ class TestURLAccess():
54
52
  @needs_aws_s3
55
53
  def test_read_from_url(self) -> None:
56
54
  import io
55
+
57
56
  output = io.BytesIO()
58
- size, executable = URLAccess.read_from_url("s3://toil-datasets/hello.txt", output)
57
+ size, executable = URLAccess.read_from_url(
58
+ "s3://toil-datasets/hello.txt", output
59
+ )
59
60
  assert isinstance(size, int)
60
61
  assert size > 0
61
62
  assert not executable
@@ -11,29 +11,18 @@
11
11
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
- import logging
15
- from typing import IO, Optional, Union
16
-
17
- from configargparse import ArgParser, ArgumentParser
18
-
19
- from toil.batchSystems.abstractBatchSystem import (
20
- AbstractBatchSystem,
21
- UpdatedBatchJobInfo,
22
- )
23
- from toil.batchSystems.cleanup_support import BatchSystemCleanupSupport
24
- from toil.batchSystems.options import OptionSetter
25
- from toil.batchSystems.registry import add_batch_system_factory
26
- from toil.common import Toil, addOptions
27
- from toil.job import JobDescription
28
-
29
14
  import io
15
+ import logging
16
+ from typing import IO
30
17
  from urllib.parse import ParseResult
31
- from toil.test import ToilTest
32
- from toil.lib.url import URLAccess
18
+
33
19
  from toil.lib.plugins import register_plugin, remove_plugin
20
+ from toil.lib.url import URLAccess
21
+ from toil.test import ToilTest
34
22
 
35
23
  logger = logging.getLogger(__name__)
36
24
 
25
+
37
26
  class FakeURLPlugin(URLAccess):
38
27
  @classmethod
39
28
  def _supports_url(cls, url: ParseResult, export: bool = False) -> bool:
@@ -66,7 +55,12 @@ class FakeURLPlugin(URLAccess):
66
55
  return io.BytesIO(b"hello world")
67
56
 
68
57
  @classmethod
69
- def _write_to_url(cls, readable: Union[IO[bytes], IO[str]], url: ParseResult, executable: bool = False) -> None:
58
+ def _write_to_url(
59
+ cls,
60
+ readable: IO[bytes] | IO[str],
61
+ url: ParseResult,
62
+ executable: bool = False,
63
+ ) -> None:
70
64
  pass
71
65
 
72
66
 
@@ -11,7 +11,6 @@
11
11
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
- import datetime
15
14
  import logging
16
15
  import os
17
16
  import subprocess
@@ -26,17 +25,15 @@ from uuid import uuid4
26
25
  import pytest
27
26
 
28
27
  import toil.lib.aws.session
29
-
30
28
  from toil.lib.aws import zone_to_region
31
29
  from toil.provisioners import cluster_factory
32
- from toil.provisioners.aws.awsProvisioner import AWSProvisioner
33
30
  from toil.provisioners.aws import (
34
31
  _get_spot_history,
35
32
  get_aws_zone_from_spot_market,
36
33
  get_best_aws_zone,
37
34
  )
35
+ from toil.provisioners.aws.awsProvisioner import AWSProvisioner
38
36
  from toil.test import (
39
- ToilTest,
40
37
  get_data,
41
38
  integrative,
42
39
  needs_aws_ec2,
@@ -63,6 +60,7 @@ if TYPE_CHECKING:
63
60
 
64
61
  log = logging.getLogger(__name__)
65
62
 
63
+
66
64
  @pytest.fixture
67
65
  def aws_zone():
68
66
  """
@@ -74,6 +72,7 @@ def aws_zone():
74
72
  ), "Could not determine AWS availability zone to test in; is TOIL_AWS_ZONE set?"
75
73
  return zone
76
74
 
75
+
77
76
  @pytest.fixture
78
77
  def aws_region(aws_zone):
79
78
  """
@@ -81,6 +80,7 @@ def aws_region(aws_zone):
81
80
  """
82
81
  return zone_to_region(aws_zone)
83
82
 
83
+
84
84
  @pytest.fixture
85
85
  def ec2_client(aws_region):
86
86
  """
@@ -147,7 +147,9 @@ class TestAWSProvisionerBenchTest:
147
147
  Make sure that we can process spot price history to pick a zone.
148
148
  """
149
149
  zone_options = ["us-west-2a", "af-south-1c"]
150
- zone_choice = get_aws_zone_from_spot_market(0.01, "t3.large", ec2_client, zone_options)
150
+ zone_choice = get_aws_zone_from_spot_market(
151
+ 0.01, "t3.large", ec2_client, zone_options
152
+ )
151
153
  assert zone_choice in zone_options
152
154
 
153
155
 
@@ -208,7 +210,7 @@ class AbstractAWSAutoscaleTest(AbstractClusterTest):
208
210
  instances.sort(key=lambda x: x.get("LaunchTime"))
209
211
  leader: "InstanceTypeDef" = instances[0] # assume leader was launched first
210
212
 
211
- bdm: Optional[list["InstanceBlockDeviceMappingTypeDef"]] = leader.get(
213
+ bdm: list["InstanceBlockDeviceMappingTypeDef"] | None = leader.get(
212
214
  "BlockDeviceMappings"
213
215
  )
214
216
  assert bdm is not None
@@ -305,7 +307,7 @@ class AbstractAWSAutoscaleTest(AbstractClusterTest):
305
307
  self.cluster.destroyCluster()
306
308
  boto3_ec2: "EC2Client" = self.aws.client(region=self.region, service_name="ec2")
307
309
  volume_filter: "FilterTypeDef" = {"Name": "volume-id", "Values": [volumeID]}
308
- volumes: Optional[list["VolumeTypeDef"]] = None
310
+ volumes: list["VolumeTypeDef"] | None = None
309
311
  for attempt in range(6):
310
312
  # https://github.com/BD2KGenomics/toil/issues/1567
311
313
  # retry this for up to 1 minute until the volume disappears
@@ -456,7 +458,7 @@ class AWSStaticAutoscaleTest(AWSAutoscaleTest):
456
458
 
457
459
  worker: "InstanceTypeDef" = next(wait_instances_running(boto3_ec2, [worker]))
458
460
 
459
- bdm: Optional[list["InstanceBlockDeviceMappingTypeDef"]] = worker.get(
461
+ bdm: list["InstanceBlockDeviceMappingTypeDef"] | None = worker.get(
460
462
  "BlockDeviceMappings"
461
463
  )
462
464
  assert bdm is not None
@@ -21,7 +21,6 @@ from argparse import Namespace
21
21
  from collections import defaultdict
22
22
  from queue import Empty, Queue
23
23
  from threading import Event, Thread
24
- from typing import Optional
25
24
  from unittest.mock import MagicMock
26
25
 
27
26
  from toil.batchSystems.abstractBatchSystem import (
@@ -975,9 +974,7 @@ class MockBatchSystemAndProvisioner(AbstractScalableBatchSystem, AbstractProvisi
975
974
  pass
976
975
 
977
976
  # AbstractProvisioner methods
978
- def setAutoscaledNodeTypes(
979
- self, node_types: list[tuple[set[Shape], Optional[float]]]
980
- ):
977
+ def setAutoscaledNodeTypes(self, node_types: list[tuple[set[Shape], float | None]]):
981
978
  self.node_shapes_for_testing = sorted(it for t in node_types for it in t[0])
982
979
  super().setAutoscaledNodeTypes(node_types)
983
980
 
@@ -1040,7 +1037,7 @@ class MockBatchSystemAndProvisioner(AbstractScalableBatchSystem, AbstractProvisi
1040
1037
  return dict(self.jobBatchSystemIDToIssuedJob).values()
1041
1038
 
1042
1039
  # AbstractScalableBatchSystem functionality
1043
- def getNodes(self, preemptible: Optional[bool] = False, timeout: int = 600):
1040
+ def getNodes(self, preemptible: bool | None = False, timeout: int = 600):
1044
1041
  nodes = dict()
1045
1042
  for node in self.nodesToWorker:
1046
1043
  if node.preemptible == preemptible:
@@ -15,7 +15,6 @@ import logging
15
15
  import os
16
16
  import subprocess
17
17
  import time
18
- from typing import Optional
19
18
  from uuid import uuid4
20
19
 
21
20
  from toil.lib.aws import zone_to_region
@@ -30,7 +29,6 @@ from toil.test import (
30
29
  needs_fetchable_appliance,
31
30
  slow,
32
31
  )
33
-
34
32
  from toil.test.cwl.cwlTest import TestCWLv12Conformance
35
33
 
36
34
  log = logging.getLogger(__name__)
@@ -193,7 +191,7 @@ class AbstractClusterTest(ToilTest):
193
191
  subprocess.check_call(cmd)
194
192
 
195
193
  @retry(errors=[subprocess.CalledProcessError], intervals=[1, 1])
196
- def createClusterUtil(self, args: Optional[list[str]] = None) -> None:
194
+ def createClusterUtil(self, args: list[str] | None = None) -> None:
197
195
  args = [] if args is None else args
198
196
 
199
197
  command = [
@@ -25,7 +25,6 @@ from typing import TYPE_CHECKING, Optional
25
25
  from urllib.parse import urlparse
26
26
 
27
27
  try:
28
- from flask import Flask
29
28
  from flask.testing import FlaskClient
30
29
  from werkzeug.test import TestResponse
31
30
  except ImportError:
@@ -33,7 +32,14 @@ except ImportError:
33
32
  # extra wasn't installed. We'll then skip them all.
34
33
  pass
35
34
 
36
- from toil.test import ToilTest, needs_aws_s3, needs_celery_broker, needs_cwl, needs_server, integrative
35
+ from toil.test import (
36
+ ToilTest,
37
+ integrative,
38
+ needs_aws_s3,
39
+ needs_celery_broker,
40
+ needs_cwl,
41
+ needs_server,
42
+ )
37
43
 
38
44
  logger = logging.getLogger(__name__)
39
45
  logging.basicConfig(level=logging.INFO)
@@ -195,10 +201,10 @@ class BucketUsingTest(ToilTest):
195
201
  from mypy_boto3_s3 import S3ServiceResource
196
202
  from mypy_boto3_s3.service_resource import Bucket
197
203
 
198
- region: Optional[str]
204
+ region: str | None
199
205
  s3_resource: Optional["S3ServiceResource"]
200
206
  bucket: Optional["Bucket"]
201
- bucket_name: Optional[str]
207
+ bucket_name: str | None
202
208
 
203
209
  @classmethod
204
210
  def setUpClass(cls) -> None:
@@ -292,12 +298,14 @@ class TrueDict(dict):
292
298
  Used as a workaround to set httpx post request encoding as recommended in
293
299
  <https://github.com/encode/httpx/discussions/2399#discussioncomment-3814186>.
294
300
  """
301
+
295
302
  def __bool__(self) -> bool:
296
303
  """
297
304
  Always say the object is truthy.
298
305
  """
299
306
  return True
300
307
 
308
+
301
309
  @needs_server
302
310
  class AbstractToilWESServerTest(ToilTest):
303
311
  """
@@ -557,6 +565,7 @@ class ToilWESServerBenchTest(AbstractToilWESServerTest):
557
565
  self.assertIn("system_state_counts", service_info)
558
566
  self.assertIn("tags", service_info)
559
567
 
568
+
560
569
  @needs_cwl
561
570
  class ToilWESServerWorkflowTest(AbstractToilWESServerTest):
562
571
  """
@@ -65,9 +65,7 @@ def down(job, inputFileStoreID, N, path, downCheckpoints, options, memory=sortMe
65
65
  length = os.path.getsize(inputFile)
66
66
  if length > N:
67
67
  # We will subdivide the file
68
- RealtimeLogger.critical(
69
- "Splitting file: %s of size: %s" % (inputFileStoreID, length)
70
- )
68
+ RealtimeLogger.critical(f"Splitting file: {inputFileStoreID} of size: {length}")
71
69
  # Split the file into two copies
72
70
  midPoint = getMidPoint(inputFile, 0, length)
73
71
  t1 = job.fileStore.getLocalTempFile()
@@ -109,9 +107,7 @@ def down(job, inputFileStoreID, N, path, downCheckpoints, options, memory=sortMe
109
107
  ).rv()
110
108
  else:
111
109
  # We can sort this bit of the file
112
- RealtimeLogger.critical(
113
- "Sorting file: %s of size: %s" % (inputFileStoreID, length)
114
- )
110
+ RealtimeLogger.critical(f"Sorting file: {inputFileStoreID} of size: {length}")
115
111
  # Sort the copy and write back to the fileStore
116
112
  shutil.copyfile(inputFile, inputFile + ".sort")
117
113
  sort(inputFile + ".sort")
toil/test/sort/sort.py CHANGED
@@ -12,8 +12,7 @@
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
14
 
15
- """A demonstration of toil. Sorts the lines of a file into ascending order by doing a parallel merge sort.
16
- """
15
+ """A demonstration of toil. Sorts the lines of a file into ascending order by doing a parallel merge sort."""
17
16
  import codecs
18
17
  import os
19
18
  import random
@@ -64,9 +63,7 @@ def down(job, inputFileStoreID, N, path, downCheckpoints, options, memory=sortMe
64
63
  length = os.path.getsize(inputFile)
65
64
  if length > N:
66
65
  # We will subdivide the file
67
- RealtimeLogger.critical(
68
- "Splitting file: %s of size: %s" % (inputFileStoreID, length)
69
- )
66
+ RealtimeLogger.critical(f"Splitting file: {inputFileStoreID} of size: {length}")
70
67
  # Split the file into two copies
71
68
  midPoint = getMidPoint(inputFile, 0, length)
72
69
  t1 = job.fileStore.getLocalTempFile()
@@ -108,9 +105,7 @@ def down(job, inputFileStoreID, N, path, downCheckpoints, options, memory=sortMe
108
105
  ).rv()
109
106
  else:
110
107
  # We can sort this bit of the file
111
- RealtimeLogger.critical(
112
- "Sorting file: %s of size: %s" % (inputFileStoreID, length)
113
- )
108
+ RealtimeLogger.critical(f"Sorting file: {inputFileStoreID} of size: {length}")
114
109
  # Sort the copy and write back to the fileStore
115
110
  shutil.copyfile(inputFile, inputFile + ".sort")
116
111
  sort(inputFile + ".sort")
@@ -11,14 +11,13 @@
11
11
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
- from argparse import Namespace
15
- from collections.abc import Callable, Generator, Sequence
16
14
  import logging
17
15
  import os
18
- from pathlib import Path
19
16
  import signal
20
17
  import time
21
- from typing import Optional
18
+ from argparse import Namespace
19
+ from collections.abc import Callable, Generator, Sequence
20
+ from pathlib import Path
22
21
  from uuid import uuid4
23
22
 
24
23
  import psutil
@@ -262,6 +261,7 @@ def _writeNonLocalFilesLambda(job: Job, files: tuple[Path, Path]) -> None:
262
261
  def lmd(x: Path, nlf: Path) -> None:
263
262
  x.unlink()
264
263
  nlf.unlink()
264
+
265
265
  for nlf in files:
266
266
  with nlf.open("wb") as nonLocalFileHandle:
267
267
  nonLocalFileHandle.write(os.urandom(1 * 1024 * 1024))
@@ -282,7 +282,7 @@ def _deferredFunctionRunsWithFailuresFn(job: Job, files: tuple[Path, Path]) -> N
282
282
  job.defer(_deleteFile, files[1])
283
283
 
284
284
 
285
- def _deleteFile(nonLocalFile: Path, nlf: Optional[Path] = None) -> None:
285
+ def _deleteFile(nonLocalFile: Path, nlf: Path | None = None) -> None:
286
286
  """
287
287
  Delete nonLocalFile and nlf
288
288
  :param nonLocalFile:
@@ -350,7 +350,7 @@ def _testNewJobsCanHandleOtherJobDeaths_C(
350
350
 
351
351
  class _deleteMethods:
352
352
  @staticmethod
353
- def _deleteFileMethod(nonLocalFile: Path, nlf: Optional[Path] = None) -> None:
353
+ def _deleteFileMethod(nonLocalFile: Path, nlf: Path | None = None) -> None:
354
354
  """
355
355
  Delete nonLocalFile and nlf
356
356
  """
@@ -360,7 +360,7 @@ class _deleteMethods:
360
360
 
361
361
  @classmethod
362
362
  def _deleteFileClassMethod(
363
- cls, nonLocalFile: Path, nlf: Optional[Path] = None
363
+ cls, nonLocalFile: Path, nlf: Path | None = None
364
364
  ) -> None:
365
365
  """
366
366
  Delete nonLocalFile and nlf
@@ -16,7 +16,6 @@ import os
16
16
  import time
17
17
  from argparse import Namespace
18
18
  from threading import Thread
19
- from typing import Optional
20
19
 
21
20
  from toil.common import Toil
22
21
  from toil.job import Job
@@ -87,7 +86,7 @@ def check_environment_repeatedly(job):
87
86
  check3 = waiter.addFollowOnJobFn(check_environment, "try3")
88
87
 
89
88
 
90
- def main(options: Optional[Namespace] = None):
89
+ def main(options: Namespace | None = None):
91
90
  """
92
91
  Run the actual workflow with the given options.
93
92
  """
@@ -95,7 +95,9 @@ class hidden:
95
95
  options.realTimeLogging = True
96
96
  options.workDir = self.work_dir
97
97
  options.clean = "always"
98
- log_fd, options.logFile = tempfile.mkstemp(dir=self.work_dir, prefix="logFile")
98
+ log_fd, options.logFile = tempfile.mkstemp(
99
+ dir=self.work_dir, prefix="logFile"
100
+ )
99
101
  os.close(log_fd)
100
102
  return options
101
103
 
@@ -375,7 +377,7 @@ class hidden:
375
377
  Ensures that files imported to the leader preserve their executable permissions
376
378
  when they are read by the fileStore.
377
379
  """
378
-
380
+
379
381
  for executable in True, False:
380
382
  file_path = self.create_file(content="Hello", executable=executable)
381
383
  initial_permissions = os.stat(file_path).st_mode & stat.S_IXUSR
@@ -537,9 +539,7 @@ class hidden:
537
539
  # The options namespace and the Config object now have the exact same behavior
538
540
  # which means parse_jobstore will be called with argparse rather than with the config object
539
541
  # so remove the prepended file: scheme
540
- jobStoreDev = os.stat(
541
- os.path.dirname(options.jobStore[5:])
542
- ).st_dev
542
+ jobStoreDev = os.stat(os.path.dirname(options.jobStore[5:])).st_dev
543
543
  else:
544
544
  jobStoreDev = os.stat(os.path.dirname(options.jobStore)).st_dev
545
545
  if workDirDev == jobStoreDev:
@@ -13,11 +13,13 @@
13
13
  # limitations under the License.
14
14
 
15
15
 
16
- from argparse import Namespace
17
16
  import os
18
- from pathlib import Path
19
17
  import stat
20
18
  import uuid
19
+ from argparse import Namespace
20
+ from pathlib import Path
21
+
22
+ import pytest
21
23
 
22
24
  from toil.common import Toil
23
25
  from toil.exceptions import FailedJobsException
@@ -26,9 +28,6 @@ from toil.fileStores.abstractFileStore import AbstractFileStore
26
28
  from toil.job import Job
27
29
  from toil.test import pslow as slow
28
30
 
29
- import pytest
30
- from pytest_subtests import SubTests
31
-
32
31
 
33
32
  def create_file(tmp_path: Path, content: str, executable: bool = False) -> Path:
34
33
  file_path = tmp_path / str(uuid.uuid4())
@@ -109,7 +108,7 @@ class TestImportExportFile:
109
108
  def test_import_export_restart_false(self, tmp_path: Path) -> None:
110
109
  self._run_import_export_workflow(tmp_path, restart=False)
111
110
 
112
- def test_basic_import_export(self, tmp_path: Path, subtests: SubTests) -> None:
111
+ def test_basic_import_export(self, tmp_path: Path, subtests: pytest.Subtests) -> None:
113
112
  """
114
113
  Ensures that uploaded files preserve their file permissions when they
115
114
  are downloaded again. This function checks that an imported executable file