toil 9.1.1__py3-none-any.whl → 9.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- toil/__init__.py +5 -9
- toil/batchSystems/abstractBatchSystem.py +23 -22
- toil/batchSystems/abstractGridEngineBatchSystem.py +17 -12
- toil/batchSystems/awsBatch.py +8 -8
- toil/batchSystems/cleanup_support.py +4 -4
- toil/batchSystems/contained_executor.py +3 -3
- toil/batchSystems/gridengine.py +3 -4
- toil/batchSystems/htcondor.py +5 -5
- toil/batchSystems/kubernetes.py +65 -63
- toil/batchSystems/local_support.py +2 -3
- toil/batchSystems/lsf.py +6 -7
- toil/batchSystems/mesos/batchSystem.py +11 -7
- toil/batchSystems/mesos/test/__init__.py +1 -2
- toil/batchSystems/options.py +9 -10
- toil/batchSystems/registry.py +3 -7
- toil/batchSystems/singleMachine.py +8 -11
- toil/batchSystems/slurm.py +49 -38
- toil/batchSystems/torque.py +3 -4
- toil/bus.py +36 -34
- toil/common.py +129 -89
- toil/cwl/cwltoil.py +857 -729
- toil/cwl/utils.py +44 -35
- toil/fileStores/__init__.py +3 -1
- toil/fileStores/abstractFileStore.py +28 -30
- toil/fileStores/cachingFileStore.py +8 -8
- toil/fileStores/nonCachingFileStore.py +10 -21
- toil/job.py +159 -158
- toil/jobStores/abstractJobStore.py +68 -69
- toil/jobStores/aws/jobStore.py +249 -213
- toil/jobStores/aws/utils.py +13 -24
- toil/jobStores/fileJobStore.py +28 -22
- toil/jobStores/googleJobStore.py +21 -17
- toil/jobStores/utils.py +3 -7
- toil/leader.py +17 -22
- toil/lib/accelerators.py +6 -4
- toil/lib/aws/__init__.py +9 -10
- toil/lib/aws/ami.py +33 -19
- toil/lib/aws/iam.py +6 -6
- toil/lib/aws/s3.py +259 -157
- toil/lib/aws/session.py +76 -76
- toil/lib/aws/utils.py +51 -43
- toil/lib/checksum.py +19 -15
- toil/lib/compatibility.py +3 -2
- toil/lib/conversions.py +45 -18
- toil/lib/directory.py +29 -26
- toil/lib/docker.py +93 -99
- toil/lib/dockstore.py +77 -50
- toil/lib/ec2.py +39 -38
- toil/lib/ec2nodes.py +11 -4
- toil/lib/exceptions.py +8 -5
- toil/lib/ftp_utils.py +9 -14
- toil/lib/generatedEC2Lists.py +161 -20
- toil/lib/history.py +141 -97
- toil/lib/history_submission.py +163 -72
- toil/lib/io.py +27 -17
- toil/lib/memoize.py +2 -1
- toil/lib/misc.py +15 -11
- toil/lib/pipes.py +40 -25
- toil/lib/plugins.py +12 -8
- toil/lib/resources.py +1 -0
- toil/lib/retry.py +32 -38
- toil/lib/threading.py +12 -12
- toil/lib/throttle.py +1 -2
- toil/lib/trs.py +113 -51
- toil/lib/url.py +14 -23
- toil/lib/web.py +7 -2
- toil/options/common.py +18 -15
- toil/options/cwl.py +2 -2
- toil/options/runner.py +9 -5
- toil/options/wdl.py +1 -3
- toil/provisioners/__init__.py +9 -9
- toil/provisioners/abstractProvisioner.py +22 -20
- toil/provisioners/aws/__init__.py +20 -14
- toil/provisioners/aws/awsProvisioner.py +10 -8
- toil/provisioners/clusterScaler.py +19 -18
- toil/provisioners/gceProvisioner.py +2 -3
- toil/provisioners/node.py +11 -13
- toil/realtimeLogger.py +4 -4
- toil/resource.py +5 -5
- toil/server/app.py +2 -2
- toil/server/cli/wes_cwl_runner.py +11 -11
- toil/server/utils.py +18 -21
- toil/server/wes/abstract_backend.py +9 -8
- toil/server/wes/amazon_wes_utils.py +3 -3
- toil/server/wes/tasks.py +3 -5
- toil/server/wes/toil_backend.py +17 -21
- toil/server/wsgi_app.py +3 -3
- toil/serviceManager.py +3 -4
- toil/statsAndLogging.py +12 -13
- toil/test/__init__.py +33 -24
- toil/test/batchSystems/batchSystemTest.py +12 -11
- toil/test/batchSystems/batch_system_plugin_test.py +3 -5
- toil/test/batchSystems/test_slurm.py +38 -24
- toil/test/cwl/conftest.py +5 -6
- toil/test/cwl/cwlTest.py +194 -78
- toil/test/cwl/download_file_uri.json +6 -0
- toil/test/cwl/download_file_uri_no_hostname.json +6 -0
- toil/test/docs/scripts/tutorial_staging.py +1 -0
- toil/test/jobStores/jobStoreTest.py +9 -7
- toil/test/lib/aws/test_iam.py +1 -3
- toil/test/lib/aws/test_s3.py +1 -1
- toil/test/lib/dockerTest.py +9 -9
- toil/test/lib/test_ec2.py +12 -11
- toil/test/lib/test_history.py +4 -4
- toil/test/lib/test_trs.py +16 -14
- toil/test/lib/test_url.py +7 -6
- toil/test/lib/url_plugin_test.py +12 -18
- toil/test/provisioners/aws/awsProvisionerTest.py +10 -8
- toil/test/provisioners/clusterScalerTest.py +2 -5
- toil/test/provisioners/clusterTest.py +1 -3
- toil/test/server/serverTest.py +13 -4
- toil/test/sort/restart_sort.py +2 -6
- toil/test/sort/sort.py +3 -8
- toil/test/src/deferredFunctionTest.py +7 -7
- toil/test/src/environmentTest.py +1 -2
- toil/test/src/fileStoreTest.py +5 -5
- toil/test/src/importExportFileTest.py +5 -6
- toil/test/src/jobServiceTest.py +22 -14
- toil/test/src/jobTest.py +121 -25
- toil/test/src/miscTests.py +5 -7
- toil/test/src/promisedRequirementTest.py +8 -7
- toil/test/src/regularLogTest.py +2 -3
- toil/test/src/resourceTest.py +5 -8
- toil/test/src/restartDAGTest.py +5 -6
- toil/test/src/resumabilityTest.py +2 -2
- toil/test/src/retainTempDirTest.py +3 -3
- toil/test/src/systemTest.py +3 -3
- toil/test/src/threadingTest.py +1 -1
- toil/test/src/workerTest.py +1 -2
- toil/test/utils/toilDebugTest.py +6 -4
- toil/test/utils/toilKillTest.py +1 -1
- toil/test/utils/utilsTest.py +15 -14
- toil/test/wdl/wdltoil_test.py +247 -124
- toil/test/wdl/wdltoil_test_kubernetes.py +2 -2
- toil/toilState.py +2 -3
- toil/utils/toilDebugFile.py +3 -8
- toil/utils/toilDebugJob.py +1 -2
- toil/utils/toilLaunchCluster.py +1 -2
- toil/utils/toilSshCluster.py +2 -0
- toil/utils/toilStats.py +19 -24
- toil/utils/toilStatus.py +11 -14
- toil/version.py +10 -10
- toil/wdl/wdltoil.py +313 -209
- toil/worker.py +18 -12
- {toil-9.1.1.dist-info → toil-9.2.0.dist-info}/METADATA +11 -14
- {toil-9.1.1.dist-info → toil-9.2.0.dist-info}/RECORD +150 -153
- {toil-9.1.1.dist-info → toil-9.2.0.dist-info}/WHEEL +1 -1
- toil/test/cwl/staging_cat.cwl +0 -27
- toil/test/cwl/staging_make_file.cwl +0 -25
- toil/test/cwl/staging_workflow.cwl +0 -43
- toil/test/cwl/zero_default.cwl +0 -61
- toil/test/utils/ABCWorkflowDebug/ABC.txt +0 -1
- {toil-9.1.1.dist-info → toil-9.2.0.dist-info}/entry_points.txt +0 -0
- {toil-9.1.1.dist-info → toil-9.2.0.dist-info}/licenses/LICENSE +0 -0
- {toil-9.1.1.dist-info → toil-9.2.0.dist-info}/top_level.txt +0 -0
toil/test/lib/test_ec2.py
CHANGED
|
@@ -13,15 +13,14 @@
|
|
|
13
13
|
# limitations under the License.
|
|
14
14
|
import logging
|
|
15
15
|
import os
|
|
16
|
-
|
|
17
16
|
from unittest import mock
|
|
18
17
|
|
|
19
18
|
from toil.lib.aws.ami import (
|
|
19
|
+
ReleaseFeedUnavailableError,
|
|
20
20
|
aws_marketplace_flatcar_ami_search,
|
|
21
21
|
feed_flatcar_ami_release,
|
|
22
22
|
flatcar_release_feed_ami,
|
|
23
23
|
get_flatcar_ami,
|
|
24
|
-
ReleaseFeedUnavailableError
|
|
25
24
|
)
|
|
26
25
|
from toil.test import ToilTest, needs_aws_ec2, needs_online
|
|
27
26
|
|
|
@@ -33,27 +32,29 @@ logging.basicConfig(level=logging.DEBUG)
|
|
|
33
32
|
class FlatcarFeedTest(ToilTest):
|
|
34
33
|
"""Test accessing the Flatcar AMI release feed, independent of the AWS API"""
|
|
35
34
|
|
|
36
|
-
# Note that we need to support getting
|
|
37
|
-
#
|
|
38
|
-
# times.
|
|
35
|
+
# Note that we need to support getting no AMI back, because sometimes the
|
|
36
|
+
# Flatcar feeds are just down, and we can't fail CI at those times.
|
|
39
37
|
|
|
40
38
|
def test_parse_archive_feed(self):
|
|
41
39
|
"""Make sure we can get a Flatcar release from the Internet Archive."""
|
|
42
40
|
ami = flatcar_release_feed_ami("us-west-2", "amd64", "archive")
|
|
43
|
-
|
|
44
|
-
|
|
41
|
+
if ami is not None:
|
|
42
|
+
self.assertEqual(len(ami), len("ami-02b46c73fed689d1c"))
|
|
43
|
+
self.assertTrue(ami.startswith("ami-"))
|
|
45
44
|
|
|
46
45
|
def test_parse_beta_feed(self):
|
|
47
46
|
"""Make sure we can get a Flatcar release from the beta channel."""
|
|
48
47
|
ami = flatcar_release_feed_ami("us-west-2", "amd64", "beta")
|
|
49
|
-
|
|
50
|
-
|
|
48
|
+
if ami is not None:
|
|
49
|
+
self.assertEqual(len(ami), len("ami-02b46c73fed689d1c"))
|
|
50
|
+
self.assertTrue(ami.startswith("ami-"))
|
|
51
51
|
|
|
52
52
|
def test_parse_stable_feed(self):
|
|
53
53
|
"""Make sure we can get a Flatcar release from the stable channel."""
|
|
54
54
|
ami = flatcar_release_feed_ami("us-west-2", "amd64", "stable")
|
|
55
|
-
|
|
56
|
-
|
|
55
|
+
if ami is not None:
|
|
56
|
+
self.assertEqual(len(ami), len("ami-02b46c73fed689d1c"))
|
|
57
|
+
self.assertTrue(ami.startswith("ami-"))
|
|
57
58
|
|
|
58
59
|
|
|
59
60
|
@needs_aws_ec2
|
toil/test/lib/test_history.py
CHANGED
|
@@ -12,15 +12,15 @@
|
|
|
12
12
|
# See the License for the specific language governing permissions and
|
|
13
13
|
# limitations under the License.
|
|
14
14
|
|
|
15
|
-
from collections.abc import Generator
|
|
16
|
-
from pathlib import Path
|
|
17
15
|
import logging
|
|
18
16
|
import time
|
|
19
|
-
|
|
20
|
-
from
|
|
17
|
+
from collections.abc import Generator
|
|
18
|
+
from pathlib import Path
|
|
21
19
|
|
|
22
20
|
import pytest
|
|
23
21
|
|
|
22
|
+
from toil.lib.history import HistoryManager
|
|
23
|
+
|
|
24
24
|
logger = logging.getLogger(__name__)
|
|
25
25
|
logging.basicConfig(level=logging.DEBUG)
|
|
26
26
|
|
toil/test/lib/test_trs.py
CHANGED
|
@@ -12,20 +12,21 @@
|
|
|
12
12
|
# See the License for the specific language governing permissions and
|
|
13
13
|
# limitations under the License.
|
|
14
14
|
|
|
15
|
-
import io
|
|
16
15
|
import logging
|
|
17
|
-
import pytest
|
|
18
|
-
from typing import IO
|
|
19
16
|
import urllib.request
|
|
17
|
+
from typing import IO
|
|
20
18
|
from urllib.error import URLError
|
|
21
19
|
|
|
20
|
+
import pytest
|
|
21
|
+
|
|
22
22
|
from toil.lib.retry import retry
|
|
23
|
-
from toil.lib.trs import
|
|
23
|
+
from toil.lib.trs import fetch_workflow, find_workflow
|
|
24
24
|
from toil.test import ToilTest, needs_online
|
|
25
25
|
|
|
26
26
|
logger = logging.getLogger(__name__)
|
|
27
27
|
logging.basicConfig(level=logging.DEBUG)
|
|
28
28
|
|
|
29
|
+
|
|
29
30
|
@pytest.mark.integrative
|
|
30
31
|
@needs_online
|
|
31
32
|
class DockstoreLookupTest(ToilTest):
|
|
@@ -57,7 +58,9 @@ class DockstoreLookupTest(ToilTest):
|
|
|
57
58
|
PAGE_URL = "https://dockstore.org/workflows/github.com/dockstore/bcc2020-training/HelloWorld:master?tab=info"
|
|
58
59
|
trs_id, trs_version, language = find_workflow(PAGE_URL)
|
|
59
60
|
|
|
60
|
-
self.assertEqual(
|
|
61
|
+
self.assertEqual(
|
|
62
|
+
trs_id, "#workflow/github.com/dockstore/bcc2020-training/HelloWorld"
|
|
63
|
+
)
|
|
61
64
|
self.assertEqual(trs_version, "master")
|
|
62
65
|
self.assertEqual(language, "WDL")
|
|
63
66
|
|
|
@@ -65,7 +68,7 @@ class DockstoreLookupTest(ToilTest):
|
|
|
65
68
|
TRS_ID = "#workflow/github.com/dockstore-testing/md5sum-checker"
|
|
66
69
|
TRS_VERSION = "master"
|
|
67
70
|
trs_id, trs_version, language = find_workflow(f"{TRS_ID}:{TRS_VERSION}")
|
|
68
|
-
|
|
71
|
+
|
|
69
72
|
self.assertEqual(trs_id, TRS_ID)
|
|
70
73
|
self.assertEqual(trs_version, TRS_VERSION)
|
|
71
74
|
self.assertEqual(language, "CWL")
|
|
@@ -90,7 +93,7 @@ class DockstoreLookupTest(ToilTest):
|
|
|
90
93
|
# checker workflow itself.
|
|
91
94
|
WORKFLOW_URL = "https://raw.githubusercontent.com/dockstore-testing/md5sum-checker/master/md5sum/md5sum-workflow.cwl"
|
|
92
95
|
looked_up = fetch_workflow(TRS_ID, TRS_VERSION, LANGUAGE)
|
|
93
|
-
|
|
96
|
+
|
|
94
97
|
data_from_lookup = self.read_result(looked_up).read()
|
|
95
98
|
data_from_source = self.read_result(WORKFLOW_URL).read()
|
|
96
99
|
self.assertEqual(data_from_lookup, data_from_source)
|
|
@@ -104,7 +107,7 @@ class DockstoreLookupTest(ToilTest):
|
|
|
104
107
|
fetch_workflow(TRS_ID, TRS_VERSION, LANGUAGE)
|
|
105
108
|
# This lookup is definitely cached
|
|
106
109
|
looked_up = fetch_workflow(TRS_ID, TRS_VERSION, LANGUAGE)
|
|
107
|
-
|
|
110
|
+
|
|
108
111
|
data_from_lookup = self.read_result(looked_up).read()
|
|
109
112
|
data_from_source = self.read_result(WORKFLOW_URL).read()
|
|
110
113
|
self.assertEqual(data_from_lookup, data_from_source)
|
|
@@ -112,7 +115,7 @@ class DockstoreLookupTest(ToilTest):
|
|
|
112
115
|
def test_lookup_from_trs_with_version(self) -> None:
|
|
113
116
|
TRS_VERSIONED_ID = "#workflow/github.com/dockstore-testing/md5sum-checker:workflowWithHTTPImport"
|
|
114
117
|
trs_id, trs_version, language = find_workflow(TRS_VERSIONED_ID)
|
|
115
|
-
|
|
118
|
+
|
|
116
119
|
parts = TRS_VERSIONED_ID.split(":")
|
|
117
120
|
|
|
118
121
|
self.assertEqual(trs_id, parts[0])
|
|
@@ -130,10 +133,12 @@ class DockstoreLookupTest(ToilTest):
|
|
|
130
133
|
looked_up = find_workflow(TRS_VERSIONED_ID)
|
|
131
134
|
|
|
132
135
|
def test_lookup_from_trs_nonexistent_version(self) -> None:
|
|
133
|
-
TRS_VERSIONED_ID =
|
|
136
|
+
TRS_VERSIONED_ID = (
|
|
137
|
+
"#workflow/github.com/dockstore-testing/md5sum-checker:notARealVersion"
|
|
138
|
+
)
|
|
134
139
|
with self.assertRaises(FileNotFoundError):
|
|
135
140
|
looked_up = find_workflow(TRS_VERSIONED_ID)
|
|
136
|
-
|
|
141
|
+
|
|
137
142
|
def test_get_nonexistent_workflow(self) -> None:
|
|
138
143
|
TRS_ID = "#workflow/github.com/adamnovak/veryfakerepo"
|
|
139
144
|
TRS_VERSION = "notARealVersion"
|
|
@@ -156,6 +161,3 @@ class DockstoreLookupTest(ToilTest):
|
|
|
156
161
|
LANGUAGE = "CWL"
|
|
157
162
|
with self.assertRaises(FileNotFoundError):
|
|
158
163
|
looked_up = fetch_workflow(TRS_ID, TRS_VERSION, LANGUAGE)
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
toil/test/lib/test_url.py
CHANGED
|
@@ -11,20 +11,18 @@
|
|
|
11
11
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
12
|
# See the License for the specific language governing permissions and
|
|
13
13
|
# limitations under the License.
|
|
14
|
-
import getpass
|
|
15
14
|
import logging
|
|
16
15
|
|
|
17
16
|
from pytest_httpserver import HTTPServer
|
|
18
17
|
|
|
19
|
-
from toil.lib.misc import get_user_name
|
|
20
18
|
from toil.lib.url import URLAccess
|
|
21
|
-
from toil.test import needs_aws_s3
|
|
22
|
-
|
|
19
|
+
from toil.test import needs_aws_s3
|
|
23
20
|
|
|
24
21
|
logger = logging.getLogger(__name__)
|
|
25
22
|
logging.basicConfig(level=logging.DEBUG)
|
|
26
23
|
|
|
27
|
-
|
|
24
|
+
|
|
25
|
+
class TestURLAccess:
|
|
28
26
|
"""
|
|
29
27
|
Test URLAccess class handling read, list,
|
|
30
28
|
and checking the size/existence of resources at given URL
|
|
@@ -54,8 +52,11 @@ class TestURLAccess():
|
|
|
54
52
|
@needs_aws_s3
|
|
55
53
|
def test_read_from_url(self) -> None:
|
|
56
54
|
import io
|
|
55
|
+
|
|
57
56
|
output = io.BytesIO()
|
|
58
|
-
size, executable = URLAccess.read_from_url(
|
|
57
|
+
size, executable = URLAccess.read_from_url(
|
|
58
|
+
"s3://toil-datasets/hello.txt", output
|
|
59
|
+
)
|
|
59
60
|
assert isinstance(size, int)
|
|
60
61
|
assert size > 0
|
|
61
62
|
assert not executable
|
toil/test/lib/url_plugin_test.py
CHANGED
|
@@ -11,29 +11,18 @@
|
|
|
11
11
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
12
|
# See the License for the specific language governing permissions and
|
|
13
13
|
# limitations under the License.
|
|
14
|
-
import logging
|
|
15
|
-
from typing import IO, Optional, Union
|
|
16
|
-
|
|
17
|
-
from configargparse import ArgParser, ArgumentParser
|
|
18
|
-
|
|
19
|
-
from toil.batchSystems.abstractBatchSystem import (
|
|
20
|
-
AbstractBatchSystem,
|
|
21
|
-
UpdatedBatchJobInfo,
|
|
22
|
-
)
|
|
23
|
-
from toil.batchSystems.cleanup_support import BatchSystemCleanupSupport
|
|
24
|
-
from toil.batchSystems.options import OptionSetter
|
|
25
|
-
from toil.batchSystems.registry import add_batch_system_factory
|
|
26
|
-
from toil.common import Toil, addOptions
|
|
27
|
-
from toil.job import JobDescription
|
|
28
|
-
|
|
29
14
|
import io
|
|
15
|
+
import logging
|
|
16
|
+
from typing import IO
|
|
30
17
|
from urllib.parse import ParseResult
|
|
31
|
-
|
|
32
|
-
from toil.lib.url import URLAccess
|
|
18
|
+
|
|
33
19
|
from toil.lib.plugins import register_plugin, remove_plugin
|
|
20
|
+
from toil.lib.url import URLAccess
|
|
21
|
+
from toil.test import ToilTest
|
|
34
22
|
|
|
35
23
|
logger = logging.getLogger(__name__)
|
|
36
24
|
|
|
25
|
+
|
|
37
26
|
class FakeURLPlugin(URLAccess):
|
|
38
27
|
@classmethod
|
|
39
28
|
def _supports_url(cls, url: ParseResult, export: bool = False) -> bool:
|
|
@@ -66,7 +55,12 @@ class FakeURLPlugin(URLAccess):
|
|
|
66
55
|
return io.BytesIO(b"hello world")
|
|
67
56
|
|
|
68
57
|
@classmethod
|
|
69
|
-
def _write_to_url(
|
|
58
|
+
def _write_to_url(
|
|
59
|
+
cls,
|
|
60
|
+
readable: IO[bytes] | IO[str],
|
|
61
|
+
url: ParseResult,
|
|
62
|
+
executable: bool = False,
|
|
63
|
+
) -> None:
|
|
70
64
|
pass
|
|
71
65
|
|
|
72
66
|
|
|
@@ -11,7 +11,6 @@
|
|
|
11
11
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
12
|
# See the License for the specific language governing permissions and
|
|
13
13
|
# limitations under the License.
|
|
14
|
-
import datetime
|
|
15
14
|
import logging
|
|
16
15
|
import os
|
|
17
16
|
import subprocess
|
|
@@ -26,17 +25,15 @@ from uuid import uuid4
|
|
|
26
25
|
import pytest
|
|
27
26
|
|
|
28
27
|
import toil.lib.aws.session
|
|
29
|
-
|
|
30
28
|
from toil.lib.aws import zone_to_region
|
|
31
29
|
from toil.provisioners import cluster_factory
|
|
32
|
-
from toil.provisioners.aws.awsProvisioner import AWSProvisioner
|
|
33
30
|
from toil.provisioners.aws import (
|
|
34
31
|
_get_spot_history,
|
|
35
32
|
get_aws_zone_from_spot_market,
|
|
36
33
|
get_best_aws_zone,
|
|
37
34
|
)
|
|
35
|
+
from toil.provisioners.aws.awsProvisioner import AWSProvisioner
|
|
38
36
|
from toil.test import (
|
|
39
|
-
ToilTest,
|
|
40
37
|
get_data,
|
|
41
38
|
integrative,
|
|
42
39
|
needs_aws_ec2,
|
|
@@ -63,6 +60,7 @@ if TYPE_CHECKING:
|
|
|
63
60
|
|
|
64
61
|
log = logging.getLogger(__name__)
|
|
65
62
|
|
|
63
|
+
|
|
66
64
|
@pytest.fixture
|
|
67
65
|
def aws_zone():
|
|
68
66
|
"""
|
|
@@ -74,6 +72,7 @@ def aws_zone():
|
|
|
74
72
|
), "Could not determine AWS availability zone to test in; is TOIL_AWS_ZONE set?"
|
|
75
73
|
return zone
|
|
76
74
|
|
|
75
|
+
|
|
77
76
|
@pytest.fixture
|
|
78
77
|
def aws_region(aws_zone):
|
|
79
78
|
"""
|
|
@@ -81,6 +80,7 @@ def aws_region(aws_zone):
|
|
|
81
80
|
"""
|
|
82
81
|
return zone_to_region(aws_zone)
|
|
83
82
|
|
|
83
|
+
|
|
84
84
|
@pytest.fixture
|
|
85
85
|
def ec2_client(aws_region):
|
|
86
86
|
"""
|
|
@@ -147,7 +147,9 @@ class TestAWSProvisionerBenchTest:
|
|
|
147
147
|
Make sure that we can process spot price history to pick a zone.
|
|
148
148
|
"""
|
|
149
149
|
zone_options = ["us-west-2a", "af-south-1c"]
|
|
150
|
-
zone_choice = get_aws_zone_from_spot_market(
|
|
150
|
+
zone_choice = get_aws_zone_from_spot_market(
|
|
151
|
+
0.01, "t3.large", ec2_client, zone_options
|
|
152
|
+
)
|
|
151
153
|
assert zone_choice in zone_options
|
|
152
154
|
|
|
153
155
|
|
|
@@ -208,7 +210,7 @@ class AbstractAWSAutoscaleTest(AbstractClusterTest):
|
|
|
208
210
|
instances.sort(key=lambda x: x.get("LaunchTime"))
|
|
209
211
|
leader: "InstanceTypeDef" = instances[0] # assume leader was launched first
|
|
210
212
|
|
|
211
|
-
bdm:
|
|
213
|
+
bdm: list["InstanceBlockDeviceMappingTypeDef"] | None = leader.get(
|
|
212
214
|
"BlockDeviceMappings"
|
|
213
215
|
)
|
|
214
216
|
assert bdm is not None
|
|
@@ -305,7 +307,7 @@ class AbstractAWSAutoscaleTest(AbstractClusterTest):
|
|
|
305
307
|
self.cluster.destroyCluster()
|
|
306
308
|
boto3_ec2: "EC2Client" = self.aws.client(region=self.region, service_name="ec2")
|
|
307
309
|
volume_filter: "FilterTypeDef" = {"Name": "volume-id", "Values": [volumeID]}
|
|
308
|
-
volumes:
|
|
310
|
+
volumes: list["VolumeTypeDef"] | None = None
|
|
309
311
|
for attempt in range(6):
|
|
310
312
|
# https://github.com/BD2KGenomics/toil/issues/1567
|
|
311
313
|
# retry this for up to 1 minute until the volume disappears
|
|
@@ -456,7 +458,7 @@ class AWSStaticAutoscaleTest(AWSAutoscaleTest):
|
|
|
456
458
|
|
|
457
459
|
worker: "InstanceTypeDef" = next(wait_instances_running(boto3_ec2, [worker]))
|
|
458
460
|
|
|
459
|
-
bdm:
|
|
461
|
+
bdm: list["InstanceBlockDeviceMappingTypeDef"] | None = worker.get(
|
|
460
462
|
"BlockDeviceMappings"
|
|
461
463
|
)
|
|
462
464
|
assert bdm is not None
|
|
@@ -21,7 +21,6 @@ from argparse import Namespace
|
|
|
21
21
|
from collections import defaultdict
|
|
22
22
|
from queue import Empty, Queue
|
|
23
23
|
from threading import Event, Thread
|
|
24
|
-
from typing import Optional
|
|
25
24
|
from unittest.mock import MagicMock
|
|
26
25
|
|
|
27
26
|
from toil.batchSystems.abstractBatchSystem import (
|
|
@@ -975,9 +974,7 @@ class MockBatchSystemAndProvisioner(AbstractScalableBatchSystem, AbstractProvisi
|
|
|
975
974
|
pass
|
|
976
975
|
|
|
977
976
|
# AbstractProvisioner methods
|
|
978
|
-
def setAutoscaledNodeTypes(
|
|
979
|
-
self, node_types: list[tuple[set[Shape], Optional[float]]]
|
|
980
|
-
):
|
|
977
|
+
def setAutoscaledNodeTypes(self, node_types: list[tuple[set[Shape], float | None]]):
|
|
981
978
|
self.node_shapes_for_testing = sorted(it for t in node_types for it in t[0])
|
|
982
979
|
super().setAutoscaledNodeTypes(node_types)
|
|
983
980
|
|
|
@@ -1040,7 +1037,7 @@ class MockBatchSystemAndProvisioner(AbstractScalableBatchSystem, AbstractProvisi
|
|
|
1040
1037
|
return dict(self.jobBatchSystemIDToIssuedJob).values()
|
|
1041
1038
|
|
|
1042
1039
|
# AbstractScalableBatchSystem functionality
|
|
1043
|
-
def getNodes(self, preemptible:
|
|
1040
|
+
def getNodes(self, preemptible: bool | None = False, timeout: int = 600):
|
|
1044
1041
|
nodes = dict()
|
|
1045
1042
|
for node in self.nodesToWorker:
|
|
1046
1043
|
if node.preemptible == preemptible:
|
|
@@ -15,7 +15,6 @@ import logging
|
|
|
15
15
|
import os
|
|
16
16
|
import subprocess
|
|
17
17
|
import time
|
|
18
|
-
from typing import Optional
|
|
19
18
|
from uuid import uuid4
|
|
20
19
|
|
|
21
20
|
from toil.lib.aws import zone_to_region
|
|
@@ -30,7 +29,6 @@ from toil.test import (
|
|
|
30
29
|
needs_fetchable_appliance,
|
|
31
30
|
slow,
|
|
32
31
|
)
|
|
33
|
-
|
|
34
32
|
from toil.test.cwl.cwlTest import TestCWLv12Conformance
|
|
35
33
|
|
|
36
34
|
log = logging.getLogger(__name__)
|
|
@@ -193,7 +191,7 @@ class AbstractClusterTest(ToilTest):
|
|
|
193
191
|
subprocess.check_call(cmd)
|
|
194
192
|
|
|
195
193
|
@retry(errors=[subprocess.CalledProcessError], intervals=[1, 1])
|
|
196
|
-
def createClusterUtil(self, args:
|
|
194
|
+
def createClusterUtil(self, args: list[str] | None = None) -> None:
|
|
197
195
|
args = [] if args is None else args
|
|
198
196
|
|
|
199
197
|
command = [
|
toil/test/server/serverTest.py
CHANGED
|
@@ -25,7 +25,6 @@ from typing import TYPE_CHECKING, Optional
|
|
|
25
25
|
from urllib.parse import urlparse
|
|
26
26
|
|
|
27
27
|
try:
|
|
28
|
-
from flask import Flask
|
|
29
28
|
from flask.testing import FlaskClient
|
|
30
29
|
from werkzeug.test import TestResponse
|
|
31
30
|
except ImportError:
|
|
@@ -33,7 +32,14 @@ except ImportError:
|
|
|
33
32
|
# extra wasn't installed. We'll then skip them all.
|
|
34
33
|
pass
|
|
35
34
|
|
|
36
|
-
from toil.test import
|
|
35
|
+
from toil.test import (
|
|
36
|
+
ToilTest,
|
|
37
|
+
integrative,
|
|
38
|
+
needs_aws_s3,
|
|
39
|
+
needs_celery_broker,
|
|
40
|
+
needs_cwl,
|
|
41
|
+
needs_server,
|
|
42
|
+
)
|
|
37
43
|
|
|
38
44
|
logger = logging.getLogger(__name__)
|
|
39
45
|
logging.basicConfig(level=logging.INFO)
|
|
@@ -195,10 +201,10 @@ class BucketUsingTest(ToilTest):
|
|
|
195
201
|
from mypy_boto3_s3 import S3ServiceResource
|
|
196
202
|
from mypy_boto3_s3.service_resource import Bucket
|
|
197
203
|
|
|
198
|
-
region:
|
|
204
|
+
region: str | None
|
|
199
205
|
s3_resource: Optional["S3ServiceResource"]
|
|
200
206
|
bucket: Optional["Bucket"]
|
|
201
|
-
bucket_name:
|
|
207
|
+
bucket_name: str | None
|
|
202
208
|
|
|
203
209
|
@classmethod
|
|
204
210
|
def setUpClass(cls) -> None:
|
|
@@ -292,12 +298,14 @@ class TrueDict(dict):
|
|
|
292
298
|
Used as a workaround to set httpx post request encoding as recommended in
|
|
293
299
|
<https://github.com/encode/httpx/discussions/2399#discussioncomment-3814186>.
|
|
294
300
|
"""
|
|
301
|
+
|
|
295
302
|
def __bool__(self) -> bool:
|
|
296
303
|
"""
|
|
297
304
|
Always say the object is truthy.
|
|
298
305
|
"""
|
|
299
306
|
return True
|
|
300
307
|
|
|
308
|
+
|
|
301
309
|
@needs_server
|
|
302
310
|
class AbstractToilWESServerTest(ToilTest):
|
|
303
311
|
"""
|
|
@@ -557,6 +565,7 @@ class ToilWESServerBenchTest(AbstractToilWESServerTest):
|
|
|
557
565
|
self.assertIn("system_state_counts", service_info)
|
|
558
566
|
self.assertIn("tags", service_info)
|
|
559
567
|
|
|
568
|
+
|
|
560
569
|
@needs_cwl
|
|
561
570
|
class ToilWESServerWorkflowTest(AbstractToilWESServerTest):
|
|
562
571
|
"""
|
toil/test/sort/restart_sort.py
CHANGED
|
@@ -65,9 +65,7 @@ def down(job, inputFileStoreID, N, path, downCheckpoints, options, memory=sortMe
|
|
|
65
65
|
length = os.path.getsize(inputFile)
|
|
66
66
|
if length > N:
|
|
67
67
|
# We will subdivide the file
|
|
68
|
-
RealtimeLogger.critical(
|
|
69
|
-
"Splitting file: %s of size: %s" % (inputFileStoreID, length)
|
|
70
|
-
)
|
|
68
|
+
RealtimeLogger.critical(f"Splitting file: {inputFileStoreID} of size: {length}")
|
|
71
69
|
# Split the file into two copies
|
|
72
70
|
midPoint = getMidPoint(inputFile, 0, length)
|
|
73
71
|
t1 = job.fileStore.getLocalTempFile()
|
|
@@ -109,9 +107,7 @@ def down(job, inputFileStoreID, N, path, downCheckpoints, options, memory=sortMe
|
|
|
109
107
|
).rv()
|
|
110
108
|
else:
|
|
111
109
|
# We can sort this bit of the file
|
|
112
|
-
RealtimeLogger.critical(
|
|
113
|
-
"Sorting file: %s of size: %s" % (inputFileStoreID, length)
|
|
114
|
-
)
|
|
110
|
+
RealtimeLogger.critical(f"Sorting file: {inputFileStoreID} of size: {length}")
|
|
115
111
|
# Sort the copy and write back to the fileStore
|
|
116
112
|
shutil.copyfile(inputFile, inputFile + ".sort")
|
|
117
113
|
sort(inputFile + ".sort")
|
toil/test/sort/sort.py
CHANGED
|
@@ -12,8 +12,7 @@
|
|
|
12
12
|
# See the License for the specific language governing permissions and
|
|
13
13
|
# limitations under the License.
|
|
14
14
|
|
|
15
|
-
"""A demonstration of toil. Sorts the lines of a file into ascending order by doing a parallel merge sort.
|
|
16
|
-
"""
|
|
15
|
+
"""A demonstration of toil. Sorts the lines of a file into ascending order by doing a parallel merge sort."""
|
|
17
16
|
import codecs
|
|
18
17
|
import os
|
|
19
18
|
import random
|
|
@@ -64,9 +63,7 @@ def down(job, inputFileStoreID, N, path, downCheckpoints, options, memory=sortMe
|
|
|
64
63
|
length = os.path.getsize(inputFile)
|
|
65
64
|
if length > N:
|
|
66
65
|
# We will subdivide the file
|
|
67
|
-
RealtimeLogger.critical(
|
|
68
|
-
"Splitting file: %s of size: %s" % (inputFileStoreID, length)
|
|
69
|
-
)
|
|
66
|
+
RealtimeLogger.critical(f"Splitting file: {inputFileStoreID} of size: {length}")
|
|
70
67
|
# Split the file into two copies
|
|
71
68
|
midPoint = getMidPoint(inputFile, 0, length)
|
|
72
69
|
t1 = job.fileStore.getLocalTempFile()
|
|
@@ -108,9 +105,7 @@ def down(job, inputFileStoreID, N, path, downCheckpoints, options, memory=sortMe
|
|
|
108
105
|
).rv()
|
|
109
106
|
else:
|
|
110
107
|
# We can sort this bit of the file
|
|
111
|
-
RealtimeLogger.critical(
|
|
112
|
-
"Sorting file: %s of size: %s" % (inputFileStoreID, length)
|
|
113
|
-
)
|
|
108
|
+
RealtimeLogger.critical(f"Sorting file: {inputFileStoreID} of size: {length}")
|
|
114
109
|
# Sort the copy and write back to the fileStore
|
|
115
110
|
shutil.copyfile(inputFile, inputFile + ".sort")
|
|
116
111
|
sort(inputFile + ".sort")
|
|
@@ -11,14 +11,13 @@
|
|
|
11
11
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
12
|
# See the License for the specific language governing permissions and
|
|
13
13
|
# limitations under the License.
|
|
14
|
-
from argparse import Namespace
|
|
15
|
-
from collections.abc import Callable, Generator, Sequence
|
|
16
14
|
import logging
|
|
17
15
|
import os
|
|
18
|
-
from pathlib import Path
|
|
19
16
|
import signal
|
|
20
17
|
import time
|
|
21
|
-
from
|
|
18
|
+
from argparse import Namespace
|
|
19
|
+
from collections.abc import Callable, Generator, Sequence
|
|
20
|
+
from pathlib import Path
|
|
22
21
|
from uuid import uuid4
|
|
23
22
|
|
|
24
23
|
import psutil
|
|
@@ -262,6 +261,7 @@ def _writeNonLocalFilesLambda(job: Job, files: tuple[Path, Path]) -> None:
|
|
|
262
261
|
def lmd(x: Path, nlf: Path) -> None:
|
|
263
262
|
x.unlink()
|
|
264
263
|
nlf.unlink()
|
|
264
|
+
|
|
265
265
|
for nlf in files:
|
|
266
266
|
with nlf.open("wb") as nonLocalFileHandle:
|
|
267
267
|
nonLocalFileHandle.write(os.urandom(1 * 1024 * 1024))
|
|
@@ -282,7 +282,7 @@ def _deferredFunctionRunsWithFailuresFn(job: Job, files: tuple[Path, Path]) -> N
|
|
|
282
282
|
job.defer(_deleteFile, files[1])
|
|
283
283
|
|
|
284
284
|
|
|
285
|
-
def _deleteFile(nonLocalFile: Path, nlf:
|
|
285
|
+
def _deleteFile(nonLocalFile: Path, nlf: Path | None = None) -> None:
|
|
286
286
|
"""
|
|
287
287
|
Delete nonLocalFile and nlf
|
|
288
288
|
:param nonLocalFile:
|
|
@@ -350,7 +350,7 @@ def _testNewJobsCanHandleOtherJobDeaths_C(
|
|
|
350
350
|
|
|
351
351
|
class _deleteMethods:
|
|
352
352
|
@staticmethod
|
|
353
|
-
def _deleteFileMethod(nonLocalFile: Path, nlf:
|
|
353
|
+
def _deleteFileMethod(nonLocalFile: Path, nlf: Path | None = None) -> None:
|
|
354
354
|
"""
|
|
355
355
|
Delete nonLocalFile and nlf
|
|
356
356
|
"""
|
|
@@ -360,7 +360,7 @@ class _deleteMethods:
|
|
|
360
360
|
|
|
361
361
|
@classmethod
|
|
362
362
|
def _deleteFileClassMethod(
|
|
363
|
-
cls, nonLocalFile: Path, nlf:
|
|
363
|
+
cls, nonLocalFile: Path, nlf: Path | None = None
|
|
364
364
|
) -> None:
|
|
365
365
|
"""
|
|
366
366
|
Delete nonLocalFile and nlf
|
toil/test/src/environmentTest.py
CHANGED
|
@@ -16,7 +16,6 @@ import os
|
|
|
16
16
|
import time
|
|
17
17
|
from argparse import Namespace
|
|
18
18
|
from threading import Thread
|
|
19
|
-
from typing import Optional
|
|
20
19
|
|
|
21
20
|
from toil.common import Toil
|
|
22
21
|
from toil.job import Job
|
|
@@ -87,7 +86,7 @@ def check_environment_repeatedly(job):
|
|
|
87
86
|
check3 = waiter.addFollowOnJobFn(check_environment, "try3")
|
|
88
87
|
|
|
89
88
|
|
|
90
|
-
def main(options:
|
|
89
|
+
def main(options: Namespace | None = None):
|
|
91
90
|
"""
|
|
92
91
|
Run the actual workflow with the given options.
|
|
93
92
|
"""
|
toil/test/src/fileStoreTest.py
CHANGED
|
@@ -95,7 +95,9 @@ class hidden:
|
|
|
95
95
|
options.realTimeLogging = True
|
|
96
96
|
options.workDir = self.work_dir
|
|
97
97
|
options.clean = "always"
|
|
98
|
-
log_fd, options.logFile = tempfile.mkstemp(
|
|
98
|
+
log_fd, options.logFile = tempfile.mkstemp(
|
|
99
|
+
dir=self.work_dir, prefix="logFile"
|
|
100
|
+
)
|
|
99
101
|
os.close(log_fd)
|
|
100
102
|
return options
|
|
101
103
|
|
|
@@ -375,7 +377,7 @@ class hidden:
|
|
|
375
377
|
Ensures that files imported to the leader preserve their executable permissions
|
|
376
378
|
when they are read by the fileStore.
|
|
377
379
|
"""
|
|
378
|
-
|
|
380
|
+
|
|
379
381
|
for executable in True, False:
|
|
380
382
|
file_path = self.create_file(content="Hello", executable=executable)
|
|
381
383
|
initial_permissions = os.stat(file_path).st_mode & stat.S_IXUSR
|
|
@@ -537,9 +539,7 @@ class hidden:
|
|
|
537
539
|
# The options namespace and the Config object now have the exact same behavior
|
|
538
540
|
# which means parse_jobstore will be called with argparse rather than with the config object
|
|
539
541
|
# so remove the prepended file: scheme
|
|
540
|
-
jobStoreDev = os.stat(
|
|
541
|
-
os.path.dirname(options.jobStore[5:])
|
|
542
|
-
).st_dev
|
|
542
|
+
jobStoreDev = os.stat(os.path.dirname(options.jobStore[5:])).st_dev
|
|
543
543
|
else:
|
|
544
544
|
jobStoreDev = os.stat(os.path.dirname(options.jobStore)).st_dev
|
|
545
545
|
if workDirDev == jobStoreDev:
|
|
@@ -13,11 +13,13 @@
|
|
|
13
13
|
# limitations under the License.
|
|
14
14
|
|
|
15
15
|
|
|
16
|
-
from argparse import Namespace
|
|
17
16
|
import os
|
|
18
|
-
from pathlib import Path
|
|
19
17
|
import stat
|
|
20
18
|
import uuid
|
|
19
|
+
from argparse import Namespace
|
|
20
|
+
from pathlib import Path
|
|
21
|
+
|
|
22
|
+
import pytest
|
|
21
23
|
|
|
22
24
|
from toil.common import Toil
|
|
23
25
|
from toil.exceptions import FailedJobsException
|
|
@@ -26,9 +28,6 @@ from toil.fileStores.abstractFileStore import AbstractFileStore
|
|
|
26
28
|
from toil.job import Job
|
|
27
29
|
from toil.test import pslow as slow
|
|
28
30
|
|
|
29
|
-
import pytest
|
|
30
|
-
from pytest_subtests import SubTests
|
|
31
|
-
|
|
32
31
|
|
|
33
32
|
def create_file(tmp_path: Path, content: str, executable: bool = False) -> Path:
|
|
34
33
|
file_path = tmp_path / str(uuid.uuid4())
|
|
@@ -109,7 +108,7 @@ class TestImportExportFile:
|
|
|
109
108
|
def test_import_export_restart_false(self, tmp_path: Path) -> None:
|
|
110
109
|
self._run_import_export_workflow(tmp_path, restart=False)
|
|
111
110
|
|
|
112
|
-
def test_basic_import_export(self, tmp_path: Path, subtests:
|
|
111
|
+
def test_basic_import_export(self, tmp_path: Path, subtests: pytest.Subtests) -> None:
|
|
113
112
|
"""
|
|
114
113
|
Ensures that uploaded files preserve their file permissions when they
|
|
115
114
|
are downloaded again. This function checks that an imported executable file
|