toil 6.1.0a1__py3-none-any.whl → 8.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- toil/__init__.py +122 -315
- toil/batchSystems/__init__.py +1 -0
- toil/batchSystems/abstractBatchSystem.py +173 -89
- toil/batchSystems/abstractGridEngineBatchSystem.py +272 -148
- toil/batchSystems/awsBatch.py +244 -135
- toil/batchSystems/cleanup_support.py +26 -16
- toil/batchSystems/contained_executor.py +31 -28
- toil/batchSystems/gridengine.py +86 -50
- toil/batchSystems/htcondor.py +166 -89
- toil/batchSystems/kubernetes.py +632 -382
- toil/batchSystems/local_support.py +20 -15
- toil/batchSystems/lsf.py +134 -81
- toil/batchSystems/lsfHelper.py +13 -11
- toil/batchSystems/mesos/__init__.py +41 -29
- toil/batchSystems/mesos/batchSystem.py +290 -151
- toil/batchSystems/mesos/executor.py +79 -50
- toil/batchSystems/mesos/test/__init__.py +31 -23
- toil/batchSystems/options.py +46 -28
- toil/batchSystems/registry.py +53 -19
- toil/batchSystems/singleMachine.py +296 -125
- toil/batchSystems/slurm.py +603 -138
- toil/batchSystems/torque.py +47 -33
- toil/bus.py +186 -76
- toil/common.py +664 -368
- toil/cwl/__init__.py +1 -1
- toil/cwl/cwltoil.py +1136 -483
- toil/cwl/utils.py +17 -22
- toil/deferred.py +63 -42
- toil/exceptions.py +5 -3
- toil/fileStores/__init__.py +5 -5
- toil/fileStores/abstractFileStore.py +140 -60
- toil/fileStores/cachingFileStore.py +717 -269
- toil/fileStores/nonCachingFileStore.py +116 -87
- toil/job.py +1225 -368
- toil/jobStores/abstractJobStore.py +416 -266
- toil/jobStores/aws/jobStore.py +863 -477
- toil/jobStores/aws/utils.py +201 -120
- toil/jobStores/conftest.py +3 -2
- toil/jobStores/fileJobStore.py +292 -154
- toil/jobStores/googleJobStore.py +140 -74
- toil/jobStores/utils.py +36 -15
- toil/leader.py +668 -272
- toil/lib/accelerators.py +115 -18
- toil/lib/aws/__init__.py +74 -31
- toil/lib/aws/ami.py +122 -87
- toil/lib/aws/iam.py +284 -108
- toil/lib/aws/s3.py +31 -0
- toil/lib/aws/session.py +214 -39
- toil/lib/aws/utils.py +287 -231
- toil/lib/bioio.py +13 -5
- toil/lib/compatibility.py +11 -6
- toil/lib/conversions.py +104 -47
- toil/lib/docker.py +131 -103
- toil/lib/ec2.py +361 -199
- toil/lib/ec2nodes.py +174 -106
- toil/lib/encryption/_dummy.py +5 -3
- toil/lib/encryption/_nacl.py +10 -6
- toil/lib/encryption/conftest.py +1 -0
- toil/lib/exceptions.py +26 -7
- toil/lib/expando.py +5 -3
- toil/lib/ftp_utils.py +217 -0
- toil/lib/generatedEC2Lists.py +127 -19
- toil/lib/humanize.py +6 -2
- toil/lib/integration.py +341 -0
- toil/lib/io.py +141 -15
- toil/lib/iterables.py +4 -2
- toil/lib/memoize.py +12 -8
- toil/lib/misc.py +66 -21
- toil/lib/objects.py +2 -2
- toil/lib/resources.py +68 -15
- toil/lib/retry.py +126 -81
- toil/lib/threading.py +299 -82
- toil/lib/throttle.py +16 -15
- toil/options/common.py +843 -409
- toil/options/cwl.py +175 -90
- toil/options/runner.py +50 -0
- toil/options/wdl.py +73 -17
- toil/provisioners/__init__.py +117 -46
- toil/provisioners/abstractProvisioner.py +332 -157
- toil/provisioners/aws/__init__.py +70 -33
- toil/provisioners/aws/awsProvisioner.py +1145 -715
- toil/provisioners/clusterScaler.py +541 -279
- toil/provisioners/gceProvisioner.py +282 -179
- toil/provisioners/node.py +155 -79
- toil/realtimeLogger.py +34 -22
- toil/resource.py +137 -75
- toil/server/app.py +128 -62
- toil/server/celery_app.py +3 -1
- toil/server/cli/wes_cwl_runner.py +82 -53
- toil/server/utils.py +54 -28
- toil/server/wes/abstract_backend.py +64 -26
- toil/server/wes/amazon_wes_utils.py +21 -15
- toil/server/wes/tasks.py +121 -63
- toil/server/wes/toil_backend.py +142 -107
- toil/server/wsgi_app.py +4 -3
- toil/serviceManager.py +58 -22
- toil/statsAndLogging.py +224 -70
- toil/test/__init__.py +282 -183
- toil/test/batchSystems/batchSystemTest.py +460 -210
- toil/test/batchSystems/batch_system_plugin_test.py +90 -0
- toil/test/batchSystems/test_gridengine.py +173 -0
- toil/test/batchSystems/test_lsf_helper.py +67 -58
- toil/test/batchSystems/test_slurm.py +110 -49
- toil/test/cactus/__init__.py +0 -0
- toil/test/cactus/test_cactus_integration.py +56 -0
- toil/test/cwl/cwlTest.py +496 -287
- toil/test/cwl/measure_default_memory.cwl +12 -0
- toil/test/cwl/not_run_required_input.cwl +29 -0
- toil/test/cwl/scatter_duplicate_outputs.cwl +40 -0
- toil/test/cwl/seqtk_seq.cwl +1 -1
- toil/test/docs/scriptsTest.py +69 -46
- toil/test/jobStores/jobStoreTest.py +427 -264
- toil/test/lib/aws/test_iam.py +118 -50
- toil/test/lib/aws/test_s3.py +16 -9
- toil/test/lib/aws/test_utils.py +5 -6
- toil/test/lib/dockerTest.py +118 -141
- toil/test/lib/test_conversions.py +113 -115
- toil/test/lib/test_ec2.py +58 -50
- toil/test/lib/test_integration.py +104 -0
- toil/test/lib/test_misc.py +12 -5
- toil/test/mesos/MesosDataStructuresTest.py +23 -10
- toil/test/mesos/helloWorld.py +7 -6
- toil/test/mesos/stress.py +25 -20
- toil/test/options/__init__.py +13 -0
- toil/test/options/options.py +42 -0
- toil/test/provisioners/aws/awsProvisionerTest.py +320 -150
- toil/test/provisioners/clusterScalerTest.py +440 -250
- toil/test/provisioners/clusterTest.py +166 -44
- toil/test/provisioners/gceProvisionerTest.py +174 -100
- toil/test/provisioners/provisionerTest.py +25 -13
- toil/test/provisioners/restartScript.py +5 -4
- toil/test/server/serverTest.py +188 -141
- toil/test/sort/restart_sort.py +137 -68
- toil/test/sort/sort.py +134 -66
- toil/test/sort/sortTest.py +91 -49
- toil/test/src/autoDeploymentTest.py +141 -101
- toil/test/src/busTest.py +20 -18
- toil/test/src/checkpointTest.py +8 -2
- toil/test/src/deferredFunctionTest.py +49 -35
- toil/test/src/dockerCheckTest.py +32 -24
- toil/test/src/environmentTest.py +135 -0
- toil/test/src/fileStoreTest.py +539 -272
- toil/test/src/helloWorldTest.py +7 -4
- toil/test/src/importExportFileTest.py +61 -31
- toil/test/src/jobDescriptionTest.py +46 -21
- toil/test/src/jobEncapsulationTest.py +2 -0
- toil/test/src/jobFileStoreTest.py +74 -50
- toil/test/src/jobServiceTest.py +187 -73
- toil/test/src/jobTest.py +121 -71
- toil/test/src/miscTests.py +19 -18
- toil/test/src/promisedRequirementTest.py +82 -36
- toil/test/src/promisesTest.py +7 -6
- toil/test/src/realtimeLoggerTest.py +10 -6
- toil/test/src/regularLogTest.py +71 -37
- toil/test/src/resourceTest.py +80 -49
- toil/test/src/restartDAGTest.py +36 -22
- toil/test/src/resumabilityTest.py +9 -2
- toil/test/src/retainTempDirTest.py +45 -14
- toil/test/src/systemTest.py +12 -8
- toil/test/src/threadingTest.py +44 -25
- toil/test/src/toilContextManagerTest.py +10 -7
- toil/test/src/userDefinedJobArgTypeTest.py +8 -5
- toil/test/src/workerTest.py +73 -23
- toil/test/utils/toilDebugTest.py +103 -33
- toil/test/utils/toilKillTest.py +4 -5
- toil/test/utils/utilsTest.py +245 -106
- toil/test/wdl/wdltoil_test.py +818 -149
- toil/test/wdl/wdltoil_test_kubernetes.py +91 -0
- toil/toilState.py +120 -35
- toil/utils/toilConfig.py +13 -4
- toil/utils/toilDebugFile.py +44 -27
- toil/utils/toilDebugJob.py +214 -27
- toil/utils/toilDestroyCluster.py +11 -6
- toil/utils/toilKill.py +8 -3
- toil/utils/toilLaunchCluster.py +256 -140
- toil/utils/toilMain.py +37 -16
- toil/utils/toilRsyncCluster.py +32 -14
- toil/utils/toilSshCluster.py +49 -22
- toil/utils/toilStats.py +356 -273
- toil/utils/toilStatus.py +292 -139
- toil/utils/toilUpdateEC2Instances.py +3 -1
- toil/version.py +12 -12
- toil/wdl/utils.py +5 -5
- toil/wdl/wdltoil.py +3913 -1033
- toil/worker.py +367 -184
- {toil-6.1.0a1.dist-info → toil-8.0.0.dist-info}/LICENSE +25 -0
- toil-8.0.0.dist-info/METADATA +173 -0
- toil-8.0.0.dist-info/RECORD +253 -0
- {toil-6.1.0a1.dist-info → toil-8.0.0.dist-info}/WHEEL +1 -1
- toil-6.1.0a1.dist-info/METADATA +0 -125
- toil-6.1.0a1.dist-info/RECORD +0 -237
- {toil-6.1.0a1.dist-info → toil-8.0.0.dist-info}/entry_points.txt +0 -0
- {toil-6.1.0a1.dist-info → toil-8.0.0.dist-info}/top_level.txt +0 -0
toil/test/server/serverTest.py
CHANGED
|
@@ -21,7 +21,7 @@ import uuid
|
|
|
21
21
|
import zipfile
|
|
22
22
|
from abc import abstractmethod
|
|
23
23
|
from io import BytesIO
|
|
24
|
-
from typing import Optional
|
|
24
|
+
from typing import TYPE_CHECKING, Optional
|
|
25
25
|
from urllib.parse import urlparse
|
|
26
26
|
|
|
27
27
|
try:
|
|
@@ -38,6 +38,7 @@ from toil.test import ToilTest, needs_aws_s3, needs_celery_broker, needs_server
|
|
|
38
38
|
logger = logging.getLogger(__name__)
|
|
39
39
|
logging.basicConfig(level=logging.INFO)
|
|
40
40
|
|
|
41
|
+
|
|
41
42
|
@needs_server
|
|
42
43
|
class ToilServerUtilsTest(ToilTest):
|
|
43
44
|
"""
|
|
@@ -51,9 +52,11 @@ class ToilServerUtilsTest(ToilTest):
|
|
|
51
52
|
away without flipping the state.
|
|
52
53
|
"""
|
|
53
54
|
|
|
54
|
-
from toil.server.utils import (
|
|
55
|
-
|
|
56
|
-
|
|
55
|
+
from toil.server.utils import (
|
|
56
|
+
MemoryStateStore,
|
|
57
|
+
WorkflowStateMachine,
|
|
58
|
+
WorkflowStateStore,
|
|
59
|
+
)
|
|
57
60
|
|
|
58
61
|
store = WorkflowStateStore(MemoryStateStore(), "test-workflow")
|
|
59
62
|
|
|
@@ -70,6 +73,7 @@ class ToilServerUtilsTest(ToilTest):
|
|
|
70
73
|
# Make sure it is now CANCELED due to timeout
|
|
71
74
|
self.assertEqual(state_machine.get_current_state(), "CANCELED")
|
|
72
75
|
|
|
76
|
+
|
|
73
77
|
class hidden:
|
|
74
78
|
# Hide abstract tests from the test loader
|
|
75
79
|
|
|
@@ -89,7 +93,6 @@ class hidden:
|
|
|
89
93
|
|
|
90
94
|
raise NotImplementedError()
|
|
91
95
|
|
|
92
|
-
|
|
93
96
|
def test_state_store(self) -> None:
|
|
94
97
|
"""
|
|
95
98
|
Make sure that the state store under test can store and load keys.
|
|
@@ -98,45 +101,46 @@ class hidden:
|
|
|
98
101
|
store = self.get_state_store()
|
|
99
102
|
|
|
100
103
|
# Should start None
|
|
101
|
-
self.assertEqual(store.get(
|
|
104
|
+
self.assertEqual(store.get("id1", "key1"), None)
|
|
102
105
|
|
|
103
106
|
# Should hold a value
|
|
104
|
-
store.set(
|
|
105
|
-
self.assertEqual(store.get(
|
|
107
|
+
store.set("id1", "key1", "value1")
|
|
108
|
+
self.assertEqual(store.get("id1", "key1"), "value1")
|
|
106
109
|
|
|
107
110
|
# Should distinguish by ID and key
|
|
108
|
-
self.assertEqual(store.get(
|
|
109
|
-
self.assertEqual(store.get(
|
|
111
|
+
self.assertEqual(store.get("id2", "key1"), None)
|
|
112
|
+
self.assertEqual(store.get("id1", "key2"), None)
|
|
110
113
|
|
|
111
|
-
store.set(
|
|
112
|
-
store.set(
|
|
113
|
-
self.assertEqual(store.get(
|
|
114
|
-
self.assertEqual(store.get(
|
|
115
|
-
self.assertEqual(store.get(
|
|
114
|
+
store.set("id2", "key1", "value2")
|
|
115
|
+
store.set("id1", "key2", "value3")
|
|
116
|
+
self.assertEqual(store.get("id1", "key1"), "value1")
|
|
117
|
+
self.assertEqual(store.get("id2", "key1"), "value2")
|
|
118
|
+
self.assertEqual(store.get("id1", "key2"), "value3")
|
|
116
119
|
|
|
117
120
|
# Should allow replacement
|
|
118
|
-
store.set(
|
|
119
|
-
self.assertEqual(store.get(
|
|
120
|
-
self.assertEqual(store.get(
|
|
121
|
-
self.assertEqual(store.get(
|
|
121
|
+
store.set("id1", "key1", "value4")
|
|
122
|
+
self.assertEqual(store.get("id1", "key1"), "value4")
|
|
123
|
+
self.assertEqual(store.get("id2", "key1"), "value2")
|
|
124
|
+
self.assertEqual(store.get("id1", "key2"), "value3")
|
|
122
125
|
|
|
123
126
|
# Should show up in another state store
|
|
124
127
|
store2 = self.get_state_store()
|
|
125
|
-
self.assertEqual(store2.get(
|
|
126
|
-
self.assertEqual(store2.get(
|
|
127
|
-
self.assertEqual(store2.get(
|
|
128
|
+
self.assertEqual(store2.get("id1", "key1"), "value4")
|
|
129
|
+
self.assertEqual(store2.get("id2", "key1"), "value2")
|
|
130
|
+
self.assertEqual(store2.get("id1", "key2"), "value3")
|
|
128
131
|
|
|
129
132
|
# Should allow clearing
|
|
130
|
-
store.set(
|
|
131
|
-
self.assertEqual(store.get(
|
|
132
|
-
self.assertEqual(store.get(
|
|
133
|
-
self.assertEqual(store.get(
|
|
133
|
+
store.set("id1", "key1", None)
|
|
134
|
+
self.assertEqual(store.get("id1", "key1"), None)
|
|
135
|
+
self.assertEqual(store.get("id2", "key1"), "value2")
|
|
136
|
+
self.assertEqual(store.get("id1", "key2"), "value3")
|
|
137
|
+
|
|
138
|
+
store.set("id2", "key1", None)
|
|
139
|
+
store.set("id1", "key2", None)
|
|
140
|
+
self.assertEqual(store.get("id1", "key1"), None)
|
|
141
|
+
self.assertEqual(store.get("id2", "key1"), None)
|
|
142
|
+
self.assertEqual(store.get("id1", "key2"), None)
|
|
134
143
|
|
|
135
|
-
store.set('id2', 'key1', None)
|
|
136
|
-
store.set('id1', 'key2', None)
|
|
137
|
-
self.assertEqual(store.get('id1', 'key1'), None)
|
|
138
|
-
self.assertEqual(store.get('id2', 'key1'), None)
|
|
139
|
-
self.assertEqual(store.get('id1', 'key2'), None)
|
|
140
144
|
|
|
141
145
|
class FileStateStoreTest(hidden.AbstractStateStoreTest):
|
|
142
146
|
"""
|
|
@@ -158,6 +162,7 @@ class FileStateStoreTest(hidden.AbstractStateStoreTest):
|
|
|
158
162
|
|
|
159
163
|
return FileStateStore(self.state_store_dir)
|
|
160
164
|
|
|
165
|
+
|
|
161
166
|
class FileStateStoreURLTest(hidden.AbstractStateStoreTest):
|
|
162
167
|
"""
|
|
163
168
|
Test file-based state storage using URLs instead of local paths.
|
|
@@ -167,7 +172,7 @@ class FileStateStoreURLTest(hidden.AbstractStateStoreTest):
|
|
|
167
172
|
|
|
168
173
|
def setUp(self) -> None:
|
|
169
174
|
super().setUp()
|
|
170
|
-
self.state_store_dir =
|
|
175
|
+
self.state_store_dir = "file://" + self._createTempDir()
|
|
171
176
|
|
|
172
177
|
def get_state_store(self) -> AbstractStateStore:
|
|
173
178
|
"""
|
|
@@ -178,24 +183,20 @@ class FileStateStoreURLTest(hidden.AbstractStateStoreTest):
|
|
|
178
183
|
|
|
179
184
|
return FileStateStore(self.state_store_dir)
|
|
180
185
|
|
|
186
|
+
|
|
181
187
|
@needs_aws_s3
|
|
182
188
|
class BucketUsingTest(ToilTest):
|
|
183
189
|
"""
|
|
184
190
|
Base class for tests that need a bucket.
|
|
185
191
|
"""
|
|
186
192
|
|
|
187
|
-
|
|
188
|
-
# We need the class to be evaluateable without the AWS modules, if not
|
|
189
|
-
# runnable
|
|
193
|
+
if TYPE_CHECKING:
|
|
190
194
|
from mypy_boto3_s3 import S3ServiceResource
|
|
191
195
|
from mypy_boto3_s3.service_resource import Bucket
|
|
192
|
-
except ImportError:
|
|
193
|
-
pass
|
|
194
|
-
|
|
195
196
|
|
|
196
197
|
region: Optional[str]
|
|
197
|
-
s3_resource: Optional[
|
|
198
|
-
bucket: Optional[
|
|
198
|
+
s3_resource: Optional["S3ServiceResource"]
|
|
199
|
+
bucket: Optional["Bucket"]
|
|
199
200
|
bucket_name: Optional[str]
|
|
200
201
|
|
|
201
202
|
@classmethod
|
|
@@ -218,14 +219,14 @@ class BucketUsingTest(ToilTest):
|
|
|
218
219
|
@classmethod
|
|
219
220
|
def tearDownClass(cls) -> None:
|
|
220
221
|
from toil.lib.aws.utils import delete_s3_bucket
|
|
222
|
+
|
|
221
223
|
if cls.bucket_name:
|
|
222
224
|
delete_s3_bucket(cls.s3_resource, cls.bucket_name, cls.region)
|
|
223
225
|
super().tearDownClass()
|
|
224
226
|
|
|
227
|
+
|
|
225
228
|
class AWSStateStoreTest(hidden.AbstractStateStoreTest, BucketUsingTest):
|
|
226
|
-
"""
|
|
227
|
-
Test AWS-based state storage.
|
|
228
|
-
"""
|
|
229
|
+
"""Test AWS-based state storage."""
|
|
229
230
|
|
|
230
231
|
from toil.server.utils import AbstractStateStore
|
|
231
232
|
|
|
@@ -238,7 +239,7 @@ class AWSStateStoreTest(hidden.AbstractStateStoreTest, BucketUsingTest):
|
|
|
238
239
|
|
|
239
240
|
from toil.server.utils import S3StateStore
|
|
240
241
|
|
|
241
|
-
return S3StateStore(
|
|
242
|
+
return S3StateStore("s3://" + self.bucket_name + "/" + self.bucket_path)
|
|
242
243
|
|
|
243
244
|
def test_state_store_paths(self) -> None:
|
|
244
245
|
"""
|
|
@@ -253,16 +254,18 @@ class AWSStateStoreTest(hidden.AbstractStateStoreTest, BucketUsingTest):
|
|
|
253
254
|
store = self.get_state_store()
|
|
254
255
|
|
|
255
256
|
# Should hold a value
|
|
256
|
-
store.set(
|
|
257
|
-
self.assertEqual(store.get(
|
|
257
|
+
store.set("testid", "testkey", "testvalue")
|
|
258
|
+
self.assertEqual(store.get("testid", "testkey"), "testvalue")
|
|
258
259
|
|
|
259
|
-
expected_url = urlparse(
|
|
260
|
-
|
|
260
|
+
expected_url = urlparse(
|
|
261
|
+
"s3://"
|
|
262
|
+
+ self.bucket_name
|
|
263
|
+
+ "/"
|
|
264
|
+
+ os.path.join(self.bucket_path, "testid", "testkey")
|
|
265
|
+
)
|
|
261
266
|
|
|
262
267
|
obj = get_object_for_url(expected_url, True)
|
|
263
|
-
self.assertEqual(obj.content_length, len(
|
|
264
|
-
|
|
265
|
-
|
|
268
|
+
self.assertEqual(obj.content_length, len("testvalue"))
|
|
266
269
|
|
|
267
270
|
|
|
268
271
|
@needs_server
|
|
@@ -286,8 +289,11 @@ class AbstractToilWESServerTest(ToilTest):
|
|
|
286
289
|
self.temp_dir = self._createTempDir()
|
|
287
290
|
|
|
288
291
|
from toil.server.app import create_app, parser_with_server_options
|
|
292
|
+
|
|
289
293
|
parser = parser_with_server_options()
|
|
290
|
-
args = parser.parse_args(
|
|
294
|
+
args = parser.parse_args(
|
|
295
|
+
self._server_args + ["--work_dir", os.path.join(self.temp_dir, "workflows")]
|
|
296
|
+
)
|
|
291
297
|
|
|
292
298
|
# Make the FlaskApp
|
|
293
299
|
server_app = create_app(args)
|
|
@@ -296,7 +302,8 @@ class AbstractToilWESServerTest(ToilTest):
|
|
|
296
302
|
self.app: Flask = server_app.app
|
|
297
303
|
self.app.testing = True
|
|
298
304
|
|
|
299
|
-
self.example_cwl = textwrap.dedent(
|
|
305
|
+
self.example_cwl = textwrap.dedent(
|
|
306
|
+
"""
|
|
300
307
|
cwlVersion: v1.0
|
|
301
308
|
class: CommandLineTool
|
|
302
309
|
baseCommand: echo
|
|
@@ -309,9 +316,11 @@ class AbstractToilWESServerTest(ToilTest):
|
|
|
309
316
|
outputs:
|
|
310
317
|
output:
|
|
311
318
|
type: stdout
|
|
312
|
-
"""
|
|
319
|
+
"""
|
|
320
|
+
)
|
|
313
321
|
|
|
314
|
-
self.slow_cwl = textwrap.dedent(
|
|
322
|
+
self.slow_cwl = textwrap.dedent(
|
|
323
|
+
"""
|
|
315
324
|
cwlVersion: v1.0
|
|
316
325
|
class: CommandLineTool
|
|
317
326
|
baseCommand: sleep
|
|
@@ -324,7 +333,8 @@ class AbstractToilWESServerTest(ToilTest):
|
|
|
324
333
|
outputs:
|
|
325
334
|
output:
|
|
326
335
|
type: stdout
|
|
327
|
-
"""
|
|
336
|
+
"""
|
|
337
|
+
)
|
|
328
338
|
|
|
329
339
|
def tearDown(self) -> None:
|
|
330
340
|
super().tearDown()
|
|
@@ -344,8 +354,7 @@ class AbstractToilWESServerTest(ToilTest):
|
|
|
344
354
|
The workflow should succeed, it should have some tasks, and they should have all succeeded.
|
|
345
355
|
"""
|
|
346
356
|
rv = self._fetch_run_log(client, run_id)
|
|
347
|
-
logger.debug(
|
|
348
|
-
self.assertIn("run_log", rv.json)
|
|
357
|
+
logger.debug("Log info: %s", rv.json)
|
|
349
358
|
run_log = rv.json.get("run_log")
|
|
350
359
|
self.assertEqual(type(run_log), dict)
|
|
351
360
|
if "exit_code" in run_log:
|
|
@@ -363,22 +372,15 @@ class AbstractToilWESServerTest(ToilTest):
|
|
|
363
372
|
self.assertEqual(task_log.get("exit_code"), 0)
|
|
364
373
|
|
|
365
374
|
def _report_log(self, client: "FlaskClient", run_id: str) -> None:
|
|
366
|
-
"""
|
|
367
|
-
Report the log for the given workflow run.
|
|
368
|
-
"""
|
|
375
|
+
"""Report the log for the given workflow run."""
|
|
369
376
|
rv = self._fetch_run_log(client, run_id)
|
|
370
|
-
|
|
377
|
+
logger.debug(f"Report log response: {rv.json}")
|
|
371
378
|
run_log = rv.json.get("run_log")
|
|
372
379
|
self.assertEqual(type(run_log), dict)
|
|
373
|
-
self.
|
|
374
|
-
|
|
375
|
-
self.
|
|
376
|
-
self.
|
|
377
|
-
stderr = run_log.get("stderr")
|
|
378
|
-
self.assertEqual(type(stderr), str)
|
|
379
|
-
logger.info("Got stdout %s and stderr %s", stdout, stderr)
|
|
380
|
-
self._report_absolute_url(client, stdout)
|
|
381
|
-
self._report_absolute_url(client, stderr)
|
|
380
|
+
self.assertEqual(type(run_log.get("stdout")), str)
|
|
381
|
+
self.assertEqual(type(run_log.get("stderr")), str)
|
|
382
|
+
self._report_absolute_url(client, run_log.get("stdout"))
|
|
383
|
+
self._report_absolute_url(client, run_log.get("stderr"))
|
|
382
384
|
|
|
383
385
|
def _report_absolute_url(self, client: "FlaskClient", url: str):
|
|
384
386
|
"""
|
|
@@ -393,21 +395,24 @@ class AbstractToilWESServerTest(ToilTest):
|
|
|
393
395
|
logger.info("Fetch %s", url)
|
|
394
396
|
rv = client.get(url)
|
|
395
397
|
self.assertEqual(rv.status_code, 200)
|
|
396
|
-
logger.info("Got %s:\n%s", url, rv.data.decode(
|
|
398
|
+
logger.info("Got %s:\n%s", url, rv.data.decode("utf-8"))
|
|
397
399
|
|
|
398
400
|
def _start_slow_workflow(self, client: "FlaskClient") -> str:
|
|
399
401
|
"""
|
|
400
402
|
Start a slow workflow and return its ID.
|
|
401
403
|
"""
|
|
402
|
-
rv = client.post(
|
|
403
|
-
"
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
404
|
+
rv = client.post(
|
|
405
|
+
"/ga4gh/wes/v1/runs",
|
|
406
|
+
data={
|
|
407
|
+
"workflow_url": "slow.cwl",
|
|
408
|
+
"workflow_type": "CWL",
|
|
409
|
+
"workflow_type_version": "v1.0",
|
|
410
|
+
"workflow_params": json.dumps({"delay": "5"}),
|
|
411
|
+
"workflow_attachment": [
|
|
412
|
+
(BytesIO(self.slow_cwl.encode()), "slow.cwl"),
|
|
413
|
+
],
|
|
414
|
+
},
|
|
415
|
+
)
|
|
411
416
|
# workflow is submitted successfully
|
|
412
417
|
self.assertEqual(rv.status_code, 200)
|
|
413
418
|
self.assertTrue(rv.is_json)
|
|
@@ -428,16 +433,30 @@ class AbstractToilWESServerTest(ToilTest):
|
|
|
428
433
|
self.assertEqual(rv.json.get("run_id"), run_id)
|
|
429
434
|
self.assertIn("state", rv.json)
|
|
430
435
|
state = rv.json.get("state")
|
|
431
|
-
self.assertIn(
|
|
432
|
-
|
|
433
|
-
|
|
436
|
+
self.assertIn(
|
|
437
|
+
state,
|
|
438
|
+
[
|
|
439
|
+
"UNKNOWN",
|
|
440
|
+
"QUEUED",
|
|
441
|
+
"INITIALIZING",
|
|
442
|
+
"RUNNING",
|
|
443
|
+
"PAUSED",
|
|
444
|
+
"COMPLETE",
|
|
445
|
+
"EXECUTOR_ERROR",
|
|
446
|
+
"SYSTEM_ERROR",
|
|
447
|
+
"CANCELED",
|
|
448
|
+
"CANCELING",
|
|
449
|
+
],
|
|
450
|
+
)
|
|
434
451
|
return state
|
|
435
452
|
|
|
436
453
|
def _cancel_workflow(self, client: "FlaskClient", run_id: str) -> None:
|
|
437
454
|
rv = client.post(f"/ga4gh/wes/v1/runs/{run_id}/cancel")
|
|
438
455
|
self.assertEqual(rv.status_code, 200)
|
|
439
456
|
|
|
440
|
-
def _wait_for_status(
|
|
457
|
+
def _wait_for_status(
|
|
458
|
+
self, client: "FlaskClient", run_id: str, target_status: str
|
|
459
|
+
) -> None:
|
|
441
460
|
"""
|
|
442
461
|
Wait for the given workflow run to reach the given state. If it reaches
|
|
443
462
|
a different terminal state, raise an exception.
|
|
@@ -470,19 +489,19 @@ class ToilWESServerBenchTest(AbstractToilWESServerTest):
|
|
|
470
489
|
"""
|
|
471
490
|
|
|
472
491
|
def test_home(self) -> None:
|
|
473
|
-
"""
|
|
492
|
+
"""Test the homepage endpoint."""
|
|
474
493
|
with self.app.test_client() as client:
|
|
475
494
|
rv = client.get("/")
|
|
476
495
|
self.assertEqual(rv.status_code, 302)
|
|
477
496
|
|
|
478
497
|
def test_health(self) -> None:
|
|
479
|
-
"""
|
|
498
|
+
"""Test the health check endpoint."""
|
|
480
499
|
with self.app.test_client() as client:
|
|
481
500
|
rv = client.get("/engine/v1/status")
|
|
482
501
|
self.assertEqual(rv.status_code, 200)
|
|
483
502
|
|
|
484
503
|
def test_get_service_info(self) -> None:
|
|
485
|
-
"""
|
|
504
|
+
"""Test the GET /service-info endpoint."""
|
|
486
505
|
with self.app.test_client() as client:
|
|
487
506
|
rv = client.get("/ga4gh/wes/v1/service-info")
|
|
488
507
|
self.assertEqual(rv.status_code, 200)
|
|
@@ -500,12 +519,15 @@ class ToilWESServerBenchTest(AbstractToilWESServerTest):
|
|
|
500
519
|
self.assertIn("system_state_counts", service_info)
|
|
501
520
|
self.assertIn("tags", service_info)
|
|
502
521
|
|
|
522
|
+
|
|
503
523
|
class ToilWESServerWorkflowTest(AbstractToilWESServerTest):
|
|
504
524
|
"""
|
|
505
525
|
Tests of the WES server running workflows.
|
|
506
526
|
"""
|
|
507
527
|
|
|
508
|
-
def run_zip_workflow(
|
|
528
|
+
def run_zip_workflow(
|
|
529
|
+
self, zip_path: str, include_message: bool = True, include_params: bool = True
|
|
530
|
+
) -> None:
|
|
509
531
|
"""
|
|
510
532
|
We have several zip file tests; this submits a zip file and makes sure it ran OK.
|
|
511
533
|
|
|
@@ -518,11 +540,13 @@ class ToilWESServerWorkflowTest(AbstractToilWESServerTest):
|
|
|
518
540
|
post_data = {
|
|
519
541
|
"workflow_url": "file://" + zip_path,
|
|
520
542
|
"workflow_type": "CWL",
|
|
521
|
-
"workflow_type_version": "v1.0"
|
|
543
|
+
"workflow_type_version": "v1.0",
|
|
522
544
|
}
|
|
523
545
|
if include_params or include_message:
|
|
524
546
|
# We need workflow_params too
|
|
525
|
-
post_data["workflow_params"] = json.dumps(
|
|
547
|
+
post_data["workflow_params"] = json.dumps(
|
|
548
|
+
{"message": "Hello, world!"} if include_message else {}
|
|
549
|
+
)
|
|
526
550
|
with self.app.test_client() as client:
|
|
527
551
|
rv = client.post("/ga4gh/wes/v1/runs", data=post_data)
|
|
528
552
|
# workflow is submitted successfully
|
|
@@ -539,28 +563,37 @@ class ToilWESServerWorkflowTest(AbstractToilWESServerTest):
|
|
|
539
563
|
def test_run_workflow_relative_url_no_attachments_fails(self) -> None:
|
|
540
564
|
"""Test run example CWL workflow from relative workflow URL but with no attachments."""
|
|
541
565
|
with self.app.test_client() as client:
|
|
542
|
-
rv = client.post(
|
|
543
|
-
"
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
566
|
+
rv = client.post(
|
|
567
|
+
"/ga4gh/wes/v1/runs",
|
|
568
|
+
data={
|
|
569
|
+
"workflow_url": "example.cwl",
|
|
570
|
+
"workflow_type": "CWL",
|
|
571
|
+
"workflow_type_version": "v1.0",
|
|
572
|
+
"workflow_params": "{}",
|
|
573
|
+
},
|
|
574
|
+
)
|
|
548
575
|
self.assertEqual(rv.status_code, 400)
|
|
549
576
|
self.assertTrue(rv.is_json)
|
|
550
|
-
self.assertEqual(
|
|
577
|
+
self.assertEqual(
|
|
578
|
+
rv.json.get("msg"),
|
|
579
|
+
"Relative 'workflow_url' but missing 'workflow_attachment'",
|
|
580
|
+
)
|
|
551
581
|
|
|
552
582
|
def test_run_workflow_relative_url(self) -> None:
|
|
553
583
|
"""Test run example CWL workflow from relative workflow URL."""
|
|
554
584
|
with self.app.test_client() as client:
|
|
555
|
-
rv = client.post(
|
|
556
|
-
"
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
585
|
+
rv = client.post(
|
|
586
|
+
"/ga4gh/wes/v1/runs",
|
|
587
|
+
data={
|
|
588
|
+
"workflow_url": "example.cwl",
|
|
589
|
+
"workflow_type": "CWL",
|
|
590
|
+
"workflow_type_version": "v1.0",
|
|
591
|
+
"workflow_params": json.dumps({"message": "Hello, world!"}),
|
|
592
|
+
"workflow_attachment": [
|
|
593
|
+
(BytesIO(self.example_cwl.encode()), "example.cwl"),
|
|
594
|
+
],
|
|
595
|
+
},
|
|
596
|
+
)
|
|
564
597
|
# workflow is submitted successfully
|
|
565
598
|
self.assertEqual(rv.status_code, 200)
|
|
566
599
|
self.assertTrue(rv.is_json)
|
|
@@ -573,13 +606,15 @@ class ToilWESServerWorkflowTest(AbstractToilWESServerTest):
|
|
|
573
606
|
def test_run_workflow_https_url(self) -> None:
|
|
574
607
|
"""Test run example CWL workflow from the Internet."""
|
|
575
608
|
with self.app.test_client() as client:
|
|
576
|
-
rv = client.post(
|
|
577
|
-
"
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
609
|
+
rv = client.post(
|
|
610
|
+
"/ga4gh/wes/v1/runs",
|
|
611
|
+
data={
|
|
612
|
+
"workflow_url": "https://raw.githubusercontent.com/DataBiosphere/toil/4cb5bb3871ac21a9793f638b83775926ed94a226/src/toil/test/cwl/echo.cwl",
|
|
613
|
+
"workflow_type": "CWL",
|
|
614
|
+
"workflow_type_version": "v1.2",
|
|
615
|
+
"workflow_params": json.dumps({"message": "Hello, world!"}),
|
|
616
|
+
},
|
|
617
|
+
)
|
|
583
618
|
# workflow is submitted successfully
|
|
584
619
|
self.assertEqual(rv.status_code, 200)
|
|
585
620
|
self.assertTrue(rv.is_json)
|
|
@@ -592,57 +627,63 @@ class ToilWESServerWorkflowTest(AbstractToilWESServerTest):
|
|
|
592
627
|
def test_run_workflow_single_file_zip(self) -> None:
|
|
593
628
|
"""Test run example CWL workflow from single-file ZIP."""
|
|
594
629
|
workdir = self._createTempDir()
|
|
595
|
-
zip_path = os.path.abspath(os.path.join(workdir,
|
|
596
|
-
with zipfile.ZipFile(zip_path,
|
|
597
|
-
zip_file.writestr(
|
|
630
|
+
zip_path = os.path.abspath(os.path.join(workdir, "workflow.zip"))
|
|
631
|
+
with zipfile.ZipFile(zip_path, "w") as zip_file:
|
|
632
|
+
zip_file.writestr("example.cwl", self.example_cwl)
|
|
598
633
|
self.run_zip_workflow(zip_path)
|
|
599
634
|
|
|
600
635
|
def test_run_workflow_multi_file_zip(self) -> None:
|
|
601
636
|
"""Test run example CWL workflow from multi-file ZIP."""
|
|
602
637
|
workdir = self._createTempDir()
|
|
603
|
-
zip_path = os.path.abspath(os.path.join(workdir,
|
|
604
|
-
with zipfile.ZipFile(zip_path,
|
|
605
|
-
zip_file.writestr(
|
|
606
|
-
zip_file.writestr(
|
|
638
|
+
zip_path = os.path.abspath(os.path.join(workdir, "workflow.zip"))
|
|
639
|
+
with zipfile.ZipFile(zip_path, "w") as zip_file:
|
|
640
|
+
zip_file.writestr("main.cwl", self.example_cwl)
|
|
641
|
+
zip_file.writestr("distraction.cwl", "Don't mind me")
|
|
607
642
|
self.run_zip_workflow(zip_path)
|
|
608
643
|
|
|
609
644
|
def test_run_workflow_manifest_zip(self) -> None:
|
|
610
645
|
"""Test run example CWL workflow from ZIP with manifest."""
|
|
611
646
|
workdir = self._createTempDir()
|
|
612
|
-
zip_path = os.path.abspath(os.path.join(workdir,
|
|
613
|
-
with zipfile.ZipFile(zip_path,
|
|
614
|
-
zip_file.writestr(
|
|
615
|
-
zip_file.writestr(
|
|
616
|
-
zip_file.writestr(
|
|
647
|
+
zip_path = os.path.abspath(os.path.join(workdir, "workflow.zip"))
|
|
648
|
+
with zipfile.ZipFile(zip_path, "w") as zip_file:
|
|
649
|
+
zip_file.writestr("actual.cwl", self.example_cwl)
|
|
650
|
+
zip_file.writestr("distraction.cwl", self.example_cwl)
|
|
651
|
+
zip_file.writestr(
|
|
652
|
+
"MANIFEST.json", json.dumps({"mainWorkflowURL": "actual.cwl"})
|
|
653
|
+
)
|
|
617
654
|
self.run_zip_workflow(zip_path)
|
|
618
655
|
|
|
619
|
-
|
|
620
656
|
def test_run_workflow_inputs_zip(self) -> None:
|
|
621
657
|
"""Test run example CWL workflow from ZIP without manifest but with inputs."""
|
|
622
658
|
workdir = self._createTempDir()
|
|
623
|
-
zip_path = os.path.abspath(os.path.join(workdir,
|
|
624
|
-
with zipfile.ZipFile(zip_path,
|
|
625
|
-
zip_file.writestr(
|
|
626
|
-
zip_file.writestr(
|
|
659
|
+
zip_path = os.path.abspath(os.path.join(workdir, "workflow.zip"))
|
|
660
|
+
with zipfile.ZipFile(zip_path, "w") as zip_file:
|
|
661
|
+
zip_file.writestr("main.cwl", self.example_cwl)
|
|
662
|
+
zip_file.writestr("inputs.json", json.dumps({"message": "Hello, world!"}))
|
|
627
663
|
self.run_zip_workflow(zip_path, include_message=False)
|
|
628
664
|
|
|
629
665
|
def test_run_workflow_manifest_and_inputs_zip(self) -> None:
|
|
630
666
|
"""Test run example CWL workflow from ZIP with manifest and inputs."""
|
|
631
667
|
workdir = self._createTempDir()
|
|
632
|
-
zip_path = os.path.abspath(os.path.join(workdir,
|
|
633
|
-
with zipfile.ZipFile(zip_path,
|
|
634
|
-
zip_file.writestr(
|
|
635
|
-
zip_file.writestr(
|
|
636
|
-
zip_file.writestr(
|
|
668
|
+
zip_path = os.path.abspath(os.path.join(workdir, "workflow.zip"))
|
|
669
|
+
with zipfile.ZipFile(zip_path, "w") as zip_file:
|
|
670
|
+
zip_file.writestr("actual.cwl", self.example_cwl)
|
|
671
|
+
zip_file.writestr("data.json", json.dumps({"message": "Hello, world!"}))
|
|
672
|
+
zip_file.writestr(
|
|
673
|
+
"MANIFEST.json",
|
|
674
|
+
json.dumps(
|
|
675
|
+
{"mainWorkflowURL": "actual.cwl", "inputFileURLs": ["data.json"]}
|
|
676
|
+
),
|
|
677
|
+
)
|
|
637
678
|
self.run_zip_workflow(zip_path, include_message=False)
|
|
638
679
|
|
|
639
680
|
def test_run_workflow_no_params_zip(self) -> None:
|
|
640
681
|
"""Test run example CWL workflow from ZIP without workflow_params."""
|
|
641
682
|
workdir = self._createTempDir()
|
|
642
|
-
zip_path = os.path.abspath(os.path.join(workdir,
|
|
643
|
-
with zipfile.ZipFile(zip_path,
|
|
644
|
-
zip_file.writestr(
|
|
645
|
-
zip_file.writestr(
|
|
683
|
+
zip_path = os.path.abspath(os.path.join(workdir, "workflow.zip"))
|
|
684
|
+
with zipfile.ZipFile(zip_path, "w") as zip_file:
|
|
685
|
+
zip_file.writestr("main.cwl", self.example_cwl)
|
|
686
|
+
zip_file.writestr("inputs.json", json.dumps({"message": "Hello, world!"}))
|
|
646
687
|
# Don't even bother sending workflow_params
|
|
647
688
|
self.run_zip_workflow(zip_path, include_message=False, include_params=False)
|
|
648
689
|
|
|
@@ -684,8 +725,10 @@ class ToilWESServerWorkflowTest(AbstractToilWESServerTest):
|
|
|
684
725
|
cancel_seconds = cancel_complete - cancel_sent
|
|
685
726
|
logger.info("Cancellation took %s seconds to complete", cancel_seconds)
|
|
686
727
|
from toil.server.wes.tasks import WAIT_FOR_DEATH_TIMEOUT
|
|
728
|
+
|
|
687
729
|
self.assertLess(cancel_seconds, WAIT_FOR_DEATH_TIMEOUT)
|
|
688
730
|
|
|
731
|
+
|
|
689
732
|
@needs_celery_broker
|
|
690
733
|
class ToilWESServerCeleryWorkflowTest(ToilWESServerWorkflowTest):
|
|
691
734
|
"""
|
|
@@ -699,8 +742,11 @@ class ToilWESServerCeleryWorkflowTest(ToilWESServerWorkflowTest):
|
|
|
699
742
|
super().__init__(*args, **kwargs)
|
|
700
743
|
self._server_args = []
|
|
701
744
|
|
|
745
|
+
|
|
702
746
|
@needs_celery_broker
|
|
703
|
-
class ToilWESServerCeleryS3StateWorkflowTest(
|
|
747
|
+
class ToilWESServerCeleryS3StateWorkflowTest(
|
|
748
|
+
ToilWESServerWorkflowTest, BucketUsingTest
|
|
749
|
+
):
|
|
704
750
|
"""
|
|
705
751
|
Test the server with Celery and state stored in S3.
|
|
706
752
|
"""
|
|
@@ -710,5 +756,6 @@ class ToilWESServerCeleryS3StateWorkflowTest(ToilWESServerWorkflowTest, BucketUs
|
|
|
710
756
|
self._server_args = ["--state_store", "s3://" + self.bucket_name + "/state"]
|
|
711
757
|
super().setUp()
|
|
712
758
|
|
|
759
|
+
|
|
713
760
|
if __name__ == "__main__":
|
|
714
761
|
unittest.main()
|