toil 6.1.0a1__py3-none-any.whl → 8.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (193) hide show
  1. toil/__init__.py +122 -315
  2. toil/batchSystems/__init__.py +1 -0
  3. toil/batchSystems/abstractBatchSystem.py +173 -89
  4. toil/batchSystems/abstractGridEngineBatchSystem.py +272 -148
  5. toil/batchSystems/awsBatch.py +244 -135
  6. toil/batchSystems/cleanup_support.py +26 -16
  7. toil/batchSystems/contained_executor.py +31 -28
  8. toil/batchSystems/gridengine.py +86 -50
  9. toil/batchSystems/htcondor.py +166 -89
  10. toil/batchSystems/kubernetes.py +632 -382
  11. toil/batchSystems/local_support.py +20 -15
  12. toil/batchSystems/lsf.py +134 -81
  13. toil/batchSystems/lsfHelper.py +13 -11
  14. toil/batchSystems/mesos/__init__.py +41 -29
  15. toil/batchSystems/mesos/batchSystem.py +290 -151
  16. toil/batchSystems/mesos/executor.py +79 -50
  17. toil/batchSystems/mesos/test/__init__.py +31 -23
  18. toil/batchSystems/options.py +46 -28
  19. toil/batchSystems/registry.py +53 -19
  20. toil/batchSystems/singleMachine.py +296 -125
  21. toil/batchSystems/slurm.py +603 -138
  22. toil/batchSystems/torque.py +47 -33
  23. toil/bus.py +186 -76
  24. toil/common.py +664 -368
  25. toil/cwl/__init__.py +1 -1
  26. toil/cwl/cwltoil.py +1136 -483
  27. toil/cwl/utils.py +17 -22
  28. toil/deferred.py +63 -42
  29. toil/exceptions.py +5 -3
  30. toil/fileStores/__init__.py +5 -5
  31. toil/fileStores/abstractFileStore.py +140 -60
  32. toil/fileStores/cachingFileStore.py +717 -269
  33. toil/fileStores/nonCachingFileStore.py +116 -87
  34. toil/job.py +1225 -368
  35. toil/jobStores/abstractJobStore.py +416 -266
  36. toil/jobStores/aws/jobStore.py +863 -477
  37. toil/jobStores/aws/utils.py +201 -120
  38. toil/jobStores/conftest.py +3 -2
  39. toil/jobStores/fileJobStore.py +292 -154
  40. toil/jobStores/googleJobStore.py +140 -74
  41. toil/jobStores/utils.py +36 -15
  42. toil/leader.py +668 -272
  43. toil/lib/accelerators.py +115 -18
  44. toil/lib/aws/__init__.py +74 -31
  45. toil/lib/aws/ami.py +122 -87
  46. toil/lib/aws/iam.py +284 -108
  47. toil/lib/aws/s3.py +31 -0
  48. toil/lib/aws/session.py +214 -39
  49. toil/lib/aws/utils.py +287 -231
  50. toil/lib/bioio.py +13 -5
  51. toil/lib/compatibility.py +11 -6
  52. toil/lib/conversions.py +104 -47
  53. toil/lib/docker.py +131 -103
  54. toil/lib/ec2.py +361 -199
  55. toil/lib/ec2nodes.py +174 -106
  56. toil/lib/encryption/_dummy.py +5 -3
  57. toil/lib/encryption/_nacl.py +10 -6
  58. toil/lib/encryption/conftest.py +1 -0
  59. toil/lib/exceptions.py +26 -7
  60. toil/lib/expando.py +5 -3
  61. toil/lib/ftp_utils.py +217 -0
  62. toil/lib/generatedEC2Lists.py +127 -19
  63. toil/lib/humanize.py +6 -2
  64. toil/lib/integration.py +341 -0
  65. toil/lib/io.py +141 -15
  66. toil/lib/iterables.py +4 -2
  67. toil/lib/memoize.py +12 -8
  68. toil/lib/misc.py +66 -21
  69. toil/lib/objects.py +2 -2
  70. toil/lib/resources.py +68 -15
  71. toil/lib/retry.py +126 -81
  72. toil/lib/threading.py +299 -82
  73. toil/lib/throttle.py +16 -15
  74. toil/options/common.py +843 -409
  75. toil/options/cwl.py +175 -90
  76. toil/options/runner.py +50 -0
  77. toil/options/wdl.py +73 -17
  78. toil/provisioners/__init__.py +117 -46
  79. toil/provisioners/abstractProvisioner.py +332 -157
  80. toil/provisioners/aws/__init__.py +70 -33
  81. toil/provisioners/aws/awsProvisioner.py +1145 -715
  82. toil/provisioners/clusterScaler.py +541 -279
  83. toil/provisioners/gceProvisioner.py +282 -179
  84. toil/provisioners/node.py +155 -79
  85. toil/realtimeLogger.py +34 -22
  86. toil/resource.py +137 -75
  87. toil/server/app.py +128 -62
  88. toil/server/celery_app.py +3 -1
  89. toil/server/cli/wes_cwl_runner.py +82 -53
  90. toil/server/utils.py +54 -28
  91. toil/server/wes/abstract_backend.py +64 -26
  92. toil/server/wes/amazon_wes_utils.py +21 -15
  93. toil/server/wes/tasks.py +121 -63
  94. toil/server/wes/toil_backend.py +142 -107
  95. toil/server/wsgi_app.py +4 -3
  96. toil/serviceManager.py +58 -22
  97. toil/statsAndLogging.py +224 -70
  98. toil/test/__init__.py +282 -183
  99. toil/test/batchSystems/batchSystemTest.py +460 -210
  100. toil/test/batchSystems/batch_system_plugin_test.py +90 -0
  101. toil/test/batchSystems/test_gridengine.py +173 -0
  102. toil/test/batchSystems/test_lsf_helper.py +67 -58
  103. toil/test/batchSystems/test_slurm.py +110 -49
  104. toil/test/cactus/__init__.py +0 -0
  105. toil/test/cactus/test_cactus_integration.py +56 -0
  106. toil/test/cwl/cwlTest.py +496 -287
  107. toil/test/cwl/measure_default_memory.cwl +12 -0
  108. toil/test/cwl/not_run_required_input.cwl +29 -0
  109. toil/test/cwl/scatter_duplicate_outputs.cwl +40 -0
  110. toil/test/cwl/seqtk_seq.cwl +1 -1
  111. toil/test/docs/scriptsTest.py +69 -46
  112. toil/test/jobStores/jobStoreTest.py +427 -264
  113. toil/test/lib/aws/test_iam.py +118 -50
  114. toil/test/lib/aws/test_s3.py +16 -9
  115. toil/test/lib/aws/test_utils.py +5 -6
  116. toil/test/lib/dockerTest.py +118 -141
  117. toil/test/lib/test_conversions.py +113 -115
  118. toil/test/lib/test_ec2.py +58 -50
  119. toil/test/lib/test_integration.py +104 -0
  120. toil/test/lib/test_misc.py +12 -5
  121. toil/test/mesos/MesosDataStructuresTest.py +23 -10
  122. toil/test/mesos/helloWorld.py +7 -6
  123. toil/test/mesos/stress.py +25 -20
  124. toil/test/options/__init__.py +13 -0
  125. toil/test/options/options.py +42 -0
  126. toil/test/provisioners/aws/awsProvisionerTest.py +320 -150
  127. toil/test/provisioners/clusterScalerTest.py +440 -250
  128. toil/test/provisioners/clusterTest.py +166 -44
  129. toil/test/provisioners/gceProvisionerTest.py +174 -100
  130. toil/test/provisioners/provisionerTest.py +25 -13
  131. toil/test/provisioners/restartScript.py +5 -4
  132. toil/test/server/serverTest.py +188 -141
  133. toil/test/sort/restart_sort.py +137 -68
  134. toil/test/sort/sort.py +134 -66
  135. toil/test/sort/sortTest.py +91 -49
  136. toil/test/src/autoDeploymentTest.py +141 -101
  137. toil/test/src/busTest.py +20 -18
  138. toil/test/src/checkpointTest.py +8 -2
  139. toil/test/src/deferredFunctionTest.py +49 -35
  140. toil/test/src/dockerCheckTest.py +32 -24
  141. toil/test/src/environmentTest.py +135 -0
  142. toil/test/src/fileStoreTest.py +539 -272
  143. toil/test/src/helloWorldTest.py +7 -4
  144. toil/test/src/importExportFileTest.py +61 -31
  145. toil/test/src/jobDescriptionTest.py +46 -21
  146. toil/test/src/jobEncapsulationTest.py +2 -0
  147. toil/test/src/jobFileStoreTest.py +74 -50
  148. toil/test/src/jobServiceTest.py +187 -73
  149. toil/test/src/jobTest.py +121 -71
  150. toil/test/src/miscTests.py +19 -18
  151. toil/test/src/promisedRequirementTest.py +82 -36
  152. toil/test/src/promisesTest.py +7 -6
  153. toil/test/src/realtimeLoggerTest.py +10 -6
  154. toil/test/src/regularLogTest.py +71 -37
  155. toil/test/src/resourceTest.py +80 -49
  156. toil/test/src/restartDAGTest.py +36 -22
  157. toil/test/src/resumabilityTest.py +9 -2
  158. toil/test/src/retainTempDirTest.py +45 -14
  159. toil/test/src/systemTest.py +12 -8
  160. toil/test/src/threadingTest.py +44 -25
  161. toil/test/src/toilContextManagerTest.py +10 -7
  162. toil/test/src/userDefinedJobArgTypeTest.py +8 -5
  163. toil/test/src/workerTest.py +73 -23
  164. toil/test/utils/toilDebugTest.py +103 -33
  165. toil/test/utils/toilKillTest.py +4 -5
  166. toil/test/utils/utilsTest.py +245 -106
  167. toil/test/wdl/wdltoil_test.py +818 -149
  168. toil/test/wdl/wdltoil_test_kubernetes.py +91 -0
  169. toil/toilState.py +120 -35
  170. toil/utils/toilConfig.py +13 -4
  171. toil/utils/toilDebugFile.py +44 -27
  172. toil/utils/toilDebugJob.py +214 -27
  173. toil/utils/toilDestroyCluster.py +11 -6
  174. toil/utils/toilKill.py +8 -3
  175. toil/utils/toilLaunchCluster.py +256 -140
  176. toil/utils/toilMain.py +37 -16
  177. toil/utils/toilRsyncCluster.py +32 -14
  178. toil/utils/toilSshCluster.py +49 -22
  179. toil/utils/toilStats.py +356 -273
  180. toil/utils/toilStatus.py +292 -139
  181. toil/utils/toilUpdateEC2Instances.py +3 -1
  182. toil/version.py +12 -12
  183. toil/wdl/utils.py +5 -5
  184. toil/wdl/wdltoil.py +3913 -1033
  185. toil/worker.py +367 -184
  186. {toil-6.1.0a1.dist-info → toil-8.0.0.dist-info}/LICENSE +25 -0
  187. toil-8.0.0.dist-info/METADATA +173 -0
  188. toil-8.0.0.dist-info/RECORD +253 -0
  189. {toil-6.1.0a1.dist-info → toil-8.0.0.dist-info}/WHEEL +1 -1
  190. toil-6.1.0a1.dist-info/METADATA +0 -125
  191. toil-6.1.0a1.dist-info/RECORD +0 -237
  192. {toil-6.1.0a1.dist-info → toil-8.0.0.dist-info}/entry_points.txt +0 -0
  193. {toil-6.1.0a1.dist-info → toil-8.0.0.dist-info}/top_level.txt +0 -0
@@ -21,7 +21,7 @@ import uuid
21
21
  import zipfile
22
22
  from abc import abstractmethod
23
23
  from io import BytesIO
24
- from typing import Optional
24
+ from typing import TYPE_CHECKING, Optional
25
25
  from urllib.parse import urlparse
26
26
 
27
27
  try:
@@ -38,6 +38,7 @@ from toil.test import ToilTest, needs_aws_s3, needs_celery_broker, needs_server
38
38
  logger = logging.getLogger(__name__)
39
39
  logging.basicConfig(level=logging.INFO)
40
40
 
41
+
41
42
  @needs_server
42
43
  class ToilServerUtilsTest(ToilTest):
43
44
  """
@@ -51,9 +52,11 @@ class ToilServerUtilsTest(ToilTest):
51
52
  away without flipping the state.
52
53
  """
53
54
 
54
- from toil.server.utils import (MemoryStateStore,
55
- WorkflowStateMachine,
56
- WorkflowStateStore)
55
+ from toil.server.utils import (
56
+ MemoryStateStore,
57
+ WorkflowStateMachine,
58
+ WorkflowStateStore,
59
+ )
57
60
 
58
61
  store = WorkflowStateStore(MemoryStateStore(), "test-workflow")
59
62
 
@@ -70,6 +73,7 @@ class ToilServerUtilsTest(ToilTest):
70
73
  # Make sure it is now CANCELED due to timeout
71
74
  self.assertEqual(state_machine.get_current_state(), "CANCELED")
72
75
 
76
+
73
77
  class hidden:
74
78
  # Hide abstract tests from the test loader
75
79
 
@@ -89,7 +93,6 @@ class hidden:
89
93
 
90
94
  raise NotImplementedError()
91
95
 
92
-
93
96
  def test_state_store(self) -> None:
94
97
  """
95
98
  Make sure that the state store under test can store and load keys.
@@ -98,45 +101,46 @@ class hidden:
98
101
  store = self.get_state_store()
99
102
 
100
103
  # Should start None
101
- self.assertEqual(store.get('id1', 'key1'), None)
104
+ self.assertEqual(store.get("id1", "key1"), None)
102
105
 
103
106
  # Should hold a value
104
- store.set('id1', 'key1', 'value1')
105
- self.assertEqual(store.get('id1', 'key1'), 'value1')
107
+ store.set("id1", "key1", "value1")
108
+ self.assertEqual(store.get("id1", "key1"), "value1")
106
109
 
107
110
  # Should distinguish by ID and key
108
- self.assertEqual(store.get('id2', 'key1'), None)
109
- self.assertEqual(store.get('id1', 'key2'), None)
111
+ self.assertEqual(store.get("id2", "key1"), None)
112
+ self.assertEqual(store.get("id1", "key2"), None)
110
113
 
111
- store.set('id2', 'key1', 'value2')
112
- store.set('id1', 'key2', 'value3')
113
- self.assertEqual(store.get('id1', 'key1'), 'value1')
114
- self.assertEqual(store.get('id2', 'key1'), 'value2')
115
- self.assertEqual(store.get('id1', 'key2'), 'value3')
114
+ store.set("id2", "key1", "value2")
115
+ store.set("id1", "key2", "value3")
116
+ self.assertEqual(store.get("id1", "key1"), "value1")
117
+ self.assertEqual(store.get("id2", "key1"), "value2")
118
+ self.assertEqual(store.get("id1", "key2"), "value3")
116
119
 
117
120
  # Should allow replacement
118
- store.set('id1', 'key1', 'value4')
119
- self.assertEqual(store.get('id1', 'key1'), 'value4')
120
- self.assertEqual(store.get('id2', 'key1'), 'value2')
121
- self.assertEqual(store.get('id1', 'key2'), 'value3')
121
+ store.set("id1", "key1", "value4")
122
+ self.assertEqual(store.get("id1", "key1"), "value4")
123
+ self.assertEqual(store.get("id2", "key1"), "value2")
124
+ self.assertEqual(store.get("id1", "key2"), "value3")
122
125
 
123
126
  # Should show up in another state store
124
127
  store2 = self.get_state_store()
125
- self.assertEqual(store2.get('id1', 'key1'), 'value4')
126
- self.assertEqual(store2.get('id2', 'key1'), 'value2')
127
- self.assertEqual(store2.get('id1', 'key2'), 'value3')
128
+ self.assertEqual(store2.get("id1", "key1"), "value4")
129
+ self.assertEqual(store2.get("id2", "key1"), "value2")
130
+ self.assertEqual(store2.get("id1", "key2"), "value3")
128
131
 
129
132
  # Should allow clearing
130
- store.set('id1', 'key1', None)
131
- self.assertEqual(store.get('id1', 'key1'), None)
132
- self.assertEqual(store.get('id2', 'key1'), 'value2')
133
- self.assertEqual(store.get('id1', 'key2'), 'value3')
133
+ store.set("id1", "key1", None)
134
+ self.assertEqual(store.get("id1", "key1"), None)
135
+ self.assertEqual(store.get("id2", "key1"), "value2")
136
+ self.assertEqual(store.get("id1", "key2"), "value3")
137
+
138
+ store.set("id2", "key1", None)
139
+ store.set("id1", "key2", None)
140
+ self.assertEqual(store.get("id1", "key1"), None)
141
+ self.assertEqual(store.get("id2", "key1"), None)
142
+ self.assertEqual(store.get("id1", "key2"), None)
134
143
 
135
- store.set('id2', 'key1', None)
136
- store.set('id1', 'key2', None)
137
- self.assertEqual(store.get('id1', 'key1'), None)
138
- self.assertEqual(store.get('id2', 'key1'), None)
139
- self.assertEqual(store.get('id1', 'key2'), None)
140
144
 
141
145
  class FileStateStoreTest(hidden.AbstractStateStoreTest):
142
146
  """
@@ -158,6 +162,7 @@ class FileStateStoreTest(hidden.AbstractStateStoreTest):
158
162
 
159
163
  return FileStateStore(self.state_store_dir)
160
164
 
165
+
161
166
  class FileStateStoreURLTest(hidden.AbstractStateStoreTest):
162
167
  """
163
168
  Test file-based state storage using URLs instead of local paths.
@@ -167,7 +172,7 @@ class FileStateStoreURLTest(hidden.AbstractStateStoreTest):
167
172
 
168
173
  def setUp(self) -> None:
169
174
  super().setUp()
170
- self.state_store_dir = 'file://' + self._createTempDir()
175
+ self.state_store_dir = "file://" + self._createTempDir()
171
176
 
172
177
  def get_state_store(self) -> AbstractStateStore:
173
178
  """
@@ -178,24 +183,20 @@ class FileStateStoreURLTest(hidden.AbstractStateStoreTest):
178
183
 
179
184
  return FileStateStore(self.state_store_dir)
180
185
 
186
+
181
187
  @needs_aws_s3
182
188
  class BucketUsingTest(ToilTest):
183
189
  """
184
190
  Base class for tests that need a bucket.
185
191
  """
186
192
 
187
- try:
188
- # We need the class to be evaluateable without the AWS modules, if not
189
- # runnable
193
+ if TYPE_CHECKING:
190
194
  from mypy_boto3_s3 import S3ServiceResource
191
195
  from mypy_boto3_s3.service_resource import Bucket
192
- except ImportError:
193
- pass
194
-
195
196
 
196
197
  region: Optional[str]
197
- s3_resource: Optional['S3ServiceResource']
198
- bucket: Optional['Bucket']
198
+ s3_resource: Optional["S3ServiceResource"]
199
+ bucket: Optional["Bucket"]
199
200
  bucket_name: Optional[str]
200
201
 
201
202
  @classmethod
@@ -218,14 +219,14 @@ class BucketUsingTest(ToilTest):
218
219
  @classmethod
219
220
  def tearDownClass(cls) -> None:
220
221
  from toil.lib.aws.utils import delete_s3_bucket
222
+
221
223
  if cls.bucket_name:
222
224
  delete_s3_bucket(cls.s3_resource, cls.bucket_name, cls.region)
223
225
  super().tearDownClass()
224
226
 
227
+
225
228
  class AWSStateStoreTest(hidden.AbstractStateStoreTest, BucketUsingTest):
226
- """
227
- Test AWS-based state storage.
228
- """
229
+ """Test AWS-based state storage."""
229
230
 
230
231
  from toil.server.utils import AbstractStateStore
231
232
 
@@ -238,7 +239,7 @@ class AWSStateStoreTest(hidden.AbstractStateStoreTest, BucketUsingTest):
238
239
 
239
240
  from toil.server.utils import S3StateStore
240
241
 
241
- return S3StateStore('s3://' + self.bucket_name + '/' + self.bucket_path)
242
+ return S3StateStore("s3://" + self.bucket_name + "/" + self.bucket_path)
242
243
 
243
244
  def test_state_store_paths(self) -> None:
244
245
  """
@@ -253,16 +254,18 @@ class AWSStateStoreTest(hidden.AbstractStateStoreTest, BucketUsingTest):
253
254
  store = self.get_state_store()
254
255
 
255
256
  # Should hold a value
256
- store.set('testid', 'testkey', 'testvalue')
257
- self.assertEqual(store.get('testid', 'testkey'), 'testvalue')
257
+ store.set("testid", "testkey", "testvalue")
258
+ self.assertEqual(store.get("testid", "testkey"), "testvalue")
258
259
 
259
- expected_url = urlparse('s3://' + self.bucket_name + '/' +
260
- os.path.join(self.bucket_path, 'testid', 'testkey'))
260
+ expected_url = urlparse(
261
+ "s3://"
262
+ + self.bucket_name
263
+ + "/"
264
+ + os.path.join(self.bucket_path, "testid", "testkey")
265
+ )
261
266
 
262
267
  obj = get_object_for_url(expected_url, True)
263
- self.assertEqual(obj.content_length, len('testvalue'))
264
-
265
-
268
+ self.assertEqual(obj.content_length, len("testvalue"))
266
269
 
267
270
 
268
271
  @needs_server
@@ -286,8 +289,11 @@ class AbstractToilWESServerTest(ToilTest):
286
289
  self.temp_dir = self._createTempDir()
287
290
 
288
291
  from toil.server.app import create_app, parser_with_server_options
292
+
289
293
  parser = parser_with_server_options()
290
- args = parser.parse_args(self._server_args + ["--work_dir", os.path.join(self.temp_dir, "workflows")])
294
+ args = parser.parse_args(
295
+ self._server_args + ["--work_dir", os.path.join(self.temp_dir, "workflows")]
296
+ )
291
297
 
292
298
  # Make the FlaskApp
293
299
  server_app = create_app(args)
@@ -296,7 +302,8 @@ class AbstractToilWESServerTest(ToilTest):
296
302
  self.app: Flask = server_app.app
297
303
  self.app.testing = True
298
304
 
299
- self.example_cwl = textwrap.dedent("""
305
+ self.example_cwl = textwrap.dedent(
306
+ """
300
307
  cwlVersion: v1.0
301
308
  class: CommandLineTool
302
309
  baseCommand: echo
@@ -309,9 +316,11 @@ class AbstractToilWESServerTest(ToilTest):
309
316
  outputs:
310
317
  output:
311
318
  type: stdout
312
- """)
319
+ """
320
+ )
313
321
 
314
- self.slow_cwl = textwrap.dedent("""
322
+ self.slow_cwl = textwrap.dedent(
323
+ """
315
324
  cwlVersion: v1.0
316
325
  class: CommandLineTool
317
326
  baseCommand: sleep
@@ -324,7 +333,8 @@ class AbstractToilWESServerTest(ToilTest):
324
333
  outputs:
325
334
  output:
326
335
  type: stdout
327
- """)
336
+ """
337
+ )
328
338
 
329
339
  def tearDown(self) -> None:
330
340
  super().tearDown()
@@ -344,8 +354,7 @@ class AbstractToilWESServerTest(ToilTest):
344
354
  The workflow should succeed, it should have some tasks, and they should have all succeeded.
345
355
  """
346
356
  rv = self._fetch_run_log(client, run_id)
347
- logger.debug('Log info: %s', rv.json)
348
- self.assertIn("run_log", rv.json)
357
+ logger.debug("Log info: %s", rv.json)
349
358
  run_log = rv.json.get("run_log")
350
359
  self.assertEqual(type(run_log), dict)
351
360
  if "exit_code" in run_log:
@@ -363,22 +372,15 @@ class AbstractToilWESServerTest(ToilTest):
363
372
  self.assertEqual(task_log.get("exit_code"), 0)
364
373
 
365
374
  def _report_log(self, client: "FlaskClient", run_id: str) -> None:
366
- """
367
- Report the log for the given workflow run.
368
- """
375
+ """Report the log for the given workflow run."""
369
376
  rv = self._fetch_run_log(client, run_id)
370
- self.assertIn("run_log", rv.json)
377
+ logger.debug(f"Report log response: {rv.json}")
371
378
  run_log = rv.json.get("run_log")
372
379
  self.assertEqual(type(run_log), dict)
373
- self.assertIn("stdout", run_log)
374
- stdout = run_log.get("stdout")
375
- self.assertEqual(type(stdout), str)
376
- self.assertIn("stderr", run_log)
377
- stderr = run_log.get("stderr")
378
- self.assertEqual(type(stderr), str)
379
- logger.info("Got stdout %s and stderr %s", stdout, stderr)
380
- self._report_absolute_url(client, stdout)
381
- self._report_absolute_url(client, stderr)
380
+ self.assertEqual(type(run_log.get("stdout")), str)
381
+ self.assertEqual(type(run_log.get("stderr")), str)
382
+ self._report_absolute_url(client, run_log.get("stdout"))
383
+ self._report_absolute_url(client, run_log.get("stderr"))
382
384
 
383
385
  def _report_absolute_url(self, client: "FlaskClient", url: str):
384
386
  """
@@ -393,21 +395,24 @@ class AbstractToilWESServerTest(ToilTest):
393
395
  logger.info("Fetch %s", url)
394
396
  rv = client.get(url)
395
397
  self.assertEqual(rv.status_code, 200)
396
- logger.info("Got %s:\n%s", url, rv.data.decode('utf-8'))
398
+ logger.info("Got %s:\n%s", url, rv.data.decode("utf-8"))
397
399
 
398
400
  def _start_slow_workflow(self, client: "FlaskClient") -> str:
399
401
  """
400
402
  Start a slow workflow and return its ID.
401
403
  """
402
- rv = client.post("/ga4gh/wes/v1/runs", data={
403
- "workflow_url": "slow.cwl",
404
- "workflow_type": "CWL",
405
- "workflow_type_version": "v1.0",
406
- "workflow_params": json.dumps({"delay": "5"}),
407
- "workflow_attachment": [
408
- (BytesIO(self.slow_cwl.encode()), "slow.cwl"),
409
- ],
410
- })
404
+ rv = client.post(
405
+ "/ga4gh/wes/v1/runs",
406
+ data={
407
+ "workflow_url": "slow.cwl",
408
+ "workflow_type": "CWL",
409
+ "workflow_type_version": "v1.0",
410
+ "workflow_params": json.dumps({"delay": "5"}),
411
+ "workflow_attachment": [
412
+ (BytesIO(self.slow_cwl.encode()), "slow.cwl"),
413
+ ],
414
+ },
415
+ )
411
416
  # workflow is submitted successfully
412
417
  self.assertEqual(rv.status_code, 200)
413
418
  self.assertTrue(rv.is_json)
@@ -428,16 +433,30 @@ class AbstractToilWESServerTest(ToilTest):
428
433
  self.assertEqual(rv.json.get("run_id"), run_id)
429
434
  self.assertIn("state", rv.json)
430
435
  state = rv.json.get("state")
431
- self.assertIn(state, ["UNKNOWN", "QUEUED", "INITIALIZING", "RUNNING",
432
- "PAUSED", "COMPLETE", "EXECUTOR_ERROR", "SYSTEM_ERROR",
433
- "CANCELED", "CANCELING"])
436
+ self.assertIn(
437
+ state,
438
+ [
439
+ "UNKNOWN",
440
+ "QUEUED",
441
+ "INITIALIZING",
442
+ "RUNNING",
443
+ "PAUSED",
444
+ "COMPLETE",
445
+ "EXECUTOR_ERROR",
446
+ "SYSTEM_ERROR",
447
+ "CANCELED",
448
+ "CANCELING",
449
+ ],
450
+ )
434
451
  return state
435
452
 
436
453
  def _cancel_workflow(self, client: "FlaskClient", run_id: str) -> None:
437
454
  rv = client.post(f"/ga4gh/wes/v1/runs/{run_id}/cancel")
438
455
  self.assertEqual(rv.status_code, 200)
439
456
 
440
- def _wait_for_status(self, client: "FlaskClient", run_id: str, target_status: str) -> None:
457
+ def _wait_for_status(
458
+ self, client: "FlaskClient", run_id: str, target_status: str
459
+ ) -> None:
441
460
  """
442
461
  Wait for the given workflow run to reach the given state. If it reaches
443
462
  a different terminal state, raise an exception.
@@ -470,19 +489,19 @@ class ToilWESServerBenchTest(AbstractToilWESServerTest):
470
489
  """
471
490
 
472
491
  def test_home(self) -> None:
473
- """ Test the homepage endpoint."""
492
+ """Test the homepage endpoint."""
474
493
  with self.app.test_client() as client:
475
494
  rv = client.get("/")
476
495
  self.assertEqual(rv.status_code, 302)
477
496
 
478
497
  def test_health(self) -> None:
479
- """ Test the health check endpoint."""
498
+ """Test the health check endpoint."""
480
499
  with self.app.test_client() as client:
481
500
  rv = client.get("/engine/v1/status")
482
501
  self.assertEqual(rv.status_code, 200)
483
502
 
484
503
  def test_get_service_info(self) -> None:
485
- """ Test the GET /service-info endpoint."""
504
+ """Test the GET /service-info endpoint."""
486
505
  with self.app.test_client() as client:
487
506
  rv = client.get("/ga4gh/wes/v1/service-info")
488
507
  self.assertEqual(rv.status_code, 200)
@@ -500,12 +519,15 @@ class ToilWESServerBenchTest(AbstractToilWESServerTest):
500
519
  self.assertIn("system_state_counts", service_info)
501
520
  self.assertIn("tags", service_info)
502
521
 
522
+
503
523
  class ToilWESServerWorkflowTest(AbstractToilWESServerTest):
504
524
  """
505
525
  Tests of the WES server running workflows.
506
526
  """
507
527
 
508
- def run_zip_workflow(self, zip_path: str, include_message: bool = True, include_params: bool = True) -> None:
528
+ def run_zip_workflow(
529
+ self, zip_path: str, include_message: bool = True, include_params: bool = True
530
+ ) -> None:
509
531
  """
510
532
  We have several zip file tests; this submits a zip file and makes sure it ran OK.
511
533
 
@@ -518,11 +540,13 @@ class ToilWESServerWorkflowTest(AbstractToilWESServerTest):
518
540
  post_data = {
519
541
  "workflow_url": "file://" + zip_path,
520
542
  "workflow_type": "CWL",
521
- "workflow_type_version": "v1.0"
543
+ "workflow_type_version": "v1.0",
522
544
  }
523
545
  if include_params or include_message:
524
546
  # We need workflow_params too
525
- post_data["workflow_params"] = json.dumps({"message": "Hello, world!"} if include_message else {})
547
+ post_data["workflow_params"] = json.dumps(
548
+ {"message": "Hello, world!"} if include_message else {}
549
+ )
526
550
  with self.app.test_client() as client:
527
551
  rv = client.post("/ga4gh/wes/v1/runs", data=post_data)
528
552
  # workflow is submitted successfully
@@ -539,28 +563,37 @@ class ToilWESServerWorkflowTest(AbstractToilWESServerTest):
539
563
  def test_run_workflow_relative_url_no_attachments_fails(self) -> None:
540
564
  """Test run example CWL workflow from relative workflow URL but with no attachments."""
541
565
  with self.app.test_client() as client:
542
- rv = client.post("/ga4gh/wes/v1/runs", data={
543
- "workflow_url": "example.cwl",
544
- "workflow_type": "CWL",
545
- "workflow_type_version": "v1.0",
546
- "workflow_params": "{}"
547
- })
566
+ rv = client.post(
567
+ "/ga4gh/wes/v1/runs",
568
+ data={
569
+ "workflow_url": "example.cwl",
570
+ "workflow_type": "CWL",
571
+ "workflow_type_version": "v1.0",
572
+ "workflow_params": "{}",
573
+ },
574
+ )
548
575
  self.assertEqual(rv.status_code, 400)
549
576
  self.assertTrue(rv.is_json)
550
- self.assertEqual(rv.json.get("msg"), "Relative 'workflow_url' but missing 'workflow_attachment'")
577
+ self.assertEqual(
578
+ rv.json.get("msg"),
579
+ "Relative 'workflow_url' but missing 'workflow_attachment'",
580
+ )
551
581
 
552
582
  def test_run_workflow_relative_url(self) -> None:
553
583
  """Test run example CWL workflow from relative workflow URL."""
554
584
  with self.app.test_client() as client:
555
- rv = client.post("/ga4gh/wes/v1/runs", data={
556
- "workflow_url": "example.cwl",
557
- "workflow_type": "CWL",
558
- "workflow_type_version": "v1.0",
559
- "workflow_params": json.dumps({"message": "Hello, world!"}),
560
- "workflow_attachment": [
561
- (BytesIO(self.example_cwl.encode()), "example.cwl"),
562
- ],
563
- })
585
+ rv = client.post(
586
+ "/ga4gh/wes/v1/runs",
587
+ data={
588
+ "workflow_url": "example.cwl",
589
+ "workflow_type": "CWL",
590
+ "workflow_type_version": "v1.0",
591
+ "workflow_params": json.dumps({"message": "Hello, world!"}),
592
+ "workflow_attachment": [
593
+ (BytesIO(self.example_cwl.encode()), "example.cwl"),
594
+ ],
595
+ },
596
+ )
564
597
  # workflow is submitted successfully
565
598
  self.assertEqual(rv.status_code, 200)
566
599
  self.assertTrue(rv.is_json)
@@ -573,13 +606,15 @@ class ToilWESServerWorkflowTest(AbstractToilWESServerTest):
573
606
  def test_run_workflow_https_url(self) -> None:
574
607
  """Test run example CWL workflow from the Internet."""
575
608
  with self.app.test_client() as client:
576
- rv = client.post("/ga4gh/wes/v1/runs", data={
577
- "workflow_url": "https://raw.githubusercontent.com/DataBiosphere/toil/releases/5.4.x/src/toil"
578
- "/test/docs/scripts/cwlExampleFiles/hello.cwl",
579
- "workflow_type": "CWL",
580
- "workflow_type_version": "v1.0",
581
- "workflow_params": json.dumps({"message": "Hello, world!"}),
582
- })
609
+ rv = client.post(
610
+ "/ga4gh/wes/v1/runs",
611
+ data={
612
+ "workflow_url": "https://raw.githubusercontent.com/DataBiosphere/toil/4cb5bb3871ac21a9793f638b83775926ed94a226/src/toil/test/cwl/echo.cwl",
613
+ "workflow_type": "CWL",
614
+ "workflow_type_version": "v1.2",
615
+ "workflow_params": json.dumps({"message": "Hello, world!"}),
616
+ },
617
+ )
583
618
  # workflow is submitted successfully
584
619
  self.assertEqual(rv.status_code, 200)
585
620
  self.assertTrue(rv.is_json)
@@ -592,57 +627,63 @@ class ToilWESServerWorkflowTest(AbstractToilWESServerTest):
592
627
  def test_run_workflow_single_file_zip(self) -> None:
593
628
  """Test run example CWL workflow from single-file ZIP."""
594
629
  workdir = self._createTempDir()
595
- zip_path = os.path.abspath(os.path.join(workdir, 'workflow.zip'))
596
- with zipfile.ZipFile(zip_path, 'w') as zip_file:
597
- zip_file.writestr('example.cwl', self.example_cwl)
630
+ zip_path = os.path.abspath(os.path.join(workdir, "workflow.zip"))
631
+ with zipfile.ZipFile(zip_path, "w") as zip_file:
632
+ zip_file.writestr("example.cwl", self.example_cwl)
598
633
  self.run_zip_workflow(zip_path)
599
634
 
600
635
  def test_run_workflow_multi_file_zip(self) -> None:
601
636
  """Test run example CWL workflow from multi-file ZIP."""
602
637
  workdir = self._createTempDir()
603
- zip_path = os.path.abspath(os.path.join(workdir, 'workflow.zip'))
604
- with zipfile.ZipFile(zip_path, 'w') as zip_file:
605
- zip_file.writestr('main.cwl', self.example_cwl)
606
- zip_file.writestr('distraction.cwl', "Don't mind me")
638
+ zip_path = os.path.abspath(os.path.join(workdir, "workflow.zip"))
639
+ with zipfile.ZipFile(zip_path, "w") as zip_file:
640
+ zip_file.writestr("main.cwl", self.example_cwl)
641
+ zip_file.writestr("distraction.cwl", "Don't mind me")
607
642
  self.run_zip_workflow(zip_path)
608
643
 
609
644
  def test_run_workflow_manifest_zip(self) -> None:
610
645
  """Test run example CWL workflow from ZIP with manifest."""
611
646
  workdir = self._createTempDir()
612
- zip_path = os.path.abspath(os.path.join(workdir, 'workflow.zip'))
613
- with zipfile.ZipFile(zip_path, 'w') as zip_file:
614
- zip_file.writestr('actual.cwl', self.example_cwl)
615
- zip_file.writestr('distraction.cwl', self.example_cwl)
616
- zip_file.writestr('MANIFEST.json', json.dumps({"mainWorkflowURL": "actual.cwl"}))
647
+ zip_path = os.path.abspath(os.path.join(workdir, "workflow.zip"))
648
+ with zipfile.ZipFile(zip_path, "w") as zip_file:
649
+ zip_file.writestr("actual.cwl", self.example_cwl)
650
+ zip_file.writestr("distraction.cwl", self.example_cwl)
651
+ zip_file.writestr(
652
+ "MANIFEST.json", json.dumps({"mainWorkflowURL": "actual.cwl"})
653
+ )
617
654
  self.run_zip_workflow(zip_path)
618
655
 
619
-
620
656
  def test_run_workflow_inputs_zip(self) -> None:
621
657
  """Test run example CWL workflow from ZIP without manifest but with inputs."""
622
658
  workdir = self._createTempDir()
623
- zip_path = os.path.abspath(os.path.join(workdir, 'workflow.zip'))
624
- with zipfile.ZipFile(zip_path, 'w') as zip_file:
625
- zip_file.writestr('main.cwl', self.example_cwl)
626
- zip_file.writestr('inputs.json', json.dumps({"message": "Hello, world!"}))
659
+ zip_path = os.path.abspath(os.path.join(workdir, "workflow.zip"))
660
+ with zipfile.ZipFile(zip_path, "w") as zip_file:
661
+ zip_file.writestr("main.cwl", self.example_cwl)
662
+ zip_file.writestr("inputs.json", json.dumps({"message": "Hello, world!"}))
627
663
  self.run_zip_workflow(zip_path, include_message=False)
628
664
 
629
665
  def test_run_workflow_manifest_and_inputs_zip(self) -> None:
630
666
  """Test run example CWL workflow from ZIP with manifest and inputs."""
631
667
  workdir = self._createTempDir()
632
- zip_path = os.path.abspath(os.path.join(workdir, 'workflow.zip'))
633
- with zipfile.ZipFile(zip_path, 'w') as zip_file:
634
- zip_file.writestr('actual.cwl', self.example_cwl)
635
- zip_file.writestr('data.json', json.dumps({"message": "Hello, world!"}))
636
- zip_file.writestr('MANIFEST.json', json.dumps({"mainWorkflowURL": "actual.cwl", "inputFileURLs": ["data.json"]}))
668
+ zip_path = os.path.abspath(os.path.join(workdir, "workflow.zip"))
669
+ with zipfile.ZipFile(zip_path, "w") as zip_file:
670
+ zip_file.writestr("actual.cwl", self.example_cwl)
671
+ zip_file.writestr("data.json", json.dumps({"message": "Hello, world!"}))
672
+ zip_file.writestr(
673
+ "MANIFEST.json",
674
+ json.dumps(
675
+ {"mainWorkflowURL": "actual.cwl", "inputFileURLs": ["data.json"]}
676
+ ),
677
+ )
637
678
  self.run_zip_workflow(zip_path, include_message=False)
638
679
 
639
680
  def test_run_workflow_no_params_zip(self) -> None:
640
681
  """Test run example CWL workflow from ZIP without workflow_params."""
641
682
  workdir = self._createTempDir()
642
- zip_path = os.path.abspath(os.path.join(workdir, 'workflow.zip'))
643
- with zipfile.ZipFile(zip_path, 'w') as zip_file:
644
- zip_file.writestr('main.cwl', self.example_cwl)
645
- zip_file.writestr('inputs.json', json.dumps({"message": "Hello, world!"}))
683
+ zip_path = os.path.abspath(os.path.join(workdir, "workflow.zip"))
684
+ with zipfile.ZipFile(zip_path, "w") as zip_file:
685
+ zip_file.writestr("main.cwl", self.example_cwl)
686
+ zip_file.writestr("inputs.json", json.dumps({"message": "Hello, world!"}))
646
687
  # Don't even bother sending workflow_params
647
688
  self.run_zip_workflow(zip_path, include_message=False, include_params=False)
648
689
 
@@ -684,8 +725,10 @@ class ToilWESServerWorkflowTest(AbstractToilWESServerTest):
684
725
  cancel_seconds = cancel_complete - cancel_sent
685
726
  logger.info("Cancellation took %s seconds to complete", cancel_seconds)
686
727
  from toil.server.wes.tasks import WAIT_FOR_DEATH_TIMEOUT
728
+
687
729
  self.assertLess(cancel_seconds, WAIT_FOR_DEATH_TIMEOUT)
688
730
 
731
+
689
732
  @needs_celery_broker
690
733
  class ToilWESServerCeleryWorkflowTest(ToilWESServerWorkflowTest):
691
734
  """
@@ -699,8 +742,11 @@ class ToilWESServerCeleryWorkflowTest(ToilWESServerWorkflowTest):
699
742
  super().__init__(*args, **kwargs)
700
743
  self._server_args = []
701
744
 
745
+
702
746
  @needs_celery_broker
703
- class ToilWESServerCeleryS3StateWorkflowTest(ToilWESServerWorkflowTest, BucketUsingTest):
747
+ class ToilWESServerCeleryS3StateWorkflowTest(
748
+ ToilWESServerWorkflowTest, BucketUsingTest
749
+ ):
704
750
  """
705
751
  Test the server with Celery and state stored in S3.
706
752
  """
@@ -710,5 +756,6 @@ class ToilWESServerCeleryS3StateWorkflowTest(ToilWESServerWorkflowTest, BucketUs
710
756
  self._server_args = ["--state_store", "s3://" + self.bucket_name + "/state"]
711
757
  super().setUp()
712
758
 
759
+
713
760
  if __name__ == "__main__":
714
761
  unittest.main()