toil 5.12.0__py3-none-any.whl → 6.1.0a1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- toil/__init__.py +18 -13
- toil/batchSystems/abstractBatchSystem.py +21 -10
- toil/batchSystems/abstractGridEngineBatchSystem.py +2 -2
- toil/batchSystems/awsBatch.py +14 -14
- toil/batchSystems/contained_executor.py +3 -3
- toil/batchSystems/htcondor.py +0 -1
- toil/batchSystems/kubernetes.py +34 -31
- toil/batchSystems/local_support.py +3 -1
- toil/batchSystems/mesos/batchSystem.py +7 -7
- toil/batchSystems/options.py +32 -83
- toil/batchSystems/registry.py +104 -23
- toil/batchSystems/singleMachine.py +16 -13
- toil/batchSystems/slurm.py +3 -3
- toil/batchSystems/torque.py +0 -1
- toil/bus.py +6 -8
- toil/common.py +532 -743
- toil/cwl/__init__.py +28 -32
- toil/cwl/cwltoil.py +523 -520
- toil/cwl/utils.py +55 -10
- toil/fileStores/__init__.py +2 -2
- toil/fileStores/abstractFileStore.py +36 -11
- toil/fileStores/cachingFileStore.py +607 -530
- toil/fileStores/nonCachingFileStore.py +43 -10
- toil/job.py +140 -75
- toil/jobStores/abstractJobStore.py +147 -79
- toil/jobStores/aws/jobStore.py +23 -9
- toil/jobStores/aws/utils.py +1 -2
- toil/jobStores/fileJobStore.py +117 -19
- toil/jobStores/googleJobStore.py +16 -7
- toil/jobStores/utils.py +5 -6
- toil/leader.py +71 -43
- toil/lib/accelerators.py +10 -5
- toil/lib/aws/__init__.py +3 -14
- toil/lib/aws/ami.py +22 -9
- toil/lib/aws/iam.py +21 -13
- toil/lib/aws/session.py +2 -16
- toil/lib/aws/utils.py +4 -5
- toil/lib/compatibility.py +1 -1
- toil/lib/conversions.py +7 -3
- toil/lib/docker.py +22 -23
- toil/lib/ec2.py +10 -6
- toil/lib/ec2nodes.py +106 -100
- toil/lib/encryption/_nacl.py +2 -1
- toil/lib/generatedEC2Lists.py +325 -18
- toil/lib/io.py +21 -0
- toil/lib/misc.py +1 -1
- toil/lib/resources.py +1 -1
- toil/lib/threading.py +74 -26
- toil/options/common.py +738 -0
- toil/options/cwl.py +336 -0
- toil/options/wdl.py +32 -0
- toil/provisioners/abstractProvisioner.py +1 -4
- toil/provisioners/aws/__init__.py +3 -6
- toil/provisioners/aws/awsProvisioner.py +6 -0
- toil/provisioners/clusterScaler.py +3 -2
- toil/provisioners/gceProvisioner.py +2 -2
- toil/realtimeLogger.py +2 -1
- toil/resource.py +24 -18
- toil/server/app.py +2 -3
- toil/server/cli/wes_cwl_runner.py +4 -4
- toil/server/utils.py +1 -1
- toil/server/wes/abstract_backend.py +3 -2
- toil/server/wes/amazon_wes_utils.py +5 -4
- toil/server/wes/tasks.py +2 -3
- toil/server/wes/toil_backend.py +2 -10
- toil/server/wsgi_app.py +2 -0
- toil/serviceManager.py +12 -10
- toil/statsAndLogging.py +5 -1
- toil/test/__init__.py +29 -54
- toil/test/batchSystems/batchSystemTest.py +11 -111
- toil/test/batchSystems/test_slurm.py +3 -2
- toil/test/cwl/cwlTest.py +213 -90
- toil/test/cwl/glob_dir.cwl +15 -0
- toil/test/cwl/preemptible.cwl +21 -0
- toil/test/cwl/preemptible_expression.cwl +28 -0
- toil/test/cwl/revsort.cwl +1 -1
- toil/test/cwl/revsort2.cwl +1 -1
- toil/test/docs/scriptsTest.py +0 -1
- toil/test/jobStores/jobStoreTest.py +27 -16
- toil/test/lib/aws/test_iam.py +4 -14
- toil/test/lib/aws/test_utils.py +0 -3
- toil/test/lib/dockerTest.py +4 -4
- toil/test/lib/test_ec2.py +11 -16
- toil/test/mesos/helloWorld.py +4 -5
- toil/test/mesos/stress.py +1 -1
- toil/test/provisioners/aws/awsProvisionerTest.py +9 -5
- toil/test/provisioners/clusterScalerTest.py +6 -4
- toil/test/provisioners/clusterTest.py +14 -3
- toil/test/provisioners/gceProvisionerTest.py +0 -6
- toil/test/provisioners/restartScript.py +3 -2
- toil/test/server/serverTest.py +1 -1
- toil/test/sort/restart_sort.py +2 -1
- toil/test/sort/sort.py +2 -1
- toil/test/sort/sortTest.py +2 -13
- toil/test/src/autoDeploymentTest.py +45 -45
- toil/test/src/busTest.py +5 -5
- toil/test/src/checkpointTest.py +2 -2
- toil/test/src/deferredFunctionTest.py +1 -1
- toil/test/src/fileStoreTest.py +32 -16
- toil/test/src/helloWorldTest.py +1 -1
- toil/test/src/importExportFileTest.py +1 -1
- toil/test/src/jobDescriptionTest.py +2 -1
- toil/test/src/jobServiceTest.py +1 -1
- toil/test/src/jobTest.py +18 -18
- toil/test/src/miscTests.py +5 -3
- toil/test/src/promisedRequirementTest.py +3 -3
- toil/test/src/realtimeLoggerTest.py +1 -1
- toil/test/src/resourceTest.py +2 -2
- toil/test/src/restartDAGTest.py +1 -1
- toil/test/src/resumabilityTest.py +36 -2
- toil/test/src/retainTempDirTest.py +1 -1
- toil/test/src/systemTest.py +2 -2
- toil/test/src/toilContextManagerTest.py +2 -2
- toil/test/src/userDefinedJobArgTypeTest.py +1 -1
- toil/test/utils/toilDebugTest.py +98 -32
- toil/test/utils/toilKillTest.py +2 -2
- toil/test/utils/utilsTest.py +20 -0
- toil/test/wdl/wdltoil_test.py +148 -45
- toil/toilState.py +7 -6
- toil/utils/toilClean.py +1 -1
- toil/utils/toilConfig.py +36 -0
- toil/utils/toilDebugFile.py +60 -33
- toil/utils/toilDebugJob.py +39 -12
- toil/utils/toilDestroyCluster.py +1 -1
- toil/utils/toilKill.py +1 -1
- toil/utils/toilLaunchCluster.py +13 -2
- toil/utils/toilMain.py +3 -2
- toil/utils/toilRsyncCluster.py +1 -1
- toil/utils/toilSshCluster.py +1 -1
- toil/utils/toilStats.py +240 -143
- toil/utils/toilStatus.py +1 -4
- toil/version.py +11 -11
- toil/wdl/utils.py +2 -122
- toil/wdl/wdltoil.py +999 -386
- toil/worker.py +25 -31
- {toil-5.12.0.dist-info → toil-6.1.0a1.dist-info}/METADATA +60 -53
- toil-6.1.0a1.dist-info/RECORD +237 -0
- {toil-5.12.0.dist-info → toil-6.1.0a1.dist-info}/WHEEL +1 -1
- {toil-5.12.0.dist-info → toil-6.1.0a1.dist-info}/entry_points.txt +0 -1
- toil/batchSystems/parasol.py +0 -379
- toil/batchSystems/tes.py +0 -459
- toil/test/batchSystems/parasolTestSupport.py +0 -117
- toil/test/wdl/builtinTest.py +0 -506
- toil/test/wdl/conftest.py +0 -23
- toil/test/wdl/toilwdlTest.py +0 -522
- toil/wdl/toilwdl.py +0 -141
- toil/wdl/versions/dev.py +0 -107
- toil/wdl/versions/draft2.py +0 -980
- toil/wdl/versions/v1.py +0 -794
- toil/wdl/wdl_analysis.py +0 -116
- toil/wdl/wdl_functions.py +0 -997
- toil/wdl/wdl_synthesis.py +0 -1011
- toil/wdl/wdl_types.py +0 -243
- toil-5.12.0.dist-info/RECORD +0 -244
- /toil/{wdl/versions → options}/__init__.py +0 -0
- {toil-5.12.0.dist-info → toil-6.1.0a1.dist-info}/LICENSE +0 -0
- {toil-5.12.0.dist-info → toil-6.1.0a1.dist-info}/top_level.txt +0 -0
toil/test/cwl/cwlTest.py
CHANGED
|
@@ -17,6 +17,7 @@ import logging
|
|
|
17
17
|
import os
|
|
18
18
|
import re
|
|
19
19
|
import shutil
|
|
20
|
+
import stat
|
|
20
21
|
import subprocess
|
|
21
22
|
import sys
|
|
22
23
|
import unittest
|
|
@@ -25,7 +26,7 @@ import zipfile
|
|
|
25
26
|
from functools import partial
|
|
26
27
|
from io import StringIO
|
|
27
28
|
from pathlib import Path
|
|
28
|
-
from typing import Dict, List, MutableMapping, Optional
|
|
29
|
+
from typing import Dict, List, MutableMapping, Optional
|
|
29
30
|
from unittest.mock import Mock, call
|
|
30
31
|
from urllib.request import urlretrieve
|
|
31
32
|
|
|
@@ -34,16 +35,16 @@ import pytest
|
|
|
34
35
|
pkg_root = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) # noqa
|
|
35
36
|
sys.path.insert(0, pkg_root) # noqa
|
|
36
37
|
|
|
38
|
+
from schema_salad.exceptions import ValidationException
|
|
39
|
+
|
|
37
40
|
from toil.cwl.utils import (download_structure,
|
|
38
41
|
visit_cwl_class_and_reduce,
|
|
39
42
|
visit_top_cwl_class)
|
|
40
43
|
from toil.exceptions import FailedJobsException
|
|
41
44
|
from toil.fileStores import FileID
|
|
42
45
|
from toil.fileStores.abstractFileStore import AbstractFileStore
|
|
43
|
-
from toil.lib.aws import zone_to_region
|
|
44
46
|
from toil.lib.threading import cpu_count
|
|
45
47
|
from toil.provisioners import cluster_factory
|
|
46
|
-
from toil.provisioners.aws import get_best_aws_zone
|
|
47
48
|
from toil.test import (ToilTest,
|
|
48
49
|
needs_aws_ec2,
|
|
49
50
|
needs_aws_s3,
|
|
@@ -57,17 +58,15 @@ from toil.test import (ToilTest,
|
|
|
57
58
|
needs_local_cuda,
|
|
58
59
|
needs_lsf,
|
|
59
60
|
needs_mesos,
|
|
60
|
-
|
|
61
|
+
needs_online,
|
|
61
62
|
needs_slurm,
|
|
62
63
|
needs_torque,
|
|
63
64
|
needs_wes_server,
|
|
64
65
|
slow)
|
|
65
|
-
from toil.test.provisioners.aws.awsProvisionerTest import \
|
|
66
|
-
AbstractAWSAutoscaleTest
|
|
67
66
|
from toil.test.provisioners.clusterTest import AbstractClusterTest
|
|
68
67
|
|
|
69
68
|
log = logging.getLogger(__name__)
|
|
70
|
-
CONFORMANCE_TEST_TIMEOUT =
|
|
69
|
+
CONFORMANCE_TEST_TIMEOUT = 10000
|
|
71
70
|
|
|
72
71
|
|
|
73
72
|
def run_conformance_tests(
|
|
@@ -137,10 +136,12 @@ def run_conformance_tests(
|
|
|
137
136
|
"--logDebug",
|
|
138
137
|
"--statusWait=10",
|
|
139
138
|
"--retryCount=2",
|
|
139
|
+
"--relax-path-checks",
|
|
140
|
+
# Defaults to 20s but we can't start hundreds of nodejs processes that fast on our CI potatoes
|
|
141
|
+
"--eval-timeout=600",
|
|
142
|
+
f"--caching={caching}"
|
|
140
143
|
]
|
|
141
144
|
|
|
142
|
-
args_passed_directly_to_runner.append(f"--caching={caching}")
|
|
143
|
-
|
|
144
145
|
if extra_args:
|
|
145
146
|
args_passed_directly_to_runner += extra_args
|
|
146
147
|
|
|
@@ -149,6 +150,10 @@ def run_conformance_tests(
|
|
|
149
150
|
"--setEnv=SINGULARITY_DOCKER_HUB_MIRROR"
|
|
150
151
|
)
|
|
151
152
|
|
|
153
|
+
if batchSystem is None or batchSystem == "single_machine":
|
|
154
|
+
# Make sure we can run on small machines
|
|
155
|
+
args_passed_directly_to_runner.append("--scale=0.1")
|
|
156
|
+
|
|
152
157
|
job_store_override = None
|
|
153
158
|
|
|
154
159
|
if batchSystem == "kubernetes":
|
|
@@ -159,7 +164,8 @@ def run_conformance_tests(
|
|
|
159
164
|
else:
|
|
160
165
|
# Run tests in parallel on the local machine. Don't run too many
|
|
161
166
|
# tests at once; we want at least a couple cores for each.
|
|
162
|
-
|
|
167
|
+
# But we need to have at least a few going in parallel or we risk hitting our timeout.
|
|
168
|
+
parallel_tests = max(int(cpu_count() / 2), 4)
|
|
163
169
|
cmd.append(f"-j{parallel_tests}")
|
|
164
170
|
|
|
165
171
|
if batchSystem:
|
|
@@ -213,15 +219,33 @@ class CWLWorkflowTest(ToilTest):
|
|
|
213
219
|
shutil.rmtree(self.outDir)
|
|
214
220
|
unittest.TestCase.tearDown(self)
|
|
215
221
|
|
|
216
|
-
def
|
|
222
|
+
def test_cwl_cmdline_input(self):
|
|
223
|
+
"""
|
|
224
|
+
Test that running a CWL workflow with inputs specified on the command line passes.
|
|
225
|
+
"""
|
|
226
|
+
from toil.cwl import cwltoil
|
|
227
|
+
cwlfile = "src/toil/test/cwl/conditional_wf.cwl"
|
|
228
|
+
args = [cwlfile, "--message", "str", "--sleep", "2"]
|
|
229
|
+
st = StringIO()
|
|
230
|
+
# If the workflow runs, it must have had options
|
|
231
|
+
cwltoil.main(args, stdout=st)
|
|
232
|
+
|
|
233
|
+
def _tester(self, cwlfile, jobfile, expect, main_args=[], out_name="output", output_here=False):
|
|
217
234
|
from toil.cwl import cwltoil
|
|
218
235
|
|
|
219
236
|
st = StringIO()
|
|
220
237
|
main_args = main_args[:]
|
|
238
|
+
if not output_here:
|
|
239
|
+
# Don't just dump output in the working directory.
|
|
240
|
+
main_args.extend(
|
|
241
|
+
[
|
|
242
|
+
"--logDebug",
|
|
243
|
+
"--outdir",
|
|
244
|
+
self.outDir
|
|
245
|
+
]
|
|
246
|
+
)
|
|
221
247
|
main_args.extend(
|
|
222
248
|
[
|
|
223
|
-
"--outdir",
|
|
224
|
-
self.outDir,
|
|
225
249
|
os.path.join(self.rootDir, cwlfile),
|
|
226
250
|
os.path.join(self.rootDir, jobfile),
|
|
227
251
|
]
|
|
@@ -233,6 +257,13 @@ class CWLWorkflowTest(ToilTest):
|
|
|
233
257
|
out.get(out_name, {}).pop("nameroot", None)
|
|
234
258
|
self.assertEqual(out, expect)
|
|
235
259
|
|
|
260
|
+
for k, v in expect.items():
|
|
261
|
+
if isinstance(v, dict) and "class" in v and v["class"] == "File" and "path" in v:
|
|
262
|
+
# This is a top-level output file.
|
|
263
|
+
# None of our output files should be executable.
|
|
264
|
+
self.assertTrue(os.path.exists(v["path"]))
|
|
265
|
+
self.assertFalse(os.stat(v["path"]).st_mode & stat.S_IXUSR)
|
|
266
|
+
|
|
236
267
|
def _debug_worker_tester(self, cwlfile, jobfile, expect):
|
|
237
268
|
from toil.cwl import cwltoil
|
|
238
269
|
|
|
@@ -365,6 +396,25 @@ class CWLWorkflowTest(ToilTest):
|
|
|
365
396
|
out_name="result",
|
|
366
397
|
)
|
|
367
398
|
|
|
399
|
+
def test_glob_dir_bypass_file_store(self):
|
|
400
|
+
self.maxDiff = 1000
|
|
401
|
+
try:
|
|
402
|
+
# We need to output to the current directory to make sure that
|
|
403
|
+
# works.
|
|
404
|
+
self._tester(
|
|
405
|
+
"src/toil/test/cwl/glob_dir.cwl",
|
|
406
|
+
"src/toil/test/cwl/empty.json",
|
|
407
|
+
self._expected_glob_dir_output(os.getcwd()),
|
|
408
|
+
main_args=["--bypass-file-store"],
|
|
409
|
+
output_here=True
|
|
410
|
+
)
|
|
411
|
+
finally:
|
|
412
|
+
# Clean up anything we made in the current directory.
|
|
413
|
+
try:
|
|
414
|
+
shutil.rmtree(os.path.join(os.getcwd(), "shouldmake"))
|
|
415
|
+
except FileNotFoundError:
|
|
416
|
+
pass
|
|
417
|
+
|
|
368
418
|
@needs_aws_s3
|
|
369
419
|
def test_download_s3(self):
|
|
370
420
|
self.download("download_s3.json", self._tester)
|
|
@@ -375,6 +425,9 @@ class CWLWorkflowTest(ToilTest):
|
|
|
375
425
|
def test_download_https(self):
|
|
376
426
|
self.download("download_https.json", self._tester)
|
|
377
427
|
|
|
428
|
+
def test_download_https_reference(self):
|
|
429
|
+
self.download("download_https.json", partial(self._tester, main_args=["--reference-inputs"]))
|
|
430
|
+
|
|
378
431
|
def test_download_file(self):
|
|
379
432
|
self.download("download_file.json", self._tester)
|
|
380
433
|
|
|
@@ -382,6 +435,10 @@ class CWLWorkflowTest(ToilTest):
|
|
|
382
435
|
def test_download_directory_s3(self):
|
|
383
436
|
self.download_directory("download_directory_s3.json", self._tester)
|
|
384
437
|
|
|
438
|
+
@needs_aws_s3
|
|
439
|
+
def test_download_directory_s3_reference(self):
|
|
440
|
+
self.download_directory("download_directory_s3.json", partial(self._tester, main_args=["--reference-inputs"]))
|
|
441
|
+
|
|
385
442
|
def test_download_directory_file(self):
|
|
386
443
|
self.download_directory("download_directory_file.json", self._tester)
|
|
387
444
|
|
|
@@ -408,6 +465,8 @@ class CWLWorkflowTest(ToilTest):
|
|
|
408
465
|
self.load_contents("download_file.json", self._tester)
|
|
409
466
|
|
|
410
467
|
@slow
|
|
468
|
+
@pytest.mark.integrative
|
|
469
|
+
@unittest.skip
|
|
411
470
|
def test_bioconda(self):
|
|
412
471
|
self._tester(
|
|
413
472
|
"src/toil/test/cwl/seqtk_seq.cwl",
|
|
@@ -418,6 +477,8 @@ class CWLWorkflowTest(ToilTest):
|
|
|
418
477
|
)
|
|
419
478
|
|
|
420
479
|
@needs_docker
|
|
480
|
+
@pytest.mark.integrative
|
|
481
|
+
@unittest.skip
|
|
421
482
|
def test_biocontainers(self):
|
|
422
483
|
self._tester(
|
|
423
484
|
"src/toil/test/cwl/seqtk_seq.cwl",
|
|
@@ -491,7 +552,7 @@ class CWLWorkflowTest(ToilTest):
|
|
|
491
552
|
pass
|
|
492
553
|
|
|
493
554
|
@needs_aws_s3
|
|
494
|
-
def test_streamable(self):
|
|
555
|
+
def test_streamable(self, extra_args: List[str] = None):
|
|
495
556
|
"""
|
|
496
557
|
Test that a file with 'streamable'=True is a named pipe.
|
|
497
558
|
This is a CWL1.2 feature.
|
|
@@ -504,12 +565,16 @@ class CWLWorkflowTest(ToilTest):
|
|
|
504
565
|
|
|
505
566
|
st = StringIO()
|
|
506
567
|
args = [
|
|
568
|
+
"--logDebug",
|
|
507
569
|
"--outdir",
|
|
508
570
|
self.outDir,
|
|
509
571
|
jobstore,
|
|
510
572
|
os.path.join(self.rootDir, cwlfile),
|
|
511
573
|
os.path.join(self.rootDir, jobfile),
|
|
512
574
|
]
|
|
575
|
+
if extra_args:
|
|
576
|
+
args = extra_args + args
|
|
577
|
+
log.info("Run CWL run: %s", " ".join(args))
|
|
513
578
|
cwltoil.main(args, stdout=st)
|
|
514
579
|
out = json.loads(st.getvalue())
|
|
515
580
|
out[out_name].pop("http://commonwl.org/cwltool#generation", None)
|
|
@@ -519,12 +584,68 @@ class CWLWorkflowTest(ToilTest):
|
|
|
519
584
|
with open(out[out_name]["location"][len("file://") :]) as f:
|
|
520
585
|
self.assertEqual(f.read().strip(), "When is s4 coming out?")
|
|
521
586
|
|
|
587
|
+
@needs_aws_s3
|
|
588
|
+
def test_streamable_reference(self):
|
|
589
|
+
"""
|
|
590
|
+
Test that a streamable file is a stream even when passed around by URI.
|
|
591
|
+
"""
|
|
592
|
+
self.test_streamable(extra_args=["--reference-inputs"])
|
|
593
|
+
|
|
594
|
+
def test_preemptible(self):
|
|
595
|
+
"""
|
|
596
|
+
Tests that the http://arvados.org/cwl#UsePreemptible extension is supported.
|
|
597
|
+
"""
|
|
598
|
+
cwlfile = "src/toil/test/cwl/preemptible.cwl"
|
|
599
|
+
jobfile = "src/toil/test/cwl/empty.json"
|
|
600
|
+
out_name = "output"
|
|
601
|
+
from toil.cwl import cwltoil
|
|
602
|
+
|
|
603
|
+
st = StringIO()
|
|
604
|
+
args = [
|
|
605
|
+
"--outdir",
|
|
606
|
+
self.outDir,
|
|
607
|
+
os.path.join(self.rootDir, cwlfile),
|
|
608
|
+
os.path.join(self.rootDir, jobfile),
|
|
609
|
+
]
|
|
610
|
+
cwltoil.main(args, stdout=st)
|
|
611
|
+
out = json.loads(st.getvalue())
|
|
612
|
+
out[out_name].pop("http://commonwl.org/cwltool#generation", None)
|
|
613
|
+
out[out_name].pop("nameext", None)
|
|
614
|
+
out[out_name].pop("nameroot", None)
|
|
615
|
+
with open(out[out_name]["location"][len("file://") :]) as f:
|
|
616
|
+
self.assertEqual(f.read().strip(), "hello")
|
|
617
|
+
|
|
618
|
+
def test_preemptible_expression(self):
|
|
619
|
+
"""
|
|
620
|
+
Tests that the http://arvados.org/cwl#UsePreemptible extension is validated.
|
|
621
|
+
"""
|
|
622
|
+
cwlfile = "src/toil/test/cwl/preemptible_expression.cwl"
|
|
623
|
+
jobfile = "src/toil/test/cwl/preemptible_expression.json"
|
|
624
|
+
from toil.cwl import cwltoil
|
|
625
|
+
|
|
626
|
+
st = StringIO()
|
|
627
|
+
args = [
|
|
628
|
+
"--outdir",
|
|
629
|
+
self.outDir,
|
|
630
|
+
os.path.join(self.rootDir, cwlfile),
|
|
631
|
+
os.path.join(self.rootDir, jobfile),
|
|
632
|
+
]
|
|
633
|
+
try:
|
|
634
|
+
cwltoil.main(args, stdout=st)
|
|
635
|
+
raise RuntimeError("Did not raise correct exception")
|
|
636
|
+
except ValidationException as e:
|
|
637
|
+
# Make sure we chastise the user appropriately.
|
|
638
|
+
assert "expressions are not allowed" in str(e)
|
|
639
|
+
|
|
640
|
+
|
|
522
641
|
@staticmethod
|
|
523
642
|
def _expected_seqtk_output(outDir):
|
|
524
|
-
|
|
643
|
+
path = os.path.join(outDir, "out")
|
|
644
|
+
loc = "file://" + path
|
|
525
645
|
return {
|
|
526
646
|
"output1": {
|
|
527
647
|
"location": loc,
|
|
648
|
+
"path": path,
|
|
528
649
|
"checksum": "sha1$322e001e5a99f19abdce9f02ad0f02a17b5066c2",
|
|
529
650
|
"basename": "out",
|
|
530
651
|
"class": "File",
|
|
@@ -534,10 +655,12 @@ class CWLWorkflowTest(ToilTest):
|
|
|
534
655
|
|
|
535
656
|
@staticmethod
|
|
536
657
|
def _expected_revsort_output(outDir):
|
|
537
|
-
|
|
658
|
+
path = os.path.join(outDir, "output.txt")
|
|
659
|
+
loc = "file://" + path
|
|
538
660
|
return {
|
|
539
661
|
"output": {
|
|
540
662
|
"location": loc,
|
|
663
|
+
"path": path,
|
|
541
664
|
"basename": "output.txt",
|
|
542
665
|
"size": 1111,
|
|
543
666
|
"class": "File",
|
|
@@ -547,10 +670,12 @@ class CWLWorkflowTest(ToilTest):
|
|
|
547
670
|
|
|
548
671
|
@staticmethod
|
|
549
672
|
def _expected_revsort_nochecksum_output(outDir):
|
|
550
|
-
|
|
673
|
+
path = os.path.join(outDir, "output.txt")
|
|
674
|
+
loc = "file://" + path
|
|
551
675
|
return {
|
|
552
676
|
"output": {
|
|
553
677
|
"location": loc,
|
|
678
|
+
"path": path,
|
|
554
679
|
"basename": "output.txt",
|
|
555
680
|
"size": 1111,
|
|
556
681
|
"class": "File",
|
|
@@ -559,7 +684,8 @@ class CWLWorkflowTest(ToilTest):
|
|
|
559
684
|
|
|
560
685
|
@staticmethod
|
|
561
686
|
def _expected_download_output(outDir):
|
|
562
|
-
|
|
687
|
+
path = os.path.join(outDir, "output.txt")
|
|
688
|
+
loc = "file://" + path
|
|
563
689
|
return {
|
|
564
690
|
"output": {
|
|
565
691
|
"location": loc,
|
|
@@ -567,6 +693,36 @@ class CWLWorkflowTest(ToilTest):
|
|
|
567
693
|
"size": 0,
|
|
568
694
|
"class": "File",
|
|
569
695
|
"checksum": "sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709",
|
|
696
|
+
"path": path
|
|
697
|
+
}
|
|
698
|
+
}
|
|
699
|
+
|
|
700
|
+
@staticmethod
|
|
701
|
+
def _expected_glob_dir_output(out_dir):
|
|
702
|
+
dir_path = os.path.join(out_dir, "shouldmake")
|
|
703
|
+
dir_loc = "file://" + dir_path
|
|
704
|
+
file_path = os.path.join(dir_path, "test.txt")
|
|
705
|
+
file_loc = os.path.join(dir_loc, "test.txt")
|
|
706
|
+
return {
|
|
707
|
+
"shouldmake": {
|
|
708
|
+
"location": dir_loc,
|
|
709
|
+
"path": dir_path,
|
|
710
|
+
"basename": "shouldmake",
|
|
711
|
+
"nameroot": "shouldmake",
|
|
712
|
+
"nameext": "",
|
|
713
|
+
"class": "Directory",
|
|
714
|
+
"listing": [
|
|
715
|
+
{
|
|
716
|
+
"class": "File",
|
|
717
|
+
"location": file_loc,
|
|
718
|
+
"path": file_path,
|
|
719
|
+
"basename": "test.txt",
|
|
720
|
+
"checksum": "sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709",
|
|
721
|
+
"size": 0,
|
|
722
|
+
"nameroot": "test",
|
|
723
|
+
"nameext": ".txt"
|
|
724
|
+
}
|
|
725
|
+
]
|
|
570
726
|
}
|
|
571
727
|
}
|
|
572
728
|
|
|
@@ -582,10 +738,12 @@ class CWLWorkflowTest(ToilTest):
|
|
|
582
738
|
|
|
583
739
|
@staticmethod
|
|
584
740
|
def _expected_colon_output(outDir):
|
|
741
|
+
path = os.path.join(outDir, "A:Gln2Cys_result")
|
|
585
742
|
loc = "file://" + os.path.join(outDir, "A%3AGln2Cys_result")
|
|
586
743
|
return {
|
|
587
744
|
"result": {
|
|
588
745
|
"location": loc,
|
|
746
|
+
"path": path,
|
|
589
747
|
"basename": "A:Gln2Cys_result",
|
|
590
748
|
"class": "Directory",
|
|
591
749
|
"listing": [
|
|
@@ -597,16 +755,19 @@ class CWLWorkflowTest(ToilTest):
|
|
|
597
755
|
"size": 1111,
|
|
598
756
|
"nameroot": "whale",
|
|
599
757
|
"nameext": ".txt",
|
|
758
|
+
"path": f"{path}/whale.txt"
|
|
600
759
|
}
|
|
601
760
|
],
|
|
602
761
|
}
|
|
603
762
|
}
|
|
604
763
|
|
|
605
764
|
def _expected_streaming_output(self, outDir):
|
|
606
|
-
|
|
765
|
+
path = os.path.join(outDir, "output.txt")
|
|
766
|
+
loc = "file://" + path
|
|
607
767
|
return {
|
|
608
768
|
"output": {
|
|
609
769
|
"location": loc,
|
|
770
|
+
"path": path,
|
|
610
771
|
"basename": "output.txt",
|
|
611
772
|
"size": 24,
|
|
612
773
|
"class": "File",
|
|
@@ -616,6 +777,7 @@ class CWLWorkflowTest(ToilTest):
|
|
|
616
777
|
|
|
617
778
|
|
|
618
779
|
@needs_cwl
|
|
780
|
+
@needs_online
|
|
619
781
|
class CWLv10Test(ToilTest):
|
|
620
782
|
"""
|
|
621
783
|
Run the CWL 1.0 conformance tests in various environments.
|
|
@@ -696,12 +858,6 @@ class CWLv10Test(ToilTest):
|
|
|
696
858
|
def test_mesos_cwl_conformance(self, **kwargs):
|
|
697
859
|
return self.test_run_conformance(batchSystem="mesos", **kwargs)
|
|
698
860
|
|
|
699
|
-
@slow
|
|
700
|
-
@needs_parasol
|
|
701
|
-
@unittest.skip
|
|
702
|
-
def test_parasol_cwl_conformance(self, **kwargs):
|
|
703
|
-
return self.test_run_conformance(batchSystem="parasol", **kwargs)
|
|
704
|
-
|
|
705
861
|
@slow
|
|
706
862
|
@needs_kubernetes
|
|
707
863
|
def test_kubernetes_cwl_conformance(self, **kwargs):
|
|
@@ -745,12 +901,6 @@ class CWLv10Test(ToilTest):
|
|
|
745
901
|
def test_mesos_cwl_conformance_with_caching(self):
|
|
746
902
|
return self.test_mesos_cwl_conformance(caching=True)
|
|
747
903
|
|
|
748
|
-
@slow
|
|
749
|
-
@needs_parasol
|
|
750
|
-
@unittest.skip
|
|
751
|
-
def test_parasol_cwl_conformance_with_caching(self):
|
|
752
|
-
return self.test_parasol_cwl_conformance(caching=True)
|
|
753
|
-
|
|
754
904
|
@slow
|
|
755
905
|
@needs_kubernetes
|
|
756
906
|
def test_kubernetes_cwl_conformance_with_caching(self):
|
|
@@ -758,6 +908,7 @@ class CWLv10Test(ToilTest):
|
|
|
758
908
|
|
|
759
909
|
|
|
760
910
|
@needs_cwl
|
|
911
|
+
@needs_online
|
|
761
912
|
class CWLv11Test(ToilTest):
|
|
762
913
|
"""
|
|
763
914
|
Run the CWL 1.1 conformance tests in various environments.
|
|
@@ -812,6 +963,7 @@ class CWLv11Test(ToilTest):
|
|
|
812
963
|
|
|
813
964
|
|
|
814
965
|
@needs_cwl
|
|
966
|
+
@needs_online
|
|
815
967
|
class CWLv12Test(ToilTest):
|
|
816
968
|
"""
|
|
817
969
|
Run the CWL 1.2 conformance tests in various environments.
|
|
@@ -825,7 +977,7 @@ class CWLv12Test(ToilTest):
|
|
|
825
977
|
cls.test_yaml = os.path.join(cls.cwlSpec, "conformance_tests.yaml")
|
|
826
978
|
# TODO: Use a commit zip in case someone decides to rewrite master's history?
|
|
827
979
|
url = "https://github.com/common-workflow-language/cwl-v1.2.git"
|
|
828
|
-
commit = "
|
|
980
|
+
commit = "0d538a0dbc5518f3c6083ce4571926f65cb84f76"
|
|
829
981
|
p = subprocess.Popen(
|
|
830
982
|
f"git clone {url} {cls.cwlSpec} && cd {cls.cwlSpec} && git checkout {commit}",
|
|
831
983
|
shell=True,
|
|
@@ -839,12 +991,21 @@ class CWLv12Test(ToilTest):
|
|
|
839
991
|
@slow
|
|
840
992
|
@pytest.mark.timeout(CONFORMANCE_TEST_TIMEOUT)
|
|
841
993
|
def test_run_conformance(self, **kwargs):
|
|
994
|
+
if "junit_file" not in kwargs:
|
|
995
|
+
kwargs["junit_file"] = os.path.join(
|
|
996
|
+
self.rootDir, "conformance-1.2.junit.xml"
|
|
997
|
+
)
|
|
842
998
|
run_conformance_tests(workDir=self.cwlSpec, yml=self.test_yaml, **kwargs)
|
|
843
999
|
|
|
844
1000
|
@slow
|
|
845
1001
|
@pytest.mark.timeout(CONFORMANCE_TEST_TIMEOUT)
|
|
846
1002
|
def test_run_conformance_with_caching(self):
|
|
847
|
-
self.test_run_conformance(
|
|
1003
|
+
self.test_run_conformance(
|
|
1004
|
+
caching=True,
|
|
1005
|
+
junit_file = os.path.join(
|
|
1006
|
+
self.rootDir, "caching-conformance-1.2.junit.xml"
|
|
1007
|
+
)
|
|
1008
|
+
)
|
|
848
1009
|
|
|
849
1010
|
@slow
|
|
850
1011
|
@pytest.mark.timeout(CONFORMANCE_TEST_TIMEOUT)
|
|
@@ -855,7 +1016,10 @@ class CWLv12Test(ToilTest):
|
|
|
855
1016
|
features.
|
|
856
1017
|
"""
|
|
857
1018
|
self.test_run_conformance(
|
|
858
|
-
extra_args=["--bypass-file-store"], must_support_all_features=True
|
|
1019
|
+
extra_args=["--bypass-file-store"], must_support_all_features=True,
|
|
1020
|
+
junit_file = os.path.join(
|
|
1021
|
+
self.rootDir, "in-place-update-conformance-1.2.junit.xml"
|
|
1022
|
+
)
|
|
859
1023
|
)
|
|
860
1024
|
|
|
861
1025
|
@slow
|
|
@@ -863,7 +1027,7 @@ class CWLv12Test(ToilTest):
|
|
|
863
1027
|
def test_kubernetes_cwl_conformance(self, **kwargs):
|
|
864
1028
|
if "junit_file" not in kwargs:
|
|
865
1029
|
kwargs["junit_file"] = os.path.join(
|
|
866
|
-
self.rootDir, "kubernetes-conformance.junit.xml"
|
|
1030
|
+
self.rootDir, "kubernetes-conformance-1.2.junit.xml"
|
|
867
1031
|
)
|
|
868
1032
|
return self.test_run_conformance(
|
|
869
1033
|
batchSystem="kubernetes",
|
|
@@ -883,7 +1047,7 @@ class CWLv12Test(ToilTest):
|
|
|
883
1047
|
return self.test_kubernetes_cwl_conformance(
|
|
884
1048
|
caching=True,
|
|
885
1049
|
junit_file=os.path.join(
|
|
886
|
-
self.rootDir, "kubernetes-caching-conformance.junit.xml"
|
|
1050
|
+
self.rootDir, "kubernetes-caching-conformance-1.2.junit.xml"
|
|
887
1051
|
),
|
|
888
1052
|
)
|
|
889
1053
|
|
|
@@ -991,6 +1155,15 @@ class CWLOnARMTest(AbstractClusterTest):
|
|
|
991
1155
|
]
|
|
992
1156
|
)
|
|
993
1157
|
|
|
1158
|
+
# We know if it succeeds it should save a junit XML for us to read.
|
|
1159
|
+
# Bring it back to be an artifact.
|
|
1160
|
+
self.rsync_util(
|
|
1161
|
+
f":{self.cwl_test_dir}/toil/conformance-1.2.junit.xml",
|
|
1162
|
+
os.path.join(
|
|
1163
|
+
self._projectRootPath(),
|
|
1164
|
+
"arm-conformance-1.2.junit.xml"
|
|
1165
|
+
)
|
|
1166
|
+
)
|
|
994
1167
|
|
|
995
1168
|
@needs_cwl
|
|
996
1169
|
@pytest.mark.cwl_small_log_dir
|
|
@@ -1187,56 +1360,6 @@ def test_pick_value_with_one_null_value(caplog):
|
|
|
1187
1360
|
assert "You had a conditional step that did not run, but you did not use pickValue to handle the skipped input." not in line
|
|
1188
1361
|
|
|
1189
1362
|
|
|
1190
|
-
@needs_cwl
|
|
1191
|
-
@pytest.mark.cwl_small
|
|
1192
|
-
def test_usage_message():
|
|
1193
|
-
"""
|
|
1194
|
-
This is purely to ensure a (more) helpful error message is printed if a user does
|
|
1195
|
-
not order their positional args correctly [cwl, cwl-job (json/yml/yaml), jobstore].
|
|
1196
|
-
"""
|
|
1197
|
-
toil = "toil-cwl-runner"
|
|
1198
|
-
cwl = "test/cwl/revsort.cwl"
|
|
1199
|
-
cwl_job_json = "test/cwl/revsort-job.json"
|
|
1200
|
-
jobstore = "delete-test-toil"
|
|
1201
|
-
random_option_1 = "--logInfo"
|
|
1202
|
-
random_option_2 = "--disableChaining"
|
|
1203
|
-
cmd_wrong_ordering_1 = [
|
|
1204
|
-
toil,
|
|
1205
|
-
cwl,
|
|
1206
|
-
cwl_job_json,
|
|
1207
|
-
jobstore,
|
|
1208
|
-
random_option_1,
|
|
1209
|
-
random_option_2,
|
|
1210
|
-
]
|
|
1211
|
-
cmd_wrong_ordering_2 = [
|
|
1212
|
-
toil,
|
|
1213
|
-
cwl,
|
|
1214
|
-
jobstore,
|
|
1215
|
-
random_option_1,
|
|
1216
|
-
random_option_2,
|
|
1217
|
-
cwl_job_json,
|
|
1218
|
-
]
|
|
1219
|
-
cmd_wrong_ordering_3 = [
|
|
1220
|
-
toil,
|
|
1221
|
-
jobstore,
|
|
1222
|
-
random_option_1,
|
|
1223
|
-
random_option_2,
|
|
1224
|
-
cwl,
|
|
1225
|
-
cwl_job_json,
|
|
1226
|
-
]
|
|
1227
|
-
|
|
1228
|
-
for cmd in [cmd_wrong_ordering_1, cmd_wrong_ordering_2, cmd_wrong_ordering_3]:
|
|
1229
|
-
p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
|
1230
|
-
stdout, stderr = p.communicate()
|
|
1231
|
-
assert (
|
|
1232
|
-
b"Usage: toil-cwl-runner [options] example.cwl example-job.yaml" in stderr
|
|
1233
|
-
)
|
|
1234
|
-
assert (
|
|
1235
|
-
b"All positional arguments [cwl, yml_or_json] "
|
|
1236
|
-
b"must always be specified last for toil-cwl-runner." in stderr
|
|
1237
|
-
)
|
|
1238
|
-
|
|
1239
|
-
|
|
1240
1363
|
@needs_cwl
|
|
1241
1364
|
@pytest.mark.cwl_small
|
|
1242
1365
|
def test_workflow_echo_string():
|
|
@@ -1249,7 +1372,7 @@ def test_workflow_echo_string():
|
|
|
1249
1372
|
cmd = [toil, jobstore, option_1, option_2, option_3, cwl]
|
|
1250
1373
|
p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
|
1251
1374
|
stdout, stderr = p.communicate()
|
|
1252
|
-
assert stdout ==
|
|
1375
|
+
assert stdout.decode("utf-8").strip() == "{}", f"Got wrong output: {stdout}\nWith error: {stderr}"
|
|
1253
1376
|
assert b"Finished toil run successfully" in stderr
|
|
1254
1377
|
assert p.returncode == 0
|
|
1255
1378
|
|
|
@@ -1476,9 +1599,9 @@ def test_download_structure(tmp_path) -> None:
|
|
|
1476
1599
|
# The file store should have been asked to do the download
|
|
1477
1600
|
file_store.readGlobalFile.assert_has_calls(
|
|
1478
1601
|
[
|
|
1479
|
-
call(fid1, os.path.join(to_dir, "dir1/dir2/f1"), symlink=
|
|
1480
|
-
call(fid1, os.path.join(to_dir, "dir1/dir2/f1again"), symlink=
|
|
1481
|
-
call(fid2, os.path.join(to_dir, "anotherfile"), symlink=
|
|
1602
|
+
call(fid1, os.path.join(to_dir, "dir1/dir2/f1"), symlink=False),
|
|
1603
|
+
call(fid1, os.path.join(to_dir, "dir1/dir2/f1again"), symlink=False),
|
|
1604
|
+
call(fid2, os.path.join(to_dir, "anotherfile"), symlink=False),
|
|
1482
1605
|
],
|
|
1483
1606
|
any_order=True,
|
|
1484
1607
|
)
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
$namespaces:
|
|
2
|
+
arv: "http://arvados.org/cwl#"
|
|
3
|
+
|
|
4
|
+
class: CommandLineTool
|
|
5
|
+
doc: "Be preemptible"
|
|
6
|
+
cwlVersion: v1.0
|
|
7
|
+
|
|
8
|
+
inputs: []
|
|
9
|
+
|
|
10
|
+
outputs:
|
|
11
|
+
output:
|
|
12
|
+
type: File
|
|
13
|
+
outputBinding:
|
|
14
|
+
glob: output.txt
|
|
15
|
+
|
|
16
|
+
hints:
|
|
17
|
+
arv:UsePreemptible:
|
|
18
|
+
usePreemptible: true
|
|
19
|
+
|
|
20
|
+
baseCommand: ["echo", "hello"]
|
|
21
|
+
stdout: output.txt
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
$namespaces:
|
|
2
|
+
arv: "http://arvados.org/cwl#"
|
|
3
|
+
|
|
4
|
+
class: CommandLineTool
|
|
5
|
+
doc: "Be preemptible"
|
|
6
|
+
cwlVersion: v1.0
|
|
7
|
+
|
|
8
|
+
inputs:
|
|
9
|
+
- id: preemptible
|
|
10
|
+
type: boolean
|
|
11
|
+
|
|
12
|
+
outputs:
|
|
13
|
+
output:
|
|
14
|
+
type: File
|
|
15
|
+
outputBinding:
|
|
16
|
+
glob: output.txt
|
|
17
|
+
|
|
18
|
+
requirements:
|
|
19
|
+
InlineJavascriptRequirement: {}
|
|
20
|
+
|
|
21
|
+
hints:
|
|
22
|
+
arv:UsePreemptible:
|
|
23
|
+
# This isn't allowed; this always has to be a real boolean and can't be a cwl:Expression. See
|
|
24
|
+
# https://github.com/arvados/arvados/blob/48a0d575e6de34bcda91c489e4aa98df291a8cca/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.1.yml#L345
|
|
25
|
+
usePreemptible: $(inputs.preemptible)
|
|
26
|
+
|
|
27
|
+
baseCommand: ["echo", "hello"]
|
|
28
|
+
stdout: output.txt
|
toil/test/cwl/revsort.cwl
CHANGED
toil/test/cwl/revsort2.cwl
CHANGED