toil 5.12.0__py3-none-any.whl → 6.1.0a1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (157) hide show
  1. toil/__init__.py +18 -13
  2. toil/batchSystems/abstractBatchSystem.py +21 -10
  3. toil/batchSystems/abstractGridEngineBatchSystem.py +2 -2
  4. toil/batchSystems/awsBatch.py +14 -14
  5. toil/batchSystems/contained_executor.py +3 -3
  6. toil/batchSystems/htcondor.py +0 -1
  7. toil/batchSystems/kubernetes.py +34 -31
  8. toil/batchSystems/local_support.py +3 -1
  9. toil/batchSystems/mesos/batchSystem.py +7 -7
  10. toil/batchSystems/options.py +32 -83
  11. toil/batchSystems/registry.py +104 -23
  12. toil/batchSystems/singleMachine.py +16 -13
  13. toil/batchSystems/slurm.py +3 -3
  14. toil/batchSystems/torque.py +0 -1
  15. toil/bus.py +6 -8
  16. toil/common.py +532 -743
  17. toil/cwl/__init__.py +28 -32
  18. toil/cwl/cwltoil.py +523 -520
  19. toil/cwl/utils.py +55 -10
  20. toil/fileStores/__init__.py +2 -2
  21. toil/fileStores/abstractFileStore.py +36 -11
  22. toil/fileStores/cachingFileStore.py +607 -530
  23. toil/fileStores/nonCachingFileStore.py +43 -10
  24. toil/job.py +140 -75
  25. toil/jobStores/abstractJobStore.py +147 -79
  26. toil/jobStores/aws/jobStore.py +23 -9
  27. toil/jobStores/aws/utils.py +1 -2
  28. toil/jobStores/fileJobStore.py +117 -19
  29. toil/jobStores/googleJobStore.py +16 -7
  30. toil/jobStores/utils.py +5 -6
  31. toil/leader.py +71 -43
  32. toil/lib/accelerators.py +10 -5
  33. toil/lib/aws/__init__.py +3 -14
  34. toil/lib/aws/ami.py +22 -9
  35. toil/lib/aws/iam.py +21 -13
  36. toil/lib/aws/session.py +2 -16
  37. toil/lib/aws/utils.py +4 -5
  38. toil/lib/compatibility.py +1 -1
  39. toil/lib/conversions.py +7 -3
  40. toil/lib/docker.py +22 -23
  41. toil/lib/ec2.py +10 -6
  42. toil/lib/ec2nodes.py +106 -100
  43. toil/lib/encryption/_nacl.py +2 -1
  44. toil/lib/generatedEC2Lists.py +325 -18
  45. toil/lib/io.py +21 -0
  46. toil/lib/misc.py +1 -1
  47. toil/lib/resources.py +1 -1
  48. toil/lib/threading.py +74 -26
  49. toil/options/common.py +738 -0
  50. toil/options/cwl.py +336 -0
  51. toil/options/wdl.py +32 -0
  52. toil/provisioners/abstractProvisioner.py +1 -4
  53. toil/provisioners/aws/__init__.py +3 -6
  54. toil/provisioners/aws/awsProvisioner.py +6 -0
  55. toil/provisioners/clusterScaler.py +3 -2
  56. toil/provisioners/gceProvisioner.py +2 -2
  57. toil/realtimeLogger.py +2 -1
  58. toil/resource.py +24 -18
  59. toil/server/app.py +2 -3
  60. toil/server/cli/wes_cwl_runner.py +4 -4
  61. toil/server/utils.py +1 -1
  62. toil/server/wes/abstract_backend.py +3 -2
  63. toil/server/wes/amazon_wes_utils.py +5 -4
  64. toil/server/wes/tasks.py +2 -3
  65. toil/server/wes/toil_backend.py +2 -10
  66. toil/server/wsgi_app.py +2 -0
  67. toil/serviceManager.py +12 -10
  68. toil/statsAndLogging.py +5 -1
  69. toil/test/__init__.py +29 -54
  70. toil/test/batchSystems/batchSystemTest.py +11 -111
  71. toil/test/batchSystems/test_slurm.py +3 -2
  72. toil/test/cwl/cwlTest.py +213 -90
  73. toil/test/cwl/glob_dir.cwl +15 -0
  74. toil/test/cwl/preemptible.cwl +21 -0
  75. toil/test/cwl/preemptible_expression.cwl +28 -0
  76. toil/test/cwl/revsort.cwl +1 -1
  77. toil/test/cwl/revsort2.cwl +1 -1
  78. toil/test/docs/scriptsTest.py +0 -1
  79. toil/test/jobStores/jobStoreTest.py +27 -16
  80. toil/test/lib/aws/test_iam.py +4 -14
  81. toil/test/lib/aws/test_utils.py +0 -3
  82. toil/test/lib/dockerTest.py +4 -4
  83. toil/test/lib/test_ec2.py +11 -16
  84. toil/test/mesos/helloWorld.py +4 -5
  85. toil/test/mesos/stress.py +1 -1
  86. toil/test/provisioners/aws/awsProvisionerTest.py +9 -5
  87. toil/test/provisioners/clusterScalerTest.py +6 -4
  88. toil/test/provisioners/clusterTest.py +14 -3
  89. toil/test/provisioners/gceProvisionerTest.py +0 -6
  90. toil/test/provisioners/restartScript.py +3 -2
  91. toil/test/server/serverTest.py +1 -1
  92. toil/test/sort/restart_sort.py +2 -1
  93. toil/test/sort/sort.py +2 -1
  94. toil/test/sort/sortTest.py +2 -13
  95. toil/test/src/autoDeploymentTest.py +45 -45
  96. toil/test/src/busTest.py +5 -5
  97. toil/test/src/checkpointTest.py +2 -2
  98. toil/test/src/deferredFunctionTest.py +1 -1
  99. toil/test/src/fileStoreTest.py +32 -16
  100. toil/test/src/helloWorldTest.py +1 -1
  101. toil/test/src/importExportFileTest.py +1 -1
  102. toil/test/src/jobDescriptionTest.py +2 -1
  103. toil/test/src/jobServiceTest.py +1 -1
  104. toil/test/src/jobTest.py +18 -18
  105. toil/test/src/miscTests.py +5 -3
  106. toil/test/src/promisedRequirementTest.py +3 -3
  107. toil/test/src/realtimeLoggerTest.py +1 -1
  108. toil/test/src/resourceTest.py +2 -2
  109. toil/test/src/restartDAGTest.py +1 -1
  110. toil/test/src/resumabilityTest.py +36 -2
  111. toil/test/src/retainTempDirTest.py +1 -1
  112. toil/test/src/systemTest.py +2 -2
  113. toil/test/src/toilContextManagerTest.py +2 -2
  114. toil/test/src/userDefinedJobArgTypeTest.py +1 -1
  115. toil/test/utils/toilDebugTest.py +98 -32
  116. toil/test/utils/toilKillTest.py +2 -2
  117. toil/test/utils/utilsTest.py +20 -0
  118. toil/test/wdl/wdltoil_test.py +148 -45
  119. toil/toilState.py +7 -6
  120. toil/utils/toilClean.py +1 -1
  121. toil/utils/toilConfig.py +36 -0
  122. toil/utils/toilDebugFile.py +60 -33
  123. toil/utils/toilDebugJob.py +39 -12
  124. toil/utils/toilDestroyCluster.py +1 -1
  125. toil/utils/toilKill.py +1 -1
  126. toil/utils/toilLaunchCluster.py +13 -2
  127. toil/utils/toilMain.py +3 -2
  128. toil/utils/toilRsyncCluster.py +1 -1
  129. toil/utils/toilSshCluster.py +1 -1
  130. toil/utils/toilStats.py +240 -143
  131. toil/utils/toilStatus.py +1 -4
  132. toil/version.py +11 -11
  133. toil/wdl/utils.py +2 -122
  134. toil/wdl/wdltoil.py +999 -386
  135. toil/worker.py +25 -31
  136. {toil-5.12.0.dist-info → toil-6.1.0a1.dist-info}/METADATA +60 -53
  137. toil-6.1.0a1.dist-info/RECORD +237 -0
  138. {toil-5.12.0.dist-info → toil-6.1.0a1.dist-info}/WHEEL +1 -1
  139. {toil-5.12.0.dist-info → toil-6.1.0a1.dist-info}/entry_points.txt +0 -1
  140. toil/batchSystems/parasol.py +0 -379
  141. toil/batchSystems/tes.py +0 -459
  142. toil/test/batchSystems/parasolTestSupport.py +0 -117
  143. toil/test/wdl/builtinTest.py +0 -506
  144. toil/test/wdl/conftest.py +0 -23
  145. toil/test/wdl/toilwdlTest.py +0 -522
  146. toil/wdl/toilwdl.py +0 -141
  147. toil/wdl/versions/dev.py +0 -107
  148. toil/wdl/versions/draft2.py +0 -980
  149. toil/wdl/versions/v1.py +0 -794
  150. toil/wdl/wdl_analysis.py +0 -116
  151. toil/wdl/wdl_functions.py +0 -997
  152. toil/wdl/wdl_synthesis.py +0 -1011
  153. toil/wdl/wdl_types.py +0 -243
  154. toil-5.12.0.dist-info/RECORD +0 -244
  155. /toil/{wdl/versions → options}/__init__.py +0 -0
  156. {toil-5.12.0.dist-info → toil-6.1.0a1.dist-info}/LICENSE +0 -0
  157. {toil-5.12.0.dist-info → toil-6.1.0a1.dist-info}/top_level.txt +0 -0
toil/test/cwl/cwlTest.py CHANGED
@@ -17,6 +17,7 @@ import logging
17
17
  import os
18
18
  import re
19
19
  import shutil
20
+ import stat
20
21
  import subprocess
21
22
  import sys
22
23
  import unittest
@@ -25,7 +26,7 @@ import zipfile
25
26
  from functools import partial
26
27
  from io import StringIO
27
28
  from pathlib import Path
28
- from typing import Dict, List, MutableMapping, Optional, Union
29
+ from typing import Dict, List, MutableMapping, Optional
29
30
  from unittest.mock import Mock, call
30
31
  from urllib.request import urlretrieve
31
32
 
@@ -34,16 +35,16 @@ import pytest
34
35
  pkg_root = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) # noqa
35
36
  sys.path.insert(0, pkg_root) # noqa
36
37
 
38
+ from schema_salad.exceptions import ValidationException
39
+
37
40
  from toil.cwl.utils import (download_structure,
38
41
  visit_cwl_class_and_reduce,
39
42
  visit_top_cwl_class)
40
43
  from toil.exceptions import FailedJobsException
41
44
  from toil.fileStores import FileID
42
45
  from toil.fileStores.abstractFileStore import AbstractFileStore
43
- from toil.lib.aws import zone_to_region
44
46
  from toil.lib.threading import cpu_count
45
47
  from toil.provisioners import cluster_factory
46
- from toil.provisioners.aws import get_best_aws_zone
47
48
  from toil.test import (ToilTest,
48
49
  needs_aws_ec2,
49
50
  needs_aws_s3,
@@ -57,17 +58,15 @@ from toil.test import (ToilTest,
57
58
  needs_local_cuda,
58
59
  needs_lsf,
59
60
  needs_mesos,
60
- needs_parasol,
61
+ needs_online,
61
62
  needs_slurm,
62
63
  needs_torque,
63
64
  needs_wes_server,
64
65
  slow)
65
- from toil.test.provisioners.aws.awsProvisionerTest import \
66
- AbstractAWSAutoscaleTest
67
66
  from toil.test.provisioners.clusterTest import AbstractClusterTest
68
67
 
69
68
  log = logging.getLogger(__name__)
70
- CONFORMANCE_TEST_TIMEOUT = 3600
69
+ CONFORMANCE_TEST_TIMEOUT = 10000
71
70
 
72
71
 
73
72
  def run_conformance_tests(
@@ -137,10 +136,12 @@ def run_conformance_tests(
137
136
  "--logDebug",
138
137
  "--statusWait=10",
139
138
  "--retryCount=2",
139
+ "--relax-path-checks",
140
+ # Defaults to 20s but we can't start hundreds of nodejs processes that fast on our CI potatoes
141
+ "--eval-timeout=600",
142
+ f"--caching={caching}"
140
143
  ]
141
144
 
142
- args_passed_directly_to_runner.append(f"--caching={caching}")
143
-
144
145
  if extra_args:
145
146
  args_passed_directly_to_runner += extra_args
146
147
 
@@ -149,6 +150,10 @@ def run_conformance_tests(
149
150
  "--setEnv=SINGULARITY_DOCKER_HUB_MIRROR"
150
151
  )
151
152
 
153
+ if batchSystem is None or batchSystem == "single_machine":
154
+ # Make sure we can run on small machines
155
+ args_passed_directly_to_runner.append("--scale=0.1")
156
+
152
157
  job_store_override = None
153
158
 
154
159
  if batchSystem == "kubernetes":
@@ -159,7 +164,8 @@ def run_conformance_tests(
159
164
  else:
160
165
  # Run tests in parallel on the local machine. Don't run too many
161
166
  # tests at once; we want at least a couple cores for each.
162
- parallel_tests = max(int(cpu_count() / 2), 1)
167
+ # But we need to have at least a few going in parallel or we risk hitting our timeout.
168
+ parallel_tests = max(int(cpu_count() / 2), 4)
163
169
  cmd.append(f"-j{parallel_tests}")
164
170
 
165
171
  if batchSystem:
@@ -213,15 +219,33 @@ class CWLWorkflowTest(ToilTest):
213
219
  shutil.rmtree(self.outDir)
214
220
  unittest.TestCase.tearDown(self)
215
221
 
216
- def _tester(self, cwlfile, jobfile, expect, main_args=[], out_name="output"):
222
+ def test_cwl_cmdline_input(self):
223
+ """
224
+ Test that running a CWL workflow with inputs specified on the command line passes.
225
+ """
226
+ from toil.cwl import cwltoil
227
+ cwlfile = "src/toil/test/cwl/conditional_wf.cwl"
228
+ args = [cwlfile, "--message", "str", "--sleep", "2"]
229
+ st = StringIO()
230
+ # If the workflow runs, it must have had options
231
+ cwltoil.main(args, stdout=st)
232
+
233
+ def _tester(self, cwlfile, jobfile, expect, main_args=[], out_name="output", output_here=False):
217
234
  from toil.cwl import cwltoil
218
235
 
219
236
  st = StringIO()
220
237
  main_args = main_args[:]
238
+ if not output_here:
239
+ # Don't just dump output in the working directory.
240
+ main_args.extend(
241
+ [
242
+ "--logDebug",
243
+ "--outdir",
244
+ self.outDir
245
+ ]
246
+ )
221
247
  main_args.extend(
222
248
  [
223
- "--outdir",
224
- self.outDir,
225
249
  os.path.join(self.rootDir, cwlfile),
226
250
  os.path.join(self.rootDir, jobfile),
227
251
  ]
@@ -233,6 +257,13 @@ class CWLWorkflowTest(ToilTest):
233
257
  out.get(out_name, {}).pop("nameroot", None)
234
258
  self.assertEqual(out, expect)
235
259
 
260
+ for k, v in expect.items():
261
+ if isinstance(v, dict) and "class" in v and v["class"] == "File" and "path" in v:
262
+ # This is a top-level output file.
263
+ # None of our output files should be executable.
264
+ self.assertTrue(os.path.exists(v["path"]))
265
+ self.assertFalse(os.stat(v["path"]).st_mode & stat.S_IXUSR)
266
+
236
267
  def _debug_worker_tester(self, cwlfile, jobfile, expect):
237
268
  from toil.cwl import cwltoil
238
269
 
@@ -365,6 +396,25 @@ class CWLWorkflowTest(ToilTest):
365
396
  out_name="result",
366
397
  )
367
398
 
399
+ def test_glob_dir_bypass_file_store(self):
400
+ self.maxDiff = 1000
401
+ try:
402
+ # We need to output to the current directory to make sure that
403
+ # works.
404
+ self._tester(
405
+ "src/toil/test/cwl/glob_dir.cwl",
406
+ "src/toil/test/cwl/empty.json",
407
+ self._expected_glob_dir_output(os.getcwd()),
408
+ main_args=["--bypass-file-store"],
409
+ output_here=True
410
+ )
411
+ finally:
412
+ # Clean up anything we made in the current directory.
413
+ try:
414
+ shutil.rmtree(os.path.join(os.getcwd(), "shouldmake"))
415
+ except FileNotFoundError:
416
+ pass
417
+
368
418
  @needs_aws_s3
369
419
  def test_download_s3(self):
370
420
  self.download("download_s3.json", self._tester)
@@ -375,6 +425,9 @@ class CWLWorkflowTest(ToilTest):
375
425
  def test_download_https(self):
376
426
  self.download("download_https.json", self._tester)
377
427
 
428
+ def test_download_https_reference(self):
429
+ self.download("download_https.json", partial(self._tester, main_args=["--reference-inputs"]))
430
+
378
431
  def test_download_file(self):
379
432
  self.download("download_file.json", self._tester)
380
433
 
@@ -382,6 +435,10 @@ class CWLWorkflowTest(ToilTest):
382
435
  def test_download_directory_s3(self):
383
436
  self.download_directory("download_directory_s3.json", self._tester)
384
437
 
438
+ @needs_aws_s3
439
+ def test_download_directory_s3_reference(self):
440
+ self.download_directory("download_directory_s3.json", partial(self._tester, main_args=["--reference-inputs"]))
441
+
385
442
  def test_download_directory_file(self):
386
443
  self.download_directory("download_directory_file.json", self._tester)
387
444
 
@@ -408,6 +465,8 @@ class CWLWorkflowTest(ToilTest):
408
465
  self.load_contents("download_file.json", self._tester)
409
466
 
410
467
  @slow
468
+ @pytest.mark.integrative
469
+ @unittest.skip
411
470
  def test_bioconda(self):
412
471
  self._tester(
413
472
  "src/toil/test/cwl/seqtk_seq.cwl",
@@ -418,6 +477,8 @@ class CWLWorkflowTest(ToilTest):
418
477
  )
419
478
 
420
479
  @needs_docker
480
+ @pytest.mark.integrative
481
+ @unittest.skip
421
482
  def test_biocontainers(self):
422
483
  self._tester(
423
484
  "src/toil/test/cwl/seqtk_seq.cwl",
@@ -491,7 +552,7 @@ class CWLWorkflowTest(ToilTest):
491
552
  pass
492
553
 
493
554
  @needs_aws_s3
494
- def test_streamable(self):
555
+ def test_streamable(self, extra_args: List[str] = None):
495
556
  """
496
557
  Test that a file with 'streamable'=True is a named pipe.
497
558
  This is a CWL1.2 feature.
@@ -504,12 +565,16 @@ class CWLWorkflowTest(ToilTest):
504
565
 
505
566
  st = StringIO()
506
567
  args = [
568
+ "--logDebug",
507
569
  "--outdir",
508
570
  self.outDir,
509
571
  jobstore,
510
572
  os.path.join(self.rootDir, cwlfile),
511
573
  os.path.join(self.rootDir, jobfile),
512
574
  ]
575
+ if extra_args:
576
+ args = extra_args + args
577
+ log.info("Run CWL run: %s", " ".join(args))
513
578
  cwltoil.main(args, stdout=st)
514
579
  out = json.loads(st.getvalue())
515
580
  out[out_name].pop("http://commonwl.org/cwltool#generation", None)
@@ -519,12 +584,68 @@ class CWLWorkflowTest(ToilTest):
519
584
  with open(out[out_name]["location"][len("file://") :]) as f:
520
585
  self.assertEqual(f.read().strip(), "When is s4 coming out?")
521
586
 
587
+ @needs_aws_s3
588
+ def test_streamable_reference(self):
589
+ """
590
+ Test that a streamable file is a stream even when passed around by URI.
591
+ """
592
+ self.test_streamable(extra_args=["--reference-inputs"])
593
+
594
+ def test_preemptible(self):
595
+ """
596
+ Tests that the http://arvados.org/cwl#UsePreemptible extension is supported.
597
+ """
598
+ cwlfile = "src/toil/test/cwl/preemptible.cwl"
599
+ jobfile = "src/toil/test/cwl/empty.json"
600
+ out_name = "output"
601
+ from toil.cwl import cwltoil
602
+
603
+ st = StringIO()
604
+ args = [
605
+ "--outdir",
606
+ self.outDir,
607
+ os.path.join(self.rootDir, cwlfile),
608
+ os.path.join(self.rootDir, jobfile),
609
+ ]
610
+ cwltoil.main(args, stdout=st)
611
+ out = json.loads(st.getvalue())
612
+ out[out_name].pop("http://commonwl.org/cwltool#generation", None)
613
+ out[out_name].pop("nameext", None)
614
+ out[out_name].pop("nameroot", None)
615
+ with open(out[out_name]["location"][len("file://") :]) as f:
616
+ self.assertEqual(f.read().strip(), "hello")
617
+
618
+ def test_preemptible_expression(self):
619
+ """
620
+ Tests that the http://arvados.org/cwl#UsePreemptible extension is validated.
621
+ """
622
+ cwlfile = "src/toil/test/cwl/preemptible_expression.cwl"
623
+ jobfile = "src/toil/test/cwl/preemptible_expression.json"
624
+ from toil.cwl import cwltoil
625
+
626
+ st = StringIO()
627
+ args = [
628
+ "--outdir",
629
+ self.outDir,
630
+ os.path.join(self.rootDir, cwlfile),
631
+ os.path.join(self.rootDir, jobfile),
632
+ ]
633
+ try:
634
+ cwltoil.main(args, stdout=st)
635
+ raise RuntimeError("Did not raise correct exception")
636
+ except ValidationException as e:
637
+ # Make sure we chastise the user appropriately.
638
+ assert "expressions are not allowed" in str(e)
639
+
640
+
522
641
  @staticmethod
523
642
  def _expected_seqtk_output(outDir):
524
- loc = "file://" + os.path.join(outDir, "out")
643
+ path = os.path.join(outDir, "out")
644
+ loc = "file://" + path
525
645
  return {
526
646
  "output1": {
527
647
  "location": loc,
648
+ "path": path,
528
649
  "checksum": "sha1$322e001e5a99f19abdce9f02ad0f02a17b5066c2",
529
650
  "basename": "out",
530
651
  "class": "File",
@@ -534,10 +655,12 @@ class CWLWorkflowTest(ToilTest):
534
655
 
535
656
  @staticmethod
536
657
  def _expected_revsort_output(outDir):
537
- loc = "file://" + os.path.join(outDir, "output.txt")
658
+ path = os.path.join(outDir, "output.txt")
659
+ loc = "file://" + path
538
660
  return {
539
661
  "output": {
540
662
  "location": loc,
663
+ "path": path,
541
664
  "basename": "output.txt",
542
665
  "size": 1111,
543
666
  "class": "File",
@@ -547,10 +670,12 @@ class CWLWorkflowTest(ToilTest):
547
670
 
548
671
  @staticmethod
549
672
  def _expected_revsort_nochecksum_output(outDir):
550
- loc = "file://" + os.path.join(outDir, "output.txt")
673
+ path = os.path.join(outDir, "output.txt")
674
+ loc = "file://" + path
551
675
  return {
552
676
  "output": {
553
677
  "location": loc,
678
+ "path": path,
554
679
  "basename": "output.txt",
555
680
  "size": 1111,
556
681
  "class": "File",
@@ -559,7 +684,8 @@ class CWLWorkflowTest(ToilTest):
559
684
 
560
685
  @staticmethod
561
686
  def _expected_download_output(outDir):
562
- loc = "file://" + os.path.join(outDir, "output.txt")
687
+ path = os.path.join(outDir, "output.txt")
688
+ loc = "file://" + path
563
689
  return {
564
690
  "output": {
565
691
  "location": loc,
@@ -567,6 +693,36 @@ class CWLWorkflowTest(ToilTest):
567
693
  "size": 0,
568
694
  "class": "File",
569
695
  "checksum": "sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709",
696
+ "path": path
697
+ }
698
+ }
699
+
700
+ @staticmethod
701
+ def _expected_glob_dir_output(out_dir):
702
+ dir_path = os.path.join(out_dir, "shouldmake")
703
+ dir_loc = "file://" + dir_path
704
+ file_path = os.path.join(dir_path, "test.txt")
705
+ file_loc = os.path.join(dir_loc, "test.txt")
706
+ return {
707
+ "shouldmake": {
708
+ "location": dir_loc,
709
+ "path": dir_path,
710
+ "basename": "shouldmake",
711
+ "nameroot": "shouldmake",
712
+ "nameext": "",
713
+ "class": "Directory",
714
+ "listing": [
715
+ {
716
+ "class": "File",
717
+ "location": file_loc,
718
+ "path": file_path,
719
+ "basename": "test.txt",
720
+ "checksum": "sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709",
721
+ "size": 0,
722
+ "nameroot": "test",
723
+ "nameext": ".txt"
724
+ }
725
+ ]
570
726
  }
571
727
  }
572
728
 
@@ -582,10 +738,12 @@ class CWLWorkflowTest(ToilTest):
582
738
 
583
739
  @staticmethod
584
740
  def _expected_colon_output(outDir):
741
+ path = os.path.join(outDir, "A:Gln2Cys_result")
585
742
  loc = "file://" + os.path.join(outDir, "A%3AGln2Cys_result")
586
743
  return {
587
744
  "result": {
588
745
  "location": loc,
746
+ "path": path,
589
747
  "basename": "A:Gln2Cys_result",
590
748
  "class": "Directory",
591
749
  "listing": [
@@ -597,16 +755,19 @@ class CWLWorkflowTest(ToilTest):
597
755
  "size": 1111,
598
756
  "nameroot": "whale",
599
757
  "nameext": ".txt",
758
+ "path": f"{path}/whale.txt"
600
759
  }
601
760
  ],
602
761
  }
603
762
  }
604
763
 
605
764
  def _expected_streaming_output(self, outDir):
606
- loc = "file://" + os.path.join(outDir, "output.txt")
765
+ path = os.path.join(outDir, "output.txt")
766
+ loc = "file://" + path
607
767
  return {
608
768
  "output": {
609
769
  "location": loc,
770
+ "path": path,
610
771
  "basename": "output.txt",
611
772
  "size": 24,
612
773
  "class": "File",
@@ -616,6 +777,7 @@ class CWLWorkflowTest(ToilTest):
616
777
 
617
778
 
618
779
  @needs_cwl
780
+ @needs_online
619
781
  class CWLv10Test(ToilTest):
620
782
  """
621
783
  Run the CWL 1.0 conformance tests in various environments.
@@ -696,12 +858,6 @@ class CWLv10Test(ToilTest):
696
858
  def test_mesos_cwl_conformance(self, **kwargs):
697
859
  return self.test_run_conformance(batchSystem="mesos", **kwargs)
698
860
 
699
- @slow
700
- @needs_parasol
701
- @unittest.skip
702
- def test_parasol_cwl_conformance(self, **kwargs):
703
- return self.test_run_conformance(batchSystem="parasol", **kwargs)
704
-
705
861
  @slow
706
862
  @needs_kubernetes
707
863
  def test_kubernetes_cwl_conformance(self, **kwargs):
@@ -745,12 +901,6 @@ class CWLv10Test(ToilTest):
745
901
  def test_mesos_cwl_conformance_with_caching(self):
746
902
  return self.test_mesos_cwl_conformance(caching=True)
747
903
 
748
- @slow
749
- @needs_parasol
750
- @unittest.skip
751
- def test_parasol_cwl_conformance_with_caching(self):
752
- return self.test_parasol_cwl_conformance(caching=True)
753
-
754
904
  @slow
755
905
  @needs_kubernetes
756
906
  def test_kubernetes_cwl_conformance_with_caching(self):
@@ -758,6 +908,7 @@ class CWLv10Test(ToilTest):
758
908
 
759
909
 
760
910
  @needs_cwl
911
+ @needs_online
761
912
  class CWLv11Test(ToilTest):
762
913
  """
763
914
  Run the CWL 1.1 conformance tests in various environments.
@@ -812,6 +963,7 @@ class CWLv11Test(ToilTest):
812
963
 
813
964
 
814
965
  @needs_cwl
966
+ @needs_online
815
967
  class CWLv12Test(ToilTest):
816
968
  """
817
969
  Run the CWL 1.2 conformance tests in various environments.
@@ -825,7 +977,7 @@ class CWLv12Test(ToilTest):
825
977
  cls.test_yaml = os.path.join(cls.cwlSpec, "conformance_tests.yaml")
826
978
  # TODO: Use a commit zip in case someone decides to rewrite master's history?
827
979
  url = "https://github.com/common-workflow-language/cwl-v1.2.git"
828
- commit = "8c3fd9d9f0209a51c5efacb1c7bc02a1164688d6"
980
+ commit = "0d538a0dbc5518f3c6083ce4571926f65cb84f76"
829
981
  p = subprocess.Popen(
830
982
  f"git clone {url} {cls.cwlSpec} && cd {cls.cwlSpec} && git checkout {commit}",
831
983
  shell=True,
@@ -839,12 +991,21 @@ class CWLv12Test(ToilTest):
839
991
  @slow
840
992
  @pytest.mark.timeout(CONFORMANCE_TEST_TIMEOUT)
841
993
  def test_run_conformance(self, **kwargs):
994
+ if "junit_file" not in kwargs:
995
+ kwargs["junit_file"] = os.path.join(
996
+ self.rootDir, "conformance-1.2.junit.xml"
997
+ )
842
998
  run_conformance_tests(workDir=self.cwlSpec, yml=self.test_yaml, **kwargs)
843
999
 
844
1000
  @slow
845
1001
  @pytest.mark.timeout(CONFORMANCE_TEST_TIMEOUT)
846
1002
  def test_run_conformance_with_caching(self):
847
- self.test_run_conformance(caching=True)
1003
+ self.test_run_conformance(
1004
+ caching=True,
1005
+ junit_file = os.path.join(
1006
+ self.rootDir, "caching-conformance-1.2.junit.xml"
1007
+ )
1008
+ )
848
1009
 
849
1010
  @slow
850
1011
  @pytest.mark.timeout(CONFORMANCE_TEST_TIMEOUT)
@@ -855,7 +1016,10 @@ class CWLv12Test(ToilTest):
855
1016
  features.
856
1017
  """
857
1018
  self.test_run_conformance(
858
- extra_args=["--bypass-file-store"], must_support_all_features=True
1019
+ extra_args=["--bypass-file-store"], must_support_all_features=True,
1020
+ junit_file = os.path.join(
1021
+ self.rootDir, "in-place-update-conformance-1.2.junit.xml"
1022
+ )
859
1023
  )
860
1024
 
861
1025
  @slow
@@ -863,7 +1027,7 @@ class CWLv12Test(ToilTest):
863
1027
  def test_kubernetes_cwl_conformance(self, **kwargs):
864
1028
  if "junit_file" not in kwargs:
865
1029
  kwargs["junit_file"] = os.path.join(
866
- self.rootDir, "kubernetes-conformance.junit.xml"
1030
+ self.rootDir, "kubernetes-conformance-1.2.junit.xml"
867
1031
  )
868
1032
  return self.test_run_conformance(
869
1033
  batchSystem="kubernetes",
@@ -883,7 +1047,7 @@ class CWLv12Test(ToilTest):
883
1047
  return self.test_kubernetes_cwl_conformance(
884
1048
  caching=True,
885
1049
  junit_file=os.path.join(
886
- self.rootDir, "kubernetes-caching-conformance.junit.xml"
1050
+ self.rootDir, "kubernetes-caching-conformance-1.2.junit.xml"
887
1051
  ),
888
1052
  )
889
1053
 
@@ -991,6 +1155,15 @@ class CWLOnARMTest(AbstractClusterTest):
991
1155
  ]
992
1156
  )
993
1157
 
1158
+ # We know if it succeeds it should save a junit XML for us to read.
1159
+ # Bring it back to be an artifact.
1160
+ self.rsync_util(
1161
+ f":{self.cwl_test_dir}/toil/conformance-1.2.junit.xml",
1162
+ os.path.join(
1163
+ self._projectRootPath(),
1164
+ "arm-conformance-1.2.junit.xml"
1165
+ )
1166
+ )
994
1167
 
995
1168
  @needs_cwl
996
1169
  @pytest.mark.cwl_small_log_dir
@@ -1187,56 +1360,6 @@ def test_pick_value_with_one_null_value(caplog):
1187
1360
  assert "You had a conditional step that did not run, but you did not use pickValue to handle the skipped input." not in line
1188
1361
 
1189
1362
 
1190
- @needs_cwl
1191
- @pytest.mark.cwl_small
1192
- def test_usage_message():
1193
- """
1194
- This is purely to ensure a (more) helpful error message is printed if a user does
1195
- not order their positional args correctly [cwl, cwl-job (json/yml/yaml), jobstore].
1196
- """
1197
- toil = "toil-cwl-runner"
1198
- cwl = "test/cwl/revsort.cwl"
1199
- cwl_job_json = "test/cwl/revsort-job.json"
1200
- jobstore = "delete-test-toil"
1201
- random_option_1 = "--logInfo"
1202
- random_option_2 = "--disableChaining"
1203
- cmd_wrong_ordering_1 = [
1204
- toil,
1205
- cwl,
1206
- cwl_job_json,
1207
- jobstore,
1208
- random_option_1,
1209
- random_option_2,
1210
- ]
1211
- cmd_wrong_ordering_2 = [
1212
- toil,
1213
- cwl,
1214
- jobstore,
1215
- random_option_1,
1216
- random_option_2,
1217
- cwl_job_json,
1218
- ]
1219
- cmd_wrong_ordering_3 = [
1220
- toil,
1221
- jobstore,
1222
- random_option_1,
1223
- random_option_2,
1224
- cwl,
1225
- cwl_job_json,
1226
- ]
1227
-
1228
- for cmd in [cmd_wrong_ordering_1, cmd_wrong_ordering_2, cmd_wrong_ordering_3]:
1229
- p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
1230
- stdout, stderr = p.communicate()
1231
- assert (
1232
- b"Usage: toil-cwl-runner [options] example.cwl example-job.yaml" in stderr
1233
- )
1234
- assert (
1235
- b"All positional arguments [cwl, yml_or_json] "
1236
- b"must always be specified last for toil-cwl-runner." in stderr
1237
- )
1238
-
1239
-
1240
1363
  @needs_cwl
1241
1364
  @pytest.mark.cwl_small
1242
1365
  def test_workflow_echo_string():
@@ -1249,7 +1372,7 @@ def test_workflow_echo_string():
1249
1372
  cmd = [toil, jobstore, option_1, option_2, option_3, cwl]
1250
1373
  p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
1251
1374
  stdout, stderr = p.communicate()
1252
- assert stdout == b"{}", f"Got wrong output: {stdout}\nWith error: {stderr}"
1375
+ assert stdout.decode("utf-8").strip() == "{}", f"Got wrong output: {stdout}\nWith error: {stderr}"
1253
1376
  assert b"Finished toil run successfully" in stderr
1254
1377
  assert p.returncode == 0
1255
1378
 
@@ -1476,9 +1599,9 @@ def test_download_structure(tmp_path) -> None:
1476
1599
  # The file store should have been asked to do the download
1477
1600
  file_store.readGlobalFile.assert_has_calls(
1478
1601
  [
1479
- call(fid1, os.path.join(to_dir, "dir1/dir2/f1"), symlink=True),
1480
- call(fid1, os.path.join(to_dir, "dir1/dir2/f1again"), symlink=True),
1481
- call(fid2, os.path.join(to_dir, "anotherfile"), symlink=True),
1602
+ call(fid1, os.path.join(to_dir, "dir1/dir2/f1"), symlink=False),
1603
+ call(fid1, os.path.join(to_dir, "dir1/dir2/f1again"), symlink=False),
1604
+ call(fid2, os.path.join(to_dir, "anotherfile"), symlink=False),
1482
1605
  ],
1483
1606
  any_order=True,
1484
1607
  )
@@ -0,0 +1,15 @@
1
+ cwlVersion: v1.2
2
+ class: CommandLineTool
3
+ inputs: []
4
+ outputs:
5
+ shouldmake:
6
+ type: Directory
7
+ outputBinding:
8
+ glob: "shouldmake"
9
+ baseCommand: [ bash, -c ]
10
+ arguments:
11
+ - |
12
+ mkdir -p shouldmake;
13
+ touch shouldmake/test.txt;
14
+
15
+
@@ -0,0 +1,21 @@
1
+ $namespaces:
2
+ arv: "http://arvados.org/cwl#"
3
+
4
+ class: CommandLineTool
5
+ doc: "Be preemptible"
6
+ cwlVersion: v1.0
7
+
8
+ inputs: []
9
+
10
+ outputs:
11
+ output:
12
+ type: File
13
+ outputBinding:
14
+ glob: output.txt
15
+
16
+ hints:
17
+ arv:UsePreemptible:
18
+ usePreemptible: true
19
+
20
+ baseCommand: ["echo", "hello"]
21
+ stdout: output.txt
@@ -0,0 +1,28 @@
1
+ $namespaces:
2
+ arv: "http://arvados.org/cwl#"
3
+
4
+ class: CommandLineTool
5
+ doc: "Be preemptible"
6
+ cwlVersion: v1.0
7
+
8
+ inputs:
9
+ - id: preemptible
10
+ type: boolean
11
+
12
+ outputs:
13
+ output:
14
+ type: File
15
+ outputBinding:
16
+ glob: output.txt
17
+
18
+ requirements:
19
+ InlineJavascriptRequirement: {}
20
+
21
+ hints:
22
+ arv:UsePreemptible:
23
+ # This isn't allowed; this always has to be a real boolean and can't be a cwl:Expression. See
24
+ # https://github.com/arvados/arvados/blob/48a0d575e6de34bcda91c489e4aa98df291a8cca/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.1.yml#L345
25
+ usePreemptible: $(inputs.preemptible)
26
+
27
+ baseCommand: ["echo", "hello"]
28
+ stdout: output.txt
toil/test/cwl/revsort.cwl CHANGED
@@ -10,7 +10,7 @@ cwlVersion: v1.0
10
10
  # in which the command line tools will execute.
11
11
  hints:
12
12
  - class: DockerRequirement
13
- dockerPull: debian:8
13
+ dockerPull: debian:12
14
14
 
15
15
 
16
16
  # The inputs array defines the structure of the input object that describes
@@ -10,7 +10,7 @@ cwlVersion: v1.0
10
10
  # in which the command line tools will execute.
11
11
  hints:
12
12
  - class: DockerRequirement
13
- dockerPull: debian:8
13
+ dockerPull: debian:12
14
14
 
15
15
 
16
16
  # The inputs array defines the structure of the input object that describes