toil 9.1.1__py3-none-any.whl → 9.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- toil/__init__.py +5 -9
- toil/batchSystems/abstractBatchSystem.py +23 -22
- toil/batchSystems/abstractGridEngineBatchSystem.py +17 -12
- toil/batchSystems/awsBatch.py +8 -8
- toil/batchSystems/cleanup_support.py +4 -4
- toil/batchSystems/contained_executor.py +3 -3
- toil/batchSystems/gridengine.py +3 -4
- toil/batchSystems/htcondor.py +5 -5
- toil/batchSystems/kubernetes.py +65 -63
- toil/batchSystems/local_support.py +2 -3
- toil/batchSystems/lsf.py +6 -7
- toil/batchSystems/mesos/batchSystem.py +11 -7
- toil/batchSystems/mesos/test/__init__.py +1 -2
- toil/batchSystems/options.py +9 -10
- toil/batchSystems/registry.py +3 -7
- toil/batchSystems/singleMachine.py +8 -11
- toil/batchSystems/slurm.py +49 -38
- toil/batchSystems/torque.py +3 -4
- toil/bus.py +36 -34
- toil/common.py +129 -89
- toil/cwl/cwltoil.py +857 -729
- toil/cwl/utils.py +44 -35
- toil/fileStores/__init__.py +3 -1
- toil/fileStores/abstractFileStore.py +28 -30
- toil/fileStores/cachingFileStore.py +8 -8
- toil/fileStores/nonCachingFileStore.py +10 -21
- toil/job.py +159 -158
- toil/jobStores/abstractJobStore.py +68 -69
- toil/jobStores/aws/jobStore.py +249 -213
- toil/jobStores/aws/utils.py +13 -24
- toil/jobStores/fileJobStore.py +28 -22
- toil/jobStores/googleJobStore.py +21 -17
- toil/jobStores/utils.py +3 -7
- toil/leader.py +17 -22
- toil/lib/accelerators.py +6 -4
- toil/lib/aws/__init__.py +9 -10
- toil/lib/aws/ami.py +33 -19
- toil/lib/aws/iam.py +6 -6
- toil/lib/aws/s3.py +259 -157
- toil/lib/aws/session.py +76 -76
- toil/lib/aws/utils.py +51 -43
- toil/lib/checksum.py +19 -15
- toil/lib/compatibility.py +3 -2
- toil/lib/conversions.py +45 -18
- toil/lib/directory.py +29 -26
- toil/lib/docker.py +93 -99
- toil/lib/dockstore.py +77 -50
- toil/lib/ec2.py +39 -38
- toil/lib/ec2nodes.py +11 -4
- toil/lib/exceptions.py +8 -5
- toil/lib/ftp_utils.py +9 -14
- toil/lib/generatedEC2Lists.py +161 -20
- toil/lib/history.py +141 -97
- toil/lib/history_submission.py +163 -72
- toil/lib/io.py +27 -17
- toil/lib/memoize.py +2 -1
- toil/lib/misc.py +15 -11
- toil/lib/pipes.py +40 -25
- toil/lib/plugins.py +12 -8
- toil/lib/resources.py +1 -0
- toil/lib/retry.py +32 -38
- toil/lib/threading.py +12 -12
- toil/lib/throttle.py +1 -2
- toil/lib/trs.py +113 -51
- toil/lib/url.py +14 -23
- toil/lib/web.py +7 -2
- toil/options/common.py +18 -15
- toil/options/cwl.py +2 -2
- toil/options/runner.py +9 -5
- toil/options/wdl.py +1 -3
- toil/provisioners/__init__.py +9 -9
- toil/provisioners/abstractProvisioner.py +22 -20
- toil/provisioners/aws/__init__.py +20 -14
- toil/provisioners/aws/awsProvisioner.py +10 -8
- toil/provisioners/clusterScaler.py +19 -18
- toil/provisioners/gceProvisioner.py +2 -3
- toil/provisioners/node.py +11 -13
- toil/realtimeLogger.py +4 -4
- toil/resource.py +5 -5
- toil/server/app.py +2 -2
- toil/server/cli/wes_cwl_runner.py +11 -11
- toil/server/utils.py +18 -21
- toil/server/wes/abstract_backend.py +9 -8
- toil/server/wes/amazon_wes_utils.py +3 -3
- toil/server/wes/tasks.py +3 -5
- toil/server/wes/toil_backend.py +17 -21
- toil/server/wsgi_app.py +3 -3
- toil/serviceManager.py +3 -4
- toil/statsAndLogging.py +12 -13
- toil/test/__init__.py +33 -24
- toil/test/batchSystems/batchSystemTest.py +12 -11
- toil/test/batchSystems/batch_system_plugin_test.py +3 -5
- toil/test/batchSystems/test_slurm.py +38 -24
- toil/test/cwl/conftest.py +5 -6
- toil/test/cwl/cwlTest.py +194 -78
- toil/test/cwl/download_file_uri.json +6 -0
- toil/test/cwl/download_file_uri_no_hostname.json +6 -0
- toil/test/docs/scripts/tutorial_staging.py +1 -0
- toil/test/jobStores/jobStoreTest.py +9 -7
- toil/test/lib/aws/test_iam.py +1 -3
- toil/test/lib/aws/test_s3.py +1 -1
- toil/test/lib/dockerTest.py +9 -9
- toil/test/lib/test_ec2.py +12 -11
- toil/test/lib/test_history.py +4 -4
- toil/test/lib/test_trs.py +16 -14
- toil/test/lib/test_url.py +7 -6
- toil/test/lib/url_plugin_test.py +12 -18
- toil/test/provisioners/aws/awsProvisionerTest.py +10 -8
- toil/test/provisioners/clusterScalerTest.py +2 -5
- toil/test/provisioners/clusterTest.py +1 -3
- toil/test/server/serverTest.py +13 -4
- toil/test/sort/restart_sort.py +2 -6
- toil/test/sort/sort.py +3 -8
- toil/test/src/deferredFunctionTest.py +7 -7
- toil/test/src/environmentTest.py +1 -2
- toil/test/src/fileStoreTest.py +5 -5
- toil/test/src/importExportFileTest.py +5 -6
- toil/test/src/jobServiceTest.py +22 -14
- toil/test/src/jobTest.py +121 -25
- toil/test/src/miscTests.py +5 -7
- toil/test/src/promisedRequirementTest.py +8 -7
- toil/test/src/regularLogTest.py +2 -3
- toil/test/src/resourceTest.py +5 -8
- toil/test/src/restartDAGTest.py +5 -6
- toil/test/src/resumabilityTest.py +2 -2
- toil/test/src/retainTempDirTest.py +3 -3
- toil/test/src/systemTest.py +3 -3
- toil/test/src/threadingTest.py +1 -1
- toil/test/src/workerTest.py +1 -2
- toil/test/utils/toilDebugTest.py +6 -4
- toil/test/utils/toilKillTest.py +1 -1
- toil/test/utils/utilsTest.py +15 -14
- toil/test/wdl/wdltoil_test.py +247 -124
- toil/test/wdl/wdltoil_test_kubernetes.py +2 -2
- toil/toilState.py +2 -3
- toil/utils/toilDebugFile.py +3 -8
- toil/utils/toilDebugJob.py +1 -2
- toil/utils/toilLaunchCluster.py +1 -2
- toil/utils/toilSshCluster.py +2 -0
- toil/utils/toilStats.py +19 -24
- toil/utils/toilStatus.py +11 -14
- toil/version.py +10 -10
- toil/wdl/wdltoil.py +313 -209
- toil/worker.py +18 -12
- {toil-9.1.1.dist-info → toil-9.2.0.dist-info}/METADATA +11 -14
- {toil-9.1.1.dist-info → toil-9.2.0.dist-info}/RECORD +150 -153
- {toil-9.1.1.dist-info → toil-9.2.0.dist-info}/WHEEL +1 -1
- toil/test/cwl/staging_cat.cwl +0 -27
- toil/test/cwl/staging_make_file.cwl +0 -25
- toil/test/cwl/staging_workflow.cwl +0 -43
- toil/test/cwl/zero_default.cwl +0 -61
- toil/test/utils/ABCWorkflowDebug/ABC.txt +0 -1
- {toil-9.1.1.dist-info → toil-9.2.0.dist-info}/entry_points.txt +0 -0
- {toil-9.1.1.dist-info → toil-9.2.0.dist-info}/licenses/LICENSE +0 -0
- {toil-9.1.1.dist-info → toil-9.2.0.dist-info}/top_level.txt +0 -0
toil/test/cwl/cwlTest.py
CHANGED
|
@@ -12,7 +12,6 @@
|
|
|
12
12
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
13
13
|
# See the License for the specific language governing permissions and
|
|
14
14
|
# limitations under the License.
|
|
15
|
-
from collections.abc import Generator
|
|
16
15
|
import json
|
|
17
16
|
import logging
|
|
18
17
|
import os
|
|
@@ -23,10 +22,11 @@ import subprocess
|
|
|
23
22
|
import sys
|
|
24
23
|
import uuid
|
|
25
24
|
import zipfile
|
|
25
|
+
from collections.abc import Callable, Generator
|
|
26
26
|
from functools import partial
|
|
27
27
|
from io import StringIO
|
|
28
28
|
from pathlib import Path
|
|
29
|
-
from typing import TYPE_CHECKING,
|
|
29
|
+
from typing import TYPE_CHECKING, cast
|
|
30
30
|
from unittest.mock import Mock, call
|
|
31
31
|
from urllib.request import urlretrieve
|
|
32
32
|
|
|
@@ -43,33 +43,29 @@ from schema_salad.exceptions import ValidationException
|
|
|
43
43
|
from toil.cwl.utils import (
|
|
44
44
|
DirectoryStructure,
|
|
45
45
|
download_structure,
|
|
46
|
+
remove_redundant_mounts,
|
|
46
47
|
visit_cwl_class_and_reduce,
|
|
47
48
|
visit_top_cwl_class,
|
|
48
|
-
remove_redundant_mounts
|
|
49
49
|
)
|
|
50
50
|
from toil.fileStores import FileID
|
|
51
51
|
from toil.fileStores.abstractFileStore import AbstractFileStore
|
|
52
52
|
from toil.job import WorkerImportJob
|
|
53
53
|
from toil.lib.threading import cpu_count
|
|
54
|
-
from toil.test import
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
from toil.test import
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
pneeds_slurm as needs_slurm,
|
|
70
|
-
pneeds_torque as needs_torque,
|
|
71
|
-
pneeds_wes_server as needs_wes_server,
|
|
72
|
-
)
|
|
54
|
+
from toil.test import get_data
|
|
55
|
+
from toil.test import pneeds_aws_s3 as needs_aws_s3
|
|
56
|
+
from toil.test import pneeds_cwl as needs_cwl
|
|
57
|
+
from toil.test import pneeds_docker as needs_docker
|
|
58
|
+
from toil.test import pneeds_docker_cuda as needs_docker_cuda
|
|
59
|
+
from toil.test import pneeds_gridengine as needs_gridengine
|
|
60
|
+
from toil.test import pneeds_kubernetes as needs_kubernetes
|
|
61
|
+
from toil.test import pneeds_local_cuda as needs_local_cuda
|
|
62
|
+
from toil.test import pneeds_lsf as needs_lsf
|
|
63
|
+
from toil.test import pneeds_mesos as needs_mesos
|
|
64
|
+
from toil.test import pneeds_online as needs_online
|
|
65
|
+
from toil.test import pneeds_slurm as needs_slurm
|
|
66
|
+
from toil.test import pneeds_torque as needs_torque
|
|
67
|
+
from toil.test import pneeds_wes_server as needs_wes_server
|
|
68
|
+
from toil.test import pslow as slow
|
|
73
69
|
|
|
74
70
|
log = logging.getLogger(__name__)
|
|
75
71
|
CONFORMANCE_TEST_TIMEOUT = 10000
|
|
@@ -78,15 +74,15 @@ CONFORMANCE_TEST_TIMEOUT = 10000
|
|
|
78
74
|
def run_conformance_tests(
|
|
79
75
|
workDir: str,
|
|
80
76
|
yml: str,
|
|
81
|
-
runner:
|
|
77
|
+
runner: str | None = None,
|
|
82
78
|
caching: bool = False,
|
|
83
|
-
batchSystem:
|
|
84
|
-
selected_tests:
|
|
85
|
-
selected_tags:
|
|
86
|
-
skipped_tests:
|
|
87
|
-
extra_args:
|
|
79
|
+
batchSystem: str | None = None,
|
|
80
|
+
selected_tests: str | None = None,
|
|
81
|
+
selected_tags: str | None = None,
|
|
82
|
+
skipped_tests: str | None = None,
|
|
83
|
+
extra_args: list[str] | None = None,
|
|
88
84
|
must_support_all_features: bool = False,
|
|
89
|
-
junit_file:
|
|
85
|
+
junit_file: str | None = None,
|
|
90
86
|
) -> None:
|
|
91
87
|
"""
|
|
92
88
|
Run the CWL conformance tests.
|
|
@@ -225,6 +221,9 @@ def run_conformance_tests(
|
|
|
225
221
|
log.info("Unsuccessful return code is OK")
|
|
226
222
|
|
|
227
223
|
|
|
224
|
+
# This is a type for a function that runs toil-cwl-runner and checks the
|
|
225
|
+
# result. See TestCWLWorkflow._tester and TestCWLWorkflow._debug_worker_tester
|
|
226
|
+
# for implementations.
|
|
228
227
|
TesterFuncType = Callable[[Path, Path, "CWLObjectType", Path], None]
|
|
229
228
|
|
|
230
229
|
|
|
@@ -256,8 +255,29 @@ class TestCWLWorkflow:
|
|
|
256
255
|
expect: "CWLObjectType",
|
|
257
256
|
outdir: Path,
|
|
258
257
|
out_name: str = "output",
|
|
259
|
-
main_args:
|
|
258
|
+
main_args: list[str] | None = None,
|
|
260
259
|
) -> None:
|
|
260
|
+
"""
|
|
261
|
+
Helper function that runs a CWL workflow and checks the result.
|
|
262
|
+
|
|
263
|
+
Implements TesterFuncType, plus a few additional parameters.
|
|
264
|
+
|
|
265
|
+
:param cwlfile: CWL workflow file path to run.
|
|
266
|
+
:param jobfile: Path to the input definition for the workflow run.
|
|
267
|
+
:param expect: Expected result of the workflow as a deserialized CWL
|
|
268
|
+
object. Should have one key per workflow output field. If output
|
|
269
|
+
files are expected from the workflow, they need to have their
|
|
270
|
+
absolute paths on disk under outdir already filled in.
|
|
271
|
+
:param outdir: Path to a directory to put the workflow's output files
|
|
272
|
+
in.
|
|
273
|
+
:param out_name: Name of the JSON key where the workflow's outputs can
|
|
274
|
+
be found in the output JSON from Toil.
|
|
275
|
+
:param main_args: Additional arguments to pass to toil-cwl-runner. This
|
|
276
|
+
is not part of TesterFuncType; you can use partial() to fill this
|
|
277
|
+
in and stamp out a TesterFuncType that runs toil-cwl-runner with
|
|
278
|
+
various closed-over arguments.
|
|
279
|
+
"""
|
|
280
|
+
|
|
261
281
|
from toil.cwl import cwltoil
|
|
262
282
|
|
|
263
283
|
st = StringIO()
|
|
@@ -293,6 +313,21 @@ class TestCWLWorkflow:
|
|
|
293
313
|
def _debug_worker_tester(
|
|
294
314
|
self, cwlfile: Path, jobfile: Path, expect: "CWLObjectType", outdir: Path
|
|
295
315
|
) -> None:
|
|
316
|
+
"""
|
|
317
|
+
Helper function that runs a CWL workflow with --debugWorker and checks
|
|
318
|
+
the result.
|
|
319
|
+
|
|
320
|
+
Implements TesterFuncType directly.
|
|
321
|
+
|
|
322
|
+
:param cwlfile: CWL workflow file path to run.
|
|
323
|
+
:param jobfile: Path to the input definition for the workflow run.
|
|
324
|
+
:param expect: Expected result of the workflow as a deserialized CWL
|
|
325
|
+
object. Should have one key per workflow output field. If output
|
|
326
|
+
files are expected from the workflow, they need to have their
|
|
327
|
+
absolute paths on disk under outdir already filled in.
|
|
328
|
+
:param outdir: Path to a directory to put the workflow's output files
|
|
329
|
+
in.
|
|
330
|
+
"""
|
|
296
331
|
from toil.cwl import cwltoil
|
|
297
332
|
|
|
298
333
|
st = StringIO()
|
|
@@ -334,6 +369,21 @@ class TestCWLWorkflow:
|
|
|
334
369
|
)
|
|
335
370
|
|
|
336
371
|
def download(self, inputs: str, tester_fn: TesterFuncType, out_dir: Path) -> None:
|
|
372
|
+
"""
|
|
373
|
+
Run a generic download test with a tester function and check the result.
|
|
374
|
+
|
|
375
|
+
Ther test is the download.cwl workflow.
|
|
376
|
+
|
|
377
|
+
The result has to match _expected_download_output on the output
|
|
378
|
+
directory, so it must contain an empty "output.txt" file.
|
|
379
|
+
|
|
380
|
+
:param inputs: Relative path to the inputs file within the Toil source
|
|
381
|
+
tree's src/toil/test/cwl directory.
|
|
382
|
+
:param tester_fn: The tester function to use to run the workflow and
|
|
383
|
+
check the result.
|
|
384
|
+
:param out_dir: Path to the output directory to save the workflow
|
|
385
|
+
output in.
|
|
386
|
+
"""
|
|
337
387
|
with get_data(f"test/cwl/{inputs}") as input_location:
|
|
338
388
|
with get_data("test/cwl/download.cwl") as cwl_file:
|
|
339
389
|
tester_fn(
|
|
@@ -490,8 +540,8 @@ class TestCWLWorkflow:
|
|
|
490
540
|
main_args = [
|
|
491
541
|
"--outdir",
|
|
492
542
|
str(tmp_path),
|
|
493
|
-
"
|
|
494
|
-
"https://
|
|
543
|
+
"github.com/mr-c/dockstore-tool-md5sum:master",
|
|
544
|
+
"https://github.com/mr-c/dockstore-tool-md5sum/raw/refs/heads/master/test.json"
|
|
495
545
|
]
|
|
496
546
|
cwltoil.main(main_args, stdout=stdout)
|
|
497
547
|
out = json.loads(stdout.getvalue())
|
|
@@ -596,6 +646,41 @@ class TestCWLWorkflow:
|
|
|
596
646
|
def test_download_file(self, tmp_path: Path) -> None:
|
|
597
647
|
self.download("download_file.json", self._tester, tmp_path)
|
|
598
648
|
|
|
649
|
+
def test_download_file_worker_import(self, tmp_path: Path) -> None:
|
|
650
|
+
self.download(
|
|
651
|
+
"download_file.json",
|
|
652
|
+
partial(self._tester, main_args=["--run-imports-on-workers"]),
|
|
653
|
+
tmp_path,
|
|
654
|
+
)
|
|
655
|
+
|
|
656
|
+
def test_download_file_uri(self, tmp_path: Path) -> None:
|
|
657
|
+
self.download("download_file_uri.json", self._tester, tmp_path)
|
|
658
|
+
|
|
659
|
+
def test_download_file_uri_worker_import(self, tmp_path: Path) -> None:
|
|
660
|
+
self.download(
|
|
661
|
+
"download_file_uri.json",
|
|
662
|
+
partial(self._tester, main_args=["--run-imports-on-workers"]),
|
|
663
|
+
tmp_path,
|
|
664
|
+
)
|
|
665
|
+
|
|
666
|
+
def test_download_file_uri_no_hostname(self, tmp_path: Path) -> None:
|
|
667
|
+
"""
|
|
668
|
+
Test if CWL handles file: URIs without even empty hostnames.
|
|
669
|
+
"""
|
|
670
|
+
# We can in fact ship an absolute file URI to an empty file if we
|
|
671
|
+
# assume /dev/null is available. So we can still use the helpers.
|
|
672
|
+
self.download("download_file_uri_no_hostname.json", self._tester, tmp_path)
|
|
673
|
+
|
|
674
|
+
def test_download_file_uri_no_hostname_worker_import(self, tmp_path: Path) -> None:
|
|
675
|
+
"""
|
|
676
|
+
Test if CWL handles file: URIs without even empty hostnames, with worker import.
|
|
677
|
+
"""
|
|
678
|
+
self.download(
|
|
679
|
+
"download_file_uri_no_hostname.json",
|
|
680
|
+
partial(self._tester, main_args=["--run-imports-on-workers"]),
|
|
681
|
+
tmp_path,
|
|
682
|
+
)
|
|
683
|
+
|
|
599
684
|
@needs_aws_s3
|
|
600
685
|
@pytest.mark.aws_s3
|
|
601
686
|
@pytest.mark.online
|
|
@@ -827,7 +912,7 @@ class TestCWLWorkflow:
|
|
|
827
912
|
@pytest.mark.aws_s3
|
|
828
913
|
@pytest.mark.online
|
|
829
914
|
def test_streamable(
|
|
830
|
-
self, tmp_path: Path, extra_args:
|
|
915
|
+
self, tmp_path: Path, extra_args: list[str] | None = None
|
|
831
916
|
) -> None:
|
|
832
917
|
"""
|
|
833
918
|
Test that a file with 'streamable'=True is a named pipe.
|
|
@@ -1114,6 +1199,7 @@ def cwl_v1_0_spec(tmp_path: Path) -> Generator[Path]:
|
|
|
1114
1199
|
finally:
|
|
1115
1200
|
pass # no cleanup
|
|
1116
1201
|
|
|
1202
|
+
|
|
1117
1203
|
@pytest.mark.integrative
|
|
1118
1204
|
@pytest.mark.conformance
|
|
1119
1205
|
@needs_cwl
|
|
@@ -1143,11 +1229,11 @@ class TestCWLv10Conformance:
|
|
|
1143
1229
|
def test_run_conformance(
|
|
1144
1230
|
self,
|
|
1145
1231
|
cwl_v1_0_spec: Path,
|
|
1146
|
-
batchSystem:
|
|
1232
|
+
batchSystem: str | None = None,
|
|
1147
1233
|
caching: bool = False,
|
|
1148
|
-
selected_tests:
|
|
1149
|
-
skipped_tests:
|
|
1150
|
-
extra_args:
|
|
1234
|
+
selected_tests: str | None = None,
|
|
1235
|
+
skipped_tests: str | None = None,
|
|
1236
|
+
extra_args: list[str] | None = None,
|
|
1151
1237
|
) -> None:
|
|
1152
1238
|
run_conformance_tests(
|
|
1153
1239
|
workDir=str(cwl_v1_0_spec / "v1.0"),
|
|
@@ -1319,9 +1405,9 @@ class TestCWLv11Conformance:
|
|
|
1319
1405
|
self,
|
|
1320
1406
|
cwl_v1_1_spec: Path,
|
|
1321
1407
|
caching: bool = False,
|
|
1322
|
-
batchSystem:
|
|
1323
|
-
skipped_tests:
|
|
1324
|
-
extra_args:
|
|
1408
|
+
batchSystem: str | None = None,
|
|
1409
|
+
skipped_tests: str | None = None,
|
|
1410
|
+
extra_args: list[str] | None = None,
|
|
1325
1411
|
) -> None:
|
|
1326
1412
|
run_conformance_tests(
|
|
1327
1413
|
workDir=str(cwl_v1_1_spec),
|
|
@@ -1408,14 +1494,14 @@ class TestCWLv12Conformance:
|
|
|
1408
1494
|
def test_run_conformance(
|
|
1409
1495
|
self,
|
|
1410
1496
|
cwl_v1_2_spec: Path,
|
|
1411
|
-
runner:
|
|
1497
|
+
runner: str | None = None,
|
|
1412
1498
|
caching: bool = False,
|
|
1413
|
-
batchSystem:
|
|
1414
|
-
selected_tests:
|
|
1415
|
-
skipped_tests:
|
|
1416
|
-
extra_args:
|
|
1499
|
+
batchSystem: str | None = None,
|
|
1500
|
+
selected_tests: str | None = None,
|
|
1501
|
+
skipped_tests: str | None = None,
|
|
1502
|
+
extra_args: list[str] | None = None,
|
|
1417
1503
|
must_support_all_features: bool = False,
|
|
1418
|
-
junit_file:
|
|
1504
|
+
junit_file: str | None = None,
|
|
1419
1505
|
) -> None:
|
|
1420
1506
|
if junit_file is None:
|
|
1421
1507
|
junit_file = os.path.abspath("conformance-1.2.junit.xml")
|
|
@@ -1486,7 +1572,7 @@ class TestCWLv12Conformance:
|
|
|
1486
1572
|
self,
|
|
1487
1573
|
cwl_v1_2_spec: Path,
|
|
1488
1574
|
caching: bool = False,
|
|
1489
|
-
junit_file:
|
|
1575
|
+
junit_file: str | None = None,
|
|
1490
1576
|
) -> None:
|
|
1491
1577
|
if junit_file is None:
|
|
1492
1578
|
junit_file = os.path.abspath("kubernetes-conformance-1.2.junit.xml")
|
|
@@ -1920,12 +2006,17 @@ def test_trim_mounts_op_nonredundant() -> None:
|
|
|
1920
2006
|
"""
|
|
1921
2007
|
Make sure we don't remove all non-duplicate listings
|
|
1922
2008
|
"""
|
|
1923
|
-
s: CWLObjectType = {
|
|
2009
|
+
s: CWLObjectType = {
|
|
2010
|
+
"class": "Directory",
|
|
2011
|
+
"basename": "directory",
|
|
2012
|
+
"listing": [{"class": "File", "basename": "file", "contents": "hello world"}],
|
|
2013
|
+
}
|
|
1924
2014
|
remove_redundant_mounts(s)
|
|
1925
2015
|
|
|
1926
2016
|
# nothing should have been removed
|
|
1927
|
-
assert isinstance(s[
|
|
1928
|
-
assert len(s[
|
|
2017
|
+
assert isinstance(s["listing"], list)
|
|
2018
|
+
assert len(s["listing"]) == 1
|
|
2019
|
+
|
|
1929
2020
|
|
|
1930
2021
|
@needs_cwl
|
|
1931
2022
|
@pytest.mark.cwl
|
|
@@ -1944,7 +2035,7 @@ def test_trim_mounts_op_redundant() -> None:
|
|
|
1944
2035
|
"location": "file:///home/heaucques/Documents/toil/test_dir/nested_dir",
|
|
1945
2036
|
"basename": "nested_dir",
|
|
1946
2037
|
"listing": [],
|
|
1947
|
-
"path": "/home/heaucques/Documents/toil/test_dir/nested_dir"
|
|
2038
|
+
"path": "/home/heaucques/Documents/toil/test_dir/nested_dir",
|
|
1948
2039
|
},
|
|
1949
2040
|
{
|
|
1950
2041
|
"class": "File",
|
|
@@ -1954,16 +2045,17 @@ def test_trim_mounts_op_redundant() -> None:
|
|
|
1954
2045
|
"nameroot": "test_file",
|
|
1955
2046
|
"nameext": "",
|
|
1956
2047
|
"path": "/home/heaucques/Documents/toil/test_dir/test_file",
|
|
1957
|
-
"checksum": "sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709"
|
|
1958
|
-
}
|
|
2048
|
+
"checksum": "sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709",
|
|
2049
|
+
},
|
|
1959
2050
|
],
|
|
1960
|
-
"path": "/home/heaucques/Documents/toil/test_dir"
|
|
2051
|
+
"path": "/home/heaucques/Documents/toil/test_dir",
|
|
1961
2052
|
}
|
|
1962
2053
|
remove_redundant_mounts(s)
|
|
1963
2054
|
|
|
1964
2055
|
# everything should have been removed
|
|
1965
|
-
assert isinstance(s[
|
|
1966
|
-
assert len(s[
|
|
2056
|
+
assert isinstance(s["listing"], list)
|
|
2057
|
+
assert len(s["listing"]) == 0
|
|
2058
|
+
|
|
1967
2059
|
|
|
1968
2060
|
@needs_cwl
|
|
1969
2061
|
@pytest.mark.cwl
|
|
@@ -1982,7 +2074,7 @@ def test_trim_mounts_op_partially_redundant() -> None:
|
|
|
1982
2074
|
"location": "file:///home/heaucques/Documents/thing",
|
|
1983
2075
|
"basename": "thing2",
|
|
1984
2076
|
"listing": [],
|
|
1985
|
-
"path": "/home/heaucques/Documents/toil/thing2"
|
|
2077
|
+
"path": "/home/heaucques/Documents/toil/thing2",
|
|
1986
2078
|
},
|
|
1987
2079
|
{
|
|
1988
2080
|
"class": "File",
|
|
@@ -1992,16 +2084,17 @@ def test_trim_mounts_op_partially_redundant() -> None:
|
|
|
1992
2084
|
"nameroot": "test_file",
|
|
1993
2085
|
"nameext": "",
|
|
1994
2086
|
"path": "/home/heaucques/Documents/toil/test_dir/test_file",
|
|
1995
|
-
"checksum": "sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709"
|
|
1996
|
-
}
|
|
2087
|
+
"checksum": "sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709",
|
|
2088
|
+
},
|
|
1997
2089
|
],
|
|
1998
|
-
"path": "/home/heaucques/Documents/toil/test_dir"
|
|
2090
|
+
"path": "/home/heaucques/Documents/toil/test_dir",
|
|
1999
2091
|
}
|
|
2000
2092
|
remove_redundant_mounts(s)
|
|
2001
2093
|
|
|
2002
2094
|
# everything except the nested directory should be removed
|
|
2003
|
-
assert isinstance(s[
|
|
2004
|
-
assert len(s[
|
|
2095
|
+
assert isinstance(s["listing"], list)
|
|
2096
|
+
assert len(s["listing"]) == 1
|
|
2097
|
+
|
|
2005
2098
|
|
|
2006
2099
|
@needs_cwl
|
|
2007
2100
|
@pytest.mark.cwl
|
|
@@ -2012,10 +2105,16 @@ def test_trim_mounts_op_mixed_urls_and_paths() -> None:
|
|
|
2012
2105
|
"""
|
|
2013
2106
|
# Edge cases around encoding:
|
|
2014
2107
|
# Ensure URL decoded file URIs match the bare path equivalent. Both of these paths should have the same shared directory
|
|
2015
|
-
s: CWLObjectType = {
|
|
2108
|
+
s: CWLObjectType = {
|
|
2109
|
+
"class": "Directory",
|
|
2110
|
+
"basename": "123",
|
|
2111
|
+
"location": "file:///tmp/%25/123",
|
|
2112
|
+
"listing": [{"class": "File", "path": "/tmp/%/123/456", "basename": "456"}],
|
|
2113
|
+
}
|
|
2016
2114
|
remove_redundant_mounts(s)
|
|
2017
|
-
assert isinstance(s[
|
|
2018
|
-
assert len(s[
|
|
2115
|
+
assert isinstance(s["listing"], list)
|
|
2116
|
+
assert len(s["listing"]) == 0
|
|
2117
|
+
|
|
2019
2118
|
|
|
2020
2119
|
@needs_cwl
|
|
2021
2120
|
@pytest.mark.cwl
|
|
@@ -2023,22 +2122,39 @@ def test_trim_mounts_op_mixed_urls_and_paths() -> None:
|
|
|
2023
2122
|
def test_trim_mounts_op_decodable_paths() -> None:
|
|
2024
2123
|
""""""
|
|
2025
2124
|
# Ensure path names don't get unnecessarily decoded
|
|
2026
|
-
s: CWLObjectType = {
|
|
2125
|
+
s: CWLObjectType = {
|
|
2126
|
+
"class": "Directory",
|
|
2127
|
+
"basename": "dir",
|
|
2128
|
+
"path": "/tmp/cat%2Ftag/dir",
|
|
2129
|
+
"listing": [
|
|
2130
|
+
{"class": "File", "path": "/tmp/cat/tag/dir/file", "basename": "file"}
|
|
2131
|
+
],
|
|
2132
|
+
}
|
|
2027
2133
|
remove_redundant_mounts(s)
|
|
2028
|
-
assert isinstance(s[
|
|
2029
|
-
assert len(s[
|
|
2134
|
+
assert isinstance(s["listing"], list)
|
|
2135
|
+
assert len(s["listing"]) == 1
|
|
2136
|
+
|
|
2030
2137
|
|
|
2031
2138
|
@needs_cwl
|
|
2032
2139
|
@pytest.mark.cwl
|
|
2033
2140
|
@pytest.mark.cwl_small
|
|
2034
2141
|
def test_trim_mounts_op_multiple_encodings() -> None:
|
|
2035
2142
|
# Ensure differently encoded URLs are properly decoded
|
|
2036
|
-
s: CWLObjectType = {
|
|
2143
|
+
s: CWLObjectType = {
|
|
2144
|
+
"class": "Directory",
|
|
2145
|
+
"basename": "dir",
|
|
2146
|
+
"location": "file:///tmp/cat%2Ftag/dir",
|
|
2147
|
+
"listing": [
|
|
2148
|
+
{
|
|
2149
|
+
"class": "File",
|
|
2150
|
+
"location": "file:///tmp/cat%2ftag/dir/file",
|
|
2151
|
+
"basename": "file",
|
|
2152
|
+
}
|
|
2153
|
+
],
|
|
2154
|
+
}
|
|
2037
2155
|
remove_redundant_mounts(s)
|
|
2038
|
-
assert isinstance(s[
|
|
2039
|
-
assert len(s[
|
|
2040
|
-
|
|
2041
|
-
|
|
2156
|
+
assert isinstance(s["listing"], list)
|
|
2157
|
+
assert len(s["listing"]) == 0
|
|
2042
2158
|
|
|
2043
2159
|
|
|
2044
2160
|
@needs_cwl
|
|
@@ -2163,6 +2279,7 @@ def test_import_on_workers() -> None:
|
|
|
2163
2279
|
|
|
2164
2280
|
assert detector.detected is True
|
|
2165
2281
|
|
|
2282
|
+
|
|
2166
2283
|
@needs_cwl
|
|
2167
2284
|
@pytest.mark.cwl
|
|
2168
2285
|
@pytest.mark.cwl_small
|
|
@@ -2180,13 +2297,14 @@ def test_missing_tmpdir_and_tmp_outdir(tmp_path: Path) -> None:
|
|
|
2180
2297
|
"toil-cwl-runner",
|
|
2181
2298
|
f"--jobStore=file:{tmp_path / 'jobstore'}",
|
|
2182
2299
|
"--strict-memory-limit",
|
|
2183
|
-
f
|
|
2184
|
-
f
|
|
2300
|
+
f"--tmpdir-prefix={tmpdir_prefix}",
|
|
2301
|
+
f"--tmp-outdir-prefix={tmp_outdir_prefix}",
|
|
2185
2302
|
str(cwl_file),
|
|
2186
2303
|
]
|
|
2187
2304
|
p = subprocess.run(cmd)
|
|
2188
2305
|
assert p.returncode == 0
|
|
2189
2306
|
|
|
2307
|
+
|
|
2190
2308
|
# StreamHandler is generic, _typeshed doesn't exist at runtime, do a bit of typing trickery, see https://github.com/python/typeshed/issues/5680
|
|
2191
2309
|
if TYPE_CHECKING:
|
|
2192
2310
|
from _typeshed import SupportsWrite
|
|
@@ -2218,7 +2336,5 @@ class ImportWorkersMessageHandler(_stream_handler):
|
|
|
2218
2336
|
f"Log message {record.msg} has wrong number of "
|
|
2219
2337
|
f"fields in {record.args}"
|
|
2220
2338
|
) from e
|
|
2221
|
-
if formatted.startswith(
|
|
2222
|
-
f"Issued job '{WorkerImportJob.__name__}'"
|
|
2223
|
-
):
|
|
2339
|
+
if formatted.startswith(f"Issued job '{WorkerImportJob.__name__}'"):
|
|
2224
2340
|
self.detected = True
|
|
@@ -24,7 +24,6 @@ import urllib.parse as urlparse
|
|
|
24
24
|
import uuid
|
|
25
25
|
from abc import ABCMeta, abstractmethod
|
|
26
26
|
from io import BytesIO
|
|
27
|
-
from itertools import chain, islice
|
|
28
27
|
from queue import Queue
|
|
29
28
|
from tempfile import mkstemp
|
|
30
29
|
from threading import Thread
|
|
@@ -34,9 +33,9 @@ from urllib.request import Request, urlopen
|
|
|
34
33
|
import pytest
|
|
35
34
|
from stubserver import FTPStubServer
|
|
36
35
|
|
|
37
|
-
from toil.common import Config
|
|
36
|
+
from toil.common import Config
|
|
38
37
|
from toil.fileStores import FileID
|
|
39
|
-
from toil.job import
|
|
38
|
+
from toil.job import JobDescription, TemporaryID
|
|
40
39
|
from toil.jobStores.abstractJobStore import NoSuchFileException, NoSuchJobException
|
|
41
40
|
from toil.jobStores.fileJobStore import FileJobStore
|
|
42
41
|
from toil.lib.io import mkdtemp
|
|
@@ -1195,12 +1194,15 @@ class AbstractEncryptedJobStoreTest:
|
|
|
1195
1194
|
with self.jobstore_initialized.read_shared_file_stream(fileName) as f:
|
|
1196
1195
|
# If the read goes through, we should fail the assert because
|
|
1197
1196
|
# we read the cyphertext
|
|
1198
|
-
assert
|
|
1199
|
-
|
|
1200
|
-
)
|
|
1197
|
+
assert (
|
|
1198
|
+
f.read() != phrase
|
|
1199
|
+
), "Managed to read plaintext content with encryption off."
|
|
1201
1200
|
except AWSBadEncryptionKeyError as e:
|
|
1202
1201
|
# If the read doesn't go through, we get this.
|
|
1203
|
-
assert
|
|
1202
|
+
assert (
|
|
1203
|
+
"Your AWS encryption key is most likely configured incorrectly"
|
|
1204
|
+
in str(e)
|
|
1205
|
+
)
|
|
1204
1206
|
|
|
1205
1207
|
|
|
1206
1208
|
class FileJobStoreTest(AbstractJobStoreTest.Test):
|
toil/test/lib/aws/test_iam.py
CHANGED
|
@@ -29,9 +29,7 @@ class IAMTest(ToilTest):
|
|
|
29
29
|
"""Check that given permissions and associated functions perform correctly"""
|
|
30
30
|
|
|
31
31
|
def test_permissions_iam(self) -> None:
|
|
32
|
-
granted_perms = {
|
|
33
|
-
"*": {"Action": ["ec2:*", "iam:*", "s3:*"], "NotAction": []}
|
|
34
|
-
}
|
|
32
|
+
granted_perms = {"*": {"Action": ["ec2:*", "iam:*", "s3:*"], "NotAction": []}}
|
|
35
33
|
assert (
|
|
36
34
|
iam.policy_permissions_allow(
|
|
37
35
|
granted_perms, iam.CLUSTER_LAUNCHING_PERMISSIONS
|
toil/test/lib/aws/test_s3.py
CHANGED
|
@@ -16,7 +16,6 @@ import os
|
|
|
16
16
|
import uuid
|
|
17
17
|
from typing import TYPE_CHECKING, Optional
|
|
18
18
|
|
|
19
|
-
from toil.jobStores.aws.jobStore import AWSJobStore
|
|
20
19
|
from toil.lib.aws.session import establish_boto3_session
|
|
21
20
|
from toil.lib.aws.utils import create_s3_bucket, delete_s3_bucket, get_bucket_region
|
|
22
21
|
from toil.test import ToilTest, needs_aws_s3
|
|
@@ -28,6 +27,7 @@ if TYPE_CHECKING:
|
|
|
28
27
|
from mypy_boto3_s3 import S3ServiceResource
|
|
29
28
|
from mypy_boto3_s3.service_resource import Bucket
|
|
30
29
|
|
|
30
|
+
|
|
31
31
|
@needs_aws_s3
|
|
32
32
|
class S3Test(ToilTest):
|
|
33
33
|
"""Confirm the workarounds for us-east-1."""
|
toil/test/lib/dockerTest.py
CHANGED
|
@@ -13,13 +13,14 @@
|
|
|
13
13
|
# limitations under the License.
|
|
14
14
|
import logging
|
|
15
15
|
import os
|
|
16
|
-
from pathlib import Path
|
|
17
16
|
import signal
|
|
18
17
|
import time
|
|
19
|
-
from typing import Optional
|
|
20
18
|
import uuid
|
|
19
|
+
from pathlib import Path
|
|
21
20
|
from threading import Thread
|
|
22
21
|
|
|
22
|
+
import pytest
|
|
23
|
+
|
|
23
24
|
from docker.errors import ContainerError # type: ignore[import-not-found]
|
|
24
25
|
from toil.common import Toil
|
|
25
26
|
from toil.exceptions import FailedJobsException
|
|
@@ -32,9 +33,8 @@ from toil.lib.docker import (
|
|
|
32
33
|
containerIsRunning,
|
|
33
34
|
dockerKill,
|
|
34
35
|
)
|
|
35
|
-
from toil.test import pneeds_docker as needs_docker
|
|
36
|
-
|
|
37
|
-
import pytest
|
|
36
|
+
from toil.test import pneeds_docker as needs_docker
|
|
37
|
+
from toil.test import pslow as slow
|
|
38
38
|
|
|
39
39
|
logger = logging.getLogger(__name__)
|
|
40
40
|
|
|
@@ -55,7 +55,7 @@ class TestDocker:
|
|
|
55
55
|
caching: bool = False,
|
|
56
56
|
detached: bool = True,
|
|
57
57
|
rm: bool = True,
|
|
58
|
-
deferParam:
|
|
58
|
+
deferParam: int | None = None,
|
|
59
59
|
) -> None:
|
|
60
60
|
"""
|
|
61
61
|
Run the test container that creates a file in the work dir, and sleeps
|
|
@@ -376,9 +376,9 @@ def _testDockerCleanFn(
|
|
|
376
376
|
job: Job,
|
|
377
377
|
working_dir: Path,
|
|
378
378
|
detached: bool = True,
|
|
379
|
-
rm:
|
|
380
|
-
deferParam:
|
|
381
|
-
containerName:
|
|
379
|
+
rm: bool | None = None,
|
|
380
|
+
deferParam: int | None = None,
|
|
381
|
+
containerName: str | None = None,
|
|
382
382
|
) -> None:
|
|
383
383
|
"""
|
|
384
384
|
Test function for test docker_clean. Runs a container with given flags and
|