toil 8.1.0b1__py3-none-any.whl → 9.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- toil/__init__.py +0 -35
- toil/batchSystems/abstractBatchSystem.py +1 -1
- toil/batchSystems/abstractGridEngineBatchSystem.py +1 -1
- toil/batchSystems/awsBatch.py +1 -1
- toil/batchSystems/cleanup_support.py +1 -1
- toil/batchSystems/kubernetes.py +53 -7
- toil/batchSystems/local_support.py +1 -1
- toil/batchSystems/mesos/batchSystem.py +13 -8
- toil/batchSystems/mesos/test/__init__.py +3 -2
- toil/batchSystems/registry.py +15 -118
- toil/batchSystems/singleMachine.py +1 -1
- toil/batchSystems/slurm.py +27 -26
- toil/bus.py +5 -3
- toil/common.py +59 -12
- toil/cwl/cwltoil.py +81 -38
- toil/cwl/utils.py +103 -3
- toil/job.py +64 -49
- toil/jobStores/abstractJobStore.py +35 -239
- toil/jobStores/aws/jobStore.py +2 -1
- toil/jobStores/fileJobStore.py +27 -2
- toil/jobStores/googleJobStore.py +110 -33
- toil/leader.py +9 -0
- toil/lib/accelerators.py +4 -2
- toil/lib/aws/utils.py.orig +504 -0
- toil/lib/bioio.py +1 -1
- toil/lib/docker.py +252 -91
- toil/lib/dockstore.py +11 -3
- toil/lib/exceptions.py +5 -3
- toil/lib/generatedEC2Lists.py +81 -19
- toil/lib/history.py +87 -13
- toil/lib/history_submission.py +23 -9
- toil/lib/io.py +34 -22
- toil/lib/misc.py +8 -2
- toil/lib/plugins.py +106 -0
- toil/lib/resources.py +2 -1
- toil/lib/threading.py +11 -10
- toil/lib/url.py +320 -0
- toil/options/common.py +8 -0
- toil/options/cwl.py +13 -1
- toil/options/runner.py +17 -10
- toil/options/wdl.py +22 -0
- toil/provisioners/aws/awsProvisioner.py +25 -2
- toil/server/api_spec/LICENSE +201 -0
- toil/server/api_spec/README.rst +5 -0
- toil/server/app.py +12 -6
- toil/server/cli/wes_cwl_runner.py +3 -2
- toil/server/wes/abstract_backend.py +21 -43
- toil/server/wes/toil_backend.py +2 -2
- toil/test/__init__.py +275 -115
- toil/test/batchSystems/batchSystemTest.py +228 -213
- toil/test/batchSystems/batch_system_plugin_test.py +7 -0
- toil/test/batchSystems/test_slurm.py +27 -0
- toil/test/cactus/pestis.tar.gz +0 -0
- toil/test/conftest.py +7 -0
- toil/test/cwl/2.fasta +11 -0
- toil/test/cwl/2.fastq +12 -0
- toil/test/cwl/conftest.py +1 -1
- toil/test/cwl/cwlTest.py +1175 -870
- toil/test/cwl/directory/directory/file.txt +15 -0
- toil/test/cwl/download_directory_file.json +4 -0
- toil/test/cwl/download_directory_s3.json +4 -0
- toil/test/cwl/download_file.json +6 -0
- toil/test/cwl/download_http.json +6 -0
- toil/test/cwl/download_https.json +6 -0
- toil/test/cwl/download_s3.json +6 -0
- toil/test/cwl/download_subdirectory_file.json +5 -0
- toil/test/cwl/download_subdirectory_s3.json +5 -0
- toil/test/cwl/empty.json +1 -0
- toil/test/cwl/mock_mpi/fake_mpi.yml +8 -0
- toil/test/cwl/mock_mpi/fake_mpi_run.py +42 -0
- toil/test/cwl/optional-file-exists.json +6 -0
- toil/test/cwl/optional-file-missing.json +6 -0
- toil/test/cwl/preemptible_expression.json +1 -0
- toil/test/cwl/revsort-job-missing.json +6 -0
- toil/test/cwl/revsort-job.json +6 -0
- toil/test/cwl/s3_secondary_file.json +16 -0
- toil/test/cwl/seqtk_seq_job.json +6 -0
- toil/test/cwl/stream.json +6 -0
- toil/test/cwl/test_filename_conflict_resolution.ms/table.dat +0 -0
- toil/test/cwl/test_filename_conflict_resolution.ms/table.f0 +0 -0
- toil/test/cwl/test_filename_conflict_resolution.ms/table.f1 +0 -0
- toil/test/cwl/test_filename_conflict_resolution.ms/table.f1i +0 -0
- toil/test/cwl/test_filename_conflict_resolution.ms/table.f2 +0 -0
- toil/test/cwl/test_filename_conflict_resolution.ms/table.f2_TSM0 +0 -0
- toil/test/cwl/test_filename_conflict_resolution.ms/table.f3 +0 -0
- toil/test/cwl/test_filename_conflict_resolution.ms/table.f3_TSM0 +0 -0
- toil/test/cwl/test_filename_conflict_resolution.ms/table.f4 +0 -0
- toil/test/cwl/test_filename_conflict_resolution.ms/table.f4_TSM0 +0 -0
- toil/test/cwl/test_filename_conflict_resolution.ms/table.f5 +0 -0
- toil/test/cwl/test_filename_conflict_resolution.ms/table.info +0 -0
- toil/test/cwl/test_filename_conflict_resolution.ms/table.lock +0 -0
- toil/test/cwl/whale.txt +16 -0
- toil/test/docs/scripts/example_alwaysfail.py +38 -0
- toil/test/docs/scripts/example_alwaysfail_with_files.wdl +33 -0
- toil/test/docs/scripts/example_cachingbenchmark.py +117 -0
- toil/test/docs/scripts/stagingExampleFiles/in.txt +1 -0
- toil/test/docs/scripts/stagingExampleFiles/out.txt +2 -0
- toil/test/docs/scripts/tutorial_arguments.py +23 -0
- toil/test/docs/scripts/tutorial_debugging.patch +12 -0
- toil/test/docs/scripts/tutorial_debugging_hangs.wdl +126 -0
- toil/test/docs/scripts/tutorial_debugging_works.wdl +129 -0
- toil/test/docs/scripts/tutorial_docker.py +20 -0
- toil/test/docs/scripts/tutorial_dynamic.py +24 -0
- toil/test/docs/scripts/tutorial_encapsulation.py +28 -0
- toil/test/docs/scripts/tutorial_encapsulation2.py +29 -0
- toil/test/docs/scripts/tutorial_helloworld.py +15 -0
- toil/test/docs/scripts/tutorial_invokeworkflow.py +27 -0
- toil/test/docs/scripts/tutorial_invokeworkflow2.py +30 -0
- toil/test/docs/scripts/tutorial_jobfunctions.py +22 -0
- toil/test/docs/scripts/tutorial_managing.py +29 -0
- toil/test/docs/scripts/tutorial_managing2.py +56 -0
- toil/test/docs/scripts/tutorial_multiplejobs.py +25 -0
- toil/test/docs/scripts/tutorial_multiplejobs2.py +21 -0
- toil/test/docs/scripts/tutorial_multiplejobs3.py +22 -0
- toil/test/docs/scripts/tutorial_promises.py +25 -0
- toil/test/docs/scripts/tutorial_promises2.py +30 -0
- toil/test/docs/scripts/tutorial_quickstart.py +22 -0
- toil/test/docs/scripts/tutorial_requirements.py +44 -0
- toil/test/docs/scripts/tutorial_services.py +45 -0
- toil/test/docs/scripts/tutorial_staging.py +45 -0
- toil/test/docs/scripts/tutorial_stats.py +64 -0
- toil/test/docs/scriptsTest.py +2 -1
- toil/test/lib/aws/test_iam.py +3 -1
- toil/test/lib/dockerTest.py +205 -122
- toil/test/lib/test_history.py +101 -77
- toil/test/lib/test_url.py +69 -0
- toil/test/lib/url_plugin_test.py +105 -0
- toil/test/provisioners/aws/awsProvisionerTest.py +13 -10
- toil/test/provisioners/clusterTest.py +17 -4
- toil/test/provisioners/gceProvisionerTest.py +17 -15
- toil/test/server/serverTest.py +78 -36
- toil/test/sort/sort.py +4 -1
- toil/test/src/busTest.py +17 -17
- toil/test/src/deferredFunctionTest.py +145 -132
- toil/test/src/importExportFileTest.py +71 -63
- toil/test/src/jobEncapsulationTest.py +27 -28
- toil/test/src/jobServiceTest.py +149 -133
- toil/test/src/jobTest.py +219 -211
- toil/test/src/miscTests.py +66 -60
- toil/test/src/promisedRequirementTest.py +163 -169
- toil/test/src/regularLogTest.py +24 -24
- toil/test/src/resourceTest.py +82 -76
- toil/test/src/restartDAGTest.py +51 -47
- toil/test/src/resumabilityTest.py +24 -19
- toil/test/src/retainTempDirTest.py +60 -57
- toil/test/src/systemTest.py +17 -13
- toil/test/src/threadingTest.py +29 -32
- toil/test/utils/ABCWorkflowDebug/B_file.txt +1 -0
- toil/test/utils/ABCWorkflowDebug/debugWorkflow.py +204 -0
- toil/test/utils/ABCWorkflowDebug/mkFile.py +16 -0
- toil/test/utils/ABCWorkflowDebug/sleep.cwl +12 -0
- toil/test/utils/ABCWorkflowDebug/sleep.yaml +1 -0
- toil/test/utils/toilDebugTest.py +117 -102
- toil/test/utils/toilKillTest.py +54 -53
- toil/test/utils/utilsTest.py +303 -229
- toil/test/wdl/lint_error.wdl +9 -0
- toil/test/wdl/md5sum/empty_file.json +1 -0
- toil/test/wdl/md5sum/md5sum-gs.json +1 -0
- toil/test/wdl/md5sum/md5sum.1.0.wdl +32 -0
- toil/test/wdl/md5sum/md5sum.input +1 -0
- toil/test/wdl/md5sum/md5sum.json +1 -0
- toil/test/wdl/md5sum/md5sum.wdl +25 -0
- toil/test/wdl/miniwdl_self_test/inputs-namespaced.json +1 -0
- toil/test/wdl/miniwdl_self_test/inputs.json +1 -0
- toil/test/wdl/miniwdl_self_test/self_test.wdl +40 -0
- toil/test/wdl/standard_library/as_map.json +16 -0
- toil/test/wdl/standard_library/as_map_as_input.wdl +23 -0
- toil/test/wdl/standard_library/as_pairs.json +7 -0
- toil/test/wdl/standard_library/as_pairs_as_input.wdl +23 -0
- toil/test/wdl/standard_library/ceil.json +3 -0
- toil/test/wdl/standard_library/ceil_as_command.wdl +16 -0
- toil/test/wdl/standard_library/ceil_as_input.wdl +16 -0
- toil/test/wdl/standard_library/collect_by_key.json +1 -0
- toil/test/wdl/standard_library/collect_by_key_as_input.wdl +23 -0
- toil/test/wdl/standard_library/cross.json +11 -0
- toil/test/wdl/standard_library/cross_as_input.wdl +19 -0
- toil/test/wdl/standard_library/flatten.json +7 -0
- toil/test/wdl/standard_library/flatten_as_input.wdl +18 -0
- toil/test/wdl/standard_library/floor.json +3 -0
- toil/test/wdl/standard_library/floor_as_command.wdl +16 -0
- toil/test/wdl/standard_library/floor_as_input.wdl +16 -0
- toil/test/wdl/standard_library/keys.json +8 -0
- toil/test/wdl/standard_library/keys_as_input.wdl +24 -0
- toil/test/wdl/standard_library/length.json +7 -0
- toil/test/wdl/standard_library/length_as_input.wdl +16 -0
- toil/test/wdl/standard_library/length_as_input_with_map.json +7 -0
- toil/test/wdl/standard_library/length_as_input_with_map.wdl +17 -0
- toil/test/wdl/standard_library/length_invalid.json +3 -0
- toil/test/wdl/standard_library/range.json +3 -0
- toil/test/wdl/standard_library/range_0.json +3 -0
- toil/test/wdl/standard_library/range_as_input.wdl +17 -0
- toil/test/wdl/standard_library/range_invalid.json +3 -0
- toil/test/wdl/standard_library/read_boolean.json +3 -0
- toil/test/wdl/standard_library/read_boolean_as_command.wdl +17 -0
- toil/test/wdl/standard_library/read_float.json +3 -0
- toil/test/wdl/standard_library/read_float_as_command.wdl +17 -0
- toil/test/wdl/standard_library/read_int.json +3 -0
- toil/test/wdl/standard_library/read_int_as_command.wdl +17 -0
- toil/test/wdl/standard_library/read_json.json +3 -0
- toil/test/wdl/standard_library/read_json_as_output.wdl +31 -0
- toil/test/wdl/standard_library/read_lines.json +3 -0
- toil/test/wdl/standard_library/read_lines_as_output.wdl +31 -0
- toil/test/wdl/standard_library/read_map.json +3 -0
- toil/test/wdl/standard_library/read_map_as_output.wdl +31 -0
- toil/test/wdl/standard_library/read_string.json +3 -0
- toil/test/wdl/standard_library/read_string_as_command.wdl +17 -0
- toil/test/wdl/standard_library/read_tsv.json +3 -0
- toil/test/wdl/standard_library/read_tsv_as_output.wdl +31 -0
- toil/test/wdl/standard_library/round.json +3 -0
- toil/test/wdl/standard_library/round_as_command.wdl +16 -0
- toil/test/wdl/standard_library/round_as_input.wdl +16 -0
- toil/test/wdl/standard_library/size.json +3 -0
- toil/test/wdl/standard_library/size_as_command.wdl +17 -0
- toil/test/wdl/standard_library/size_as_output.wdl +36 -0
- toil/test/wdl/standard_library/stderr.json +3 -0
- toil/test/wdl/standard_library/stderr_as_output.wdl +30 -0
- toil/test/wdl/standard_library/stdout.json +3 -0
- toil/test/wdl/standard_library/stdout_as_output.wdl +30 -0
- toil/test/wdl/standard_library/sub.json +3 -0
- toil/test/wdl/standard_library/sub_as_input.wdl +17 -0
- toil/test/wdl/standard_library/sub_as_input_with_file.wdl +17 -0
- toil/test/wdl/standard_library/transpose.json +6 -0
- toil/test/wdl/standard_library/transpose_as_input.wdl +18 -0
- toil/test/wdl/standard_library/write_json.json +6 -0
- toil/test/wdl/standard_library/write_json_as_command.wdl +17 -0
- toil/test/wdl/standard_library/write_lines.json +7 -0
- toil/test/wdl/standard_library/write_lines_as_command.wdl +17 -0
- toil/test/wdl/standard_library/write_map.json +6 -0
- toil/test/wdl/standard_library/write_map_as_command.wdl +17 -0
- toil/test/wdl/standard_library/write_tsv.json +6 -0
- toil/test/wdl/standard_library/write_tsv_as_command.wdl +17 -0
- toil/test/wdl/standard_library/zip.json +12 -0
- toil/test/wdl/standard_library/zip_as_input.wdl +19 -0
- toil/test/wdl/test.csv +3 -0
- toil/test/wdl/test.tsv +3 -0
- toil/test/wdl/testfiles/croo.wdl +38 -0
- toil/test/wdl/testfiles/drop_files.wdl +62 -0
- toil/test/wdl/testfiles/drop_files_subworkflow.wdl +13 -0
- toil/test/wdl/testfiles/empty.txt +0 -0
- toil/test/wdl/testfiles/not_enough_outputs.wdl +33 -0
- toil/test/wdl/testfiles/random.wdl +66 -0
- toil/test/wdl/testfiles/read_file.wdl +18 -0
- toil/test/wdl/testfiles/string_file_coercion.json +1 -0
- toil/test/wdl/testfiles/string_file_coercion.wdl +35 -0
- toil/test/wdl/testfiles/test.json +4 -0
- toil/test/wdl/testfiles/test_boolean.txt +1 -0
- toil/test/wdl/testfiles/test_float.txt +1 -0
- toil/test/wdl/testfiles/test_int.txt +1 -0
- toil/test/wdl/testfiles/test_lines.txt +5 -0
- toil/test/wdl/testfiles/test_map.txt +2 -0
- toil/test/wdl/testfiles/test_string.txt +1 -0
- toil/test/wdl/testfiles/url_to_file.wdl +13 -0
- toil/test/wdl/testfiles/url_to_optional_file.wdl +14 -0
- toil/test/wdl/testfiles/vocab.json +1 -0
- toil/test/wdl/testfiles/vocab.wdl +66 -0
- toil/test/wdl/testfiles/wait.wdl +34 -0
- toil/test/wdl/wdl_specification/type_pair.json +23 -0
- toil/test/wdl/wdl_specification/type_pair_basic.wdl +36 -0
- toil/test/wdl/wdl_specification/type_pair_with_files.wdl +36 -0
- toil/test/wdl/wdl_specification/v1_spec.json +1 -0
- toil/test/wdl/wdl_specification/v1_spec_declaration.wdl +39 -0
- toil/test/wdl/wdltoil_test.py +751 -529
- toil/test/wdl/wdltoil_test_kubernetes.py +2 -2
- toil/utils/toilSshCluster.py +23 -0
- toil/utils/toilUpdateEC2Instances.py +1 -0
- toil/version.py +5 -5
- toil/wdl/wdltoil.py +518 -437
- toil/worker.py +11 -6
- {toil-8.1.0b1.dist-info → toil-9.0.0.dist-info}/METADATA +25 -24
- toil-9.0.0.dist-info/RECORD +444 -0
- {toil-8.1.0b1.dist-info → toil-9.0.0.dist-info}/WHEEL +1 -1
- toil-8.1.0b1.dist-info/RECORD +0 -259
- {toil-8.1.0b1.dist-info → toil-9.0.0.dist-info}/entry_points.txt +0 -0
- {toil-8.1.0b1.dist-info → toil-9.0.0.dist-info/licenses}/LICENSE +0 -0
- {toil-8.1.0b1.dist-info → toil-9.0.0.dist-info}/top_level.txt +0 -0
toil/test/src/miscTests.py
CHANGED
|
@@ -14,39 +14,42 @@
|
|
|
14
14
|
import inspect
|
|
15
15
|
import logging
|
|
16
16
|
import os
|
|
17
|
+
import re
|
|
18
|
+
from pathlib import Path
|
|
17
19
|
import random
|
|
18
20
|
import sys
|
|
21
|
+
from types import FrameType
|
|
19
22
|
from uuid import uuid4
|
|
23
|
+
from typing import cast, Union
|
|
20
24
|
|
|
21
25
|
from toil.common import getNodeID
|
|
22
26
|
from toil.lib.exceptions import panic, raise_
|
|
23
27
|
from toil.lib.io import AtomicFileCreate, atomic_install, atomic_tmp_file, mkdtemp
|
|
24
|
-
from toil.lib.misc import CalledProcessErrorStderr, call_command
|
|
25
|
-
from toil.test import
|
|
28
|
+
from toil.lib.misc import CalledProcessErrorStderr, StrPath, call_command
|
|
29
|
+
from toil.test import pslow as slow
|
|
30
|
+
|
|
31
|
+
import pytest
|
|
26
32
|
|
|
27
33
|
log = logging.getLogger(__name__)
|
|
28
34
|
logging.basicConfig()
|
|
29
35
|
|
|
30
36
|
|
|
31
|
-
class
|
|
37
|
+
class TestMisc:
|
|
32
38
|
"""
|
|
33
39
|
This class contains miscellaneous tests that don't have enough content to be their own test
|
|
34
40
|
file, and that don't logically fit in with any of the other test suites.
|
|
35
41
|
"""
|
|
36
42
|
|
|
37
|
-
def
|
|
38
|
-
super().setUp()
|
|
39
|
-
self.testDir = self._createTempDir()
|
|
40
|
-
|
|
41
|
-
def testIDStability(self):
|
|
43
|
+
def testIDStability(self) -> None:
|
|
42
44
|
prevNodeID = None
|
|
43
45
|
for i in range(10, 1):
|
|
44
46
|
nodeID = getNodeID()
|
|
45
|
-
|
|
47
|
+
assert nodeID == prevNodeID
|
|
46
48
|
prevNodeID = nodeID
|
|
47
49
|
|
|
48
50
|
@slow
|
|
49
|
-
|
|
51
|
+
@pytest.mark.slow
|
|
52
|
+
def testGetSizeOfDirectoryWorks(self, tmp_path: Path) -> None:
|
|
50
53
|
"""A test to make sure toil.common.getDirSizeRecursively does not
|
|
51
54
|
underestimate the amount of disk space needed.
|
|
52
55
|
|
|
@@ -58,9 +61,9 @@ class MiscTests(ToilTest):
|
|
|
58
61
|
from toil.common import getDirSizeRecursively
|
|
59
62
|
|
|
60
63
|
# a list of the directories used in the test
|
|
61
|
-
directories = [
|
|
62
|
-
# A dict of {FILENAME: FILESIZE} for all files used in the test
|
|
63
|
-
files = {}
|
|
64
|
+
directories: list[StrPath] = [tmp_path]
|
|
65
|
+
# A dict of {FILENAME: FILESIZE or FILELINK} for all files used in the test
|
|
66
|
+
files: dict[Path, Union[int, str]] = {}
|
|
64
67
|
# Create a random directory structure
|
|
65
68
|
for i in range(0, 10):
|
|
66
69
|
directories.append(mkdtemp(dir=random.choice(directories), prefix="test"))
|
|
@@ -68,11 +71,11 @@ class MiscTests(ToilTest):
|
|
|
68
71
|
# these are fresh files of size [1, 10] MB and 25% of the time they are hard links to old
|
|
69
72
|
# files.
|
|
70
73
|
while len(files) <= 50:
|
|
71
|
-
fileName =
|
|
74
|
+
fileName = Path(random.choice(directories)) / self._getRandomName()
|
|
72
75
|
if random.randint(0, 100) < 75:
|
|
73
76
|
# Create a fresh file in the range of 1-10 MB
|
|
74
77
|
fileSize = int(round(random.random(), 2) * 10 * 1024 * 1024)
|
|
75
|
-
with open(
|
|
78
|
+
with fileName.open("wb") as fileHandle:
|
|
76
79
|
fileHandle.write(os.urandom(fileSize))
|
|
77
80
|
files[fileName] = fileSize
|
|
78
81
|
else:
|
|
@@ -83,94 +86,96 @@ class MiscTests(ToilTest):
|
|
|
83
86
|
os.link(linkSrc, fileName)
|
|
84
87
|
files[fileName] = "Link to %s" % linkSrc
|
|
85
88
|
|
|
86
|
-
computedDirectorySize = getDirSizeRecursively(
|
|
89
|
+
computedDirectorySize = getDirSizeRecursively(tmp_path)
|
|
87
90
|
totalExpectedSize = sum(x for x in list(files.values()) if isinstance(x, int))
|
|
88
|
-
|
|
91
|
+
assert computedDirectorySize >= totalExpectedSize
|
|
89
92
|
|
|
90
93
|
@staticmethod
|
|
91
|
-
def _getRandomName():
|
|
92
|
-
return uuid4().hex
|
|
94
|
+
def _getRandomName() -> str:
|
|
95
|
+
return str(uuid4().hex)
|
|
93
96
|
|
|
94
|
-
def _get_test_out_file(self, tail):
|
|
95
|
-
outf =
|
|
96
|
-
if
|
|
97
|
-
|
|
97
|
+
def _get_test_out_file(self, testDir: Path, tail: str) -> Path:
|
|
98
|
+
outf = testDir / f"test.{tail}"
|
|
99
|
+
if outf.exists():
|
|
100
|
+
outf.unlink()
|
|
98
101
|
return outf
|
|
99
102
|
|
|
100
|
-
def _write_test_file(self, outf_tmp):
|
|
103
|
+
def _write_test_file(self, outf_tmp: str) -> None:
|
|
101
104
|
with open(outf_tmp, "w") as fh:
|
|
102
|
-
fh.write(
|
|
105
|
+
fh.write(Path(outf_tmp).as_uri() + "\n")
|
|
103
106
|
|
|
104
|
-
def test_atomic_install(self):
|
|
105
|
-
outf = self._get_test_out_file(".foo.gz")
|
|
107
|
+
def test_atomic_install(self, tmp_path: Path) -> None:
|
|
108
|
+
outf = self._get_test_out_file(tmp_path, ".foo.gz")
|
|
106
109
|
outf_tmp = atomic_tmp_file(outf)
|
|
107
110
|
self._write_test_file(outf_tmp)
|
|
108
111
|
atomic_install(outf_tmp, outf)
|
|
109
|
-
|
|
112
|
+
assert outf.exists()
|
|
110
113
|
|
|
111
|
-
def test_atomic_install_dev(self):
|
|
114
|
+
def test_atomic_install_dev(self) -> None:
|
|
112
115
|
devn = "/dev/null"
|
|
113
116
|
tmp = atomic_tmp_file(devn)
|
|
114
|
-
|
|
117
|
+
assert tmp == devn
|
|
115
118
|
atomic_install(tmp, devn)
|
|
116
119
|
|
|
117
|
-
def test_atomic_context_ok(self):
|
|
118
|
-
outf = self._get_test_out_file(".tar")
|
|
120
|
+
def test_atomic_context_ok(self, tmp_path: Path) -> None:
|
|
121
|
+
outf = self._get_test_out_file(tmp_path, ".tar")
|
|
119
122
|
with AtomicFileCreate(outf) as outf_tmp:
|
|
120
123
|
self._write_test_file(outf_tmp)
|
|
121
|
-
|
|
124
|
+
assert outf.exists()
|
|
122
125
|
|
|
123
|
-
def test_atomic_context_error(self):
|
|
124
|
-
outf = self._get_test_out_file(".tar")
|
|
126
|
+
def test_atomic_context_error(self, tmp_path: Path) -> None:
|
|
127
|
+
outf = self._get_test_out_file(tmp_path, ".tar")
|
|
125
128
|
try:
|
|
126
129
|
with AtomicFileCreate(outf) as outf_tmp:
|
|
127
130
|
self._write_test_file(outf_tmp)
|
|
128
131
|
raise Exception("stop!")
|
|
129
132
|
except Exception as ex:
|
|
130
|
-
|
|
131
|
-
|
|
133
|
+
assert str(ex) == "stop!"
|
|
134
|
+
assert not os.path.exists(outf)
|
|
132
135
|
|
|
133
|
-
def test_call_command_ok(self):
|
|
136
|
+
def test_call_command_ok(self) -> None:
|
|
134
137
|
o = call_command(["echo", "Fred"])
|
|
135
|
-
|
|
136
|
-
|
|
138
|
+
assert "Fred\n" == o
|
|
139
|
+
assert isinstance(o, str), str(type(o))
|
|
137
140
|
|
|
138
|
-
def test_call_command_err(self):
|
|
139
|
-
with
|
|
141
|
+
def test_call_command_err(self) -> None:
|
|
142
|
+
with pytest.raises(
|
|
140
143
|
CalledProcessErrorStderr,
|
|
141
|
-
|
|
144
|
+
match=re.escape(
|
|
145
|
+
"Command '['cat', '/dev/Frankenheimer']' exit status 1: cat: /dev/Frankenheimer: No such file or directory"
|
|
146
|
+
),
|
|
142
147
|
):
|
|
143
148
|
call_command(["cat", "/dev/Frankenheimer"])
|
|
144
149
|
|
|
145
150
|
|
|
146
|
-
class TestPanic
|
|
147
|
-
def test_panic_by_hand(self):
|
|
151
|
+
class TestPanic:
|
|
152
|
+
def test_panic_by_hand(self) -> None:
|
|
148
153
|
try:
|
|
149
154
|
self.try_and_panic_by_hand()
|
|
150
155
|
except:
|
|
151
156
|
self.__assert_raised_exception_is_primary()
|
|
152
157
|
|
|
153
|
-
def test_panic(self):
|
|
158
|
+
def test_panic(self) -> None:
|
|
154
159
|
try:
|
|
155
160
|
self.try_and_panic()
|
|
156
161
|
except:
|
|
157
162
|
self.__assert_raised_exception_is_primary()
|
|
158
163
|
|
|
159
|
-
def test_panic_with_secondary(self):
|
|
164
|
+
def test_panic_with_secondary(self) -> None:
|
|
160
165
|
try:
|
|
161
166
|
self.try_and_panic_with_secondary()
|
|
162
167
|
except:
|
|
163
168
|
self.__assert_raised_exception_is_primary()
|
|
164
169
|
|
|
165
|
-
def test_nested_panic(self):
|
|
170
|
+
def test_nested_panic(self) -> None:
|
|
166
171
|
try:
|
|
167
172
|
self.try_and_nested_panic_with_secondary()
|
|
168
173
|
except:
|
|
169
174
|
self.__assert_raised_exception_is_primary()
|
|
170
175
|
|
|
171
|
-
def try_and_panic_by_hand(self):
|
|
176
|
+
def try_and_panic_by_hand(self) -> None:
|
|
172
177
|
try:
|
|
173
|
-
self.line_of_primary_exc = inspect.currentframe().f_lineno + 1
|
|
178
|
+
self.line_of_primary_exc = (inspect.currentframe()).f_lineno + 1 # type: ignore[union-attr]
|
|
174
179
|
raise ValueError("primary")
|
|
175
180
|
except Exception:
|
|
176
181
|
exc_type, exc_value, traceback = sys.exc_info()
|
|
@@ -180,35 +185,36 @@ class TestPanic(ToilTest):
|
|
|
180
185
|
pass
|
|
181
186
|
raise_(exc_type, exc_value, traceback)
|
|
182
187
|
|
|
183
|
-
def try_and_panic(self):
|
|
188
|
+
def try_and_panic(self) -> None:
|
|
184
189
|
try:
|
|
185
|
-
self.line_of_primary_exc = inspect.currentframe().f_lineno + 1
|
|
190
|
+
self.line_of_primary_exc = (inspect.currentframe()).f_lineno + 1 # type: ignore[union-attr]
|
|
186
191
|
raise ValueError("primary")
|
|
187
192
|
except:
|
|
188
193
|
with panic(log):
|
|
189
194
|
pass
|
|
190
195
|
|
|
191
|
-
def try_and_panic_with_secondary(self):
|
|
196
|
+
def try_and_panic_with_secondary(self) -> None:
|
|
192
197
|
try:
|
|
193
|
-
self.line_of_primary_exc = inspect.currentframe().f_lineno + 1
|
|
198
|
+
self.line_of_primary_exc = (inspect.currentframe()).f_lineno + 1 # type: ignore[union-attr]
|
|
194
199
|
raise ValueError("primary")
|
|
195
200
|
except:
|
|
196
201
|
with panic(log):
|
|
197
202
|
raise RuntimeError("secondary")
|
|
198
203
|
|
|
199
|
-
def try_and_nested_panic_with_secondary(self):
|
|
204
|
+
def try_and_nested_panic_with_secondary(self) -> None:
|
|
200
205
|
try:
|
|
201
|
-
self.line_of_primary_exc = inspect.currentframe().f_lineno + 1
|
|
206
|
+
self.line_of_primary_exc = (inspect.currentframe()).f_lineno + 1 # type: ignore[union-attr]
|
|
202
207
|
raise ValueError("primary")
|
|
203
208
|
except:
|
|
204
209
|
with panic(log):
|
|
205
210
|
with panic(log):
|
|
206
211
|
raise RuntimeError("secondary")
|
|
207
212
|
|
|
208
|
-
def __assert_raised_exception_is_primary(self):
|
|
213
|
+
def __assert_raised_exception_is_primary(self) -> None:
|
|
209
214
|
exc_type, exc_value, exc_traceback = sys.exc_info()
|
|
210
|
-
|
|
211
|
-
|
|
215
|
+
assert exc_type == ValueError
|
|
216
|
+
assert str(exc_value) == "primary"
|
|
217
|
+
assert exc_traceback is not None
|
|
212
218
|
while exc_traceback.tb_next is not None:
|
|
213
219
|
exc_traceback = exc_traceback.tb_next
|
|
214
|
-
|
|
220
|
+
assert exc_traceback.tb_lineno == self.line_of_primary_exc
|
|
@@ -12,172 +12,162 @@
|
|
|
12
12
|
# See the License for the specific language governing permissions and
|
|
13
13
|
# limitations under the License.
|
|
14
14
|
|
|
15
|
+
from collections.abc import Generator
|
|
16
|
+
import argparse
|
|
15
17
|
import logging
|
|
16
18
|
import os
|
|
19
|
+
from pathlib import Path
|
|
17
20
|
import time
|
|
21
|
+
from typing import Any
|
|
18
22
|
|
|
19
|
-
|
|
23
|
+
from toil.test.batchSystems import batchSystemTest
|
|
20
24
|
from toil.batchSystems.mesos.test import MesosTestSupport
|
|
21
|
-
from toil.
|
|
25
|
+
from toil.fileStores import FileID
|
|
26
|
+
from toil.job import Job, PromisedRequirement, JobFunctionWrappingJob, Promise
|
|
22
27
|
from toil.lib.retry import retry_flaky_test
|
|
23
|
-
from toil.test import needs_mesos, slow
|
|
28
|
+
from toil.test import pneeds_mesos as needs_mesos, pslow as slow
|
|
24
29
|
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
class hidden:
|
|
29
|
-
"""
|
|
30
|
-
Hide abstract base class from unittest's test case loader.
|
|
31
|
-
|
|
32
|
-
http://stackoverflow.com/questions/1323455/python-unit-test-with-base-and-sub-class#answer-25695512
|
|
33
|
-
"""
|
|
30
|
+
import pytest
|
|
34
31
|
|
|
35
|
-
|
|
36
|
-
batchSystemTest.hidden.AbstractBatchSystemJobTest
|
|
37
|
-
):
|
|
38
|
-
"""An abstract base class for testing Toil workflows with promised requirements."""
|
|
39
|
-
|
|
40
|
-
@slow
|
|
41
|
-
def testConcurrencyDynamic(self):
|
|
42
|
-
"""
|
|
43
|
-
Asserts that promised core resources are allocated properly using a dynamic Toil workflow
|
|
44
|
-
"""
|
|
45
|
-
for coresPerJob in self.allocatedCores:
|
|
46
|
-
log.debug(
|
|
47
|
-
"Testing %d cores per job with CPU count %d",
|
|
48
|
-
coresPerJob,
|
|
49
|
-
self.cpuCount,
|
|
50
|
-
)
|
|
51
|
-
tempDir = self._createTempDir("testFiles")
|
|
52
|
-
counterPath = self.getCounterPath(tempDir)
|
|
53
|
-
|
|
54
|
-
root = Job.wrapJobFn(
|
|
55
|
-
maxConcurrency,
|
|
56
|
-
self.cpuCount,
|
|
57
|
-
counterPath,
|
|
58
|
-
coresPerJob,
|
|
59
|
-
cores=1,
|
|
60
|
-
memory="1M",
|
|
61
|
-
disk="1M",
|
|
62
|
-
)
|
|
63
|
-
values = Job.Runner.startToil(root, self.getOptions(tempDir))
|
|
64
|
-
maxValue = max(values)
|
|
65
|
-
self.assertLessEqual(maxValue, self.cpuCount // coresPerJob)
|
|
66
|
-
|
|
67
|
-
@slow
|
|
68
|
-
@retry_flaky_test(
|
|
69
|
-
prepare=[
|
|
70
|
-
batchSystemTest.hidden.AbstractBatchSystemJobTest.tearDown,
|
|
71
|
-
batchSystemTest.hidden.AbstractBatchSystemJobTest.setUp,
|
|
72
|
-
]
|
|
73
|
-
)
|
|
74
|
-
def testConcurrencyStatic(self):
|
|
75
|
-
"""
|
|
76
|
-
Asserts that promised core resources are allocated properly using a static DAG
|
|
77
|
-
"""
|
|
78
|
-
for coresPerJob in self.allocatedCores:
|
|
79
|
-
log.debug(
|
|
80
|
-
"Testing %d cores per job with CPU count %d",
|
|
81
|
-
coresPerJob,
|
|
82
|
-
self.cpuCount,
|
|
83
|
-
)
|
|
84
|
-
tempDir = self._createTempDir("testFiles")
|
|
85
|
-
counterPath = self.getCounterPath(tempDir)
|
|
32
|
+
log = logging.getLogger(__name__)
|
|
86
33
|
|
|
87
|
-
root = Job()
|
|
88
|
-
one = Job.wrapFn(getOne, cores=0.1, memory="32M", disk="1M")
|
|
89
|
-
thirtyTwoMb = Job.wrapFn(
|
|
90
|
-
getThirtyTwoMb, cores=0.1, memory="32M", disk="1M"
|
|
91
|
-
)
|
|
92
|
-
root.addChild(one)
|
|
93
|
-
root.addChild(thirtyTwoMb)
|
|
94
|
-
for _ in range(self.cpuCount):
|
|
95
|
-
root.addFollowOn(
|
|
96
|
-
Job.wrapFn(
|
|
97
|
-
batchSystemTest.measureConcurrency,
|
|
98
|
-
counterPath,
|
|
99
|
-
cores=PromisedRequirement(
|
|
100
|
-
lambda x: x * coresPerJob, one.rv()
|
|
101
|
-
),
|
|
102
|
-
memory=PromisedRequirement(thirtyTwoMb.rv()),
|
|
103
|
-
disk="1M",
|
|
104
|
-
)
|
|
105
|
-
)
|
|
106
|
-
Job.Runner.startToil(root, self.getOptions(tempDir))
|
|
107
|
-
_, maxValue = batchSystemTest.getCounters(counterPath)
|
|
108
|
-
self.assertLessEqual(maxValue, self.cpuCount // coresPerJob)
|
|
109
|
-
|
|
110
|
-
def getOptions(self, tempDir, caching=True):
|
|
111
|
-
options = super().getOptions(tempDir)
|
|
112
|
-
# defaultCores defaults to 1 - this is coincidentally the core requirement relied upon by this
|
|
113
|
-
# test, so we change defaultCores to 2 to make the test more strict
|
|
114
|
-
options.defaultCores = 2
|
|
115
|
-
options.caching = caching
|
|
116
|
-
return options
|
|
117
|
-
|
|
118
|
-
def getCounterPath(self, tempDir):
|
|
119
|
-
"""
|
|
120
|
-
Returns path to a counter file
|
|
121
|
-
:param str tempDir: path to test directory
|
|
122
|
-
:return: path to counter file
|
|
123
|
-
"""
|
|
124
|
-
counterPath = os.path.join(tempDir, "counter")
|
|
125
|
-
batchSystemTest.resetCounters(counterPath)
|
|
126
|
-
minValue, maxValue = batchSystemTest.getCounters(counterPath)
|
|
127
|
-
assert (minValue, maxValue) == (0, 0)
|
|
128
|
-
return counterPath
|
|
129
|
-
|
|
130
|
-
def testPromisesWithJobStoreFileObjects(self, caching=True):
|
|
131
|
-
"""
|
|
132
|
-
Check whether FileID objects are being pickled properly when used as return
|
|
133
|
-
values of functions. Then ensure that lambdas of promised FileID objects can be
|
|
134
|
-
used to describe the requirements of a subsequent job. This type of operation will be
|
|
135
|
-
used commonly in Toil scripts.
|
|
136
|
-
:return: None
|
|
137
|
-
"""
|
|
138
|
-
file1 = 1024
|
|
139
|
-
file2 = 512
|
|
140
|
-
F1 = Job.wrapJobFn(_writer, file1)
|
|
141
|
-
F2 = Job.wrapJobFn(_writer, file2)
|
|
142
|
-
G = Job.wrapJobFn(
|
|
143
|
-
_follower,
|
|
144
|
-
file1 + file2,
|
|
145
|
-
disk=PromisedRequirement(
|
|
146
|
-
lambda x, y: x.size + y.size, F1.rv(), F2.rv()
|
|
147
|
-
),
|
|
148
|
-
)
|
|
149
|
-
F1.addChild(F2)
|
|
150
|
-
F2.addChild(G)
|
|
151
34
|
|
|
152
|
-
|
|
153
|
-
|
|
35
|
+
class AbstractPromisedRequirementsTest(batchSystemTest.AbstractBatchSystemJobTest):
|
|
36
|
+
"""An abstract base class for testing Toil workflows with promised requirements."""
|
|
37
|
+
|
|
38
|
+
@slow
|
|
39
|
+
@pytest.mark.slow
|
|
40
|
+
def testConcurrencyDynamic(self, tmp_path: Path) -> None:
|
|
41
|
+
"""
|
|
42
|
+
Asserts that promised core resources are allocated properly using a dynamic Toil workflow
|
|
43
|
+
"""
|
|
44
|
+
for coresPerJob in self.allocatedCores:
|
|
45
|
+
log.debug(
|
|
46
|
+
"Testing %d cores per job with CPU count %d",
|
|
47
|
+
coresPerJob,
|
|
48
|
+
self.cpuCount,
|
|
154
49
|
)
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
self.
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
50
|
+
tempDir = tmp_path / f"testFiles_{coresPerJob}"
|
|
51
|
+
tempDir.mkdir()
|
|
52
|
+
counterPath = self.getCounterPath(tempDir)
|
|
53
|
+
|
|
54
|
+
root = Job.wrapJobFn(
|
|
55
|
+
maxConcurrency,
|
|
56
|
+
self.cpuCount,
|
|
57
|
+
counterPath,
|
|
58
|
+
coresPerJob,
|
|
59
|
+
cores=1,
|
|
60
|
+
memory="1M",
|
|
61
|
+
disk="1M",
|
|
166
62
|
)
|
|
167
|
-
|
|
168
|
-
|
|
63
|
+
values = Job.Runner.startToil(root, self.getOptions(tempDir))
|
|
64
|
+
maxValue = max(values)
|
|
65
|
+
assert maxValue <= (self.cpuCount // coresPerJob)
|
|
66
|
+
|
|
67
|
+
@slow
|
|
68
|
+
@pytest.mark.slow
|
|
69
|
+
@retry_flaky_test(prepare=[])
|
|
70
|
+
def testConcurrencyStatic(self, tmp_path: Path) -> None:
|
|
71
|
+
"""
|
|
72
|
+
Asserts that promised core resources are allocated properly using a static DAG
|
|
73
|
+
"""
|
|
74
|
+
for coresPerJob in self.allocatedCores:
|
|
75
|
+
log.debug(
|
|
76
|
+
"Testing %d cores per job with CPU count %d",
|
|
77
|
+
coresPerJob,
|
|
78
|
+
self.cpuCount,
|
|
169
79
|
)
|
|
170
|
-
|
|
171
|
-
|
|
80
|
+
tempDir = tmp_path / f"testFiles_{coresPerJob}"
|
|
81
|
+
tempDir.mkdir()
|
|
82
|
+
counterPath = self.getCounterPath(tempDir)
|
|
83
|
+
|
|
84
|
+
root = Job()
|
|
85
|
+
one = Job.wrapFn(getOne, cores=0.1, memory="32M", disk="1M")
|
|
86
|
+
thirtyTwoMb = Job.wrapFn(getThirtyTwoMb, cores=0.1, memory="32M", disk="1M")
|
|
87
|
+
root.addChild(one)
|
|
88
|
+
root.addChild(thirtyTwoMb)
|
|
89
|
+
for _ in range(self.cpuCount):
|
|
90
|
+
root.addFollowOn(
|
|
91
|
+
Job.wrapFn(
|
|
92
|
+
batchSystemTest.measureConcurrency,
|
|
93
|
+
counterPath,
|
|
94
|
+
cores=PromisedRequirement(lambda x: x * coresPerJob, one.rv()),
|
|
95
|
+
memory=PromisedRequirement(thirtyTwoMb.rv()),
|
|
96
|
+
disk="1M",
|
|
97
|
+
)
|
|
98
|
+
)
|
|
99
|
+
Job.Runner.startToil(root, self.getOptions(tempDir))
|
|
100
|
+
_, maxValue = batchSystemTest.getCounters(counterPath)
|
|
101
|
+
assert maxValue <= (self.cpuCount // coresPerJob)
|
|
102
|
+
|
|
103
|
+
def getOptions(self, tempDir: Path, caching: bool = True) -> argparse.Namespace:
|
|
104
|
+
options = super().getOptions(tempDir)
|
|
105
|
+
# defaultCores defaults to 1 - this is coincidentally the core requirement relied upon by this
|
|
106
|
+
# test, so we change defaultCores to 2 to make the test more strict
|
|
107
|
+
options.defaultCores = 2
|
|
108
|
+
options.caching = caching
|
|
109
|
+
return options
|
|
110
|
+
|
|
111
|
+
def getCounterPath(self, tempDir: Path) -> Path:
|
|
112
|
+
"""
|
|
113
|
+
Returns path to a counter file
|
|
114
|
+
:param tempDir: path to test directory
|
|
115
|
+
:return: path to counter file
|
|
116
|
+
"""
|
|
117
|
+
counterPath = tempDir / "counter"
|
|
118
|
+
batchSystemTest.resetCounters(counterPath)
|
|
119
|
+
minValue, maxValue = batchSystemTest.getCounters(counterPath)
|
|
120
|
+
assert (minValue, maxValue) == (0, 0)
|
|
121
|
+
return counterPath
|
|
122
|
+
|
|
123
|
+
def testPromisesWithJobStoreFileObjects(
|
|
124
|
+
self, tmp_path: Path, caching: bool = True
|
|
125
|
+
) -> None:
|
|
126
|
+
"""
|
|
127
|
+
Check whether FileID objects are being pickled properly when used as return
|
|
128
|
+
values of functions. Then ensure that lambdas of promised FileID objects can be
|
|
129
|
+
used to describe the requirements of a subsequent job. This type of operation will be
|
|
130
|
+
used commonly in Toil scripts.
|
|
131
|
+
:return: None
|
|
132
|
+
"""
|
|
133
|
+
file1 = 1024
|
|
134
|
+
file2 = 512
|
|
135
|
+
F1 = Job.wrapJobFn(_writer, file1)
|
|
136
|
+
F2 = Job.wrapJobFn(_writer, file2)
|
|
137
|
+
G = Job.wrapJobFn(
|
|
138
|
+
_follower,
|
|
139
|
+
file1 + file2,
|
|
140
|
+
disk=PromisedRequirement(lambda x, y: x.size + y.size, F1.rv(), F2.rv()),
|
|
141
|
+
)
|
|
142
|
+
F1.addChild(F2)
|
|
143
|
+
F2.addChild(G)
|
|
144
|
+
|
|
145
|
+
Job.Runner.startToil(F1, self.getOptions(tmp_path, caching=caching))
|
|
146
|
+
|
|
147
|
+
def testPromisesWithNonCachingFileStore(self, tmp_path: Path) -> None:
|
|
148
|
+
self.testPromisesWithJobStoreFileObjects(tmp_path, caching=False)
|
|
149
|
+
|
|
150
|
+
@slow
|
|
151
|
+
@pytest.mark.slow
|
|
152
|
+
def testPromiseRequirementRaceStatic(self, tmp_path: Path) -> None:
|
|
153
|
+
"""
|
|
154
|
+
Checks for a race condition when using promised requirements and child job functions.
|
|
155
|
+
"""
|
|
156
|
+
A = Job.wrapJobFn(logDiskUsage, "A", sleep=5, disk=PromisedRequirement(1024))
|
|
157
|
+
B = Job.wrapJobFn(
|
|
158
|
+
logDiskUsage, "B", disk=PromisedRequirement(lambda x: x + 1024, A.rv())
|
|
159
|
+
)
|
|
160
|
+
A.addChild(B)
|
|
161
|
+
Job.Runner.startToil(A, self.getOptions(tmp_path))
|
|
172
162
|
|
|
173
163
|
|
|
174
|
-
def _writer(job, fileSize):
|
|
164
|
+
def _writer(job: JobFunctionWrappingJob, fileSize: int) -> FileID:
|
|
175
165
|
"""
|
|
176
166
|
Write a local file and return the FileID obtained from running writeGlobalFile on
|
|
177
167
|
it.
|
|
178
168
|
|
|
179
|
-
:param job
|
|
180
|
-
:param
|
|
169
|
+
:param job: job
|
|
170
|
+
:param fileSize: Size of the file in bytes
|
|
181
171
|
:returns: the result of writeGlobalFile on a locally created file
|
|
182
172
|
:rtype: job.FileID
|
|
183
173
|
"""
|
|
@@ -186,26 +176,28 @@ def _writer(job, fileSize):
|
|
|
186
176
|
return job.fileStore.writeGlobalFile(fH.name)
|
|
187
177
|
|
|
188
178
|
|
|
189
|
-
def _follower(job, expectedDisk):
|
|
179
|
+
def _follower(job: Job, expectedDisk: Any) -> None:
|
|
190
180
|
"""
|
|
191
181
|
This job follows the _writer jobs and ensures the expected disk is used.
|
|
192
182
|
|
|
193
|
-
:param job
|
|
183
|
+
:param job: job
|
|
194
184
|
:param expectedDisk: Expect disk to be used by this job
|
|
195
185
|
:return: None
|
|
196
186
|
"""
|
|
197
187
|
assert job.disk == expectedDisk
|
|
198
188
|
|
|
199
189
|
|
|
200
|
-
def maxConcurrency(
|
|
190
|
+
def maxConcurrency(
|
|
191
|
+
job: Job, cpuCount: int, filename: Path, coresPerJob: int
|
|
192
|
+
) -> list[Promise]:
|
|
201
193
|
"""
|
|
202
194
|
Returns the max number of concurrent tasks when using a PromisedRequirement instance
|
|
203
195
|
to allocate the number of cores per job.
|
|
204
196
|
|
|
205
|
-
:param
|
|
206
|
-
:param
|
|
207
|
-
:param
|
|
208
|
-
:return
|
|
197
|
+
:param cpuCount: number of available cpus
|
|
198
|
+
:param filename: path to counter file
|
|
199
|
+
:param coresPerJob: number of cores assigned to each job
|
|
200
|
+
:return: max concurrency value per CPU
|
|
209
201
|
"""
|
|
210
202
|
one = job.addChildFn(getOne, cores=0.1, memory="32M", disk="1M")
|
|
211
203
|
thirtyTwoMb = job.addChildFn(getThirtyTwoMb, cores=0.1, memory="32M", disk="1M")
|
|
@@ -223,15 +215,15 @@ def maxConcurrency(job, cpuCount, filename, coresPerJob):
|
|
|
223
215
|
return values
|
|
224
216
|
|
|
225
217
|
|
|
226
|
-
def getOne():
|
|
218
|
+
def getOne() -> int:
|
|
227
219
|
return 1
|
|
228
220
|
|
|
229
221
|
|
|
230
|
-
def getThirtyTwoMb():
|
|
222
|
+
def getThirtyTwoMb() -> str:
|
|
231
223
|
return "32M"
|
|
232
224
|
|
|
233
225
|
|
|
234
|
-
def logDiskUsage(job, funcName, sleep=0):
|
|
226
|
+
def logDiskUsage(job: JobFunctionWrappingJob, funcName: str, sleep: int = 0) -> int:
|
|
235
227
|
"""
|
|
236
228
|
Logs the job's disk usage to master and sleeps for specified amount of time.
|
|
237
229
|
|
|
@@ -243,34 +235,36 @@ def logDiskUsage(job, funcName, sleep=0):
|
|
|
243
235
|
return diskUsage
|
|
244
236
|
|
|
245
237
|
|
|
246
|
-
class
|
|
238
|
+
class TestSingleMachinePromisedRequirements(AbstractPromisedRequirementsTest):
|
|
247
239
|
"""
|
|
248
240
|
Tests against the SingleMachine batch system
|
|
249
241
|
"""
|
|
250
242
|
|
|
251
|
-
def getBatchSystemName(self):
|
|
243
|
+
def getBatchSystemName(self) -> str:
|
|
252
244
|
return "single_machine"
|
|
253
245
|
|
|
254
|
-
def tearDown(self):
|
|
246
|
+
def tearDown(self) -> None:
|
|
255
247
|
pass
|
|
256
248
|
|
|
257
249
|
|
|
258
250
|
@needs_mesos
|
|
259
|
-
class
|
|
260
|
-
hidden.AbstractPromisedRequirementsTest, MesosTestSupport
|
|
261
|
-
):
|
|
251
|
+
class TestMesosPromisedRequirements(AbstractPromisedRequirementsTest, MesosTestSupport):
|
|
262
252
|
"""
|
|
263
253
|
Tests against the Mesos batch system
|
|
264
254
|
"""
|
|
265
255
|
|
|
266
|
-
|
|
256
|
+
@pytest.fixture(autouse=True)
|
|
257
|
+
def mesos_support(self) -> Generator[None]:
|
|
258
|
+
try:
|
|
259
|
+
self._startMesos(self.cpuCount)
|
|
260
|
+
yield
|
|
261
|
+
finally:
|
|
262
|
+
self._stopMesos()
|
|
263
|
+
|
|
264
|
+
def getOptions(self, tempDir: Path, caching: bool = True) -> argparse.Namespace:
|
|
267
265
|
options = super().getOptions(tempDir, caching=caching)
|
|
268
266
|
options.mesos_endpoint = "localhost:5050"
|
|
269
267
|
return options
|
|
270
268
|
|
|
271
|
-
def getBatchSystemName(self):
|
|
272
|
-
self._startMesos(self.cpuCount)
|
|
269
|
+
def getBatchSystemName(self) -> str:
|
|
273
270
|
return "mesos"
|
|
274
|
-
|
|
275
|
-
def tearDown(self):
|
|
276
|
-
self._stopMesos()
|