toil 8.2.0__py3-none-any.whl → 9.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- toil/batchSystems/abstractBatchSystem.py +13 -5
- toil/batchSystems/abstractGridEngineBatchSystem.py +17 -5
- toil/batchSystems/kubernetes.py +13 -2
- toil/batchSystems/mesos/batchSystem.py +33 -2
- toil/batchSystems/registry.py +15 -118
- toil/batchSystems/slurm.py +191 -16
- toil/common.py +20 -1
- toil/cwl/cwltoil.py +97 -119
- toil/cwl/utils.py +103 -3
- toil/fileStores/__init__.py +1 -1
- toil/fileStores/abstractFileStore.py +5 -2
- toil/fileStores/cachingFileStore.py +1 -1
- toil/job.py +30 -14
- toil/jobStores/abstractJobStore.py +35 -255
- toil/jobStores/aws/jobStore.py +864 -1964
- toil/jobStores/aws/utils.py +24 -270
- toil/jobStores/fileJobStore.py +2 -1
- toil/jobStores/googleJobStore.py +32 -13
- toil/jobStores/utils.py +0 -327
- toil/leader.py +27 -22
- toil/lib/accelerators.py +1 -1
- toil/lib/aws/config.py +22 -0
- toil/lib/aws/s3.py +477 -9
- toil/lib/aws/utils.py +22 -33
- toil/lib/checksum.py +88 -0
- toil/lib/conversions.py +33 -31
- toil/lib/directory.py +217 -0
- toil/lib/ec2.py +97 -29
- toil/lib/exceptions.py +2 -1
- toil/lib/expando.py +2 -2
- toil/lib/generatedEC2Lists.py +138 -19
- toil/lib/io.py +33 -2
- toil/lib/memoize.py +21 -7
- toil/lib/misc.py +1 -1
- toil/lib/pipes.py +385 -0
- toil/lib/plugins.py +106 -0
- toil/lib/retry.py +1 -1
- toil/lib/threading.py +1 -1
- toil/lib/url.py +320 -0
- toil/lib/web.py +4 -5
- toil/options/cwl.py +13 -1
- toil/options/runner.py +17 -10
- toil/options/wdl.py +12 -1
- toil/provisioners/__init__.py +5 -2
- toil/provisioners/aws/__init__.py +43 -36
- toil/provisioners/aws/awsProvisioner.py +47 -15
- toil/provisioners/node.py +60 -12
- toil/resource.py +3 -13
- toil/server/app.py +12 -6
- toil/server/cli/wes_cwl_runner.py +2 -2
- toil/server/wes/abstract_backend.py +21 -43
- toil/server/wes/toil_backend.py +2 -2
- toil/test/__init__.py +16 -18
- toil/test/batchSystems/batchSystemTest.py +2 -9
- toil/test/batchSystems/batch_system_plugin_test.py +7 -0
- toil/test/batchSystems/test_slurm.py +103 -14
- toil/test/cwl/cwlTest.py +181 -8
- toil/test/cwl/staging_cat.cwl +27 -0
- toil/test/cwl/staging_make_file.cwl +25 -0
- toil/test/cwl/staging_workflow.cwl +43 -0
- toil/test/cwl/zero_default.cwl +61 -0
- toil/test/docs/scripts/tutorial_staging.py +17 -8
- toil/test/docs/scriptsTest.py +2 -1
- toil/test/jobStores/jobStoreTest.py +23 -133
- toil/test/lib/aws/test_iam.py +7 -7
- toil/test/lib/aws/test_s3.py +30 -33
- toil/test/lib/aws/test_utils.py +9 -9
- toil/test/lib/test_url.py +69 -0
- toil/test/lib/url_plugin_test.py +105 -0
- toil/test/provisioners/aws/awsProvisionerTest.py +60 -7
- toil/test/provisioners/clusterTest.py +15 -2
- toil/test/provisioners/gceProvisionerTest.py +1 -1
- toil/test/server/serverTest.py +78 -36
- toil/test/src/autoDeploymentTest.py +2 -3
- toil/test/src/fileStoreTest.py +89 -87
- toil/test/utils/ABCWorkflowDebug/ABC.txt +1 -0
- toil/test/utils/ABCWorkflowDebug/debugWorkflow.py +4 -4
- toil/test/utils/toilKillTest.py +35 -28
- toil/test/wdl/md5sum/md5sum-gs.json +1 -1
- toil/test/wdl/md5sum/md5sum.json +1 -1
- toil/test/wdl/testfiles/read_file.wdl +18 -0
- toil/test/wdl/testfiles/url_to_optional_file.wdl +2 -1
- toil/test/wdl/wdltoil_test.py +171 -162
- toil/test/wdl/wdltoil_test_kubernetes.py +9 -0
- toil/utils/toilDebugFile.py +6 -3
- toil/utils/toilSshCluster.py +23 -0
- toil/utils/toilStats.py +17 -2
- toil/utils/toilUpdateEC2Instances.py +1 -0
- toil/version.py +10 -10
- toil/wdl/wdltoil.py +1179 -825
- toil/worker.py +16 -8
- {toil-8.2.0.dist-info → toil-9.1.0.dist-info}/METADATA +32 -32
- {toil-8.2.0.dist-info → toil-9.1.0.dist-info}/RECORD +97 -85
- {toil-8.2.0.dist-info → toil-9.1.0.dist-info}/WHEEL +1 -1
- toil/lib/iterables.py +0 -112
- toil/test/docs/scripts/stagingExampleFiles/in.txt +0 -1
- {toil-8.2.0.dist-info → toil-9.1.0.dist-info}/entry_points.txt +0 -0
- {toil-8.2.0.dist-info → toil-9.1.0.dist-info}/licenses/LICENSE +0 -0
- {toil-8.2.0.dist-info → toil-9.1.0.dist-info}/top_level.txt +0 -0
toil/test/wdl/wdltoil_test.py
CHANGED
|
@@ -13,6 +13,8 @@ from unittest.mock import patch
|
|
|
13
13
|
from uuid import uuid4
|
|
14
14
|
|
|
15
15
|
import pytest
|
|
16
|
+
from pytest_httpserver import HTTPServer
|
|
17
|
+
|
|
16
18
|
import WDL.Error
|
|
17
19
|
import WDL.Expr
|
|
18
20
|
|
|
@@ -23,6 +25,7 @@ from toil.test import (
|
|
|
23
25
|
needs_docker,
|
|
24
26
|
needs_docker_cuda,
|
|
25
27
|
needs_google_storage,
|
|
28
|
+
needs_online,
|
|
26
29
|
needs_singularity_or_docker,
|
|
27
30
|
needs_wdl,
|
|
28
31
|
slow,
|
|
@@ -32,14 +35,13 @@ from toil.wdl.wdltoil import (
|
|
|
32
35
|
WDLSectionJob,
|
|
33
36
|
WDLWorkflowGraph,
|
|
34
37
|
parse_disks,
|
|
35
|
-
remove_common_leading_whitespace,
|
|
36
38
|
)
|
|
37
39
|
|
|
38
40
|
logger = logging.getLogger(__name__)
|
|
39
41
|
|
|
40
42
|
|
|
41
43
|
WDL_CONFORMANCE_TEST_REPO = "https://github.com/DataBiosphere/wdl-conformance-tests.git"
|
|
42
|
-
WDL_CONFORMANCE_TEST_COMMIT = "
|
|
44
|
+
WDL_CONFORMANCE_TEST_COMMIT = "46b5f85ee38ec60d0b8b9c35928b5104a2af83d5"
|
|
43
45
|
# These tests are known to require things not implemented by
|
|
44
46
|
# Toil and will not be run in CI.
|
|
45
47
|
WDL_CONFORMANCE_TESTS_UNSUPPORTED_BY_TOIL = [
|
|
@@ -108,16 +110,25 @@ class TestWDLConformance:
|
|
|
108
110
|
"Failed process standard error: %s",
|
|
109
111
|
p.stderr.decode("utf-8", errors="replace"),
|
|
110
112
|
)
|
|
113
|
+
else:
|
|
114
|
+
logger.debug(
|
|
115
|
+
"Successful process standard output: %s",
|
|
116
|
+
p.stdout.decode("utf-8", errors="replace"),
|
|
117
|
+
)
|
|
118
|
+
logger.debug(
|
|
119
|
+
"Successful process standard error: %s",
|
|
120
|
+
p.stderr.decode("utf-8", errors="replace"),
|
|
121
|
+
)
|
|
111
122
|
|
|
112
123
|
p.check_returncode()
|
|
113
124
|
|
|
114
125
|
@slow
|
|
115
126
|
def test_unit_tests_v11(self, wdl_conformance_test_repo: Path) -> None:
|
|
116
|
-
#
|
|
117
|
-
#
|
|
127
|
+
# TODO: Using a branch lets Toil commits that formerly passed start to
|
|
128
|
+
# fail CI when the branch moves.
|
|
118
129
|
os.chdir(wdl_conformance_test_repo)
|
|
119
|
-
repo_url = "https://github.com/
|
|
120
|
-
repo_branch = "wdl-1.1
|
|
130
|
+
repo_url = "https://github.com/openwdl/wdl.git"
|
|
131
|
+
repo_branch = "wdl-1.1"
|
|
121
132
|
commands1 = [
|
|
122
133
|
exactPython,
|
|
123
134
|
"setup_unit_tests.py",
|
|
@@ -193,6 +204,30 @@ class TestWDLConformance:
|
|
|
193
204
|
|
|
194
205
|
self.check(p)
|
|
195
206
|
|
|
207
|
+
# estimated running time: 10 minutes (once all the appropriate tests get
|
|
208
|
+
# marked as "development")
|
|
209
|
+
@slow
|
|
210
|
+
def test_conformance_tests_development(self, wdl_conformance_test_repo: Path) -> None:
|
|
211
|
+
os.chdir(wdl_conformance_test_repo)
|
|
212
|
+
commands = [
|
|
213
|
+
exactPython,
|
|
214
|
+
"run.py",
|
|
215
|
+
"--runner",
|
|
216
|
+
"toil-wdl-runner",
|
|
217
|
+
"--conformance-file",
|
|
218
|
+
"conformance.yaml",
|
|
219
|
+
"-v",
|
|
220
|
+
"development",
|
|
221
|
+
]
|
|
222
|
+
if WDL_CONFORMANCE_TESTS_UNSUPPORTED_BY_TOIL:
|
|
223
|
+
commands.append("--exclude-numbers")
|
|
224
|
+
commands.append(
|
|
225
|
+
",".join([str(t) for t in WDL_CONFORMANCE_TESTS_UNSUPPORTED_BY_TOIL])
|
|
226
|
+
)
|
|
227
|
+
p = subprocess.run(commands, capture_output=True)
|
|
228
|
+
|
|
229
|
+
self.check(p)
|
|
230
|
+
|
|
196
231
|
@slow
|
|
197
232
|
def test_conformance_tests_integration(
|
|
198
233
|
self, wdl_conformance_test_repo: Path
|
|
@@ -249,6 +284,7 @@ class TestWDL:
|
|
|
249
284
|
assert os.path.exists(result["ga4ghMd5.value"])
|
|
250
285
|
assert os.path.basename(result["ga4ghMd5.value"]) == "md5sum.txt"
|
|
251
286
|
|
|
287
|
+
@needs_online
|
|
252
288
|
def test_url_to_file(self, tmp_path: Path) -> None:
|
|
253
289
|
"""
|
|
254
290
|
Test if web URL strings can be coerced to usable Files.
|
|
@@ -308,6 +344,59 @@ class TestWDL:
|
|
|
308
344
|
assert isinstance(result["wait.result"], str)
|
|
309
345
|
assert result["wait.result"] == "waited"
|
|
310
346
|
|
|
347
|
+
def test_restart(self, tmp_path: Path) -> None:
|
|
348
|
+
"""
|
|
349
|
+
Test if a WDL workflow can be restarted and finish successfully.
|
|
350
|
+
"""
|
|
351
|
+
with get_data("test/wdl/testfiles/read_file.wdl") as wdl:
|
|
352
|
+
out_dir = tmp_path / "out"
|
|
353
|
+
file_path = tmp_path / "file"
|
|
354
|
+
jobstore_path = tmp_path / "tree"
|
|
355
|
+
command = (
|
|
356
|
+
self.base_command
|
|
357
|
+
+ [
|
|
358
|
+
str(wdl),
|
|
359
|
+
"-o",
|
|
360
|
+
str(out_dir),
|
|
361
|
+
"-i",
|
|
362
|
+
json.dumps({"read_file.input_string": str(file_path)}),
|
|
363
|
+
"--jobStore",
|
|
364
|
+
str(jobstore_path),
|
|
365
|
+
"--retryCount=0"
|
|
366
|
+
]
|
|
367
|
+
)
|
|
368
|
+
with pytest.raises(subprocess.CalledProcessError):
|
|
369
|
+
# The first time we run it, it should fail because it's trying
|
|
370
|
+
# to work on a nonexistent file from a string path.
|
|
371
|
+
result_json = subprocess.check_output(
|
|
372
|
+
command + ["--logCritical"]
|
|
373
|
+
)
|
|
374
|
+
|
|
375
|
+
# Then create the file
|
|
376
|
+
with open(file_path, "w") as f:
|
|
377
|
+
f.write("This is a line\n")
|
|
378
|
+
f.write("This is a different line")
|
|
379
|
+
|
|
380
|
+
# Now it should work
|
|
381
|
+
result_json = subprocess.check_output(
|
|
382
|
+
command + ["--restart"]
|
|
383
|
+
)
|
|
384
|
+
result = json.loads(result_json)
|
|
385
|
+
|
|
386
|
+
assert "read_file.lines" in result
|
|
387
|
+
assert isinstance(result["read_file.lines"], list)
|
|
388
|
+
assert result["read_file.lines"] == [
|
|
389
|
+
"This is a line",
|
|
390
|
+
"This is a different line"
|
|
391
|
+
]
|
|
392
|
+
|
|
393
|
+
# Since we were catching
|
|
394
|
+
# <https://github.com/DataBiosphere/toil/issues/5247> at file
|
|
395
|
+
# export, make sure we actually exported a file.
|
|
396
|
+
assert "read_file.remade_file" in result
|
|
397
|
+
assert isinstance(result["read_file.remade_file"], str)
|
|
398
|
+
assert os.path.exists(result["read_file.remade_file"])
|
|
399
|
+
|
|
311
400
|
@needs_singularity_or_docker
|
|
312
401
|
def test_workflow_file_deletion(self, tmp_path: Path) -> None:
|
|
313
402
|
"""
|
|
@@ -597,7 +686,7 @@ class TestWDL:
|
|
|
597
686
|
!= result_not_cached["random.value_written"]
|
|
598
687
|
)
|
|
599
688
|
|
|
600
|
-
def test_url_to_optional_file(self, tmp_path: Path) -> None:
|
|
689
|
+
def test_url_to_optional_file(self, tmp_path: Path, httpserver: HTTPServer) -> None:
|
|
601
690
|
"""
|
|
602
691
|
Test if missing and error-producing URLs are handled correctly for optional File? values.
|
|
603
692
|
"""
|
|
@@ -610,7 +699,15 @@ class TestWDL:
|
|
|
610
699
|
Return the parsed output.
|
|
611
700
|
"""
|
|
612
701
|
logger.info("Test optional file with HTTP code %s", code)
|
|
613
|
-
|
|
702
|
+
httpserver.expect_request(
|
|
703
|
+
"/" + str(code)
|
|
704
|
+
).respond_with_data(
|
|
705
|
+
"Some data",
|
|
706
|
+
status=code,
|
|
707
|
+
content_type="text/plain"
|
|
708
|
+
)
|
|
709
|
+
base_url = httpserver.url_for("/")
|
|
710
|
+
json_value = '{"url_to_optional_file.http_code": %d, "url_to_optional_file.base_url": "%s"}' % (code, base_url)
|
|
614
711
|
result_json = subprocess.check_output(
|
|
615
712
|
self.base_command
|
|
616
713
|
+ [
|
|
@@ -792,25 +889,29 @@ class TestWDL:
|
|
|
792
889
|
env["TOIL_DOCKSTORE_TOKEN"] = "99cf5578ebe94b194d7864630a86258fa3d6cedcc17d757b5dd49e64ee3b68c3"
|
|
793
890
|
# Enable history for when <https://github.com/DataBiosphere/toil/pull/5258> merges
|
|
794
891
|
env["TOIL_HISTORY"] = "True"
|
|
892
|
+
|
|
893
|
+
try:
|
|
894
|
+
output_log = subprocess.check_output(
|
|
895
|
+
self.base_command
|
|
896
|
+
+ [
|
|
897
|
+
wdl_file,
|
|
898
|
+
json_input,
|
|
899
|
+
"--logDebug",
|
|
900
|
+
"-o",
|
|
901
|
+
str(tmp_path),
|
|
902
|
+
"--outputDialect",
|
|
903
|
+
"miniwdl",
|
|
904
|
+
"--publishWorkflowMetrics=current",
|
|
905
|
+
]
|
|
906
|
+
+ (extra_args or []),
|
|
907
|
+
stderr=subprocess.STDOUT,
|
|
908
|
+
env=env,
|
|
909
|
+
).decode("utf-8", errors="replace")
|
|
910
|
+
except subprocess.CalledProcessError as e:
|
|
911
|
+
logger.error("Test run of Toil failed: %s", e.stdout.decode("utf-8", errors="replace"))
|
|
912
|
+
raise
|
|
795
913
|
|
|
796
|
-
output_log
|
|
797
|
-
self.base_command
|
|
798
|
-
+ [
|
|
799
|
-
wdl_file,
|
|
800
|
-
json_input,
|
|
801
|
-
"--logDebug",
|
|
802
|
-
"-o",
|
|
803
|
-
str(tmp_path),
|
|
804
|
-
"--outputDialect",
|
|
805
|
-
"miniwdl",
|
|
806
|
-
"--publishWorkflowMetrics=current",
|
|
807
|
-
]
|
|
808
|
-
+ (extra_args or []),
|
|
809
|
-
stderr=subprocess.STDOUT,
|
|
810
|
-
env=env,
|
|
811
|
-
)
|
|
812
|
-
|
|
813
|
-
assert b'Workflow metrics were accepted by Dockstore.' in output_log
|
|
914
|
+
assert "Workflow metrics were accepted by Dockstore." in output_log, f"No acceptance message in log: {output_log}"
|
|
814
915
|
|
|
815
916
|
@slow
|
|
816
917
|
@needs_docker_cuda
|
|
@@ -1055,7 +1156,7 @@ class TestWDLToilBench(unittest.TestCase):
|
|
|
1055
1156
|
assert "decl2" in result[0]
|
|
1056
1157
|
assert "successor" in result[1]
|
|
1057
1158
|
|
|
1058
|
-
def make_string_expr(self, to_parse: str) -> WDL.Expr.String:
|
|
1159
|
+
def make_string_expr(self, to_parse: str, expr_type: type[WDL.Expr.String] = WDL.Expr.String) -> WDL.Expr.String:
|
|
1059
1160
|
"""
|
|
1060
1161
|
Parse pseudo-WDL for testing whitespace removal.
|
|
1061
1162
|
"""
|
|
@@ -1066,122 +1167,7 @@ class TestWDLToilBench(unittest.TestCase):
|
|
|
1066
1167
|
for i in range(1, len(parts), 2):
|
|
1067
1168
|
parts[i] = WDL.Expr.Placeholder(pos, {}, WDL.Expr.Null(pos))
|
|
1068
1169
|
|
|
1069
|
-
return
|
|
1070
|
-
|
|
1071
|
-
def test_remove_common_leading_whitespace(self) -> None:
|
|
1072
|
-
"""
|
|
1073
|
-
Make sure leading whitespace removal works properly.
|
|
1074
|
-
"""
|
|
1075
|
-
|
|
1076
|
-
# For a single line, we remove its leading whitespace
|
|
1077
|
-
expr = self.make_string_expr(" a ~{b} c")
|
|
1078
|
-
trimmed = remove_common_leading_whitespace(expr)
|
|
1079
|
-
assert len(trimmed.parts) == 3
|
|
1080
|
-
assert trimmed.parts[0] == "a "
|
|
1081
|
-
assert trimmed.parts[2] == " c"
|
|
1082
|
-
|
|
1083
|
-
# Whitespace removed isn't affected by totally blank lines
|
|
1084
|
-
expr = self.make_string_expr(" \n\n a\n ~{stuff}\n b\n\n")
|
|
1085
|
-
trimmed = remove_common_leading_whitespace(expr)
|
|
1086
|
-
assert len(trimmed.parts) == 3
|
|
1087
|
-
assert trimmed.parts[0] == "\n\na\n"
|
|
1088
|
-
assert trimmed.parts[2] == "\nb\n\n"
|
|
1089
|
-
|
|
1090
|
-
# Unless blank toleration is off
|
|
1091
|
-
expr = self.make_string_expr(" \n\n a\n ~{stuff}\n b\n\n")
|
|
1092
|
-
trimmed = remove_common_leading_whitespace(expr, tolerate_blanks=False)
|
|
1093
|
-
assert len(trimmed.parts) == 3
|
|
1094
|
-
assert trimmed.parts[0] == " \n\n a\n "
|
|
1095
|
-
assert trimmed.parts[2] == "\n b\n\n"
|
|
1096
|
-
|
|
1097
|
-
# Whitespace is still removed if the first line doesn't have it before the newline
|
|
1098
|
-
expr = self.make_string_expr("\n a\n ~{stuff}\n b\n")
|
|
1099
|
-
trimmed = remove_common_leading_whitespace(expr)
|
|
1100
|
-
assert len(trimmed.parts) == 3
|
|
1101
|
-
assert trimmed.parts[0] == "\na\n"
|
|
1102
|
-
assert trimmed.parts[2] == "\nb\n"
|
|
1103
|
-
|
|
1104
|
-
# Whitespace is not removed if actual content is dedented
|
|
1105
|
-
expr = self.make_string_expr(" \n\n a\n ~{stuff}\nuhoh\n b\n\n")
|
|
1106
|
-
trimmed = remove_common_leading_whitespace(expr)
|
|
1107
|
-
assert len(trimmed.parts) == 3
|
|
1108
|
-
assert trimmed.parts[0] == " \n\n a\n "
|
|
1109
|
-
assert trimmed.parts[2] == "\nuhoh\n b\n\n"
|
|
1110
|
-
|
|
1111
|
-
# Unless dedents are tolerated
|
|
1112
|
-
expr = self.make_string_expr(" \n\n a\n ~{stuff}\nuhoh\n b\n\n")
|
|
1113
|
-
trimmed = remove_common_leading_whitespace(expr, tolerate_dedents=True)
|
|
1114
|
-
assert len(trimmed.parts) == 3
|
|
1115
|
-
assert trimmed.parts[0] == "\n\na\n"
|
|
1116
|
-
assert trimmed.parts[2] == "\nuhoh\nb\n\n"
|
|
1117
|
-
|
|
1118
|
-
# Whitespace is still removed if all-whitespace lines have less of it
|
|
1119
|
-
expr = self.make_string_expr("\n a\n ~{stuff}\n \n b\n")
|
|
1120
|
-
trimmed = remove_common_leading_whitespace(expr)
|
|
1121
|
-
assert len(trimmed.parts) == 3
|
|
1122
|
-
assert trimmed.parts[0] == "\na\n"
|
|
1123
|
-
assert trimmed.parts[2] == "\n\nb\n"
|
|
1124
|
-
|
|
1125
|
-
# Unless all-whitespace lines are not tolerated
|
|
1126
|
-
expr = self.make_string_expr("\n a\n ~{stuff}\n \n b\n")
|
|
1127
|
-
trimmed = remove_common_leading_whitespace(expr, tolerate_all_whitespace=False)
|
|
1128
|
-
assert len(trimmed.parts) == 3
|
|
1129
|
-
assert trimmed.parts[0] == "\n a\n "
|
|
1130
|
-
assert trimmed.parts[2] == "\n\n b\n"
|
|
1131
|
-
|
|
1132
|
-
# When mixed tabs and spaces are detected, nothing is changed.
|
|
1133
|
-
expr = self.make_string_expr("\n a\n\t~{stuff}\n b\n")
|
|
1134
|
-
trimmed = remove_common_leading_whitespace(expr)
|
|
1135
|
-
assert len(trimmed.parts) == 3
|
|
1136
|
-
assert trimmed.parts[0] == "\n a\n\t"
|
|
1137
|
-
assert trimmed.parts[2] == "\n b\n"
|
|
1138
|
-
|
|
1139
|
-
# When mixed tabs and spaces are not in the prefix, whitespace is removed.
|
|
1140
|
-
expr = self.make_string_expr("\n\ta\n\t~{stuff} \n\tb\n")
|
|
1141
|
-
trimmed = remove_common_leading_whitespace(expr)
|
|
1142
|
-
assert len(trimmed.parts) == 3
|
|
1143
|
-
assert trimmed.parts[0] == "\na\n"
|
|
1144
|
-
assert trimmed.parts[2] == " \nb\n"
|
|
1145
|
-
|
|
1146
|
-
# An empty string works
|
|
1147
|
-
expr = self.make_string_expr("")
|
|
1148
|
-
trimmed = remove_common_leading_whitespace(expr)
|
|
1149
|
-
assert len(trimmed.parts) == 1
|
|
1150
|
-
assert trimmed.parts[0] == ""
|
|
1151
|
-
|
|
1152
|
-
# A string of only whitespace is preserved as an all-whitespece line
|
|
1153
|
-
expr = self.make_string_expr("\t\t\t")
|
|
1154
|
-
trimmed = remove_common_leading_whitespace(expr)
|
|
1155
|
-
assert len(trimmed.parts) == 1
|
|
1156
|
-
assert trimmed.parts[0] == "\t\t\t"
|
|
1157
|
-
|
|
1158
|
-
# A string of only whitespace is trimmed when all-whitespace lines are not tolerated
|
|
1159
|
-
expr = self.make_string_expr("\t\t\t")
|
|
1160
|
-
trimmed = remove_common_leading_whitespace(expr, tolerate_all_whitespace=False)
|
|
1161
|
-
assert len(trimmed.parts) == 1
|
|
1162
|
-
assert trimmed.parts[0] == ""
|
|
1163
|
-
|
|
1164
|
-
# An empty expression works
|
|
1165
|
-
expr = WDL.Expr.String(
|
|
1166
|
-
WDL.Error.SourcePosition("nowhere", "nowhere", 0, 0, 0, 0), []
|
|
1167
|
-
)
|
|
1168
|
-
trimmed = remove_common_leading_whitespace(expr)
|
|
1169
|
-
assert len(trimmed.parts) == 0
|
|
1170
|
-
|
|
1171
|
-
# An expression of only placeholders works
|
|
1172
|
-
expr = self.make_string_expr("~{AAA}")
|
|
1173
|
-
trimmed = remove_common_leading_whitespace(expr)
|
|
1174
|
-
assert len(trimmed.parts) == 3
|
|
1175
|
-
assert trimmed.parts[0] == ""
|
|
1176
|
-
assert trimmed.parts[2] == ""
|
|
1177
|
-
|
|
1178
|
-
# The command flag is preserved
|
|
1179
|
-
expr = self.make_string_expr(" a ~{b} c")
|
|
1180
|
-
trimmed = remove_common_leading_whitespace(expr)
|
|
1181
|
-
assert trimmed.command == False
|
|
1182
|
-
expr.command = True
|
|
1183
|
-
trimmed = remove_common_leading_whitespace(expr)
|
|
1184
|
-
assert trimmed.command == True
|
|
1170
|
+
return expr_type(pos, parts)
|
|
1185
1171
|
|
|
1186
1172
|
def test_choose_human_readable_directory(self) -> None:
|
|
1187
1173
|
"""
|
|
@@ -1189,34 +1175,57 @@ class TestWDLToilBench(unittest.TestCase):
|
|
|
1189
1175
|
"""
|
|
1190
1176
|
|
|
1191
1177
|
from toil.wdl.wdltoil import (
|
|
1192
|
-
DirectoryNamingStateDict,
|
|
1193
1178
|
choose_human_readable_directory,
|
|
1194
1179
|
)
|
|
1195
1180
|
|
|
1196
|
-
|
|
1197
|
-
|
|
1198
|
-
# The first time we should get apath with the task name and without the ID
|
|
1181
|
+
# The first time we should get a path with the task name
|
|
1199
1182
|
first_chosen = choose_human_readable_directory(
|
|
1200
|
-
"root", "taskname", "
|
|
1183
|
+
"root", "taskname", "https://example.com/some/directory"
|
|
1201
1184
|
)
|
|
1202
1185
|
assert first_chosen.startswith("root")
|
|
1203
|
-
assert "taskname" in first_chosen
|
|
1204
|
-
assert "111-222-333" not in first_chosen
|
|
1205
1186
|
|
|
1206
|
-
# If we use the same
|
|
1207
|
-
|
|
1208
|
-
"root", "taskname", "
|
|
1187
|
+
# If we use the same parent we should get the same result
|
|
1188
|
+
same_parent = choose_human_readable_directory(
|
|
1189
|
+
"root", "taskname", "https://example.com/some/directory"
|
|
1190
|
+
)
|
|
1191
|
+
assert same_parent == first_chosen
|
|
1192
|
+
|
|
1193
|
+
# If we use a lower parent with a URL, we do not necessarily need to be
|
|
1194
|
+
# inside the higher parent.
|
|
1195
|
+
|
|
1196
|
+
# If we use a URL with a creative number of slashes, it should be distinct.
|
|
1197
|
+
slash_parent = choose_human_readable_directory(
|
|
1198
|
+
"root", "taskname", "https://example.com/some/directory//////"
|
|
1199
|
+
)
|
|
1200
|
+
assert slash_parent != first_chosen
|
|
1201
|
+
|
|
1202
|
+
# If we use the same parent URL but a different task we should get the same result
|
|
1203
|
+
other_task = choose_human_readable_directory(
|
|
1204
|
+
"root", "taskname2", "https://example.com/some/directory"
|
|
1205
|
+
)
|
|
1206
|
+
assert other_task == first_chosen
|
|
1207
|
+
|
|
1208
|
+
# If we use a different parent we should get a different result still obeying the constraints
|
|
1209
|
+
diff_parent = choose_human_readable_directory(
|
|
1210
|
+
"root", "taskname", "/data/tmp/files/somewhere"
|
|
1211
|
+
)
|
|
1212
|
+
assert diff_parent != first_chosen
|
|
1213
|
+
assert diff_parent.startswith("root")
|
|
1214
|
+
assert "taskname" in diff_parent
|
|
1215
|
+
|
|
1216
|
+
# If we use a subpath parent with a filename we should get a path inside it.
|
|
1217
|
+
diff_parent_subpath = choose_human_readable_directory(
|
|
1218
|
+
"root", "taskname", "/data/tmp/files/somewhere/else"
|
|
1209
1219
|
)
|
|
1210
|
-
assert
|
|
1220
|
+
assert os.path.dirname(diff_parent_subpath) == diff_parent
|
|
1211
1221
|
|
|
1212
|
-
# If we use a different
|
|
1213
|
-
|
|
1214
|
-
"root", "
|
|
1222
|
+
# If we use the same parent path but a different task we should get a different result.
|
|
1223
|
+
other_task_directory = choose_human_readable_directory(
|
|
1224
|
+
"root", "taskname2", "/data/tmp/files/somewhere"
|
|
1215
1225
|
)
|
|
1216
|
-
assert
|
|
1217
|
-
assert
|
|
1218
|
-
assert "
|
|
1219
|
-
assert "222-333-444" not in diff_id
|
|
1226
|
+
assert other_task_directory != diff_parent
|
|
1227
|
+
assert other_task_directory.startswith("root")
|
|
1228
|
+
assert "taskname2" in other_task_directory
|
|
1220
1229
|
|
|
1221
1230
|
def test_uri_packing(self) -> None:
|
|
1222
1231
|
"""
|
|
@@ -1232,7 +1241,7 @@ class TestWDLToilBench(unittest.TestCase):
|
|
|
1232
1241
|
file_basename = "thefile.txt"
|
|
1233
1242
|
|
|
1234
1243
|
# Pack and unpack it
|
|
1235
|
-
uri = pack_toil_uri(file_id, task_path, dir_id, file_basename)
|
|
1244
|
+
uri = pack_toil_uri(file_id, task_path, str(dir_id), file_basename)
|
|
1236
1245
|
unpacked = unpack_toil_uri(uri)
|
|
1237
1246
|
|
|
1238
1247
|
# Make sure we got what we put in
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import unittest
|
|
2
2
|
from uuid import uuid4
|
|
3
3
|
|
|
4
|
+
import logging
|
|
4
5
|
import pytest
|
|
5
6
|
|
|
6
7
|
from toil.provisioners import cluster_factory
|
|
@@ -12,6 +13,8 @@ from toil.test.wdl.wdltoil_test import (
|
|
|
12
13
|
)
|
|
13
14
|
|
|
14
15
|
|
|
16
|
+
logger = logging.getLogger(__name__)
|
|
17
|
+
|
|
15
18
|
@integrative
|
|
16
19
|
@slow
|
|
17
20
|
@pytest.mark.timeout(1800)
|
|
@@ -52,6 +55,7 @@ class WDLKubernetesClusterTest(AbstractClusterTest):
|
|
|
52
55
|
workflow that performs an image pull on the worker.
|
|
53
56
|
:return:
|
|
54
57
|
"""
|
|
58
|
+
|
|
55
59
|
self.numWorkers = "1"
|
|
56
60
|
self.requestedLeaderStorage = 30
|
|
57
61
|
# create the cluster
|
|
@@ -64,6 +68,8 @@ class WDLKubernetesClusterTest(AbstractClusterTest):
|
|
|
64
68
|
|
|
65
69
|
wdl_dir = "wdl_conformance_tests"
|
|
66
70
|
|
|
71
|
+
logger.info("Cloning WDL tests onto cluster...")
|
|
72
|
+
|
|
67
73
|
# get the wdl-conformance-tests repo to get WDL tasks to run
|
|
68
74
|
self.sshUtil(
|
|
69
75
|
[
|
|
@@ -78,6 +84,9 @@ class WDLKubernetesClusterTest(AbstractClusterTest):
|
|
|
78
84
|
|
|
79
85
|
# run WDL workflow that will run singularity
|
|
80
86
|
test_options = [f"tests/md5sum/md5sum.wdl", f"tests/md5sum/md5sum.json"]
|
|
87
|
+
|
|
88
|
+
logger.info("Running workflow...")
|
|
89
|
+
|
|
81
90
|
self.sshUtil(
|
|
82
91
|
[
|
|
83
92
|
"bash",
|
toil/utils/toilDebugFile.py
CHANGED
|
@@ -47,12 +47,12 @@ def fetchJobStoreFiles(jobStore: FileJobStore, options: argparse.Namespace) -> N
|
|
|
47
47
|
jobStoreHits = glob(directoryname=options.jobStore, glob_pattern=jobStoreFile)
|
|
48
48
|
for jobStoreFileID in jobStoreHits:
|
|
49
49
|
logger.debug(
|
|
50
|
-
f"Copying job store file: {jobStoreFileID} to {options.localFilePath
|
|
50
|
+
f"Copying job store file: {jobStoreFileID} to {options.localFilePath}"
|
|
51
51
|
)
|
|
52
52
|
jobStore.read_file(
|
|
53
53
|
jobStoreFileID,
|
|
54
54
|
os.path.join(
|
|
55
|
-
options.localFilePath
|
|
55
|
+
options.localFilePath, os.path.basename(jobStoreFileID)
|
|
56
56
|
),
|
|
57
57
|
symlink=options.useSymlinks,
|
|
58
58
|
)
|
|
@@ -97,7 +97,10 @@ def printContentsOfJobStore(
|
|
|
97
97
|
def main() -> None:
|
|
98
98
|
parser = parser_with_common_options(jobstore_option=True, prog="toil debug-file")
|
|
99
99
|
parser.add_argument(
|
|
100
|
-
"--localFilePath",
|
|
100
|
+
"--localFilePath",
|
|
101
|
+
type=str,
|
|
102
|
+
default=".",
|
|
103
|
+
help="Location to which to copy job store files."
|
|
101
104
|
)
|
|
102
105
|
parser.add_argument(
|
|
103
106
|
"--fetch",
|
toil/utils/toilSshCluster.py
CHANGED
|
@@ -14,6 +14,7 @@
|
|
|
14
14
|
"""SSH into the toil appliance container running on the leader of the cluster."""
|
|
15
15
|
import argparse
|
|
16
16
|
import logging
|
|
17
|
+
import socket
|
|
17
18
|
import sys
|
|
18
19
|
|
|
19
20
|
from toil.common import parser_with_common_options
|
|
@@ -22,6 +23,22 @@ from toil.statsAndLogging import set_logging_from_options
|
|
|
22
23
|
|
|
23
24
|
logger = logging.getLogger(__name__)
|
|
24
25
|
|
|
26
|
+
def have_ipv6() -> bool:
|
|
27
|
+
"""
|
|
28
|
+
Return True if the IPv6 loopback interface is useable.
|
|
29
|
+
"""
|
|
30
|
+
|
|
31
|
+
# We sniff for actual IPv6 like in
|
|
32
|
+
# https://github.com/urllib3/urllib3/pull/840/files in urllib3 (which we
|
|
33
|
+
# don't depend on)
|
|
34
|
+
if socket.has_ipv6:
|
|
35
|
+
# Built with IPv6 support
|
|
36
|
+
try:
|
|
37
|
+
socket.socket(socket.AF_INET6).bind(("::1", 0))
|
|
38
|
+
return True
|
|
39
|
+
except Exception:
|
|
40
|
+
pass
|
|
41
|
+
return False
|
|
25
42
|
|
|
26
43
|
def main() -> None:
|
|
27
44
|
parser = parser_with_common_options(
|
|
@@ -73,6 +90,12 @@ def main() -> None:
|
|
|
73
90
|
["-L", f"{options.grafana_port}:localhost:3000", "-L", "9090:localhost:9090"]
|
|
74
91
|
)
|
|
75
92
|
|
|
93
|
+
if not have_ipv6():
|
|
94
|
+
# If we try to do SSH port forwarding without any other options, but
|
|
95
|
+
# IPv6 is turned off on the host, we might get complaints that we
|
|
96
|
+
# "Cannot assign requested address" on ports on [::1].
|
|
97
|
+
sshOptions.append("-4")
|
|
98
|
+
|
|
76
99
|
try:
|
|
77
100
|
cluster.getLeader().sshAppliance(
|
|
78
101
|
*command,
|
toil/utils/toilStats.py
CHANGED
|
@@ -326,6 +326,8 @@ def sprint_tag(
|
|
|
326
326
|
out_str += header + "\n"
|
|
327
327
|
out_str += sub_header + "\n"
|
|
328
328
|
out_str += tag_str + "\n"
|
|
329
|
+
if tag.excess_cpu > 0:
|
|
330
|
+
out_str += f" ({tag.excess_cpu} used more CPU than requested!)\n"
|
|
329
331
|
return out_str
|
|
330
332
|
|
|
331
333
|
|
|
@@ -507,13 +509,25 @@ def build_element(
|
|
|
507
509
|
float(item.get(category_key, defaults[category])), category
|
|
508
510
|
)
|
|
509
511
|
values.append(category_value)
|
|
510
|
-
|
|
512
|
+
|
|
513
|
+
excess_cpu_items = 0
|
|
511
514
|
for index in range(0, len(item_values[CATEGORIES[0]])):
|
|
512
515
|
# For each item, compute the computed categories
|
|
513
|
-
|
|
516
|
+
|
|
517
|
+
# Compute wait time (allocated CPU time wasted).
|
|
518
|
+
# Note that if any item uses *more* CPU cores than requested, at any
|
|
519
|
+
# time, that decreases the amount of wait we're able to see from that
|
|
520
|
+
# item. If it hapens a lot, our computed wait could go negative, so we
|
|
521
|
+
# bound it below at 0.
|
|
522
|
+
wait_value = (
|
|
514
523
|
item_values["time"][index] * item_values["cores"][index]
|
|
515
524
|
- item_values["clock"][index]
|
|
516
525
|
)
|
|
526
|
+
if wait_value < 0:
|
|
527
|
+
# Remember an item used more CPU than allocated.
|
|
528
|
+
excess_cpu_items += 1
|
|
529
|
+
wait_value = 0
|
|
530
|
+
item_values["wait"].append(wait_value)
|
|
517
531
|
|
|
518
532
|
for category, values in item_values.items():
|
|
519
533
|
values.sort()
|
|
@@ -531,6 +545,7 @@ def build_element(
|
|
|
531
545
|
item_element["average_" + category] = float(sum(values) / len(values))
|
|
532
546
|
item_element["min_" + category] = float(min(values))
|
|
533
547
|
item_element["max_" + category] = float(max(values))
|
|
548
|
+
item_element["excess_cpu"] = excess_cpu_items
|
|
534
549
|
|
|
535
550
|
element[item_name] = item_element
|
|
536
551
|
|
toil/version.py
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
|
-
baseVersion = '
|
|
1
|
+
baseVersion = '9.1.0'
|
|
2
2
|
cgcloudVersion = '1.6.0a1.dev393'
|
|
3
|
-
version = '
|
|
4
|
-
cacheTag = 'cache-local-py3.
|
|
5
|
-
mainCacheTag = 'cache-master-py3.
|
|
6
|
-
distVersion = '
|
|
7
|
-
exactPython = 'python3.
|
|
8
|
-
python = 'python3.
|
|
9
|
-
dockerTag = '
|
|
10
|
-
currentCommit = '
|
|
3
|
+
version = '9.1.0-e341bb669efe78f93308e5ff1f02f7e375973511'
|
|
4
|
+
cacheTag = 'cache-local-py3.9'
|
|
5
|
+
mainCacheTag = 'cache-master-py3.9'
|
|
6
|
+
distVersion = '9.1.0'
|
|
7
|
+
exactPython = 'python3.9'
|
|
8
|
+
python = 'python3.9'
|
|
9
|
+
dockerTag = '9.1.0-e341bb669efe78f93308e5ff1f02f7e375973511-py3.9'
|
|
10
|
+
currentCommit = 'e341bb669efe78f93308e5ff1f02f7e375973511'
|
|
11
11
|
dockerRegistry = 'quay.io/ucsc_cgl'
|
|
12
12
|
dockerName = 'toil'
|
|
13
13
|
dirty = False
|
|
14
|
-
cwltool_version = '3.1.
|
|
14
|
+
cwltool_version = '3.1.20250715140722'
|