toil 8.2.0__py3-none-any.whl → 9.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (99) hide show
  1. toil/batchSystems/abstractBatchSystem.py +13 -5
  2. toil/batchSystems/abstractGridEngineBatchSystem.py +17 -5
  3. toil/batchSystems/kubernetes.py +13 -2
  4. toil/batchSystems/mesos/batchSystem.py +33 -2
  5. toil/batchSystems/registry.py +15 -118
  6. toil/batchSystems/slurm.py +191 -16
  7. toil/common.py +20 -1
  8. toil/cwl/cwltoil.py +97 -119
  9. toil/cwl/utils.py +103 -3
  10. toil/fileStores/__init__.py +1 -1
  11. toil/fileStores/abstractFileStore.py +5 -2
  12. toil/fileStores/cachingFileStore.py +1 -1
  13. toil/job.py +30 -14
  14. toil/jobStores/abstractJobStore.py +35 -255
  15. toil/jobStores/aws/jobStore.py +864 -1964
  16. toil/jobStores/aws/utils.py +24 -270
  17. toil/jobStores/fileJobStore.py +2 -1
  18. toil/jobStores/googleJobStore.py +32 -13
  19. toil/jobStores/utils.py +0 -327
  20. toil/leader.py +27 -22
  21. toil/lib/accelerators.py +1 -1
  22. toil/lib/aws/config.py +22 -0
  23. toil/lib/aws/s3.py +477 -9
  24. toil/lib/aws/utils.py +22 -33
  25. toil/lib/checksum.py +88 -0
  26. toil/lib/conversions.py +33 -31
  27. toil/lib/directory.py +217 -0
  28. toil/lib/ec2.py +97 -29
  29. toil/lib/exceptions.py +2 -1
  30. toil/lib/expando.py +2 -2
  31. toil/lib/generatedEC2Lists.py +138 -19
  32. toil/lib/io.py +33 -2
  33. toil/lib/memoize.py +21 -7
  34. toil/lib/misc.py +1 -1
  35. toil/lib/pipes.py +385 -0
  36. toil/lib/plugins.py +106 -0
  37. toil/lib/retry.py +1 -1
  38. toil/lib/threading.py +1 -1
  39. toil/lib/url.py +320 -0
  40. toil/lib/web.py +4 -5
  41. toil/options/cwl.py +13 -1
  42. toil/options/runner.py +17 -10
  43. toil/options/wdl.py +12 -1
  44. toil/provisioners/__init__.py +5 -2
  45. toil/provisioners/aws/__init__.py +43 -36
  46. toil/provisioners/aws/awsProvisioner.py +47 -15
  47. toil/provisioners/node.py +60 -12
  48. toil/resource.py +3 -13
  49. toil/server/app.py +12 -6
  50. toil/server/cli/wes_cwl_runner.py +2 -2
  51. toil/server/wes/abstract_backend.py +21 -43
  52. toil/server/wes/toil_backend.py +2 -2
  53. toil/test/__init__.py +16 -18
  54. toil/test/batchSystems/batchSystemTest.py +2 -9
  55. toil/test/batchSystems/batch_system_plugin_test.py +7 -0
  56. toil/test/batchSystems/test_slurm.py +103 -14
  57. toil/test/cwl/cwlTest.py +181 -8
  58. toil/test/cwl/staging_cat.cwl +27 -0
  59. toil/test/cwl/staging_make_file.cwl +25 -0
  60. toil/test/cwl/staging_workflow.cwl +43 -0
  61. toil/test/cwl/zero_default.cwl +61 -0
  62. toil/test/docs/scripts/tutorial_staging.py +17 -8
  63. toil/test/docs/scriptsTest.py +2 -1
  64. toil/test/jobStores/jobStoreTest.py +23 -133
  65. toil/test/lib/aws/test_iam.py +7 -7
  66. toil/test/lib/aws/test_s3.py +30 -33
  67. toil/test/lib/aws/test_utils.py +9 -9
  68. toil/test/lib/test_url.py +69 -0
  69. toil/test/lib/url_plugin_test.py +105 -0
  70. toil/test/provisioners/aws/awsProvisionerTest.py +60 -7
  71. toil/test/provisioners/clusterTest.py +15 -2
  72. toil/test/provisioners/gceProvisionerTest.py +1 -1
  73. toil/test/server/serverTest.py +78 -36
  74. toil/test/src/autoDeploymentTest.py +2 -3
  75. toil/test/src/fileStoreTest.py +89 -87
  76. toil/test/utils/ABCWorkflowDebug/ABC.txt +1 -0
  77. toil/test/utils/ABCWorkflowDebug/debugWorkflow.py +4 -4
  78. toil/test/utils/toilKillTest.py +35 -28
  79. toil/test/wdl/md5sum/md5sum-gs.json +1 -1
  80. toil/test/wdl/md5sum/md5sum.json +1 -1
  81. toil/test/wdl/testfiles/read_file.wdl +18 -0
  82. toil/test/wdl/testfiles/url_to_optional_file.wdl +2 -1
  83. toil/test/wdl/wdltoil_test.py +171 -162
  84. toil/test/wdl/wdltoil_test_kubernetes.py +9 -0
  85. toil/utils/toilDebugFile.py +6 -3
  86. toil/utils/toilSshCluster.py +23 -0
  87. toil/utils/toilStats.py +17 -2
  88. toil/utils/toilUpdateEC2Instances.py +1 -0
  89. toil/version.py +10 -10
  90. toil/wdl/wdltoil.py +1179 -825
  91. toil/worker.py +16 -8
  92. {toil-8.2.0.dist-info → toil-9.1.0.dist-info}/METADATA +32 -32
  93. {toil-8.2.0.dist-info → toil-9.1.0.dist-info}/RECORD +97 -85
  94. {toil-8.2.0.dist-info → toil-9.1.0.dist-info}/WHEEL +1 -1
  95. toil/lib/iterables.py +0 -112
  96. toil/test/docs/scripts/stagingExampleFiles/in.txt +0 -1
  97. {toil-8.2.0.dist-info → toil-9.1.0.dist-info}/entry_points.txt +0 -0
  98. {toil-8.2.0.dist-info → toil-9.1.0.dist-info}/licenses/LICENSE +0 -0
  99. {toil-8.2.0.dist-info → toil-9.1.0.dist-info}/top_level.txt +0 -0
@@ -13,6 +13,8 @@ from unittest.mock import patch
13
13
  from uuid import uuid4
14
14
 
15
15
  import pytest
16
+ from pytest_httpserver import HTTPServer
17
+
16
18
  import WDL.Error
17
19
  import WDL.Expr
18
20
 
@@ -23,6 +25,7 @@ from toil.test import (
23
25
  needs_docker,
24
26
  needs_docker_cuda,
25
27
  needs_google_storage,
28
+ needs_online,
26
29
  needs_singularity_or_docker,
27
30
  needs_wdl,
28
31
  slow,
@@ -32,14 +35,13 @@ from toil.wdl.wdltoil import (
32
35
  WDLSectionJob,
33
36
  WDLWorkflowGraph,
34
37
  parse_disks,
35
- remove_common_leading_whitespace,
36
38
  )
37
39
 
38
40
  logger = logging.getLogger(__name__)
39
41
 
40
42
 
41
43
  WDL_CONFORMANCE_TEST_REPO = "https://github.com/DataBiosphere/wdl-conformance-tests.git"
42
- WDL_CONFORMANCE_TEST_COMMIT = "baf44bcc7e6f6927540adf77d91b26a5558ae4b7"
44
+ WDL_CONFORMANCE_TEST_COMMIT = "46b5f85ee38ec60d0b8b9c35928b5104a2af83d5"
43
45
  # These tests are known to require things not implemented by
44
46
  # Toil and will not be run in CI.
45
47
  WDL_CONFORMANCE_TESTS_UNSUPPORTED_BY_TOIL = [
@@ -108,16 +110,25 @@ class TestWDLConformance:
108
110
  "Failed process standard error: %s",
109
111
  p.stderr.decode("utf-8", errors="replace"),
110
112
  )
113
+ else:
114
+ logger.debug(
115
+ "Successful process standard output: %s",
116
+ p.stdout.decode("utf-8", errors="replace"),
117
+ )
118
+ logger.debug(
119
+ "Successful process standard error: %s",
120
+ p.stderr.decode("utf-8", errors="replace"),
121
+ )
111
122
 
112
123
  p.check_returncode()
113
124
 
114
125
  @slow
115
126
  def test_unit_tests_v11(self, wdl_conformance_test_repo: Path) -> None:
116
- # There are still some bugs with the WDL spec, use a fixed version until
117
- # See comments of https://github.com/openwdl/wdl/pull/669
127
+ # TODO: Using a branch lets Toil commits that formerly passed start to
128
+ # fail CI when the branch moves.
118
129
  os.chdir(wdl_conformance_test_repo)
119
- repo_url = "https://github.com/stxue1/wdl.git"
120
- repo_branch = "wdl-1.1.3-fixes"
130
+ repo_url = "https://github.com/openwdl/wdl.git"
131
+ repo_branch = "wdl-1.1"
121
132
  commands1 = [
122
133
  exactPython,
123
134
  "setup_unit_tests.py",
@@ -193,6 +204,30 @@ class TestWDLConformance:
193
204
 
194
205
  self.check(p)
195
206
 
207
+ # estimated running time: 10 minutes (once all the appropriate tests get
208
+ # marked as "development")
209
+ @slow
210
+ def test_conformance_tests_development(self, wdl_conformance_test_repo: Path) -> None:
211
+ os.chdir(wdl_conformance_test_repo)
212
+ commands = [
213
+ exactPython,
214
+ "run.py",
215
+ "--runner",
216
+ "toil-wdl-runner",
217
+ "--conformance-file",
218
+ "conformance.yaml",
219
+ "-v",
220
+ "development",
221
+ ]
222
+ if WDL_CONFORMANCE_TESTS_UNSUPPORTED_BY_TOIL:
223
+ commands.append("--exclude-numbers")
224
+ commands.append(
225
+ ",".join([str(t) for t in WDL_CONFORMANCE_TESTS_UNSUPPORTED_BY_TOIL])
226
+ )
227
+ p = subprocess.run(commands, capture_output=True)
228
+
229
+ self.check(p)
230
+
196
231
  @slow
197
232
  def test_conformance_tests_integration(
198
233
  self, wdl_conformance_test_repo: Path
@@ -249,6 +284,7 @@ class TestWDL:
249
284
  assert os.path.exists(result["ga4ghMd5.value"])
250
285
  assert os.path.basename(result["ga4ghMd5.value"]) == "md5sum.txt"
251
286
 
287
+ @needs_online
252
288
  def test_url_to_file(self, tmp_path: Path) -> None:
253
289
  """
254
290
  Test if web URL strings can be coerced to usable Files.
@@ -308,6 +344,59 @@ class TestWDL:
308
344
  assert isinstance(result["wait.result"], str)
309
345
  assert result["wait.result"] == "waited"
310
346
 
347
+ def test_restart(self, tmp_path: Path) -> None:
348
+ """
349
+ Test if a WDL workflow can be restarted and finish successfully.
350
+ """
351
+ with get_data("test/wdl/testfiles/read_file.wdl") as wdl:
352
+ out_dir = tmp_path / "out"
353
+ file_path = tmp_path / "file"
354
+ jobstore_path = tmp_path / "tree"
355
+ command = (
356
+ self.base_command
357
+ + [
358
+ str(wdl),
359
+ "-o",
360
+ str(out_dir),
361
+ "-i",
362
+ json.dumps({"read_file.input_string": str(file_path)}),
363
+ "--jobStore",
364
+ str(jobstore_path),
365
+ "--retryCount=0"
366
+ ]
367
+ )
368
+ with pytest.raises(subprocess.CalledProcessError):
369
+ # The first time we run it, it should fail because it's trying
370
+ # to work on a nonexistent file from a string path.
371
+ result_json = subprocess.check_output(
372
+ command + ["--logCritical"]
373
+ )
374
+
375
+ # Then create the file
376
+ with open(file_path, "w") as f:
377
+ f.write("This is a line\n")
378
+ f.write("This is a different line")
379
+
380
+ # Now it should work
381
+ result_json = subprocess.check_output(
382
+ command + ["--restart"]
383
+ )
384
+ result = json.loads(result_json)
385
+
386
+ assert "read_file.lines" in result
387
+ assert isinstance(result["read_file.lines"], list)
388
+ assert result["read_file.lines"] == [
389
+ "This is a line",
390
+ "This is a different line"
391
+ ]
392
+
393
+ # Since we were catching
394
+ # <https://github.com/DataBiosphere/toil/issues/5247> at file
395
+ # export, make sure we actually exported a file.
396
+ assert "read_file.remade_file" in result
397
+ assert isinstance(result["read_file.remade_file"], str)
398
+ assert os.path.exists(result["read_file.remade_file"])
399
+
311
400
  @needs_singularity_or_docker
312
401
  def test_workflow_file_deletion(self, tmp_path: Path) -> None:
313
402
  """
@@ -597,7 +686,7 @@ class TestWDL:
597
686
  != result_not_cached["random.value_written"]
598
687
  )
599
688
 
600
- def test_url_to_optional_file(self, tmp_path: Path) -> None:
689
+ def test_url_to_optional_file(self, tmp_path: Path, httpserver: HTTPServer) -> None:
601
690
  """
602
691
  Test if missing and error-producing URLs are handled correctly for optional File? values.
603
692
  """
@@ -610,7 +699,15 @@ class TestWDL:
610
699
  Return the parsed output.
611
700
  """
612
701
  logger.info("Test optional file with HTTP code %s", code)
613
- json_value = '{"url_to_optional_file.http_code": %d}' % code
702
+ httpserver.expect_request(
703
+ "/" + str(code)
704
+ ).respond_with_data(
705
+ "Some data",
706
+ status=code,
707
+ content_type="text/plain"
708
+ )
709
+ base_url = httpserver.url_for("/")
710
+ json_value = '{"url_to_optional_file.http_code": %d, "url_to_optional_file.base_url": "%s"}' % (code, base_url)
614
711
  result_json = subprocess.check_output(
615
712
  self.base_command
616
713
  + [
@@ -792,25 +889,29 @@ class TestWDL:
792
889
  env["TOIL_DOCKSTORE_TOKEN"] = "99cf5578ebe94b194d7864630a86258fa3d6cedcc17d757b5dd49e64ee3b68c3"
793
890
  # Enable history for when <https://github.com/DataBiosphere/toil/pull/5258> merges
794
891
  env["TOIL_HISTORY"] = "True"
892
+
893
+ try:
894
+ output_log = subprocess.check_output(
895
+ self.base_command
896
+ + [
897
+ wdl_file,
898
+ json_input,
899
+ "--logDebug",
900
+ "-o",
901
+ str(tmp_path),
902
+ "--outputDialect",
903
+ "miniwdl",
904
+ "--publishWorkflowMetrics=current",
905
+ ]
906
+ + (extra_args or []),
907
+ stderr=subprocess.STDOUT,
908
+ env=env,
909
+ ).decode("utf-8", errors="replace")
910
+ except subprocess.CalledProcessError as e:
911
+ logger.error("Test run of Toil failed: %s", e.stdout.decode("utf-8", errors="replace"))
912
+ raise
795
913
 
796
- output_log = subprocess.check_output(
797
- self.base_command
798
- + [
799
- wdl_file,
800
- json_input,
801
- "--logDebug",
802
- "-o",
803
- str(tmp_path),
804
- "--outputDialect",
805
- "miniwdl",
806
- "--publishWorkflowMetrics=current",
807
- ]
808
- + (extra_args or []),
809
- stderr=subprocess.STDOUT,
810
- env=env,
811
- )
812
-
813
- assert b'Workflow metrics were accepted by Dockstore.' in output_log
914
+ assert "Workflow metrics were accepted by Dockstore." in output_log, f"No acceptance message in log: {output_log}"
814
915
 
815
916
  @slow
816
917
  @needs_docker_cuda
@@ -1055,7 +1156,7 @@ class TestWDLToilBench(unittest.TestCase):
1055
1156
  assert "decl2" in result[0]
1056
1157
  assert "successor" in result[1]
1057
1158
 
1058
- def make_string_expr(self, to_parse: str) -> WDL.Expr.String:
1159
+ def make_string_expr(self, to_parse: str, expr_type: type[WDL.Expr.String] = WDL.Expr.String) -> WDL.Expr.String:
1059
1160
  """
1060
1161
  Parse pseudo-WDL for testing whitespace removal.
1061
1162
  """
@@ -1066,122 +1167,7 @@ class TestWDLToilBench(unittest.TestCase):
1066
1167
  for i in range(1, len(parts), 2):
1067
1168
  parts[i] = WDL.Expr.Placeholder(pos, {}, WDL.Expr.Null(pos))
1068
1169
 
1069
- return WDL.Expr.String(pos, parts)
1070
-
1071
- def test_remove_common_leading_whitespace(self) -> None:
1072
- """
1073
- Make sure leading whitespace removal works properly.
1074
- """
1075
-
1076
- # For a single line, we remove its leading whitespace
1077
- expr = self.make_string_expr(" a ~{b} c")
1078
- trimmed = remove_common_leading_whitespace(expr)
1079
- assert len(trimmed.parts) == 3
1080
- assert trimmed.parts[0] == "a "
1081
- assert trimmed.parts[2] == " c"
1082
-
1083
- # Whitespace removed isn't affected by totally blank lines
1084
- expr = self.make_string_expr(" \n\n a\n ~{stuff}\n b\n\n")
1085
- trimmed = remove_common_leading_whitespace(expr)
1086
- assert len(trimmed.parts) == 3
1087
- assert trimmed.parts[0] == "\n\na\n"
1088
- assert trimmed.parts[2] == "\nb\n\n"
1089
-
1090
- # Unless blank toleration is off
1091
- expr = self.make_string_expr(" \n\n a\n ~{stuff}\n b\n\n")
1092
- trimmed = remove_common_leading_whitespace(expr, tolerate_blanks=False)
1093
- assert len(trimmed.parts) == 3
1094
- assert trimmed.parts[0] == " \n\n a\n "
1095
- assert trimmed.parts[2] == "\n b\n\n"
1096
-
1097
- # Whitespace is still removed if the first line doesn't have it before the newline
1098
- expr = self.make_string_expr("\n a\n ~{stuff}\n b\n")
1099
- trimmed = remove_common_leading_whitespace(expr)
1100
- assert len(trimmed.parts) == 3
1101
- assert trimmed.parts[0] == "\na\n"
1102
- assert trimmed.parts[2] == "\nb\n"
1103
-
1104
- # Whitespace is not removed if actual content is dedented
1105
- expr = self.make_string_expr(" \n\n a\n ~{stuff}\nuhoh\n b\n\n")
1106
- trimmed = remove_common_leading_whitespace(expr)
1107
- assert len(trimmed.parts) == 3
1108
- assert trimmed.parts[0] == " \n\n a\n "
1109
- assert trimmed.parts[2] == "\nuhoh\n b\n\n"
1110
-
1111
- # Unless dedents are tolerated
1112
- expr = self.make_string_expr(" \n\n a\n ~{stuff}\nuhoh\n b\n\n")
1113
- trimmed = remove_common_leading_whitespace(expr, tolerate_dedents=True)
1114
- assert len(trimmed.parts) == 3
1115
- assert trimmed.parts[0] == "\n\na\n"
1116
- assert trimmed.parts[2] == "\nuhoh\nb\n\n"
1117
-
1118
- # Whitespace is still removed if all-whitespace lines have less of it
1119
- expr = self.make_string_expr("\n a\n ~{stuff}\n \n b\n")
1120
- trimmed = remove_common_leading_whitespace(expr)
1121
- assert len(trimmed.parts) == 3
1122
- assert trimmed.parts[0] == "\na\n"
1123
- assert trimmed.parts[2] == "\n\nb\n"
1124
-
1125
- # Unless all-whitespace lines are not tolerated
1126
- expr = self.make_string_expr("\n a\n ~{stuff}\n \n b\n")
1127
- trimmed = remove_common_leading_whitespace(expr, tolerate_all_whitespace=False)
1128
- assert len(trimmed.parts) == 3
1129
- assert trimmed.parts[0] == "\n a\n "
1130
- assert trimmed.parts[2] == "\n\n b\n"
1131
-
1132
- # When mixed tabs and spaces are detected, nothing is changed.
1133
- expr = self.make_string_expr("\n a\n\t~{stuff}\n b\n")
1134
- trimmed = remove_common_leading_whitespace(expr)
1135
- assert len(trimmed.parts) == 3
1136
- assert trimmed.parts[0] == "\n a\n\t"
1137
- assert trimmed.parts[2] == "\n b\n"
1138
-
1139
- # When mixed tabs and spaces are not in the prefix, whitespace is removed.
1140
- expr = self.make_string_expr("\n\ta\n\t~{stuff} \n\tb\n")
1141
- trimmed = remove_common_leading_whitespace(expr)
1142
- assert len(trimmed.parts) == 3
1143
- assert trimmed.parts[0] == "\na\n"
1144
- assert trimmed.parts[2] == " \nb\n"
1145
-
1146
- # An empty string works
1147
- expr = self.make_string_expr("")
1148
- trimmed = remove_common_leading_whitespace(expr)
1149
- assert len(trimmed.parts) == 1
1150
- assert trimmed.parts[0] == ""
1151
-
1152
- # A string of only whitespace is preserved as an all-whitespece line
1153
- expr = self.make_string_expr("\t\t\t")
1154
- trimmed = remove_common_leading_whitespace(expr)
1155
- assert len(trimmed.parts) == 1
1156
- assert trimmed.parts[0] == "\t\t\t"
1157
-
1158
- # A string of only whitespace is trimmed when all-whitespace lines are not tolerated
1159
- expr = self.make_string_expr("\t\t\t")
1160
- trimmed = remove_common_leading_whitespace(expr, tolerate_all_whitespace=False)
1161
- assert len(trimmed.parts) == 1
1162
- assert trimmed.parts[0] == ""
1163
-
1164
- # An empty expression works
1165
- expr = WDL.Expr.String(
1166
- WDL.Error.SourcePosition("nowhere", "nowhere", 0, 0, 0, 0), []
1167
- )
1168
- trimmed = remove_common_leading_whitespace(expr)
1169
- assert len(trimmed.parts) == 0
1170
-
1171
- # An expression of only placeholders works
1172
- expr = self.make_string_expr("~{AAA}")
1173
- trimmed = remove_common_leading_whitespace(expr)
1174
- assert len(trimmed.parts) == 3
1175
- assert trimmed.parts[0] == ""
1176
- assert trimmed.parts[2] == ""
1177
-
1178
- # The command flag is preserved
1179
- expr = self.make_string_expr(" a ~{b} c")
1180
- trimmed = remove_common_leading_whitespace(expr)
1181
- assert trimmed.command == False
1182
- expr.command = True
1183
- trimmed = remove_common_leading_whitespace(expr)
1184
- assert trimmed.command == True
1170
+ return expr_type(pos, parts)
1185
1171
 
1186
1172
  def test_choose_human_readable_directory(self) -> None:
1187
1173
  """
@@ -1189,34 +1175,57 @@ class TestWDLToilBench(unittest.TestCase):
1189
1175
  """
1190
1176
 
1191
1177
  from toil.wdl.wdltoil import (
1192
- DirectoryNamingStateDict,
1193
1178
  choose_human_readable_directory,
1194
1179
  )
1195
1180
 
1196
- state: DirectoryNamingStateDict = {}
1197
-
1198
- # The first time we should get apath with the task name and without the ID
1181
+ # The first time we should get a path with the task name
1199
1182
  first_chosen = choose_human_readable_directory(
1200
- "root", "taskname", "111-222-333", state
1183
+ "root", "taskname", "https://example.com/some/directory"
1201
1184
  )
1202
1185
  assert first_chosen.startswith("root")
1203
- assert "taskname" in first_chosen
1204
- assert "111-222-333" not in first_chosen
1205
1186
 
1206
- # If we use the same ID we should get the same result
1207
- same_id = choose_human_readable_directory(
1208
- "root", "taskname", "111-222-333", state
1187
+ # If we use the same parent we should get the same result
1188
+ same_parent = choose_human_readable_directory(
1189
+ "root", "taskname", "https://example.com/some/directory"
1190
+ )
1191
+ assert same_parent == first_chosen
1192
+
1193
+ # If we use a lower parent with a URL, we do not necessarily need to be
1194
+ # inside the higher parent.
1195
+
1196
+ # If we use a URL with a creative number of slashes, it should be distinct.
1197
+ slash_parent = choose_human_readable_directory(
1198
+ "root", "taskname", "https://example.com/some/directory//////"
1199
+ )
1200
+ assert slash_parent != first_chosen
1201
+
1202
+ # If we use the same parent URL but a different task we should get the same result
1203
+ other_task = choose_human_readable_directory(
1204
+ "root", "taskname2", "https://example.com/some/directory"
1205
+ )
1206
+ assert other_task == first_chosen
1207
+
1208
+ # If we use a different parent we should get a different result still obeying the constraints
1209
+ diff_parent = choose_human_readable_directory(
1210
+ "root", "taskname", "/data/tmp/files/somewhere"
1211
+ )
1212
+ assert diff_parent != first_chosen
1213
+ assert diff_parent.startswith("root")
1214
+ assert "taskname" in diff_parent
1215
+
1216
+ # If we use a subpath parent with a filename we should get a path inside it.
1217
+ diff_parent_subpath = choose_human_readable_directory(
1218
+ "root", "taskname", "/data/tmp/files/somewhere/else"
1209
1219
  )
1210
- assert same_id == first_chosen
1220
+ assert os.path.dirname(diff_parent_subpath) == diff_parent
1211
1221
 
1212
- # If we use a different ID we should get a different result still obeying the constraints
1213
- diff_id = choose_human_readable_directory(
1214
- "root", "taskname", "222-333-444", state
1222
+ # If we use the same parent path but a different task we should get a different result.
1223
+ other_task_directory = choose_human_readable_directory(
1224
+ "root", "taskname2", "/data/tmp/files/somewhere"
1215
1225
  )
1216
- assert diff_id != first_chosen
1217
- assert diff_id.startswith("root")
1218
- assert "taskname" in diff_id
1219
- assert "222-333-444" not in diff_id
1226
+ assert other_task_directory != diff_parent
1227
+ assert other_task_directory.startswith("root")
1228
+ assert "taskname2" in other_task_directory
1220
1229
 
1221
1230
  def test_uri_packing(self) -> None:
1222
1231
  """
@@ -1232,7 +1241,7 @@ class TestWDLToilBench(unittest.TestCase):
1232
1241
  file_basename = "thefile.txt"
1233
1242
 
1234
1243
  # Pack and unpack it
1235
- uri = pack_toil_uri(file_id, task_path, dir_id, file_basename)
1244
+ uri = pack_toil_uri(file_id, task_path, str(dir_id), file_basename)
1236
1245
  unpacked = unpack_toil_uri(uri)
1237
1246
 
1238
1247
  # Make sure we got what we put in
@@ -1,6 +1,7 @@
1
1
  import unittest
2
2
  from uuid import uuid4
3
3
 
4
+ import logging
4
5
  import pytest
5
6
 
6
7
  from toil.provisioners import cluster_factory
@@ -12,6 +13,8 @@ from toil.test.wdl.wdltoil_test import (
12
13
  )
13
14
 
14
15
 
16
+ logger = logging.getLogger(__name__)
17
+
15
18
  @integrative
16
19
  @slow
17
20
  @pytest.mark.timeout(1800)
@@ -52,6 +55,7 @@ class WDLKubernetesClusterTest(AbstractClusterTest):
52
55
  workflow that performs an image pull on the worker.
53
56
  :return:
54
57
  """
58
+
55
59
  self.numWorkers = "1"
56
60
  self.requestedLeaderStorage = 30
57
61
  # create the cluster
@@ -64,6 +68,8 @@ class WDLKubernetesClusterTest(AbstractClusterTest):
64
68
 
65
69
  wdl_dir = "wdl_conformance_tests"
66
70
 
71
+ logger.info("Cloning WDL tests onto cluster...")
72
+
67
73
  # get the wdl-conformance-tests repo to get WDL tasks to run
68
74
  self.sshUtil(
69
75
  [
@@ -78,6 +84,9 @@ class WDLKubernetesClusterTest(AbstractClusterTest):
78
84
 
79
85
  # run WDL workflow that will run singularity
80
86
  test_options = [f"tests/md5sum/md5sum.wdl", f"tests/md5sum/md5sum.json"]
87
+
88
+ logger.info("Running workflow...")
89
+
81
90
  self.sshUtil(
82
91
  [
83
92
  "bash",
@@ -47,12 +47,12 @@ def fetchJobStoreFiles(jobStore: FileJobStore, options: argparse.Namespace) -> N
47
47
  jobStoreHits = glob(directoryname=options.jobStore, glob_pattern=jobStoreFile)
48
48
  for jobStoreFileID in jobStoreHits:
49
49
  logger.debug(
50
- f"Copying job store file: {jobStoreFileID} to {options.localFilePath[0]}"
50
+ f"Copying job store file: {jobStoreFileID} to {options.localFilePath}"
51
51
  )
52
52
  jobStore.read_file(
53
53
  jobStoreFileID,
54
54
  os.path.join(
55
- options.localFilePath[0], os.path.basename(jobStoreFileID)
55
+ options.localFilePath, os.path.basename(jobStoreFileID)
56
56
  ),
57
57
  symlink=options.useSymlinks,
58
58
  )
@@ -97,7 +97,10 @@ def printContentsOfJobStore(
97
97
  def main() -> None:
98
98
  parser = parser_with_common_options(jobstore_option=True, prog="toil debug-file")
99
99
  parser.add_argument(
100
- "--localFilePath", nargs=1, help="Location to which to copy job store files."
100
+ "--localFilePath",
101
+ type=str,
102
+ default=".",
103
+ help="Location to which to copy job store files."
101
104
  )
102
105
  parser.add_argument(
103
106
  "--fetch",
@@ -14,6 +14,7 @@
14
14
  """SSH into the toil appliance container running on the leader of the cluster."""
15
15
  import argparse
16
16
  import logging
17
+ import socket
17
18
  import sys
18
19
 
19
20
  from toil.common import parser_with_common_options
@@ -22,6 +23,22 @@ from toil.statsAndLogging import set_logging_from_options
22
23
 
23
24
  logger = logging.getLogger(__name__)
24
25
 
26
+ def have_ipv6() -> bool:
27
+ """
28
+ Return True if the IPv6 loopback interface is useable.
29
+ """
30
+
31
+ # We sniff for actual IPv6 like in
32
+ # https://github.com/urllib3/urllib3/pull/840/files in urllib3 (which we
33
+ # don't depend on)
34
+ if socket.has_ipv6:
35
+ # Built with IPv6 support
36
+ try:
37
+ socket.socket(socket.AF_INET6).bind(("::1", 0))
38
+ return True
39
+ except Exception:
40
+ pass
41
+ return False
25
42
 
26
43
  def main() -> None:
27
44
  parser = parser_with_common_options(
@@ -73,6 +90,12 @@ def main() -> None:
73
90
  ["-L", f"{options.grafana_port}:localhost:3000", "-L", "9090:localhost:9090"]
74
91
  )
75
92
 
93
+ if not have_ipv6():
94
+ # If we try to do SSH port forwarding without any other options, but
95
+ # IPv6 is turned off on the host, we might get complaints that we
96
+ # "Cannot assign requested address" on ports on [::1].
97
+ sshOptions.append("-4")
98
+
76
99
  try:
77
100
  cluster.getLeader().sshAppliance(
78
101
  *command,
toil/utils/toilStats.py CHANGED
@@ -326,6 +326,8 @@ def sprint_tag(
326
326
  out_str += header + "\n"
327
327
  out_str += sub_header + "\n"
328
328
  out_str += tag_str + "\n"
329
+ if tag.excess_cpu > 0:
330
+ out_str += f" ({tag.excess_cpu} used more CPU than requested!)\n"
329
331
  return out_str
330
332
 
331
333
 
@@ -507,13 +509,25 @@ def build_element(
507
509
  float(item.get(category_key, defaults[category])), category
508
510
  )
509
511
  values.append(category_value)
510
-
512
+
513
+ excess_cpu_items = 0
511
514
  for index in range(0, len(item_values[CATEGORIES[0]])):
512
515
  # For each item, compute the computed categories
513
- item_values["wait"].append(
516
+
517
+ # Compute wait time (allocated CPU time wasted).
518
+ # Note that if any item uses *more* CPU cores than requested, at any
519
+ # time, that decreases the amount of wait we're able to see from that
520
+ # item. If it hapens a lot, our computed wait could go negative, so we
521
+ # bound it below at 0.
522
+ wait_value = (
514
523
  item_values["time"][index] * item_values["cores"][index]
515
524
  - item_values["clock"][index]
516
525
  )
526
+ if wait_value < 0:
527
+ # Remember an item used more CPU than allocated.
528
+ excess_cpu_items += 1
529
+ wait_value = 0
530
+ item_values["wait"].append(wait_value)
517
531
 
518
532
  for category, values in item_values.items():
519
533
  values.sort()
@@ -531,6 +545,7 @@ def build_element(
531
545
  item_element["average_" + category] = float(sum(values) / len(values))
532
546
  item_element["min_" + category] = float(min(values))
533
547
  item_element["max_" + category] = float(max(values))
548
+ item_element["excess_cpu"] = excess_cpu_items
534
549
 
535
550
  element[item_name] = item_element
536
551
 
@@ -1,3 +1,4 @@
1
+ #!/usr/bin/env python3
1
2
  # Copyright (C) 2015-2021 Regents of the University of California
2
3
  #
3
4
  # Licensed under the Apache License, Version 2.0 (the "License");
toil/version.py CHANGED
@@ -1,14 +1,14 @@
1
- baseVersion = '8.2.0'
1
+ baseVersion = '9.1.0'
2
2
  cgcloudVersion = '1.6.0a1.dev393'
3
- version = '8.2.0-3efbe6f84b89a1eeab1d7b5913bbd01084d6490e'
4
- cacheTag = 'cache-local-py3.10'
5
- mainCacheTag = 'cache-master-py3.10'
6
- distVersion = '8.2.0'
7
- exactPython = 'python3.10'
8
- python = 'python3.10'
9
- dockerTag = '8.2.0-3efbe6f84b89a1eeab1d7b5913bbd01084d6490e-py3.10'
10
- currentCommit = '3efbe6f84b89a1eeab1d7b5913bbd01084d6490e'
3
+ version = '9.1.0-e341bb669efe78f93308e5ff1f02f7e375973511'
4
+ cacheTag = 'cache-local-py3.9'
5
+ mainCacheTag = 'cache-master-py3.9'
6
+ distVersion = '9.1.0'
7
+ exactPython = 'python3.9'
8
+ python = 'python3.9'
9
+ dockerTag = '9.1.0-e341bb669efe78f93308e5ff1f02f7e375973511-py3.9'
10
+ currentCommit = 'e341bb669efe78f93308e5ff1f02f7e375973511'
11
11
  dockerRegistry = 'quay.io/ucsc_cgl'
12
12
  dockerName = 'toil'
13
13
  dirty = False
14
- cwltool_version = '3.1.20250110105449'
14
+ cwltool_version = '3.1.20250715140722'