hpcflow-new2 0.2.0a188__py3-none-any.whl → 0.2.0a190__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (115) hide show
  1. hpcflow/__pyinstaller/hook-hpcflow.py +8 -6
  2. hpcflow/_version.py +1 -1
  3. hpcflow/app.py +1 -0
  4. hpcflow/data/scripts/main_script_test_hdf5_in_obj.py +1 -1
  5. hpcflow/data/scripts/main_script_test_hdf5_out_obj.py +1 -1
  6. hpcflow/sdk/__init__.py +21 -15
  7. hpcflow/sdk/app.py +2133 -770
  8. hpcflow/sdk/cli.py +281 -250
  9. hpcflow/sdk/cli_common.py +6 -2
  10. hpcflow/sdk/config/__init__.py +1 -1
  11. hpcflow/sdk/config/callbacks.py +77 -42
  12. hpcflow/sdk/config/cli.py +126 -103
  13. hpcflow/sdk/config/config.py +578 -311
  14. hpcflow/sdk/config/config_file.py +131 -95
  15. hpcflow/sdk/config/errors.py +112 -85
  16. hpcflow/sdk/config/types.py +145 -0
  17. hpcflow/sdk/core/actions.py +1054 -994
  18. hpcflow/sdk/core/app_aware.py +24 -0
  19. hpcflow/sdk/core/cache.py +81 -63
  20. hpcflow/sdk/core/command_files.py +275 -185
  21. hpcflow/sdk/core/commands.py +111 -107
  22. hpcflow/sdk/core/element.py +724 -503
  23. hpcflow/sdk/core/enums.py +192 -0
  24. hpcflow/sdk/core/environment.py +74 -93
  25. hpcflow/sdk/core/errors.py +398 -51
  26. hpcflow/sdk/core/json_like.py +540 -272
  27. hpcflow/sdk/core/loop.py +380 -334
  28. hpcflow/sdk/core/loop_cache.py +160 -43
  29. hpcflow/sdk/core/object_list.py +370 -207
  30. hpcflow/sdk/core/parameters.py +728 -600
  31. hpcflow/sdk/core/rule.py +59 -41
  32. hpcflow/sdk/core/run_dir_files.py +33 -22
  33. hpcflow/sdk/core/task.py +1546 -1325
  34. hpcflow/sdk/core/task_schema.py +240 -196
  35. hpcflow/sdk/core/test_utils.py +126 -88
  36. hpcflow/sdk/core/types.py +387 -0
  37. hpcflow/sdk/core/utils.py +410 -305
  38. hpcflow/sdk/core/validation.py +82 -9
  39. hpcflow/sdk/core/workflow.py +1192 -1028
  40. hpcflow/sdk/core/zarr_io.py +98 -137
  41. hpcflow/sdk/demo/cli.py +46 -33
  42. hpcflow/sdk/helper/cli.py +18 -16
  43. hpcflow/sdk/helper/helper.py +75 -63
  44. hpcflow/sdk/helper/watcher.py +61 -28
  45. hpcflow/sdk/log.py +83 -59
  46. hpcflow/sdk/persistence/__init__.py +8 -31
  47. hpcflow/sdk/persistence/base.py +988 -586
  48. hpcflow/sdk/persistence/defaults.py +6 -0
  49. hpcflow/sdk/persistence/discovery.py +38 -0
  50. hpcflow/sdk/persistence/json.py +408 -153
  51. hpcflow/sdk/persistence/pending.py +158 -123
  52. hpcflow/sdk/persistence/store_resource.py +37 -22
  53. hpcflow/sdk/persistence/types.py +307 -0
  54. hpcflow/sdk/persistence/utils.py +14 -11
  55. hpcflow/sdk/persistence/zarr.py +477 -420
  56. hpcflow/sdk/runtime.py +44 -41
  57. hpcflow/sdk/submission/{jobscript_info.py → enums.py} +39 -12
  58. hpcflow/sdk/submission/jobscript.py +444 -404
  59. hpcflow/sdk/submission/schedulers/__init__.py +133 -40
  60. hpcflow/sdk/submission/schedulers/direct.py +97 -71
  61. hpcflow/sdk/submission/schedulers/sge.py +132 -126
  62. hpcflow/sdk/submission/schedulers/slurm.py +263 -268
  63. hpcflow/sdk/submission/schedulers/utils.py +7 -2
  64. hpcflow/sdk/submission/shells/__init__.py +14 -15
  65. hpcflow/sdk/submission/shells/base.py +102 -29
  66. hpcflow/sdk/submission/shells/bash.py +72 -55
  67. hpcflow/sdk/submission/shells/os_version.py +31 -30
  68. hpcflow/sdk/submission/shells/powershell.py +37 -29
  69. hpcflow/sdk/submission/submission.py +203 -257
  70. hpcflow/sdk/submission/types.py +143 -0
  71. hpcflow/sdk/typing.py +163 -12
  72. hpcflow/tests/conftest.py +8 -6
  73. hpcflow/tests/schedulers/slurm/test_slurm_submission.py +5 -2
  74. hpcflow/tests/scripts/test_main_scripts.py +60 -30
  75. hpcflow/tests/shells/wsl/test_wsl_submission.py +6 -4
  76. hpcflow/tests/unit/test_action.py +86 -75
  77. hpcflow/tests/unit/test_action_rule.py +9 -4
  78. hpcflow/tests/unit/test_app.py +13 -6
  79. hpcflow/tests/unit/test_cli.py +1 -1
  80. hpcflow/tests/unit/test_command.py +71 -54
  81. hpcflow/tests/unit/test_config.py +20 -15
  82. hpcflow/tests/unit/test_config_file.py +21 -18
  83. hpcflow/tests/unit/test_element.py +58 -62
  84. hpcflow/tests/unit/test_element_iteration.py +3 -1
  85. hpcflow/tests/unit/test_element_set.py +29 -19
  86. hpcflow/tests/unit/test_group.py +4 -2
  87. hpcflow/tests/unit/test_input_source.py +116 -93
  88. hpcflow/tests/unit/test_input_value.py +29 -24
  89. hpcflow/tests/unit/test_json_like.py +44 -35
  90. hpcflow/tests/unit/test_loop.py +65 -58
  91. hpcflow/tests/unit/test_object_list.py +17 -12
  92. hpcflow/tests/unit/test_parameter.py +16 -7
  93. hpcflow/tests/unit/test_persistence.py +48 -35
  94. hpcflow/tests/unit/test_resources.py +20 -18
  95. hpcflow/tests/unit/test_run.py +8 -3
  96. hpcflow/tests/unit/test_runtime.py +2 -1
  97. hpcflow/tests/unit/test_schema_input.py +23 -15
  98. hpcflow/tests/unit/test_shell.py +3 -2
  99. hpcflow/tests/unit/test_slurm.py +8 -7
  100. hpcflow/tests/unit/test_submission.py +39 -19
  101. hpcflow/tests/unit/test_task.py +352 -247
  102. hpcflow/tests/unit/test_task_schema.py +33 -20
  103. hpcflow/tests/unit/test_utils.py +9 -11
  104. hpcflow/tests/unit/test_value_sequence.py +15 -12
  105. hpcflow/tests/unit/test_workflow.py +114 -83
  106. hpcflow/tests/unit/test_workflow_template.py +0 -1
  107. hpcflow/tests/workflows/test_jobscript.py +2 -1
  108. hpcflow/tests/workflows/test_workflows.py +18 -13
  109. {hpcflow_new2-0.2.0a188.dist-info → hpcflow_new2-0.2.0a190.dist-info}/METADATA +2 -1
  110. hpcflow_new2-0.2.0a190.dist-info/RECORD +165 -0
  111. hpcflow/sdk/core/parallel.py +0 -21
  112. hpcflow_new2-0.2.0a188.dist-info/RECORD +0 -158
  113. {hpcflow_new2-0.2.0a188.dist-info → hpcflow_new2-0.2.0a190.dist-info}/LICENSE +0 -0
  114. {hpcflow_new2-0.2.0a188.dist-info → hpcflow_new2-0.2.0a190.dist-info}/WHEEL +0 -0
  115. {hpcflow_new2-0.2.0a188.dist-info → hpcflow_new2-0.2.0a190.dist-info}/entry_points.txt +0 -0
@@ -1,37 +1,38 @@
1
+ from __future__ import annotations
1
2
  from hpcflow.sdk.submission.schedulers.slurm import SlurmPosix
2
3
 
3
4
 
4
- def test_parse_job_ID_simple():
5
+ def test_parse_job_ID_simple() -> None:
5
6
  assert SlurmPosix._parse_job_IDs("123") == ("123", None)
6
7
 
7
8
 
8
- def test_parse_job_ID_simple_array_item():
9
+ def test_parse_job_ID_simple_array_item() -> None:
9
10
  assert SlurmPosix._parse_job_IDs("123_10") == ("123", [9])
10
11
 
11
12
 
12
- def test_parse_job_ID_array_simple_range():
13
+ def test_parse_job_ID_array_simple_range() -> None:
13
14
  assert SlurmPosix._parse_job_IDs("3397752_[9-11]") == ("3397752", [8, 9, 10])
14
15
 
15
16
 
16
- def test_parse_job_ID_array_simple_multiple_range():
17
+ def test_parse_job_ID_array_simple_multiple_range() -> None:
17
18
  assert SlurmPosix._parse_job_IDs("49203_[3-5,9-11]") == (
18
19
  "49203",
19
20
  [2, 3, 4, 8, 9, 10],
20
21
  )
21
22
 
22
23
 
23
- def test_parse_job_ID_array_simple_mixed_range():
24
+ def test_parse_job_ID_array_simple_mixed_range() -> None:
24
25
  assert SlurmPosix._parse_job_IDs("30627658_[5,8-10]") == (
25
26
  "30627658",
26
27
  [4, 7, 8, 9],
27
28
  )
28
29
 
29
30
 
30
- def test_parse_job_ID_array_simple_range_with_max_concurrent():
31
+ def test_parse_job_ID_array_simple_range_with_max_concurrent() -> None:
31
32
  assert SlurmPosix._parse_job_IDs("3397752_[9-11%2]") == ("3397752", [8, 9, 10])
32
33
 
33
34
 
34
- def test_parse_job_ID_array_simple_multiple_range_max_concurrent():
35
+ def test_parse_job_ID_array_simple_multiple_range_max_concurrent() -> None:
35
36
  assert SlurmPosix._parse_job_IDs("49203_[3-5%1,9-11%2]") == (
36
37
  "49203",
37
38
  [2, 3, 4, 8, 9, 10],
@@ -1,5 +1,7 @@
1
+ from __future__ import annotations
1
2
  from datetime import timedelta
2
-
3
+ from typing import Any
4
+ from typing_extensions import TypedDict
3
5
  import pytest
4
6
 
5
7
  from hpcflow.app import app as hf
@@ -8,8 +10,8 @@ from hpcflow.sdk.core.errors import (
8
10
  MissingEnvironmentExecutableError,
9
11
  MissingEnvironmentExecutableInstanceError,
10
12
  )
13
+ from hpcflow.sdk.core.utils import timedelta_format, timedelta_parse
11
14
  from hpcflow.sdk.submission.jobscript import group_resource_map_into_jobscripts
12
- from hpcflow.sdk.submission.submission import timedelta_format, timedelta_parse
13
15
 
14
16
 
15
17
  @pytest.fixture
@@ -18,9 +20,14 @@ def null_config(tmp_path):
18
20
  hf.load_config(config_dir=tmp_path)
19
21
 
20
22
 
21
- def test_group_resource_map_into_jobscripts(null_config):
23
+ class _Example(TypedDict):
24
+ resources: list[list[int]]
25
+ expected: list[dict[str, Any]]
26
+
27
+
28
+ def test_group_resource_map_into_jobscripts(null_config) -> None:
22
29
  # x-axis corresponds to elements; y-axis corresponds to actions:
23
- examples = (
30
+ examples: tuple[_Example, ...] = (
24
31
  {
25
32
  "resources": [
26
33
  [1, 1, 1, 2, -1, 2, 4, -1, 1],
@@ -178,13 +185,13 @@ def test_group_resource_map_into_jobscripts(null_config):
178
185
  assert jobscripts_i == i["expected"]
179
186
 
180
187
 
181
- def test_timedelta_parse_format_round_trip(null_config):
188
+ def test_timedelta_parse_format_round_trip(null_config) -> None:
182
189
  td = timedelta(days=2, hours=25, minutes=92, seconds=77)
183
190
  td_str = timedelta_format(td)
184
191
  assert td_str == timedelta_format(timedelta_parse(td_str))
185
192
 
186
193
 
187
- def test_raise_missing_env_executable(new_null_config, tmp_path):
194
+ def test_raise_missing_env_executable(new_null_config, tmp_path) -> None:
188
195
  exec_name = (
189
196
  "my_executable" # null_env (the default) has no executable "my_executable"
190
197
  )
@@ -202,7 +209,7 @@ def test_raise_missing_env_executable(new_null_config, tmp_path):
202
209
  wk.add_submission()
203
210
 
204
211
 
205
- def test_raise_missing_matching_env_executable(new_null_config, tmp_path):
212
+ def test_raise_missing_matching_env_executable(new_null_config, tmp_path) -> None:
206
213
  """The executable label exists, but no a matching instance."""
207
214
  env_name = "my_hpcflow_env"
208
215
  exec_label = "my_exec_name"
@@ -241,7 +248,7 @@ def test_raise_missing_matching_env_executable(new_null_config, tmp_path):
241
248
  wk.add_submission()
242
249
 
243
250
 
244
- def test_no_raise_matching_env_executable(new_null_config, tmp_path):
251
+ def test_no_raise_matching_env_executable(new_null_config, tmp_path) -> None:
245
252
  env_name = "my_hpcflow_env"
246
253
  exec_label = "my_exec_name"
247
254
  env = hf.Environment(
@@ -278,7 +285,7 @@ def test_no_raise_matching_env_executable(new_null_config, tmp_path):
278
285
  wk.add_submission()
279
286
 
280
287
 
281
- def test_raise_missing_env(new_null_config, tmp_path):
288
+ def test_raise_missing_env(new_null_config, tmp_path) -> None:
282
289
  env_name = "my_hpcflow_env"
283
290
  ts = hf.TaskSchema(
284
291
  objective="test_sub",
@@ -294,7 +301,7 @@ def test_raise_missing_env(new_null_config, tmp_path):
294
301
  wk.add_submission()
295
302
 
296
303
 
297
- def test_custom_env_and_executable(new_null_config, tmp_path):
304
+ def test_custom_env_and_executable(new_null_config, tmp_path) -> None:
298
305
  env_name = "my_hpcflow_env"
299
306
  exec_label = "my_exec_name"
300
307
  env = hf.Environment(
@@ -330,7 +337,7 @@ def test_custom_env_and_executable(new_null_config, tmp_path):
330
337
  wk.add_submission()
331
338
 
332
339
 
333
- def test_abort_EARs_file_creation(null_config, tmp_path):
340
+ def test_abort_EARs_file_creation(null_config, tmp_path) -> None:
334
341
  wk_name = "temp"
335
342
  t1 = hf.Task(
336
343
  schema=hf.task_schemas.test_t1_conditional_OS,
@@ -342,6 +349,7 @@ def test_abort_EARs_file_creation(null_config, tmp_path):
342
349
  path=tmp_path,
343
350
  )
344
351
  sub = wk.add_submission()
352
+ assert sub is not None
345
353
  wk.submissions_path.mkdir(exist_ok=True, parents=True)
346
354
  sub.path.mkdir(exist_ok=True)
347
355
  sub._write_abort_EARs_file()
@@ -352,7 +360,7 @@ def test_abort_EARs_file_creation(null_config, tmp_path):
352
360
 
353
361
 
354
362
  @pytest.mark.parametrize("run_id", [0, 1, 2])
355
- def test_abort_EARs_file_update(null_config, tmp_path, run_id):
363
+ def test_abort_EARs_file_update(null_config, tmp_path, run_id) -> None:
356
364
  wk_name = "temp"
357
365
  t1 = hf.Task(
358
366
  schema=hf.task_schemas.test_t1_conditional_OS,
@@ -364,6 +372,7 @@ def test_abort_EARs_file_update(null_config, tmp_path, run_id):
364
372
  path=tmp_path,
365
373
  )
366
374
  sub = wk.add_submission()
375
+ assert sub is not None
367
376
  wk.submissions_path.mkdir(exist_ok=True, parents=True)
368
377
  sub.path.mkdir(exist_ok=True)
369
378
  sub._write_abort_EARs_file()
@@ -378,7 +387,7 @@ def test_abort_EARs_file_update(null_config, tmp_path, run_id):
378
387
  assert lines == "\n".join(lines_exp) + "\n"
379
388
 
380
389
 
381
- def test_abort_EARs_file_update_with_existing_abort(null_config, tmp_path):
390
+ def test_abort_EARs_file_update_with_existing_abort(null_config, tmp_path) -> None:
382
391
  wk_name = "temp"
383
392
  t1 = hf.Task(
384
393
  schema=hf.task_schemas.test_t1_conditional_OS,
@@ -390,6 +399,7 @@ def test_abort_EARs_file_update_with_existing_abort(null_config, tmp_path):
390
399
  path=tmp_path,
391
400
  )
392
401
  sub = wk.add_submission()
402
+ assert sub is not None
393
403
  wk.submissions_path.mkdir(exist_ok=True, parents=True)
394
404
  sub.path.mkdir(exist_ok=True)
395
405
  sub._write_abort_EARs_file()
@@ -404,7 +414,7 @@ def test_abort_EARs_file_update_with_existing_abort(null_config, tmp_path):
404
414
  assert lines == "\n".join(lines_exp) + "\n"
405
415
 
406
416
 
407
- def test_unique_schedulers_one_direct(new_null_config, tmp_path):
417
+ def test_unique_schedulers_one_direct(new_null_config, tmp_path) -> None:
408
418
  t1 = hf.Task(
409
419
  schema=hf.task_schemas.test_t1_conditional_OS,
410
420
  inputs={"p1": 1},
@@ -419,12 +429,15 @@ def test_unique_schedulers_one_direct(new_null_config, tmp_path):
419
429
  path=tmp_path,
420
430
  )
421
431
  sub = wk.add_submission()
432
+ assert sub is not None
422
433
  scheds = sub.get_unique_schedulers()
423
434
 
424
435
  assert len(scheds) == 1
425
436
 
426
437
 
427
- def test_unique_schedulers_one_direct_distinct_resources(new_null_config, tmp_path):
438
+ def test_unique_schedulers_one_direct_distinct_resources(
439
+ new_null_config, tmp_path
440
+ ) -> None:
428
441
  t1 = hf.Task(
429
442
  schema=hf.task_schemas.test_t1_conditional_OS,
430
443
  inputs={"p1": 1},
@@ -441,13 +454,14 @@ def test_unique_schedulers_one_direct_distinct_resources(new_null_config, tmp_pa
441
454
  path=tmp_path,
442
455
  )
443
456
  sub = wk.add_submission()
457
+ assert sub is not None
444
458
  scheds = sub.get_unique_schedulers()
445
459
 
446
460
  assert len(scheds) == 1
447
461
 
448
462
 
449
463
  @pytest.mark.slurm
450
- def test_unique_schedulers_one_SLURM(new_null_config, tmp_path):
464
+ def test_unique_schedulers_one_SLURM(new_null_config, tmp_path) -> None:
451
465
  hf.config.add_scheduler("slurm")
452
466
  t1 = hf.Task(
453
467
  schema=hf.task_schemas.test_t1_conditional_OS,
@@ -465,13 +479,16 @@ def test_unique_schedulers_one_SLURM(new_null_config, tmp_path):
465
479
  path=tmp_path,
466
480
  )
467
481
  sub = wk.add_submission()
482
+ assert sub is not None
468
483
  scheds = sub.get_unique_schedulers()
469
484
 
470
485
  assert len(scheds) == 1
471
486
 
472
487
 
473
488
  @pytest.mark.slurm
474
- def test_unique_schedulers_one_SLURM_distinct_resources(new_null_config, tmp_path):
489
+ def test_unique_schedulers_one_SLURM_distinct_resources(
490
+ new_null_config, tmp_path
491
+ ) -> None:
475
492
  hf.config.add_scheduler("slurm")
476
493
  t1 = hf.Task(
477
494
  schema=hf.task_schemas.test_t1_conditional_OS,
@@ -489,13 +506,14 @@ def test_unique_schedulers_one_SLURM_distinct_resources(new_null_config, tmp_pat
489
506
  path=tmp_path,
490
507
  )
491
508
  sub = wk.add_submission()
509
+ assert sub is not None
492
510
  scheds = sub.get_unique_schedulers()
493
511
 
494
512
  assert len(scheds) == 1
495
513
 
496
514
 
497
515
  @pytest.mark.slurm
498
- def test_unique_schedulers_two_direct_and_SLURM(new_null_config, tmp_path):
516
+ def test_unique_schedulers_two_direct_and_SLURM(new_null_config, tmp_path) -> None:
499
517
  hf.config.add_scheduler("slurm")
500
518
  t1 = hf.Task(
501
519
  schema=hf.task_schemas.test_t1_conditional_OS,
@@ -513,12 +531,13 @@ def test_unique_schedulers_two_direct_and_SLURM(new_null_config, tmp_path):
513
531
  path=tmp_path,
514
532
  )
515
533
  sub = wk.add_submission()
534
+ assert sub is not None
516
535
  scheds = sub.get_unique_schedulers()
517
536
 
518
537
  assert len(scheds) == 2
519
538
 
520
539
 
521
- def test_scheduler_config_defaults(new_null_config, tmp_path):
540
+ def test_scheduler_config_defaults(new_null_config, tmp_path) -> None:
522
541
  """Check default options defined in the config are merged into jobscript resources."""
523
542
  hf.config.set("schedulers.direct.defaults.options", {"a": "c"})
524
543
 
@@ -540,5 +559,6 @@ def test_scheduler_config_defaults(new_null_config, tmp_path):
540
559
  path=tmp_path,
541
560
  )
542
561
  sub = wk.add_submission()
562
+ assert sub is not None
543
563
  assert sub.jobscripts[0].resources.scheduler_args == {"options": {"a": "c"}}
544
564
  assert sub.jobscripts[1].resources.scheduler_args == {"options": {"a": "b"}}