hpcflow-new2 0.2.0a188__py3-none-any.whl → 0.2.0a190__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- hpcflow/__pyinstaller/hook-hpcflow.py +8 -6
- hpcflow/_version.py +1 -1
- hpcflow/app.py +1 -0
- hpcflow/data/scripts/main_script_test_hdf5_in_obj.py +1 -1
- hpcflow/data/scripts/main_script_test_hdf5_out_obj.py +1 -1
- hpcflow/sdk/__init__.py +21 -15
- hpcflow/sdk/app.py +2133 -770
- hpcflow/sdk/cli.py +281 -250
- hpcflow/sdk/cli_common.py +6 -2
- hpcflow/sdk/config/__init__.py +1 -1
- hpcflow/sdk/config/callbacks.py +77 -42
- hpcflow/sdk/config/cli.py +126 -103
- hpcflow/sdk/config/config.py +578 -311
- hpcflow/sdk/config/config_file.py +131 -95
- hpcflow/sdk/config/errors.py +112 -85
- hpcflow/sdk/config/types.py +145 -0
- hpcflow/sdk/core/actions.py +1054 -994
- hpcflow/sdk/core/app_aware.py +24 -0
- hpcflow/sdk/core/cache.py +81 -63
- hpcflow/sdk/core/command_files.py +275 -185
- hpcflow/sdk/core/commands.py +111 -107
- hpcflow/sdk/core/element.py +724 -503
- hpcflow/sdk/core/enums.py +192 -0
- hpcflow/sdk/core/environment.py +74 -93
- hpcflow/sdk/core/errors.py +398 -51
- hpcflow/sdk/core/json_like.py +540 -272
- hpcflow/sdk/core/loop.py +380 -334
- hpcflow/sdk/core/loop_cache.py +160 -43
- hpcflow/sdk/core/object_list.py +370 -207
- hpcflow/sdk/core/parameters.py +728 -600
- hpcflow/sdk/core/rule.py +59 -41
- hpcflow/sdk/core/run_dir_files.py +33 -22
- hpcflow/sdk/core/task.py +1546 -1325
- hpcflow/sdk/core/task_schema.py +240 -196
- hpcflow/sdk/core/test_utils.py +126 -88
- hpcflow/sdk/core/types.py +387 -0
- hpcflow/sdk/core/utils.py +410 -305
- hpcflow/sdk/core/validation.py +82 -9
- hpcflow/sdk/core/workflow.py +1192 -1028
- hpcflow/sdk/core/zarr_io.py +98 -137
- hpcflow/sdk/demo/cli.py +46 -33
- hpcflow/sdk/helper/cli.py +18 -16
- hpcflow/sdk/helper/helper.py +75 -63
- hpcflow/sdk/helper/watcher.py +61 -28
- hpcflow/sdk/log.py +83 -59
- hpcflow/sdk/persistence/__init__.py +8 -31
- hpcflow/sdk/persistence/base.py +988 -586
- hpcflow/sdk/persistence/defaults.py +6 -0
- hpcflow/sdk/persistence/discovery.py +38 -0
- hpcflow/sdk/persistence/json.py +408 -153
- hpcflow/sdk/persistence/pending.py +158 -123
- hpcflow/sdk/persistence/store_resource.py +37 -22
- hpcflow/sdk/persistence/types.py +307 -0
- hpcflow/sdk/persistence/utils.py +14 -11
- hpcflow/sdk/persistence/zarr.py +477 -420
- hpcflow/sdk/runtime.py +44 -41
- hpcflow/sdk/submission/{jobscript_info.py → enums.py} +39 -12
- hpcflow/sdk/submission/jobscript.py +444 -404
- hpcflow/sdk/submission/schedulers/__init__.py +133 -40
- hpcflow/sdk/submission/schedulers/direct.py +97 -71
- hpcflow/sdk/submission/schedulers/sge.py +132 -126
- hpcflow/sdk/submission/schedulers/slurm.py +263 -268
- hpcflow/sdk/submission/schedulers/utils.py +7 -2
- hpcflow/sdk/submission/shells/__init__.py +14 -15
- hpcflow/sdk/submission/shells/base.py +102 -29
- hpcflow/sdk/submission/shells/bash.py +72 -55
- hpcflow/sdk/submission/shells/os_version.py +31 -30
- hpcflow/sdk/submission/shells/powershell.py +37 -29
- hpcflow/sdk/submission/submission.py +203 -257
- hpcflow/sdk/submission/types.py +143 -0
- hpcflow/sdk/typing.py +163 -12
- hpcflow/tests/conftest.py +8 -6
- hpcflow/tests/schedulers/slurm/test_slurm_submission.py +5 -2
- hpcflow/tests/scripts/test_main_scripts.py +60 -30
- hpcflow/tests/shells/wsl/test_wsl_submission.py +6 -4
- hpcflow/tests/unit/test_action.py +86 -75
- hpcflow/tests/unit/test_action_rule.py +9 -4
- hpcflow/tests/unit/test_app.py +13 -6
- hpcflow/tests/unit/test_cli.py +1 -1
- hpcflow/tests/unit/test_command.py +71 -54
- hpcflow/tests/unit/test_config.py +20 -15
- hpcflow/tests/unit/test_config_file.py +21 -18
- hpcflow/tests/unit/test_element.py +58 -62
- hpcflow/tests/unit/test_element_iteration.py +3 -1
- hpcflow/tests/unit/test_element_set.py +29 -19
- hpcflow/tests/unit/test_group.py +4 -2
- hpcflow/tests/unit/test_input_source.py +116 -93
- hpcflow/tests/unit/test_input_value.py +29 -24
- hpcflow/tests/unit/test_json_like.py +44 -35
- hpcflow/tests/unit/test_loop.py +65 -58
- hpcflow/tests/unit/test_object_list.py +17 -12
- hpcflow/tests/unit/test_parameter.py +16 -7
- hpcflow/tests/unit/test_persistence.py +48 -35
- hpcflow/tests/unit/test_resources.py +20 -18
- hpcflow/tests/unit/test_run.py +8 -3
- hpcflow/tests/unit/test_runtime.py +2 -1
- hpcflow/tests/unit/test_schema_input.py +23 -15
- hpcflow/tests/unit/test_shell.py +3 -2
- hpcflow/tests/unit/test_slurm.py +8 -7
- hpcflow/tests/unit/test_submission.py +39 -19
- hpcflow/tests/unit/test_task.py +352 -247
- hpcflow/tests/unit/test_task_schema.py +33 -20
- hpcflow/tests/unit/test_utils.py +9 -11
- hpcflow/tests/unit/test_value_sequence.py +15 -12
- hpcflow/tests/unit/test_workflow.py +114 -83
- hpcflow/tests/unit/test_workflow_template.py +0 -1
- hpcflow/tests/workflows/test_jobscript.py +2 -1
- hpcflow/tests/workflows/test_workflows.py +18 -13
- {hpcflow_new2-0.2.0a188.dist-info → hpcflow_new2-0.2.0a190.dist-info}/METADATA +2 -1
- hpcflow_new2-0.2.0a190.dist-info/RECORD +165 -0
- hpcflow/sdk/core/parallel.py +0 -21
- hpcflow_new2-0.2.0a188.dist-info/RECORD +0 -158
- {hpcflow_new2-0.2.0a188.dist-info → hpcflow_new2-0.2.0a190.dist-info}/LICENSE +0 -0
- {hpcflow_new2-0.2.0a188.dist-info → hpcflow_new2-0.2.0a190.dist-info}/WHEEL +0 -0
- {hpcflow_new2-0.2.0a188.dist-info → hpcflow_new2-0.2.0a190.dist-info}/entry_points.txt +0 -0
hpcflow/tests/unit/test_slurm.py
CHANGED
@@ -1,37 +1,38 @@
|
|
1
|
+
from __future__ import annotations
|
1
2
|
from hpcflow.sdk.submission.schedulers.slurm import SlurmPosix
|
2
3
|
|
3
4
|
|
4
|
-
def test_parse_job_ID_simple():
|
5
|
+
def test_parse_job_ID_simple() -> None:
|
5
6
|
assert SlurmPosix._parse_job_IDs("123") == ("123", None)
|
6
7
|
|
7
8
|
|
8
|
-
def test_parse_job_ID_simple_array_item():
|
9
|
+
def test_parse_job_ID_simple_array_item() -> None:
|
9
10
|
assert SlurmPosix._parse_job_IDs("123_10") == ("123", [9])
|
10
11
|
|
11
12
|
|
12
|
-
def test_parse_job_ID_array_simple_range():
|
13
|
+
def test_parse_job_ID_array_simple_range() -> None:
|
13
14
|
assert SlurmPosix._parse_job_IDs("3397752_[9-11]") == ("3397752", [8, 9, 10])
|
14
15
|
|
15
16
|
|
16
|
-
def test_parse_job_ID_array_simple_multiple_range():
|
17
|
+
def test_parse_job_ID_array_simple_multiple_range() -> None:
|
17
18
|
assert SlurmPosix._parse_job_IDs("49203_[3-5,9-11]") == (
|
18
19
|
"49203",
|
19
20
|
[2, 3, 4, 8, 9, 10],
|
20
21
|
)
|
21
22
|
|
22
23
|
|
23
|
-
def test_parse_job_ID_array_simple_mixed_range():
|
24
|
+
def test_parse_job_ID_array_simple_mixed_range() -> None:
|
24
25
|
assert SlurmPosix._parse_job_IDs("30627658_[5,8-10]") == (
|
25
26
|
"30627658",
|
26
27
|
[4, 7, 8, 9],
|
27
28
|
)
|
28
29
|
|
29
30
|
|
30
|
-
def test_parse_job_ID_array_simple_range_with_max_concurrent():
|
31
|
+
def test_parse_job_ID_array_simple_range_with_max_concurrent() -> None:
|
31
32
|
assert SlurmPosix._parse_job_IDs("3397752_[9-11%2]") == ("3397752", [8, 9, 10])
|
32
33
|
|
33
34
|
|
34
|
-
def test_parse_job_ID_array_simple_multiple_range_max_concurrent():
|
35
|
+
def test_parse_job_ID_array_simple_multiple_range_max_concurrent() -> None:
|
35
36
|
assert SlurmPosix._parse_job_IDs("49203_[3-5%1,9-11%2]") == (
|
36
37
|
"49203",
|
37
38
|
[2, 3, 4, 8, 9, 10],
|
@@ -1,5 +1,7 @@
|
|
1
|
+
from __future__ import annotations
|
1
2
|
from datetime import timedelta
|
2
|
-
|
3
|
+
from typing import Any
|
4
|
+
from typing_extensions import TypedDict
|
3
5
|
import pytest
|
4
6
|
|
5
7
|
from hpcflow.app import app as hf
|
@@ -8,8 +10,8 @@ from hpcflow.sdk.core.errors import (
|
|
8
10
|
MissingEnvironmentExecutableError,
|
9
11
|
MissingEnvironmentExecutableInstanceError,
|
10
12
|
)
|
13
|
+
from hpcflow.sdk.core.utils import timedelta_format, timedelta_parse
|
11
14
|
from hpcflow.sdk.submission.jobscript import group_resource_map_into_jobscripts
|
12
|
-
from hpcflow.sdk.submission.submission import timedelta_format, timedelta_parse
|
13
15
|
|
14
16
|
|
15
17
|
@pytest.fixture
|
@@ -18,9 +20,14 @@ def null_config(tmp_path):
|
|
18
20
|
hf.load_config(config_dir=tmp_path)
|
19
21
|
|
20
22
|
|
21
|
-
|
23
|
+
class _Example(TypedDict):
|
24
|
+
resources: list[list[int]]
|
25
|
+
expected: list[dict[str, Any]]
|
26
|
+
|
27
|
+
|
28
|
+
def test_group_resource_map_into_jobscripts(null_config) -> None:
|
22
29
|
# x-axis corresponds to elements; y-axis corresponds to actions:
|
23
|
-
examples = (
|
30
|
+
examples: tuple[_Example, ...] = (
|
24
31
|
{
|
25
32
|
"resources": [
|
26
33
|
[1, 1, 1, 2, -1, 2, 4, -1, 1],
|
@@ -178,13 +185,13 @@ def test_group_resource_map_into_jobscripts(null_config):
|
|
178
185
|
assert jobscripts_i == i["expected"]
|
179
186
|
|
180
187
|
|
181
|
-
def test_timedelta_parse_format_round_trip(null_config):
|
188
|
+
def test_timedelta_parse_format_round_trip(null_config) -> None:
|
182
189
|
td = timedelta(days=2, hours=25, minutes=92, seconds=77)
|
183
190
|
td_str = timedelta_format(td)
|
184
191
|
assert td_str == timedelta_format(timedelta_parse(td_str))
|
185
192
|
|
186
193
|
|
187
|
-
def test_raise_missing_env_executable(new_null_config, tmp_path):
|
194
|
+
def test_raise_missing_env_executable(new_null_config, tmp_path) -> None:
|
188
195
|
exec_name = (
|
189
196
|
"my_executable" # null_env (the default) has no executable "my_executable"
|
190
197
|
)
|
@@ -202,7 +209,7 @@ def test_raise_missing_env_executable(new_null_config, tmp_path):
|
|
202
209
|
wk.add_submission()
|
203
210
|
|
204
211
|
|
205
|
-
def test_raise_missing_matching_env_executable(new_null_config, tmp_path):
|
212
|
+
def test_raise_missing_matching_env_executable(new_null_config, tmp_path) -> None:
|
206
213
|
"""The executable label exists, but no a matching instance."""
|
207
214
|
env_name = "my_hpcflow_env"
|
208
215
|
exec_label = "my_exec_name"
|
@@ -241,7 +248,7 @@ def test_raise_missing_matching_env_executable(new_null_config, tmp_path):
|
|
241
248
|
wk.add_submission()
|
242
249
|
|
243
250
|
|
244
|
-
def test_no_raise_matching_env_executable(new_null_config, tmp_path):
|
251
|
+
def test_no_raise_matching_env_executable(new_null_config, tmp_path) -> None:
|
245
252
|
env_name = "my_hpcflow_env"
|
246
253
|
exec_label = "my_exec_name"
|
247
254
|
env = hf.Environment(
|
@@ -278,7 +285,7 @@ def test_no_raise_matching_env_executable(new_null_config, tmp_path):
|
|
278
285
|
wk.add_submission()
|
279
286
|
|
280
287
|
|
281
|
-
def test_raise_missing_env(new_null_config, tmp_path):
|
288
|
+
def test_raise_missing_env(new_null_config, tmp_path) -> None:
|
282
289
|
env_name = "my_hpcflow_env"
|
283
290
|
ts = hf.TaskSchema(
|
284
291
|
objective="test_sub",
|
@@ -294,7 +301,7 @@ def test_raise_missing_env(new_null_config, tmp_path):
|
|
294
301
|
wk.add_submission()
|
295
302
|
|
296
303
|
|
297
|
-
def test_custom_env_and_executable(new_null_config, tmp_path):
|
304
|
+
def test_custom_env_and_executable(new_null_config, tmp_path) -> None:
|
298
305
|
env_name = "my_hpcflow_env"
|
299
306
|
exec_label = "my_exec_name"
|
300
307
|
env = hf.Environment(
|
@@ -330,7 +337,7 @@ def test_custom_env_and_executable(new_null_config, tmp_path):
|
|
330
337
|
wk.add_submission()
|
331
338
|
|
332
339
|
|
333
|
-
def test_abort_EARs_file_creation(null_config, tmp_path):
|
340
|
+
def test_abort_EARs_file_creation(null_config, tmp_path) -> None:
|
334
341
|
wk_name = "temp"
|
335
342
|
t1 = hf.Task(
|
336
343
|
schema=hf.task_schemas.test_t1_conditional_OS,
|
@@ -342,6 +349,7 @@ def test_abort_EARs_file_creation(null_config, tmp_path):
|
|
342
349
|
path=tmp_path,
|
343
350
|
)
|
344
351
|
sub = wk.add_submission()
|
352
|
+
assert sub is not None
|
345
353
|
wk.submissions_path.mkdir(exist_ok=True, parents=True)
|
346
354
|
sub.path.mkdir(exist_ok=True)
|
347
355
|
sub._write_abort_EARs_file()
|
@@ -352,7 +360,7 @@ def test_abort_EARs_file_creation(null_config, tmp_path):
|
|
352
360
|
|
353
361
|
|
354
362
|
@pytest.mark.parametrize("run_id", [0, 1, 2])
|
355
|
-
def test_abort_EARs_file_update(null_config, tmp_path, run_id):
|
363
|
+
def test_abort_EARs_file_update(null_config, tmp_path, run_id) -> None:
|
356
364
|
wk_name = "temp"
|
357
365
|
t1 = hf.Task(
|
358
366
|
schema=hf.task_schemas.test_t1_conditional_OS,
|
@@ -364,6 +372,7 @@ def test_abort_EARs_file_update(null_config, tmp_path, run_id):
|
|
364
372
|
path=tmp_path,
|
365
373
|
)
|
366
374
|
sub = wk.add_submission()
|
375
|
+
assert sub is not None
|
367
376
|
wk.submissions_path.mkdir(exist_ok=True, parents=True)
|
368
377
|
sub.path.mkdir(exist_ok=True)
|
369
378
|
sub._write_abort_EARs_file()
|
@@ -378,7 +387,7 @@ def test_abort_EARs_file_update(null_config, tmp_path, run_id):
|
|
378
387
|
assert lines == "\n".join(lines_exp) + "\n"
|
379
388
|
|
380
389
|
|
381
|
-
def test_abort_EARs_file_update_with_existing_abort(null_config, tmp_path):
|
390
|
+
def test_abort_EARs_file_update_with_existing_abort(null_config, tmp_path) -> None:
|
382
391
|
wk_name = "temp"
|
383
392
|
t1 = hf.Task(
|
384
393
|
schema=hf.task_schemas.test_t1_conditional_OS,
|
@@ -390,6 +399,7 @@ def test_abort_EARs_file_update_with_existing_abort(null_config, tmp_path):
|
|
390
399
|
path=tmp_path,
|
391
400
|
)
|
392
401
|
sub = wk.add_submission()
|
402
|
+
assert sub is not None
|
393
403
|
wk.submissions_path.mkdir(exist_ok=True, parents=True)
|
394
404
|
sub.path.mkdir(exist_ok=True)
|
395
405
|
sub._write_abort_EARs_file()
|
@@ -404,7 +414,7 @@ def test_abort_EARs_file_update_with_existing_abort(null_config, tmp_path):
|
|
404
414
|
assert lines == "\n".join(lines_exp) + "\n"
|
405
415
|
|
406
416
|
|
407
|
-
def test_unique_schedulers_one_direct(new_null_config, tmp_path):
|
417
|
+
def test_unique_schedulers_one_direct(new_null_config, tmp_path) -> None:
|
408
418
|
t1 = hf.Task(
|
409
419
|
schema=hf.task_schemas.test_t1_conditional_OS,
|
410
420
|
inputs={"p1": 1},
|
@@ -419,12 +429,15 @@ def test_unique_schedulers_one_direct(new_null_config, tmp_path):
|
|
419
429
|
path=tmp_path,
|
420
430
|
)
|
421
431
|
sub = wk.add_submission()
|
432
|
+
assert sub is not None
|
422
433
|
scheds = sub.get_unique_schedulers()
|
423
434
|
|
424
435
|
assert len(scheds) == 1
|
425
436
|
|
426
437
|
|
427
|
-
def test_unique_schedulers_one_direct_distinct_resources(
|
438
|
+
def test_unique_schedulers_one_direct_distinct_resources(
|
439
|
+
new_null_config, tmp_path
|
440
|
+
) -> None:
|
428
441
|
t1 = hf.Task(
|
429
442
|
schema=hf.task_schemas.test_t1_conditional_OS,
|
430
443
|
inputs={"p1": 1},
|
@@ -441,13 +454,14 @@ def test_unique_schedulers_one_direct_distinct_resources(new_null_config, tmp_pa
|
|
441
454
|
path=tmp_path,
|
442
455
|
)
|
443
456
|
sub = wk.add_submission()
|
457
|
+
assert sub is not None
|
444
458
|
scheds = sub.get_unique_schedulers()
|
445
459
|
|
446
460
|
assert len(scheds) == 1
|
447
461
|
|
448
462
|
|
449
463
|
@pytest.mark.slurm
|
450
|
-
def test_unique_schedulers_one_SLURM(new_null_config, tmp_path):
|
464
|
+
def test_unique_schedulers_one_SLURM(new_null_config, tmp_path) -> None:
|
451
465
|
hf.config.add_scheduler("slurm")
|
452
466
|
t1 = hf.Task(
|
453
467
|
schema=hf.task_schemas.test_t1_conditional_OS,
|
@@ -465,13 +479,16 @@ def test_unique_schedulers_one_SLURM(new_null_config, tmp_path):
|
|
465
479
|
path=tmp_path,
|
466
480
|
)
|
467
481
|
sub = wk.add_submission()
|
482
|
+
assert sub is not None
|
468
483
|
scheds = sub.get_unique_schedulers()
|
469
484
|
|
470
485
|
assert len(scheds) == 1
|
471
486
|
|
472
487
|
|
473
488
|
@pytest.mark.slurm
|
474
|
-
def test_unique_schedulers_one_SLURM_distinct_resources(
|
489
|
+
def test_unique_schedulers_one_SLURM_distinct_resources(
|
490
|
+
new_null_config, tmp_path
|
491
|
+
) -> None:
|
475
492
|
hf.config.add_scheduler("slurm")
|
476
493
|
t1 = hf.Task(
|
477
494
|
schema=hf.task_schemas.test_t1_conditional_OS,
|
@@ -489,13 +506,14 @@ def test_unique_schedulers_one_SLURM_distinct_resources(new_null_config, tmp_pat
|
|
489
506
|
path=tmp_path,
|
490
507
|
)
|
491
508
|
sub = wk.add_submission()
|
509
|
+
assert sub is not None
|
492
510
|
scheds = sub.get_unique_schedulers()
|
493
511
|
|
494
512
|
assert len(scheds) == 1
|
495
513
|
|
496
514
|
|
497
515
|
@pytest.mark.slurm
|
498
|
-
def test_unique_schedulers_two_direct_and_SLURM(new_null_config, tmp_path):
|
516
|
+
def test_unique_schedulers_two_direct_and_SLURM(new_null_config, tmp_path) -> None:
|
499
517
|
hf.config.add_scheduler("slurm")
|
500
518
|
t1 = hf.Task(
|
501
519
|
schema=hf.task_schemas.test_t1_conditional_OS,
|
@@ -513,12 +531,13 @@ def test_unique_schedulers_two_direct_and_SLURM(new_null_config, tmp_path):
|
|
513
531
|
path=tmp_path,
|
514
532
|
)
|
515
533
|
sub = wk.add_submission()
|
534
|
+
assert sub is not None
|
516
535
|
scheds = sub.get_unique_schedulers()
|
517
536
|
|
518
537
|
assert len(scheds) == 2
|
519
538
|
|
520
539
|
|
521
|
-
def test_scheduler_config_defaults(new_null_config, tmp_path):
|
540
|
+
def test_scheduler_config_defaults(new_null_config, tmp_path) -> None:
|
522
541
|
"""Check default options defined in the config are merged into jobscript resources."""
|
523
542
|
hf.config.set("schedulers.direct.defaults.options", {"a": "c"})
|
524
543
|
|
@@ -540,5 +559,6 @@ def test_scheduler_config_defaults(new_null_config, tmp_path):
|
|
540
559
|
path=tmp_path,
|
541
560
|
)
|
542
561
|
sub = wk.add_submission()
|
562
|
+
assert sub is not None
|
543
563
|
assert sub.jobscripts[0].resources.scheduler_args == {"options": {"a": "c"}}
|
544
564
|
assert sub.jobscripts[1].resources.scheduler_args == {"options": {"a": "b"}}
|