hpcflow-new2 0.2.0a190__py3-none-any.whl → 0.2.0a200__py3-none-any.whl

hpcflow/tests/unit/test_persistence.py

@@ -1,16 +1,20 @@
 from __future__ import annotations
 from pathlib import Path
+import sys
+from typing import Any, Dict, List, Tuple
 from typing import cast, TYPE_CHECKING
 import numpy as np
 import zarr  # type: ignore
 import pytest
-from hpcflow.sdk.core.test_utils import make_test_data_YAML_workflow
+from hpcflow.sdk.core.test_utils import make_test_data_YAML_workflow, make_workflow
 from hpcflow.sdk.persistence.json import (
     JSONPersistentStore,
     JsonStoreElement,
     JsonStoreElementIter,
     JsonStoreEAR,
 )
+from hpcflow.sdk.persistence.zarr import ZarrPersistentStore
+
 from hpcflow.app import app as hf
 
 if TYPE_CHECKING:
@@ -258,6 +262,9 @@ def test_make_zarr_store_no_compressor(null_config, tmp_path: Path):
 
 
 @pytest.mark.integration
+@pytest.mark.skipif(
+    sys.version_info < (3, 9), reason="Python 3.8 support is being removed anyway."
+)
 def test_zarr_rechunk_data_equivalent(null_config, tmp_path: Path):
     t1 = hf.Task(
         schema=hf.task_schemas.test_t1_conditional_OS,
@@ -276,10 +283,10 @@ def test_zarr_rechunk_data_equivalent(null_config, tmp_path: Path):
     arr = cast("ZarrPersistentStore", wk._store)._get_EARs_arr()
     assert arr.chunks == arr.shape
 
-    bak_path = (Path(wk.path) / arr.path).with_suffix(".bak")
+    bak_path = (Path(arr.store.path) / arr.path).with_suffix(".bak")
     arr_bak = zarr.open(bak_path)
 
-    assert arr_bak.chunks == (1,)
+    assert arr_bak.chunks == (1, 1)  # runs array is 2D
 
     # check backup and new runs data are equal:
     assert np.all(arr[:] == arr_bak[:])
@@ -289,6 +296,9 @@ def test_zarr_rechunk_data_equivalent(null_config, tmp_path: Path):
 
 
 @pytest.mark.integration
+@pytest.mark.skipif(
+    sys.version_info < (3, 9), reason="Python 3.8 support is being removed anyway."
+)
 def test_zarr_rechunk_data_equivalent_custom_chunk_size(null_config, tmp_path: Path):
     t1 = hf.Task(
         schema=hf.task_schemas.test_t1_conditional_OS,
@@ -305,12 +315,12 @@ def test_zarr_rechunk_data_equivalent_custom_chunk_size(null_config, tmp_path: P
     wk.rechunk_runs(backup=True, status=False, chunk_size=2)
 
     arr = cast("ZarrPersistentStore", wk._store)._get_EARs_arr()
-    assert arr.chunks == (2,)
+    assert arr.chunks == (2, 2)  # runs array is 2D
 
-    bak_path = (Path(wk.path) / arr.path).with_suffix(".bak")
+    bak_path = (Path(arr.store.path) / arr.path).with_suffix(".bak")
     arr_bak = zarr.open(bak_path)
 
-    assert arr_bak.chunks == (1,)
+    assert arr_bak.chunks == (1, 1)  # runs array is 2D
 
     # check backup and new runs data are equal:
     assert np.all(arr[:] == arr_bak[:])
@@ -334,7 +344,7 @@ def test_zarr_rechunk_data_no_backup_load_runs(null_config, tmp_path: Path):
 
     arr = cast("ZarrPersistentStore", wk._store)._get_EARs_arr()
 
-    bak_path = (Path(wk.path) / arr.path).with_suffix(".bak")
+    bak_path = (Path(arr.store.path) / arr.path).with_suffix(".bak")
     assert not bak_path.is_file()
 
     # check we can load runs:
@@ -358,14 +368,186 @@ def test_zarr_rechunk_data_no_backup_load_parameter_base(null_config, tmp_path:
         path=tmp_path,
     )
     wk.submit(wait=True, status=False, add_to_known=False)
+
+    params_old = wk.get_all_parameter_data()
     wk.rechunk_parameter_base(backup=False, status=False)
 
+    wk = wk.reload()
+    params_new = wk.get_all_parameter_data()
+    assert params_new == params_old
+
     arr = cast("ZarrPersistentStore", wk._store)._get_parameter_base_array()
 
-    bak_path = (Path(wk.path) / arr.path).with_suffix(".bak")
+    bak_path = (Path(arr.store.path) / arr.path).with_suffix(".bak")
     assert not bak_path.is_file()
 
     # check we can load parameters:
     param_IDs = []
     for i in wk.get_all_parameters():
         param_IDs.append(i.id_)
+
+
+def test_get_parameter_sources_duplicate_ids(null_config, tmp_path):
+    wk = make_workflow(
+        schemas_spec=[[{"p1": None}, ("p1",), "t1"]],
+        local_inputs={0: ("p1",)},
+        path=tmp_path,
+    )
+    id_lst = [0, 1, 1, 2, 0]
+    src = wk._store.get_parameter_sources(id_lst)
+    assert len(src) == len(id_lst)
+    assert src[0] == src[4]
+    assert src[1] == src[2]
+
+
+def _transform_jobscript_dependencies_to_encodable(
+    deps: dict[tuple[int, int], dict[tuple[int, int], dict[str, Any]]],
+) -> dict[str, list[dict[str, Any]]]:
+    """Transform a dict of jobscript dependencies written in a more testing-friendly/
+    convenient format into the format expected by the method
+    `ZarrPersistentStore._encode_jobscript_block_dependencies`.
+
+    """
+    max_js_idx = max(i[0] for i in deps)
+    sub_js: dict[str, list[dict[str, Any]]] = {
+        "jobscripts": [
+            {"blocks": [], "index": js_idx} for js_idx in range(max_js_idx + 1)
+        ]
+    }
+    for (js_idx, blk_idx), deps_i in deps.items():
+        sub_js["jobscripts"][js_idx]["blocks"].append(
+            {
+                "dependencies": [[[k[0], k[1]], v] for k, v in deps_i.items()],
+                "index": blk_idx,
+            }
+        )
+    return sub_js
+
+
+def test_zarr_encode_jobscript_block_dependencies_element_mapping_array_non_array_equivalence():
+    deps_1 = {
+        (0, 0): {},
+        (1, 0): {(0, 0): {"js_element_mapping": {0: [0]}, "is_array": True}},
+    }
+    deps_2 = {
+        (0, 0): {},
+        (1, 0): {(0, 0): {"js_element_mapping": {0: np.array([0])}, "is_array": True}},
+    }
+    deps_1 = _transform_jobscript_dependencies_to_encodable(deps_1)
+    deps_2 = _transform_jobscript_dependencies_to_encodable(deps_2)
+    arr_1 = ZarrPersistentStore._encode_jobscript_block_dependencies(deps_1)
+    arr_2 = ZarrPersistentStore._encode_jobscript_block_dependencies(deps_2)
+    assert np.array_equal(arr_1, arr_2)
+
+
+def test_zarr_encode_decode_jobscript_block_dependencies():
+
+    deps = {
+        (0, 0): {},
+        (1, 0): {
+            (0, 0): {
+                "js_element_mapping": {0: [0], 1: [1]},
+                "is_array": True,
+            }
+        },
+        (2, 0): {
+            (1, 0): {
+                "js_element_mapping": {0: [0, 1], 1: [0, 1]},
+                "is_array": False,
+            }
+        },
+        (2, 1): {
+            (0, 0): {"js_element_mapping": {0: [0, 1]}, "is_array": False},
+            (2, 0): {"js_element_mapping": {0: [0, 1]}, "is_array": False},
+        },
+    }
+    deps_t = _transform_jobscript_dependencies_to_encodable(deps)
+    arr = ZarrPersistentStore._encode_jobscript_block_dependencies(deps_t)
+    assert np.array_equal(
+        arr,
+        np.array(
+            [
+                2,
+                0,
+                0,
+                12,
+                1,
+                0,
+                9,
+                0,
+                0,
+                1,
+                2,
+                0,
+                0,
+                2,
+                1,
+                1,
+                14,
+                2,
+                0,
+                11,
+                1,
+                0,
+                0,
+                3,
+                0,
+                0,
+                1,
+                3,
+                1,
+                0,
+                1,
+                18,
+                2,
+                1,
+                7,
+                0,
+                0,
+                0,
+                3,
+                0,
+                0,
+                1,
+                7,
+                2,
+                0,
+                0,
+                3,
+                0,
+                0,
+                1,
+            ]
+        ),
+    )
+    deps_rt = ZarrPersistentStore._decode_jobscript_block_dependencies(arr)
+    assert deps_rt == deps
+
+
+def test_zarr_encode_decode_jobscript_block_dependencies_large_many_to_one():
+    deps = {
+        (0, 0): {},
+        (1, 0): {
+            (0, 0): {"js_element_mapping": {0: list(range(1_000_000))}, "is_array": False}
+        },
+    }
+    deps_t = _transform_jobscript_dependencies_to_encodable(deps)
+    arr = ZarrPersistentStore._encode_jobscript_block_dependencies(deps_t)
+    deps_rt = ZarrPersistentStore._decode_jobscript_block_dependencies(arr)
+    assert deps_rt == deps
+
+
+def test_zarr_encode_decode_jobscript_block_dependencies_large_one_to_one():
+    deps = {
+        (0, 0): {},
+        (1, 0): {
+            (0, 0): {
+                "js_element_mapping": {i: [i] for i in range(1_000_000)},
+                "is_array": False,
+            }
+        },
+    }
+    deps_t = _transform_jobscript_dependencies_to_encodable(deps)
+    arr = ZarrPersistentStore._encode_jobscript_block_dependencies(deps_t)
+    deps_rt = ZarrPersistentStore._decode_jobscript_block_dependencies(arr)
+    assert deps_rt == deps

hpcflow/tests/unit/test_run.py

@@ -1,6 +1,13 @@
 from __future__ import annotations
+
+import os
+
+import pytest
+
 from pathlib import Path
 from hpcflow.app import app as hf
+from hpcflow.sdk.core.actions import SkipReason
+from hpcflow.sdk.core.test_utils import make_workflow_to_run_command
 
 
 def test_compose_commands_no_shell_var(null_config, tmp_path: Path):
@@ -17,7 +24,7 @@ def test_compose_commands_no_shell_var(null_config, tmp_path: Path):
     assert sub is not None
     js = sub.jobscripts[0]
     run = wk.tasks[0].elements[0].iterations[0].action_runs[0]
-    _, shell_vars = run.compose_commands(jobscript=js, JS_action_idx=0)
+    _, shell_vars = run.compose_commands(environments=sub.environments, shell=js.shell)
     assert shell_vars == {0: []}
 
 
@@ -46,7 +53,7 @@ def test_compose_commands_single_shell_var(null_config, tmp_path: Path):
     assert sub is not None
     js = sub.jobscripts[0]
     run = wk.tasks[0].elements[0].iterations[0].action_runs[0]
-    _, shell_vars = run.compose_commands(jobscript=js, JS_action_idx=0)
+    _, shell_vars = run.compose_commands(environments=sub.environments, shell=js.shell)
     assert shell_vars == {0: [("outputs.p1", "parameter_p1", "stdout")]}
 
 
@@ -76,5 +83,104 @@ def test_compose_commands_multi_single_shell_var(null_config, tmp_path: Path):
     assert sub is not None
     js = sub.jobscripts[0]
     run = wk.tasks[0].elements[0].iterations[0].action_runs[0]
-    _, shell_vars = run.compose_commands(jobscript=js, JS_action_idx=0)
+    _, shell_vars = run.compose_commands(environments=sub.environments, shell=js.shell)
     assert shell_vars == {0: [], 1: [("outputs.p1", "parameter_p1", "stdout")]}
+
+
+@pytest.mark.integration
+def test_run_dir_diff_new_file(null_config, tmp_path):
+    if os.name == "nt":
+        command = "New-Item -Path 'new_file.txt' -ItemType File"
+    else:
+        command = "touch new_file.txt"
+    wk = make_workflow_to_run_command(
+        command=command,
+        requires_dir=True,
+        path=tmp_path,
+        name="w2",
+        overwrite=True,
+    )
+    wk.submit(wait=True, add_to_known=False, status=False)
+    assert wk.get_all_EARs()[0].dir_diff.files_created == ["new_file.txt"]
+
+
+@pytest.mark.integration
+def test_run_skip_reason_upstream_failure(null_config, tmp_path):
+    ts = hf.TaskSchema(
+        objective="t1",
+        inputs=[hf.SchemaInput("p1")],
+        outputs=[hf.SchemaInput("p2")],
+        actions=[
+            hf.Action(
+                commands=[
+                    hf.Command(
+                        command="echo $(( <<parameter:p1>> + 100 ))",
+                        stdout="<<parameter:p2>>",
+                    ),
+                    hf.Command(command="exit 1"),
+                ]
+            ),
+            hf.Action(
+                commands=[
+                    hf.Command(
+                        command="echo $(( <<parameter:p2>> + 100 ))",
+                        stdout="<<parameter:p2>>",
+                    ),
+                ]
+            ),  # should be skipped due to failure of action 0
+        ],
+    )
+    wk = hf.Workflow.from_template_data(
+        template_name="test_skip_reason",
+        path=tmp_path,
+        tasks=[hf.Task(schema=ts, inputs={"p1": 100})],
+    )
+    wk.submit(wait=True, add_to_known=False, status=False)
+    runs = wk.get_all_EARs()
+    assert not runs[0].success
+    assert not runs[1].success
+    assert runs[0].skip_reason is SkipReason.NOT_SKIPPED
+    assert runs[1].skip_reason is SkipReason.UPSTREAM_FAILURE
+
+
+@pytest.mark.integration
+def test_run_skip_reason_loop_termination(null_config, tmp_path):
+    ts = hf.TaskSchema(
+        objective="t1",
+        inputs=[hf.SchemaInput("p1")],
+        outputs=[hf.SchemaInput("p1")],
+        actions=[
+            hf.Action(
+                commands=[
+                    hf.Command(
+                        command="echo $(( <<parameter:p1>> + 100 ))",
+                        stdout="<<int(parameter:p1)>>",
+                    ),
+                ]
+            ),
+        ],
+    )
+    loop_term = hf.Rule(path="outputs.p1", condition={"value.equal_to": 300})
+    wk = hf.Workflow.from_template_data(
+        template_name="test_skip_reason",
+        path=tmp_path,
+        tasks=[hf.Task(schema=ts, inputs={"p1": 100})],
+        loops=[
+            hf.Loop(name="my_loop", tasks=[0], termination=loop_term, num_iterations=3)
+        ],
+    )
+    # loop should terminate after the second iteration
+    wk.submit(wait=True, add_to_known=False, status=False)
+    runs = wk.get_all_EARs()
+
+    assert runs[0].get("outputs.p1") == 200
+    assert runs[1].get("outputs.p1") == 300
+    assert not runs[2].get("outputs.p1")
+
+    assert runs[0].success
+    assert runs[1].success
+    assert not runs[2].success
+
+    assert runs[0].skip_reason is SkipReason.NOT_SKIPPED
+    assert runs[1].skip_reason is SkipReason.NOT_SKIPPED
+    assert runs[2].skip_reason is SkipReason.LOOP_TERMINATION

hpcflow/tests/unit/test_run_directories.py

@@ -0,0 +1,29 @@
+from pathlib import Path
+import pytest
+from hpcflow.app import app as hf
+from hpcflow.sdk.core.test_utils import make_workflow
+
+
+@pytest.mark.parametrize("store", ["json", "zarr"])
+def test_run_directories(null_config, tmp_path, store):
+    wk = make_workflow(
+        schemas_spec=[
+            [{"p1": None}, ("p1",), "t1"],
+            [{"p2": None}, ("p2",), "t2", {"requires_dir": True}],
+        ],
+        local_inputs={0: ("p1",)},
+        local_sequences={1: [("inputs.p2", 2, 0)]},
+        path=tmp_path,
+        store=store,
+    )
+    lp_0 = hf.Loop(name="my_loop", tasks=[1], num_iterations=2)
+    wk.add_loop(lp_0)
+    sub = wk.add_submission()  # populates run directories
+
+    run_dirs = wk.get_run_directories()
+
+    assert run_dirs[0] is None
+    assert str(run_dirs[1]) == str(Path(wk.path).joinpath("execute/t_1/e_0/i_0"))
+    assert str(run_dirs[2]) == str(Path(wk.path).joinpath("execute/t_1/e_1/i_0"))
+    assert str(run_dirs[3]) == str(Path(wk.path).joinpath("execute/t_1/e_0/i_1"))
+    assert str(run_dirs[4]) == str(Path(wk.path).joinpath("execute/t_1/e_1/i_1"))

hpcflow/tests/unit/test_shell.py

@@ -77,3 +77,23 @@ def test_process_JS_header_args_app_invoc_bash() -> None:
     for i, j in zip(app_invocs, expected):
         processed = shell.process_JS_header_args({"app_invoc": i})
         assert processed["app_invoc"] == j
+
+
+def test_format_array_powershell():
+    shell = ALL_SHELLS["powershell"]["nt"]()
+    assert shell.format_array([1, 2, 3]) == "@(1, 2, 3)"
+
+
+def test_format_array_get_item_powershell():
+    shell = ALL_SHELLS["powershell"]["nt"]()
+    assert shell.format_array_get_item("my_arr", 3) == "$my_arr[3]"
+
+
+def test_format_array_bash():
+    shell = ALL_SHELLS["bash"]["posix"]()
+    assert shell.format_array([1, 2, 3]) == "(1 2 3)"
+
+
+def test_format_array_get_item_bash():
+    shell = ALL_SHELLS["bash"]["posix"]()
+    assert shell.format_array_get_item("my_arr", 3) == r"${my_arr[3]}"

hpcflow/tests/unit/test_submission.py

@@ -247,6 +247,8 @@ def test_raise_missing_matching_env_executable(new_null_config, tmp_path) -> Non
     with pytest.raises(MissingEnvironmentExecutableInstanceError):
         wk.add_submission()
 
+    hf.reload_template_components()  # remove extra envs
+
 
 def test_no_raise_matching_env_executable(new_null_config, tmp_path) -> None:
     env_name = "my_hpcflow_env"
@@ -284,6 +286,8 @@ def test_no_raise_matching_env_executable(new_null_config, tmp_path) -> None:
     wk = hf.Workflow.from_template(wkt, path=tmp_path)
     wk.add_submission()
 
+    hf.reload_template_components()  # remove extra envs
+
 
 def test_raise_missing_env(new_null_config, tmp_path) -> None:
     env_name = "my_hpcflow_env"
@@ -336,82 +340,7 @@ def test_custom_env_and_executable(new_null_config, tmp_path) -> None:
     wk = hf.Workflow.from_template(wkt, path=tmp_path)
     wk.add_submission()
 
-
-def test_abort_EARs_file_creation(null_config, tmp_path) -> None:
-    wk_name = "temp"
-    t1 = hf.Task(
-        schema=hf.task_schemas.test_t1_conditional_OS,
-        sequences=[hf.ValueSequence("inputs.p1", values=[1, 2, 3])],
-    )
-    wkt = hf.WorkflowTemplate(name=wk_name, tasks=[t1])
-    wk = hf.Workflow.from_template(
-        template=wkt,
-        path=tmp_path,
-    )
-    sub = wk.add_submission()
-    assert sub is not None
-    wk.submissions_path.mkdir(exist_ok=True, parents=True)
-    sub.path.mkdir(exist_ok=True)
-    sub._write_abort_EARs_file()
-    with sub.abort_EARs_file_path.open("rt") as fp:
-        lines = fp.read()
-
-    assert lines == "0\n0\n0\n"
-
-
-@pytest.mark.parametrize("run_id", [0, 1, 2])
-def test_abort_EARs_file_update(null_config, tmp_path, run_id) -> None:
-    wk_name = "temp"
-    t1 = hf.Task(
-        schema=hf.task_schemas.test_t1_conditional_OS,
-        sequences=[hf.ValueSequence("inputs.p1", values=[1, 2, 3])],
-    )
-    wkt = hf.WorkflowTemplate(name=wk_name, tasks=[t1])
-    wk = hf.Workflow.from_template(
-        template=wkt,
-        path=tmp_path,
-    )
-    sub = wk.add_submission()
-    assert sub is not None
-    wk.submissions_path.mkdir(exist_ok=True, parents=True)
-    sub.path.mkdir(exist_ok=True)
-    sub._write_abort_EARs_file()
-
-    sub._set_run_abort(run_ID=run_id)
-
-    with sub.abort_EARs_file_path.open("rt") as fp:
-        lines = fp.read()
-
-    lines_exp = ["0", "0", "0"]
-    lines_exp[run_id] = "1"
-    assert lines == "\n".join(lines_exp) + "\n"
-
-
-def test_abort_EARs_file_update_with_existing_abort(null_config, tmp_path) -> None:
-    wk_name = "temp"
-    t1 = hf.Task(
-        schema=hf.task_schemas.test_t1_conditional_OS,
-        sequences=[hf.ValueSequence("inputs.p1", values=[1, 2, 3])],
-    )
-    wkt = hf.WorkflowTemplate(name=wk_name, tasks=[t1])
-    wk = hf.Workflow.from_template(
-        template=wkt,
-        path=tmp_path,
-    )
-    sub = wk.add_submission()
-    assert sub is not None
-    wk.submissions_path.mkdir(exist_ok=True, parents=True)
-    sub.path.mkdir(exist_ok=True)
-    sub._write_abort_EARs_file()
-
-    sub._set_run_abort(run_ID=1)
-    sub._set_run_abort(run_ID=2)
-
-    with sub.abort_EARs_file_path.open("rt") as fp:
-        lines = fp.read()
-
-    lines_exp = ["0", "1", "1"]
-    assert lines == "\n".join(lines_exp) + "\n"
+    hf.reload_template_components()  # remove extra envs
 
 
 def test_unique_schedulers_one_direct(new_null_config, tmp_path) -> None:

hpcflow/tests/unit/test_workflow_template.py

@@ -1,5 +1,7 @@
+from textwrap import dedent
 import pytest
 from hpcflow.app import app as hf
+from hpcflow.sdk.core.errors import MissingVariableSubstitutionError
 from hpcflow.sdk.core.test_utils import (
     make_test_data_YAML_workflow_template,
 )
@@ -38,6 +40,35 @@ def test_workflow_template_vars(tmp_path, new_null_config):
     assert wkt.tasks[0].element_sets[0].repeats[0]["number"] == num_repeats
 
 
+def test_workflow_template_vars_raise_no_vars(tmp_path, new_null_config):
+    # no default value for the variable is provided in `benchmark_N_elements`, so should
+    # raise if the variables dict is not passed:
+    with pytest.raises(MissingVariableSubstitutionError):
+        make_test_data_YAML_workflow_template("benchmark_N_elements.yaml")
+
+
+def test_workflow_template_vars_defaults_used(tmp_path, new_null_config):
+    # `benchmark_script_runner` contains a default value for the variable `N`, so that
+    # should be used, since we don't pass any variables:
+    wkt = make_test_data_YAML_workflow_template("benchmark_script_runner.yaml")
+    assert wkt.tasks[0].element_sets[0].repeats[0]["number"] == 1
+
+
+def test_workflow_template_vars_False_no_substitution(tmp_path, new_null_config):
+    # read a yaml template, check variables are not substituted, when `variables=False`:
+    wkt_yaml = dedent(
+        """\
+        name: workflow_1
+        tasks:
+          - schema: test_t1_conditional_OS            
+            inputs:
+              p1: <<var:my_var>>
+    """
+    )
+    wkt = hf.WorkflowTemplate.from_YAML_string(wkt_yaml, variables=False)
+    assert wkt.tasks[0].element_sets[0].inputs[0].value == "<<var:my_var>>"
+
+
 def test_env_preset_merge_simple(null_config):
     s1 = hf.TaskSchema(
         objective="s1",

hpcflow/tests/unit/utils/test_arrays.py

@@ -0,0 +1,40 @@
+import numpy as np
+
+from hpcflow.sdk.utils.arrays import get_1D_idx, get_2D_idx, split_arr
+
+
+def test_get_2D_idx():
+    assert get_2D_idx(0, num_cols=10) == (0, 0)
+    assert get_2D_idx(4, num_cols=10) == (0, 4)
+    assert get_2D_idx(9, num_cols=10) == (0, 9)
+    assert get_2D_idx(10, num_cols=10) == (1, 0)
+    assert get_2D_idx(13, num_cols=10) == (1, 3)
+    assert get_2D_idx(20, num_cols=10) == (2, 0)
+    arr_r, arr_c = get_2D_idx(np.array([0, 4, 9, 10, 13, 20]), num_cols=10)
+    assert np.array_equal(arr_r, np.array([0, 0, 0, 1, 1, 2]))
+    assert np.array_equal(arr_c, np.array([0, 4, 9, 0, 3, 0]))
+
+
+def test_get_1D_idx():
+    assert get_1D_idx(*(0, 0), num_cols=10) == 0
+    assert get_1D_idx(*(0, 4), num_cols=10) == 4
+    assert get_1D_idx(*(0, 9), num_cols=10) == 9
+    assert get_1D_idx(*(1, 0), num_cols=10) == 10
+    assert get_1D_idx(*(1, 3), num_cols=10) == 13
+    assert get_1D_idx(*(2, 0), num_cols=10) == 20
+
+    assert np.array_equal(
+        get_1D_idx(
+            np.array([0, 0, 0, 1, 1, 2]), np.array([0, 4, 9, 0, 3, 0]), num_cols=10
+        ),
+        np.array([0, 4, 9, 10, 13, 20]),
+    )
+
+
+def test_split_arr():
+    splt = split_arr(np.array([4, 0, 1, 2, 3, 4, 1, 4, 5, 6]), metadata_size=1)
+    assert len(splt) == 2
+    assert np.array_equal(splt[0][0], np.array([0]))
+    assert np.array_equal(splt[0][1], np.array([1, 2, 3]))
+    assert np.array_equal(splt[1][0], np.array([1]))
+    assert np.array_equal(splt[1][1], np.array([4, 5, 6]))

hpcflow/tests/unit/utils/test_deferred_file_writer.py

@@ -0,0 +1,34 @@
+from hpcflow.sdk.utils.deferred_file import DeferredFileWriter
+
+
+def test_file_not_created(tmp_path):
+    file_name = tmp_path / "test.txt"
+    assert not file_name.is_file()
+    with DeferredFileWriter(file_name, mode="w") as fp:
+        assert not fp._is_open
+    assert not file_name.is_file()
+
+
+def test_append_file_not_opened(tmp_path):
+    file_name = tmp_path / "test.txt"
+    with DeferredFileWriter(file_name, mode="a") as fp:
+        assert not fp._is_open
+    assert not file_name.is_file()
+
+
+def test_file_created_write(tmp_path):
+    file_name = tmp_path / "test.txt"
+    assert not file_name.is_file()
+    with DeferredFileWriter(file_name, mode="w") as fp:
+        fp.write("contents\n")
+        assert fp._is_open
+    assert file_name.is_file()
+
+
+def test_file_created_writelines(tmp_path):
+    file_name = tmp_path / "test.txt"
+    assert not file_name.is_file()
+    with DeferredFileWriter(file_name, mode="w") as fp:
+        fp.writelines(["contents\n"])
+        assert fp._is_open
+    assert file_name.is_file()