hpcflow 0.1.9__py3-none-any.whl → 0.2.0a271__py3-none-any.whl

hpcflow/tests/unit/test_workflow_template.py

@@ -0,0 +1,186 @@
+from textwrap import dedent
+import pytest
+from hpcflow.app import app as hf
+from hpcflow.sdk.core.errors import MissingVariableSubstitutionError
+from hpcflow.sdk.core.test_utils import (
+    make_test_data_YAML_workflow_template,
+)
+
+
+def test_merge_template_level_resources_into_element_set(null_config):
+    wkt = hf.WorkflowTemplate(
+        name="w1",
+        tasks=[hf.Task(schema=[hf.task_schemas.test_t1_ps])],
+        resources={"any": {"num_cores": 1}},
+    )
+    assert wkt.tasks[0].element_sets[0].resources == hf.ResourceList.from_json_like(
+        {"any": {"num_cores": 1}}
+    )
+
+
+def test_equivalence_from_YAML_and_JSON_files(null_config):
+    wkt_yaml = make_test_data_YAML_workflow_template("workflow_1.yaml")
+    wkt_json = make_test_data_YAML_workflow_template("workflow_1.json")
+    assert wkt_json == wkt_yaml
+
+
+def test_reuse(null_config, tmp_path):
+    """Test we can re-use a template that has already been made persistent."""
+    wkt = hf.WorkflowTemplate(name="test", tasks=[])
+    wk1 = hf.Workflow.from_template(wkt, name="test_1", path=tmp_path)
+    wk2 = hf.Workflow.from_template(wkt, name="test_2", path=tmp_path)
+
+
+def test_workflow_template_vars(tmp_path, new_null_config):
+    num_repeats = 2
+    wkt = make_test_data_YAML_workflow_template(
+        workflow_name="benchmark_N_elements.yaml",
+        variables={"N": num_repeats},
+    )
+    assert wkt.tasks[0].element_sets[0].repeats[0]["number"] == num_repeats
+
+
+def test_workflow_template_vars_raise_no_vars(tmp_path, new_null_config):
+    # no default value for the variable is provided in `benchmark_N_elements`, so should
+    # raise if the variables dict is not passed:
+    with pytest.raises(MissingVariableSubstitutionError):
+        make_test_data_YAML_workflow_template("benchmark_N_elements.yaml")
+
+
+def test_workflow_template_vars_defaults_used(tmp_path, new_null_config):
+    # `benchmark_script_runner` contains a default value for the variable `N`, so that
+    # should be used, since we don't pass any variables:
+    wkt = make_test_data_YAML_workflow_template("benchmark_script_runner.yaml")
+    assert wkt.tasks[0].element_sets[0].repeats[0]["number"] == 1
+
+
+def test_workflow_template_vars_False_no_substitution(tmp_path, new_null_config):
+    # read a yaml template, check variables are not substituted, when `variables=False`:
+    wkt_yaml = dedent(
+        """\
+        name: workflow_1
+        tasks:
+          - schema: test_t1_conditional_OS            
+            inputs:
+              p1: <<var:my_var>>
+    """
+    )
+    wkt = hf.WorkflowTemplate.from_YAML_string(wkt_yaml, variables=False)
+    assert wkt.tasks[0].element_sets[0].inputs[0].value == "<<var:my_var>>"
+
+
+def test_env_preset_merge_simple(null_config):
+    s1 = hf.TaskSchema(
+        objective="s1",
+        actions=[hf.Action(environments=[hf.ActionEnvironment("my_env")])],
+        environment_presets={"my_env_preset": {"my_env": {"version": 1}}},
+    )
+    wkt = hf.WorkflowTemplate(
+        name="test",
+        env_presets="my_env_preset",
+        tasks=[hf.Task(schema=s1)],
+    )
+    assert wkt.tasks[0].element_sets[0].env_preset == "my_env_preset"
+    assert wkt.tasks[0].element_sets[0].resources[0].environments == {
+        "my_env": {"version": 1}
+    }
+
+
+def test_env_preset_merge_simple_list(null_config):
+    s1 = hf.TaskSchema(
+        objective="s1",
+        actions=[hf.Action(environments=[hf.ActionEnvironment("my_env")])],
+        environment_presets={"my_env_preset": {"my_env": {"version": 1}}},
+    )
+    wkt = hf.WorkflowTemplate(
+        name="test",
+        env_presets=["my_env_preset", "my_other_env_preset"],
+        tasks=[hf.Task(schema=s1)],
+    )
+    assert wkt.tasks[0].element_sets[0].env_preset == "my_env_preset"
+    assert wkt.tasks[0].element_sets[0].resources[0].environments == {
+        "my_env": {"version": 1}
+    }
+
+
+def test_env_preset_no_merge_existing_env_preset(null_config):
+    s1 = hf.TaskSchema(
+        objective="s1",
+        actions=[hf.Action(environments=[hf.ActionEnvironment("my_env")])],
+        environment_presets={
+            "env_preset_1": {"my_env": {"version": 1}},
+            "env_preset_2": {"my_env": {"version": 2}},
+        },
+    )
+    wkt = hf.WorkflowTemplate(
+        name="test",
+        env_presets="env_preset_1",
+        tasks=[hf.Task(schema=s1, env_preset="env_preset_2")],
+    )
+    assert wkt.tasks[0].element_sets[0].env_preset == "env_preset_2"
+    assert wkt.tasks[0].element_sets[0].resources[0].environments == {
+        "my_env": {"version": 2}
+    }
+
+
+def test_environments_merge_simple(null_config):
+    s1 = hf.TaskSchema(
+        objective="s1",
+        actions=[hf.Action(environments=[hf.ActionEnvironment("my_env")])],
+    )
+    wkt = hf.WorkflowTemplate(
+        name="test",
+        environments={"my_env": {"version": 1}, "my_other_env": {"version": 2}},
+        tasks=[hf.Task(schema=s1)],
+    )
+    assert wkt.tasks[0].element_sets[0].environments == {"my_env": {"version": 1}}
+    assert wkt.tasks[0].element_sets[0].resources[0].environments == {
+        "my_env": {"version": 1}
+    }
+
+
+def test_environments_no_merge_existing_envs(null_config):
+    s1 = hf.TaskSchema(
+        objective="s1",
+        actions=[hf.Action(environments=[hf.ActionEnvironment("my_env")])],
+    )
+    wkt = hf.WorkflowTemplate(
+        name="test",
+        environments={"my_env": {"version": 1}, "my_other_env": {"version": 2}},
+        tasks=[hf.Task(schema=s1, environments={"my_env": {"version": 2}})],
+    )
+    assert wkt.tasks[0].element_sets[0].environments == {"my_env": {"version": 2}}
+    assert wkt.tasks[0].element_sets[0].resources[0].environments == {
+        "my_env": {"version": 2}
+    }
+
+
+def test_raise_on_env_preset_and_environments(null_config):
+    with pytest.raises(ValueError):
+        wkt = hf.WorkflowTemplate(
+            name="test",
+            env_presets="my_env_preset",
+            environments={"my_env": {"version": 1}},
+        )
+
+
+def test_default_env_preset_used_if_available(null_config):
+    """Test that if no env_presets or environments are specified at template-level or task
+    level, the default (named as an empty string) env preset is used if available."""
+
+    s1 = hf.TaskSchema(
+        objective="s1",
+        actions=[hf.Action(environments=[hf.ActionEnvironment("my_env")])],
+        environment_presets={
+            "": {"my_env": {"version": 1}},
+            "env_preset_1": {"my_env": {"version": 2}},
+        },
+    )
+    wkt = hf.WorkflowTemplate(
+        name="test",
+        tasks=[hf.Task(schema=s1)],
+    )
+    assert wkt.tasks[0].element_sets[0].env_preset == ""
+    assert wkt.tasks[0].element_sets[0].resources[0].environments == {
+        "my_env": {"version": 1}
+    }

hpcflow/tests/unit/utils/test_arrays.py

@@ -0,0 +1,40 @@
+import numpy as np
+
+from hpcflow.sdk.utils.arrays import get_1D_idx, get_2D_idx, split_arr
+
+
+def test_get_2D_idx():
+    assert get_2D_idx(0, num_cols=10) == (0, 0)
+    assert get_2D_idx(4, num_cols=10) == (0, 4)
+    assert get_2D_idx(9, num_cols=10) == (0, 9)
+    assert get_2D_idx(10, num_cols=10) == (1, 0)
+    assert get_2D_idx(13, num_cols=10) == (1, 3)
+    assert get_2D_idx(20, num_cols=10) == (2, 0)
+    arr_r, arr_c = get_2D_idx(np.array([0, 4, 9, 10, 13, 20]), num_cols=10)
+    assert np.array_equal(arr_r, np.array([0, 0, 0, 1, 1, 2]))
+    assert np.array_equal(arr_c, np.array([0, 4, 9, 0, 3, 0]))
+
+
+def test_get_1D_idx():
+    assert get_1D_idx(*(0, 0), num_cols=10) == 0
+    assert get_1D_idx(*(0, 4), num_cols=10) == 4
+    assert get_1D_idx(*(0, 9), num_cols=10) == 9
+    assert get_1D_idx(*(1, 0), num_cols=10) == 10
+    assert get_1D_idx(*(1, 3), num_cols=10) == 13
+    assert get_1D_idx(*(2, 0), num_cols=10) == 20
+
+    assert np.array_equal(
+        get_1D_idx(
+            np.array([0, 0, 0, 1, 1, 2]), np.array([0, 4, 9, 0, 3, 0]), num_cols=10
+        ),
+        np.array([0, 4, 9, 10, 13, 20]),
+    )
+
+
+def test_split_arr():
+    splt = split_arr(np.array([4, 0, 1, 2, 3, 4, 1, 4, 5, 6]), metadata_size=1)
+    assert len(splt) == 2
+    assert np.array_equal(splt[0][0], np.array([0]))
+    assert np.array_equal(splt[0][1], np.array([1, 2, 3]))
+    assert np.array_equal(splt[1][0], np.array([1]))
+    assert np.array_equal(splt[1][1], np.array([4, 5, 6]))

hpcflow/tests/unit/utils/test_deferred_file_writer.py

@@ -0,0 +1,34 @@
+from hpcflow.sdk.utils.deferred_file import DeferredFileWriter
+
+
+def test_file_not_created(tmp_path):
+    file_name = tmp_path / "test.txt"
+    assert not file_name.is_file()
+    with DeferredFileWriter(file_name, mode="w") as fp:
+        assert not fp._is_open
+    assert not file_name.is_file()
+
+
+def test_append_file_not_opened(tmp_path):
+    file_name = tmp_path / "test.txt"
+    with DeferredFileWriter(file_name, mode="a") as fp:
+        assert not fp._is_open
+    assert not file_name.is_file()
+
+
+def test_file_created_write(tmp_path):
+    file_name = tmp_path / "test.txt"
+    assert not file_name.is_file()
+    with DeferredFileWriter(file_name, mode="w") as fp:
+        fp.write("contents\n")
+        assert fp._is_open
+    assert file_name.is_file()
+
+
+def test_file_created_writelines(tmp_path):
+    file_name = tmp_path / "test.txt"
+    assert not file_name.is_file()
+    with DeferredFileWriter(file_name, mode="w") as fp:
+        fp.writelines(["contents\n"])
+        assert fp._is_open
+    assert file_name.is_file()

hpcflow/tests/unit/utils/test_hashing.py

@@ -0,0 +1,65 @@
+from hpcflow.sdk.utils.hashing import get_hash
+
+
+def test_get_hash_simple_types_is_int():
+    assert isinstance(get_hash(1), int)
+    assert isinstance(get_hash(3.2), int)
+    assert isinstance(get_hash("a"), int)
+    assert isinstance(get_hash("abc"), int)
+
+
+def test_get_hash_compound_types_is_int():
+    assert isinstance(get_hash([1, 2, 3]), int)
+    assert isinstance(get_hash((1, 2, 3)), int)
+    assert isinstance(get_hash({1, 2, 3}), int)
+    assert isinstance(get_hash({"a": 1, "b": 2, "c": 3}), int)
+
+
+def test_get_hash_nested_dict_is_int():
+    assert isinstance(get_hash({"a": {"b": {"c": [1, 2, 3, ("4", 5, 6)]}}}), int)
+
+
+def test_get_hash_distinct_simple_types():
+    assert get_hash(1) != get_hash(2)
+    assert get_hash(2.2) != get_hash(2.3)
+    assert get_hash("a") != get_hash("b")
+    assert get_hash("abc") != get_hash("ABC")
+
+
+def test_get_hash_distinct_compound_types():
+    assert get_hash([1, 2, 3]) != get_hash([1, 2, 4])
+    assert get_hash((1, 2, 3)) != get_hash((1, 2, 4))
+    assert get_hash({1, 2, 3}) != get_hash({1, 2, 4})
+    assert get_hash({"a": 1, "b": 2, "c": 3}) != get_hash({"a": 1, "b": 2, "c": 4})
+    assert get_hash({"a": {"b": {"c": [1, 2, 3, ("4", 5, 7)]}}}) == get_hash(
+        {"a": {"b": {"c": [1, 2, 3, ("4", 5, 7)]}}}
+    )
+    assert get_hash({"a": 1}) != get_hash(1) != get_hash("a")
+
+
+def test_get_hash_equal_simple_types():
+    assert get_hash(1) == get_hash(1)
+    assert get_hash(2.2) == get_hash(2.2)
+    assert get_hash("a") == get_hash("a")
+    assert get_hash("abc") == get_hash("abc")
+
+
+def test_get_hash_equal_compound_types():
+    assert get_hash([1, 2, 3]) == get_hash([1, 2, 3])
+    assert get_hash((1, 2, 3)) == get_hash((1, 2, 3))
+    assert get_hash({1, 2, 3}) == get_hash({1, 2, 3})
+    assert get_hash({"a": 1, "b": 2, "c": 3}) == get_hash({"a": 1, "b": 2, "c": 3})
+    assert get_hash({"a": {"b": {"c": [1, 2, 3, ("4", 5, 6)]}}}) == get_hash(
+        {"a": {"b": {"c": [1, 2, 3, ("4", 5, 6)]}}}
+    )
+
+
+def test_get_hash_order_insensitivity():
+    assert get_hash({"a": 1, "b": 2}) == get_hash({"b": 2, "a": 1})
+    assert get_hash({1, 2, 3}) == get_hash({2, 3, 1})
+
+
+def test_get_hash_order_sensitivity():
+    assert get_hash([1, 2, 3]) != get_hash([2, 3, 1])
+    assert get_hash((1, 2, 3)) != get_hash((2, 3, 1))
+    assert get_hash("abc") != get_hash("cba")

hpcflow/tests/unit/utils/test_patches.py

@@ -0,0 +1,5 @@
+from hpcflow.sdk.utils.patches import resolve_path
+
+
+def test_absolute_path():
+    assert resolve_path("my_file_path").is_absolute()

hpcflow/tests/unit/utils/test_redirect_std.py

@@ -0,0 +1,50 @@
+import sys
+
+import pytest
+from hpcflow.sdk.core.utils import redirect_std_to_file
+
+
+def test_stdout_redirect(tmp_path):
+    file_name = tmp_path / "test.txt"
+    expected = "stdout"
+    with redirect_std_to_file(file_name, mode="w"):
+        print(expected)
+    with file_name.open("r") as fp:
+        contents = fp.read().strip()
+    assert contents == expected
+
+
+def test_stderr_redirect(tmp_path):
+    file_name = tmp_path / "test.txt"
+    expected = "stderr"
+    with redirect_std_to_file(file_name, mode="w"):
+        print(expected, file=sys.stderr)
+    with file_name.open("r") as fp:
+        contents = fp.read().strip()
+    assert contents == expected
+
+
+def test_exception_exits_with_code(tmp_path):
+    file_name = tmp_path / "test.txt"
+    with pytest.raises(SystemExit) as exc:
+        with redirect_std_to_file(file_name, mode="w"):
+            raise ValueError("oh no!")
+    assert exc.value.code == 1
+
+
+def test_exception_prints_to_file(tmp_path):
+    file_name = tmp_path / "test.txt"
+    with pytest.raises(SystemExit):
+        with redirect_std_to_file(file_name, mode="w"):
+            raise ValueError("oh no!")
+    with file_name.open("r") as fp:
+        contents = fp.read().strip()
+    assert 'ValueError("oh no!")' in contents
+
+
+def test_file_not_created(tmp_path):
+    file_name = tmp_path / "test.txt"
+    assert not file_name.is_file()
+    with redirect_std_to_file(file_name, mode="w"):
+        pass
+    assert not file_name.is_file()

hpcflow/tests/unit/utils/test_strings.py

@@ -0,0 +1,97 @@
+from textwrap import dedent
+
+from hpcflow.sdk.utils.strings import extract_py_from_future_imports
+
+
+def test_extract_py_from_future_imports_none():
+    py_str = dedent(
+        """\
+
+    def my_function():
+        print("blah!")
+    """
+    )
+    new_str, imports = extract_py_from_future_imports(py_str)
+    assert imports == set()
+    assert new_str == py_str
+
+
+def test_extract_py_from_future_imports_single():
+    py_str = dedent(
+        """\
+    from __future__ import annotations
+
+    def my_function():
+        print("blah!")
+    """
+    )
+    new_str, imports = extract_py_from_future_imports(py_str)
+    assert imports == {"annotations"}
+    assert new_str == dedent(
+        """\
+
+    def my_function():
+        print("blah!")
+    """
+    )
+
+
+def test_extract_py_from_future_imports_multi():
+    py_str = dedent(
+        """\
+    from __future__ import annotations, feature_2
+
+    def my_function():
+        print("blah!")
+    """
+    )
+    new_str, imports = extract_py_from_future_imports(py_str)
+    assert imports == {"annotations", "feature_2"}
+    assert new_str == dedent(
+        """\
+
+    def my_function():
+        print("blah!")
+    """
+    )
+
+
+def test_extract_py_from_future_imports_trailing_comma():
+    py_str = dedent(
+        """\
+    from __future__ import annotations,
+
+    def my_function():
+        print("blah!")
+    """
+    )
+    new_str, imports = extract_py_from_future_imports(py_str)
+    assert imports == {"annotations"}
+    assert new_str == dedent(
+        """\
+
+    def my_function():
+        print("blah!")
+    """
+    )
+
+
+def test_extract_py_from_future_imports_multi_lines():
+    py_str = dedent(
+        """\
+    from __future__ import annotations, feature_2
+    from __future__ import feature_2, feature_3,
+
+    def my_function():
+        print("blah!")
+    """
+    )
+    new_str, imports = extract_py_from_future_imports(py_str)
+    assert imports == {"annotations", "feature_2", "feature_3"}
+    assert new_str == dedent(
+        """\
+
+    def my_function():
+        print("blah!")
+    """
+    )

hpcflow/tests/workflows/test_directory_structure.py

@@ -0,0 +1,31 @@
+"""Tests concerning the directory structure of a created or submitted workflow"""
+
+import os
+from pathlib import Path
+import pytest
+
+from hpcflow.sdk.core.test_utils import (
+    make_test_data_YAML_workflow,
+    make_workflow_to_run_command,
+)
+
+
+@pytest.mark.integration
+def test_std_stream_file_not_created(tmp_path, new_null_config):
+    """Normally, the app standard stream file should not be written."""
+    wk = make_test_data_YAML_workflow("workflow_1.yaml", path=tmp_path)
+    wk.submit(wait=True, add_to_known=False)
+    run = wk.get_all_EARs()[0]
+    std_stream_path = run.get_app_std_path()
+    assert not std_stream_path.is_file()
+
+
+@pytest.mark.integration
+def test_std_stream_file_created_on_exception_raised(tmp_path, new_null_config):
+    command = 'wkflow_app --std-stream "$HPCFLOW_RUN_STD_PATH" internal noop --raise'
+    wk = make_workflow_to_run_command(command=command, path=tmp_path)
+    wk.submit(wait=True, add_to_known=False)
+    run = wk.get_all_EARs()[0]
+    std_stream_path = run.get_app_std_path()
+    assert std_stream_path.is_file()
+    assert "ValueError: internal noop raised!" in std_stream_path.read_text()