hpcflow 0.1.9__py3-none-any.whl → 0.2.0a271__py3-none-any.whl

hpcflow/tests/conftest.py

@@ -0,0 +1,123 @@
+from __future__ import annotations
+from pathlib import Path
+import pytest
+from hpcflow.app import app as hf
+
+
+def pytest_addoption(parser: pytest.Parser):
+    parser.addoption(
+        "--slurm",
+        action="store_true",
+        default=False,
+        help="run slurm tests",
+    )
+    parser.addoption(
+        "--wsl",
+        action="store_true",
+        default=False,
+        help="run Windows Subsystem for Linux tests",
+    )
+    parser.addoption(
+        "--direct-linux",
+        action="store_true",
+        default=False,
+        help="run direct-linux submission tests",
+    )
+    parser.addoption(
+        "--integration",
+        action="store_true",
+        default=False,
+        help="run integration-like workflow submission tests",
+    )
+    parser.addoption(
+        "--repeat",
+        action="store",
+        default=1,
+        type=int,
+        help="number of times to repeat each test",
+    )
+
+
+def pytest_configure(config: pytest.Config):
+    config.addinivalue_line("markers", "slurm: mark test as slurm to run")
+    config.addinivalue_line("markers", "wsl: mark test as wsl to run")
+    config.addinivalue_line(
+        "markers", "direct_linux: mark test as a direct-linux submission test to run"
+    )
+    config.addinivalue_line(
+        "markers",
+        "integration: mark test as an integration-like workflow submission test to run",
+    )
+    hf.run_time_info.in_pytest = True
+
+
+def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]):
+    if config.getoption("--slurm"):
+        # --slurm given in cli: only run slurm tests
+        for item in items:
+            if "slurm" not in item.keywords:
+                item.add_marker(pytest.mark.skip(reason="need no --slurm option to run"))
+    elif config.getoption("--wsl"):
+        # --wsl given in CLI: only run wsl tests
+        for item in items:
+            if "wsl" not in item.keywords:
+                item.add_marker(pytest.mark.skip(reason="need no --wsl option to run"))
+    elif config.getoption("--direct-linux"):
+        # --direct-linux in CLI: only run these tests
+        for item in items:
+            if "direct_linux" not in item.keywords:
+                item.add_marker(
+                    pytest.mark.skip(reason="remove --direct-linux option to run")
+                )
+    elif config.getoption("--integration"):
+        # --integration in CLI: only run these tests
+        for item in items:
+            if "integration" not in item.keywords:
+                item.add_marker(
+                    pytest.mark.skip(reason="remove --integration option to run")
+                )
+    else:
+        # --slurm not given in cli: skip slurm tests and do not skip other tests
+        for item in items:
+            if "slurm" in item.keywords:
+                item.add_marker(pytest.mark.skip(reason="need --slurm option to run"))
+            elif "wsl" in item.keywords:
+                item.add_marker(pytest.mark.skip(reason="need --wsl option to run"))
+            elif "direct_linux" in item.keywords:
+                item.add_marker(
+                    pytest.mark.skip(reason="add --direct_linux option to run")
+                )
+            elif "integration" in item.keywords:
+                item.add_marker(
+                    pytest.mark.skip(reason="add --integration option to run")
+                )
+
+
+def pytest_unconfigure(config: pytest.Config):
+    hf.run_time_info.in_pytest = False
+
+
+@pytest.fixture
+def null_config(tmp_path: Path):
+    if not hf.is_config_loaded:
+        hf.load_config(config_dir=tmp_path)
+    hf.run_time_info.in_pytest = True
+
+
+@pytest.fixture
+def new_null_config(tmp_path: Path):
+    hf.load_config(config_dir=tmp_path, warn=False)
+    hf.load_template_components(warn=False)
+    hf.run_time_info.in_pytest = True
+
+
+@pytest.fixture
+def unload_config():
+    hf.unload_config()
+
+
+def pytest_generate_tests(metafunc):
+    repeats_num = int(metafunc.config.getoption("--repeat"))
+    if repeats_num > 1:
+        metafunc.fixturenames.append("tmp_ct")
+        metafunc.parametrize("tmp_ct", range(repeats_num))

hpcflow/tests/data/benchmark_N_elements.yaml

@@ -0,0 +1,6 @@
+name: workflow_1
+tasks:
+  - schema: test_t1_conditional_OS
+    repeats: <<var:N>>
+    inputs:
+      p1: 101

hpcflow/tests/data/benchmark_script_runner.yaml

@@ -0,0 +1,26 @@
+doc: |
+  A workflow for benchmarking the overhead introduced by hpcflow in running a Python
+  script `N` times.
+
+template_components:
+  task_schemas:
+    - objective: run_script
+      inputs:
+        - parameter: p1
+      outputs:
+        - parameter: p2
+      actions:
+        - environments:
+            - scope:
+                type: any
+              environment: python_env
+          script: <<script:main_script_test_direct_in_direct_out.py>>
+          script_exe: python_script
+          script_data_in: direct
+          script_data_out: direct
+
+tasks:
+  - schema: run_script
+    inputs:
+      p1: 101
+    repeats: <<var:N[default=1]>>

hpcflow/tests/data/multi_path_sequences.yaml

@@ -0,0 +1,29 @@
+template_components:
+  task_schemas:
+    - objective: add_p1_components
+      inputs:
+        - parameter: p1
+      outputs:
+        - parameter: p2
+      actions:
+        - rules:
+            - path: resources.os_name
+              condition: { value.equal_to: posix }
+          commands:
+            - command: echo "$((<<parameter:p1.a>> + <<parameter:p1.b>>))"
+              stdout: <<int(parameter:p2)>>
+        - rules:
+            - path: resources.os_name
+              condition: { value.equal_to: nt }
+          commands:
+            - command: Write-Output ((<<parameter:p1.a>> + <<parameter:p1.b>>))
+              stdout: <<int(parameter:p2)>>
+tasks:
+  - schema: add_p1_components
+    inputs:
+      p1: {}
+    multi_path_sequences:
+      - paths: [inputs.p1.a, inputs.p1.b]
+        values:
+          - [101, 102]
+          - [201, 202]

hpcflow/tests/data/workflow_1.json

@@ -0,0 +1,10 @@
+{
+  "tasks": [
+    {
+      "schema": "test_t1_conditional_OS",
+      "inputs": {
+        "p1": 101
+      }
+    }
+  ]
+}

hpcflow/tests/data/workflow_1.yaml

@@ -0,0 +1,5 @@
+name: workflow_1
+tasks:
+  - schema: test_t1_conditional_OS
+    inputs:
+      p1: 101

hpcflow/tests/data/workflow_1_slurm.yaml

@@ -0,0 +1,8 @@
+name: workflow_1
+resources:
+  any:
+    scheduler: slurm
+tasks:
+  - schema: test_t1_conditional_OS
+    inputs:
+      p1: 101

hpcflow/tests/data/workflow_1_wsl.yaml

@@ -0,0 +1,8 @@
+name: workflow_1
+resources:
+  any:
+    shell: WSL
+tasks:
+  - schema: test_t1_bash
+    inputs:
+      p1: 101

hpcflow/tests/data/workflow_test_run_abort.yaml

@@ -0,0 +1,42 @@
+doc: |
+  A workflow to sleep a specified number of seconds, and then set an output parameter 
+  `is_finished` to the string "true". This is used for testing the run-abort
+  functionality.
+
+template_components:
+  task_schemas:
+    - objective: sleep
+      inputs:
+        - parameter: sleep_time_seconds
+      outputs:
+        - parameter: is_finished
+      actions:
+        - abortable: true
+          rules:
+            - path: resources.os_name
+              condition: { value.equal_to: posix }
+          commands:
+            - command: sleep <<parameter:sleep_time_seconds>>
+        - abortable: true
+          rules:
+            - path: resources.os_name
+              condition: { value.equal_to: nt }
+          commands:
+            - command: Start-Sleep <<parameter:sleep_time_seconds>>
+        - rules:
+            - path: resources.os_name
+              condition: { value.equal_to: posix }
+          commands:
+            - command: echo "true"
+              stdout: <<parameter:is_finished>>
+        - rules:
+            - path: resources.os_name
+              condition: { value.equal_to: nt }
+          commands:
+            - command: Write-Output "true"
+              stdout: <<parameter:is_finished>>
+
+tasks:
+  - schema: sleep
+    inputs:
+      sleep_time_seconds: 300 # 5 minutes

hpcflow/tests/jinja_templates/test_jinja_templates.py

@@ -0,0 +1,161 @@
+import os
+from textwrap import dedent
+import hpcflow.app as hf
+
+import pytest
+
+
+@pytest.mark.integration
+def test_basic_jinja_template(null_config, tmp_path):
+    jinja_template_name = "test_template.txt"
+    s1 = hf.TaskSchema(
+        objective="t1",
+        inputs=[
+            hf.SchemaInput(parameter=hf.Parameter("name")),
+            hf.SchemaInput(parameter=hf.Parameter("fruits")),
+        ],
+        actions=[hf.Action(jinja_template=f"test/{jinja_template_name}")],
+    )
+    t1 = hf.Task(schema=s1, inputs={"name": "George", "fruits": ["apple", "orange"]})
+    wk = hf.Workflow.from_template_data(
+        tasks=[t1],
+        template_name="jinja_template_test",
+        path=tmp_path,
+    )
+    wk.submit(wait=True, add_to_known=False, status=False)
+    run_dir = wk.get_all_EARs()[0].get_directory()
+    rendered = run_dir.joinpath(jinja_template_name)
+
+    expected = dedent(
+        """\
+        Hola, George!
+
+        This is a template, with a loop. Here are your specified fruits:
+        - apple
+        - orange
+        """
+    )
+
+    assert rendered.is_file()
+    assert rendered.read_text() == expected
+
+
+@pytest.mark.integration
+def test_jinja_template_path_with_resource_var(null_config, tmp_path):
+    jinja_template_name = "test_template.txt"
+    s1 = hf.TaskSchema(
+        objective="t1",
+        inputs=[
+            hf.SchemaInput(parameter=hf.Parameter("name")),
+            hf.SchemaInput(parameter=hf.Parameter("fruits")),
+        ],
+        actions=[
+            hf.Action(jinja_template=f"<<resource:resources_id>>/{jinja_template_name}")
+        ],
+    )
+    t1 = hf.Task(schema=s1, inputs={"name": "George", "fruits": ["apple", "orange"]})
+    wk = hf.Workflow.from_template_data(
+        tasks=[t1],
+        template_name="jinja_template_test",
+        path=tmp_path,
+        resources={"any": {"resources_id": "test"}},
+    )
+    wk.submit(wait=True, add_to_known=False, status=False)
+    run_dir = wk.get_all_EARs()[0].get_directory()
+    rendered = run_dir.joinpath(jinja_template_name)
+
+    expected = dedent(
+        """\
+        Hola, George!
+
+        This is a template, with a loop. Here are your specified fruits:
+        - apple
+        - orange
+        """
+    )
+
+    assert rendered.is_file()
+    assert rendered.read_text() == expected
+
+
+@pytest.mark.integration
+def test_builtin_jinja_template_path_with_env_var(new_null_config, tmp_path):
+    env = hf.Environment(
+        name="null_env",
+        specifiers={"key": "test"},  # reference this specifier in the jinja template path
+        executables=[],
+    )
+    hf.envs.add_object(env, skip_duplicates=True)
+
+    jinja_template_name = "test_template.txt"
+    s1 = hf.TaskSchema(
+        objective="t1",
+        inputs=[
+            hf.SchemaInput(parameter=hf.Parameter("name")),
+            hf.SchemaInput(parameter=hf.Parameter("fruits")),
+        ],
+        actions=[hf.Action(jinja_template=f"<<env:key>>/{jinja_template_name}")],
+    )
+    t1 = hf.Task(schema=s1, inputs={"name": "George", "fruits": ["apple", "orange"]})
+    wk = hf.Workflow.from_template_data(
+        tasks=[t1],
+        template_name="jinja_template_test",
+        path=tmp_path,
+        environments={"null_env": {"key": "test"}},
+    )
+    wk.submit(wait=True, add_to_known=False, status=False)
+    run_dir = wk.get_all_EARs()[0].get_directory()
+    rendered = run_dir.joinpath(jinja_template_name)
+
+    expected = dedent(
+        """\
+        Hola, George!
+
+        This is a template, with a loop. Here are your specified fruits:
+        - apple
+        - orange
+        """
+    )
+
+    assert rendered.is_file()
+    assert rendered.read_text() == expected
+
+    hf.reload_template_components()  # remove extra env
+
+
+@pytest.mark.integration
+def test_builtin_jinja_template_path_with_param_var(new_null_config, tmp_path):
+    jinja_template_name = "test_template.txt"
+    s1 = hf.TaskSchema(
+        objective="t1",
+        inputs=[
+            hf.SchemaInput(parameter=hf.Parameter("name")),
+            hf.SchemaInput(parameter=hf.Parameter("fruits")),
+            hf.SchemaInput(parameter=hf.Parameter("key")),
+        ],
+        actions=[hf.Action(jinja_template=f"<<parameter:key>>/{jinja_template_name}")],
+    )
+    t1 = hf.Task(
+        schema=s1, inputs={"name": "George", "fruits": ["apple", "orange"], "key": "test"}
+    )
+    wk = hf.Workflow.from_template_data(
+        tasks=[t1],
+        template_name="jinja_template_test",
+        path=tmp_path,
+    )
+    wk.submit(wait=True, add_to_known=False, status=False)
+    run_dir = wk.get_all_EARs()[0].get_directory()
+    rendered = run_dir.joinpath(jinja_template_name)
+
+    expected = dedent(
+        """\
+        Hola, George!
+
+        This is a template, with a loop. Here are your specified fruits:
+        - apple
+        - orange
+        """
+    )
+
+    assert rendered.is_file()
+    assert rendered.read_text() == expected

hpcflow/tests/programs/test_programs.py

@@ -0,0 +1,180 @@
+import os
+import pytest
+
+import hpcflow.app as hf
+
+
+@pytest.mark.integration
+def test_builtin_program_no_args_resource_var(new_null_config, tmp_path):
+    # run a builtin program
+    env_cmd = ("& " if os.name == "nt" else "") + "<<program_path>> <<args>>"
+    env = hf.Environment(
+        name="program_env",
+        executables=[
+            hf.Executable(
+                label="hello_world",
+                instances=[
+                    hf.ExecutableInstance(
+                        command=env_cmd,
+                        num_cores=1,
+                        parallel_mode=None,
+                    )
+                ],
+            )
+        ],
+    )
+    hf.envs.add_object(env, skip_duplicates=True)
+
+    act = hf.Action(
+        program="hello_world/<<resource:platform>>/hello_world<<resource:executable_extension>>",
+        program_exe="hello_world",
+        environments=[hf.ActionEnvironment("program_env")],
+    )
+    s1 = hf.TaskSchema(objective="hello", actions=[act])
+    tasks = [hf.Task(s1)]
+    wk = hf.Workflow.from_template_data(
+        template_name="test_program_no_args",
+        tasks=tasks,
+        path=tmp_path,
+    )
+    wk.submit(wait=True, status=False)
+    assert wk.submissions[0].jobscripts[0].get_stdout().strip() == "hello, world"
+
+    hf.reload_template_components()  # remove extra env
+
+
+@pytest.mark.integration
+def test_builtin_program_no_args_env_var(new_null_config, tmp_path):
+    # run a builtin program
+    env_cmd = ("& " if os.name == "nt" else "") + "<<program_path>> <<args>>"
+    default_platform = hf.ElementResources.get_default_platform()
+    env = hf.Environment(
+        name="program_env",
+        specifiers={
+            "platform": default_platform
+        },  # reference this specifier in the program path
+        executables=[
+            hf.Executable(
+                label="hello_world",
+                instances=[
+                    hf.ExecutableInstance(
+                        command=env_cmd,
+                        num_cores=1,
+                        parallel_mode=None,
+                    )
+                ],
+            )
+        ],
+    )
+    hf.envs.add_object(env, skip_duplicates=True)
+
+    act = hf.Action(
+        program="hello_world/<<env:platform>>/hello_world<<resource:executable_extension>>",
+        program_exe="hello_world",
+        environments=[hf.ActionEnvironment("program_env")],
+    )
+    s1 = hf.TaskSchema(objective="hello", actions=[act])
+    tasks = [hf.Task(s1)]
+    wk = hf.Workflow.from_template_data(
+        template_name="test_program_no_args",
+        environments={"program_env": {"platform": default_platform}},
+        tasks=tasks,
+        path=tmp_path,
+    )
+    wk.submit(wait=True, status=False)
+    assert wk.submissions[0].jobscripts[0].get_stdout().strip() == "hello, world"
+
+    hf.reload_template_components()  # remove extra env
+
+
+@pytest.mark.integration
+def test_builtin_program_no_args_param_var(new_null_config, tmp_path):
+    # run a builtin program
+    env_cmd = ("& " if os.name == "nt" else "") + "<<program_path>> <<args>>"
+    default_platform = hf.ElementResources.get_default_platform()
+    env = hf.Environment(
+        name="program_env",
+        executables=[
+            hf.Executable(
+                label="hello_world",
+                instances=[
+                    hf.ExecutableInstance(
+                        command=env_cmd,
+                        num_cores=1,
+                        parallel_mode=None,
+                    )
+                ],
+            )
+        ],
+    )
+    hf.envs.add_object(env, skip_duplicates=True)
+
+    act = hf.Action(
+        program="hello_world/<<parameter:platform>>/hello_world<<resource:executable_extension>>",
+        program_exe="hello_world",
+        environments=[hf.ActionEnvironment("program_env")],
+    )
+    s1 = hf.TaskSchema(
+        objective="hello",
+        actions=[act],
+        inputs=[hf.SchemaInput("platform")],
+    )
+    tasks = [hf.Task(s1, inputs={"platform": default_platform})]
+    wk = hf.Workflow.from_template_data(
+        template_name="test_program_no_args",
+        tasks=tasks,
+        path=tmp_path,
+    )
+    wk.submit(wait=True, status=False)
+    assert wk.submissions[0].jobscripts[0].get_stdout().strip() == "hello, world"
+
+    hf.reload_template_components()  # remove extra env
+
+
+@pytest.mark.integration
+def test_builtin_program_input_output_JSON_resource_var(new_null_config, tmp_path):
+    # run a builtin program that expects input and output JSON file paths as a cmdline arguments
+    env_cmd = ("& " if os.name == "nt" else "") + "<<program_path>> <<args>>"
+    env = hf.Environment(
+        name="program_env",
+        executables=[
+            hf.Executable(
+                label="hello_world",
+                instances=[
+                    hf.ExecutableInstance(
+                        command=env_cmd,
+                        num_cores=1,
+                        parallel_mode=None,
+                    )
+                ],
+            )
+        ],
+    )
+    hf.envs.add_object(env, skip_duplicates=True)
+
+    act = hf.Action(
+        program="hello_world/<<resource:platform>>/hello_world<<resource:executable_extension>>",
+        program_exe="hello_world",
+        program_data_in="json",
+        program_data_out="json",
+        requires_dir=True,
+        environments=[hf.ActionEnvironment("program_env")],
+    )
+    s1 = hf.TaskSchema(
+        objective="hello",
+        inputs=[hf.SchemaInput("p1"), hf.SchemaInput("p2"), hf.SchemaInput("p3")],
+        outputs=[hf.SchemaInput("p4")],
+        actions=[act],
+    )
+    p1, p2, p3 = 1, 2, 3
+    tasks = [hf.Task(s1, inputs={"p1": p1, "p2": p2, "p3": p3})]
+    wk = hf.Workflow.from_template_data(
+        template_name="test_program_input_output_files",
+        tasks=tasks,
+        path=tmp_path,
+    )
+    wk.submit(wait=True, status=False)
+    assert wk.submissions[0].jobscripts[0].get_stdout().strip() == "hello, world"
+    assert wk.tasks[0].elements[0].get("outputs.p4") == p1 + p2 + p3
+
+    hf.reload_template_components()  # remove extra env

hpcflow/tests/schedulers/direct_linux/test_direct_linux_submission.py

@@ -0,0 +1,12 @@
+from importlib import resources
+import pytest
+from hpcflow.app import app as hf
+
+
+@pytest.mark.direct_linux
+def test_workflow_1(tmp_path, null_config):
+    act = hf.Action(commands=[hf.Command("echo 'Buenas!'")])
+    ts = hf.TaskSchema(objective="hello", actions=[act])
+    wkt = hf.WorkflowTemplate(name="greetings", tasks=[hf.Task(schema=ts)])
+    wf = hf.Workflow.from_template(name="saludos", template=wkt)
+    wf.submit(wait=True, add_to_known=False)

hpcflow/tests/schedulers/sge/test_sge_submission.py

@@ -0,0 +1,36 @@
+import pytest
+from hpcflow.app import app as hf
+
+
+def test_SGE_process_resources_multi_core_with_parallel_env(null_config):
+
+    scheduler_config = {
+        "parallel_environments": {
+            None: {"num_cores": [1, 1, 1]},  # [start, step, stop]
+            "my_parallel_env": {"num_cores": [2, 1, 32]},
+        }
+    }
+
+    scheduler = hf.SGEPosix()
+    resources = hf.ElementResources(num_cores=2, SGE_parallel_env="my_parallel_env")
+
+    scheduler.process_resources(resources, scheduler_config)
+
+    assert resources.num_cores == 2
+    assert resources.SGE_parallel_env == "my_parallel_env"
+
+
+def test_SGE_process_resources_raises_on_single_core_with_parallel_env(null_config):
+
+    scheduler_config = {
+        "parallel_environments": {
+            None: {"num_cores": [1, 1, 1]},  # [start, step, stop]
+            "my_parallel_env": {"num_cores": [2, 1, 32]},
+        }
+    }
+
+    scheduler = hf.SGEPosix()
+    resources = hf.ElementResources(num_cores=1, SGE_parallel_env="my_parallel_env")
+
+    with pytest.raises(ValueError):
+        scheduler.process_resources(resources, scheduler_config)

hpcflow/tests/schedulers/slurm/test_slurm_submission.py

@@ -0,0 +1,14 @@
+from pathlib import Path
+import pytest
+from hpcflow.app import app as hf
+from hpcflow.sdk.core.test_utils import make_test_data_YAML_workflow
+
+
+@pytest.mark.slurm
+def test_workflow_1(tmp_path: Path, null_config):
+    hf.config.add_scheduler("slurm")
+    wk = make_test_data_YAML_workflow("workflow_1_slurm.yaml", path=tmp_path)
+    wk.submit(wait=True, add_to_known=False)
+    p2 = wk.tasks[0].elements[0].outputs.p2
+    assert isinstance(p2, hf.ElementParameter)
+    assert p2.value == "201"