hpcflow-new2 0.2.0a189__py3-none-any.whl → 0.2.0a199__py3-none-any.whl

hpcflow/tests/data/workflow_test_run_abort.yaml

@@ -2,42 +2,41 @@ doc: |
   A workflow to sleep a specified number of seconds, and then set an output parameter 
   `is_finished` to the string "true". This is used for testing the run-abort
   functionality.
-parameters:
-  - type: sleep_time_seconds
-  - type: is_finished
-task_schemas:
-  - objective: sleep
-    inputs:
-      - parameter: sleep_time_seconds
-    outputs:
-      - parameter: is_finished
-    actions:
-      - abortable: true
-        rules:
-          - path: resources.os_name
-            condition: { value.equal_to: posix }
-        commands:
-          - command: sleep <<parameter:sleep_time_seconds>>
-      - abortable: true
-        rules:
-          - path: resources.os_name
-            condition: { value.equal_to: nt }
-        commands:
-          - command: Start-Sleep <<parameter:sleep_time_seconds>>
-      - rules:
-          - path: resources.os_name
-            condition: { value.equal_to: posix }
-        commands:
-          - command: echo "true"
-            stdout: <<parameter:is_finished>>
-      - rules:
-          - path: resources.os_name
-            condition: { value.equal_to: nt }
-        commands:
-          - command: Write-Output "true"
-            stdout: <<parameter:is_finished>>
+
+template_components:
+  task_schemas:
+    - objective: sleep
+      inputs:
+        - parameter: sleep_time_seconds
+      outputs:
+        - parameter: is_finished
+      actions:
+        - abortable: true
+          rules:
+            - path: resources.os_name
+              condition: { value.equal_to: posix }
+          commands:
+            - command: sleep <<parameter:sleep_time_seconds>>
+        - abortable: true
+          rules:
+            - path: resources.os_name
+              condition: { value.equal_to: nt }
+          commands:
+            - command: Start-Sleep <<parameter:sleep_time_seconds>>
+        - rules:
+            - path: resources.os_name
+              condition: { value.equal_to: posix }
+          commands:
+            - command: echo "true"
+              stdout: <<parameter:is_finished>>
+        - rules:
+            - path: resources.os_name
+              condition: { value.equal_to: nt }
+          commands:
+            - command: Write-Output "true"
+              stdout: <<parameter:is_finished>>
 
 tasks:
   - schema: sleep
     inputs:
-      sleep_time_seconds: 3600 # 1 hour
+      sleep_time_seconds: 300 # 5 minutes

hpcflow/tests/schedulers/sge/test_sge_submission.py

@@ -0,0 +1,36 @@
+import pytest
+from hpcflow.app import app as hf
+
+
+def test_SGE_process_resources_multi_core_with_parallel_env(null_config):
+
+    scheduler_config = {
+        "parallel_environments": {
+            None: {"num_cores": [1, 1, 1]},  # [start, step, stop]
+            "my_parallel_env": {"num_cores": [2, 1, 32]},
+        }
+    }
+
+    scheduler = hf.SGEPosix()
+    resources = hf.ElementResources(num_cores=2, SGE_parallel_env="my_parallel_env")
+
+    scheduler.process_resources(resources, scheduler_config)
+
+    assert resources.num_cores == 2
+    assert resources.SGE_parallel_env == "my_parallel_env"
+
+
+def test_SGE_process_resources_raises_on_single_core_with_parallel_env(null_config):
+
+    scheduler_config = {
+        "parallel_environments": {
+            None: {"num_cores": [1, 1, 1]},  # [start, step, stop]
+            "my_parallel_env": {"num_cores": [2, 1, 32]},
+        }
+    }
+
+    scheduler = hf.SGEPosix()
+    resources = hf.ElementResources(num_cores=1, SGE_parallel_env="my_parallel_env")
+
+    with pytest.raises(ValueError):
+        scheduler.process_resources(resources, scheduler_config)

hpcflow/tests/schedulers/slurm/test_slurm_submission.py

@@ -1,11 +1,14 @@
+from pathlib import Path
 import pytest
 from hpcflow.app import app as hf
 from hpcflow.sdk.core.test_utils import make_test_data_YAML_workflow
 
 
 @pytest.mark.slurm
-def test_workflow_1(tmp_path, null_config):
+def test_workflow_1(tmp_path: Path, null_config):
     hf.config.add_scheduler("slurm")
     wk = make_test_data_YAML_workflow("workflow_1_slurm.yaml", path=tmp_path)
     wk.submit(wait=True, add_to_known=False)
-    assert wk.tasks[0].elements[0].outputs.p2.value == "201"
+    p2 = wk.tasks[0].elements[0].outputs.p2
+    assert isinstance(p2, hf.ElementParameter)
+    assert p2.value == "201"

hpcflow/tests/scripts/test_input_file_generators.py

@@ -0,0 +1,282 @@
+import os
+import time
+
+import pytest
+from hpcflow.app import app as hf
+
+
+@pytest.mark.integration
+@pytest.mark.skipif("hf.run_time_info.is_frozen")
+def test_input_file_generator_creates_file(null_config, tmp_path):
+
+    inp_file = hf.FileSpec(label="my_input_file", name="my_input_file.txt")
+
+    if os.name == "nt":
+        cmd = "Get-Content <<file:my_input_file>>"
+    else:
+        cmd = "cat <<file:my_input_file>>"
+
+    s1 = hf.TaskSchema(
+        objective="t1",
+        inputs=[hf.SchemaInput(parameter=hf.Parameter("p1"))],
+        actions=[
+            hf.Action(
+                commands=[hf.Command(cmd)],
+                input_file_generators=[
+                    hf.InputFileGenerator(
+                        input_file=inp_file,
+                        inputs=[hf.Parameter("p1")],
+                        script="<<script:input_file_generator_basic.py>>",
+                    ),
+                ],
+                environments=[hf.ActionEnvironment(environment="python_env")],
+            )
+        ],
+    )
+    p1_val = 101
+    t1 = hf.Task(schema=s1, inputs={"p1": p1_val})
+    wk = hf.Workflow.from_template_data(
+        tasks=[t1],
+        template_name="input_file_generator_test",
+        path=tmp_path,
+    )
+    wk.submit(wait=True, add_to_known=False)
+
+    # check the input file is written
+    run_0 = wk.get_all_EARs()[0]
+    exec_path = run_0.get_directory()
+    inp_file_path = exec_path.joinpath(inp_file.name.name)
+    inp_file_contents = inp_file_path.read_text()
+    assert inp_file_contents.strip() == str(p1_val)
+
+    # check the command successfully printed the file contents to stdout:
+    std_out = wk.submissions[0].jobscripts[0].direct_stdout_path.read_text()
+    assert std_out.strip() == str(p1_val)
+
+
+@pytest.mark.integration
+@pytest.mark.skipif("hf.run_time_info.is_frozen")
+def test_IFG_std_stream_redirect_on_exception(new_null_config, tmp_path):
+    """Test exceptions raised by the app during execution of a IFG script are printed to the
+    std-stream redirect file (and not the jobscript's standard error file)."""
+
+    # define a custom python environment which redefines the `WK_PATH` shell variable to
+    # a nonsense value so the app cannot load the workflow and thus raises an exception
+
+    app_caps = hf.package_name.upper()
+    if os.name == "nt":
+        env_cmd = f'$env:{app_caps}_WK_PATH = "nonsense_path"'
+    else:
+        env_cmd = f'export {app_caps}_WK_PATH="nonsense_path"'
+
+    env_cmd += "; python <<script_path>> <<args>>"
+    bad_env = hf.Environment(
+        name="bad_python_env",
+        executables=[
+            hf.Executable(
+                label="python_script",
+                instances=[
+                    hf.ExecutableInstance(
+                        command=env_cmd,
+                        num_cores=1,
+                        parallel_mode=None,
+                    )
+                ],
+            )
+        ],
+    )
+    hf.envs.add_object(bad_env, skip_duplicates=True)
+
+    inp_file = hf.FileSpec(label="my_input_file", name="my_input_file.txt")
+    s1 = hf.TaskSchema(
+        objective="t1",
+        inputs=[hf.SchemaInput(parameter=hf.Parameter("p1"))],
+        actions=[
+            hf.Action(
+                input_file_generators=[
+                    hf.InputFileGenerator(
+                        input_file=inp_file,
+                        inputs=[hf.Parameter("p1")],
+                        script="<<script:input_file_generator_basic.py>>",
+                    ),
+                ],
+                environments=[hf.ActionEnvironment(environment="bad_python_env")],
+            )
+        ],
+    )
+
+    p1_val = 101
+    t1 = hf.Task(schema=s1, inputs={"p1": p1_val})
+    wk = hf.Workflow.from_template_data(
+        tasks=[t1], template_name="input_file_generator_test", path=tmp_path
+    )
+    wk.submit(wait=True, add_to_known=False, status=False)
+
+    # jobscript stderr should be empty
+    assert not wk.submissions[0].jobscripts[0].direct_stderr_path.read_text()
+
+    # std stream file has workflow not found traceback
+    run = wk.get_all_EARs()[0]
+    std_stream_path = run.get_app_std_path()
+    assert std_stream_path.is_file()
+    assert "WorkflowNotFoundError" in std_stream_path.read_text()
+
+    hf.reload_template_components()  # remove extra envs
+
+
+@pytest.mark.integration
+@pytest.mark.skipif("hf.run_time_info.is_frozen")
+def test_IFG_std_out_std_err_not_redirected(null_config, tmp_path):
+    """Test that standard error and output streams from an IFG script are written to the jobscript
+    standard error and output files."""
+    inp_file = hf.FileSpec(label="my_input_file", name="my_input_file.txt")
+    s1 = hf.TaskSchema(
+        objective="t1",
+        inputs=[hf.SchemaInput(parameter=hf.Parameter("p1"))],
+        actions=[
+            hf.Action(
+                input_file_generators=[
+                    hf.InputFileGenerator(
+                        input_file=inp_file,
+                        inputs=[hf.Parameter("p1")],
+                        script="<<script:input_file_generator_test_stdout_stderr.py>>",
+                    ),
+                ],
+                environments=[hf.ActionEnvironment(environment="python_env")],
+            )
+        ],
+    )
+    p1_val = 101
+    stdout_msg = str(p1_val)
+    stderr_msg = str(p1_val)
+    t1 = hf.Task(schema=s1, inputs={"p1": p1_val})
+    wk = hf.Workflow.from_template_data(
+        tasks=[t1],
+        template_name="input_file_generator_test",
+        path=tmp_path,
+    )
+    wk.submit(wait=True, add_to_known=False)
+
+    if wk.submissions[0].jobscripts[0].resources.combine_jobscript_std:
+        std_out_err = wk.submissions[0].jobscripts[0].direct_std_out_err_path.read_text()
+        assert std_out_err.strip() == f"{stdout_msg}\n{stderr_msg}"
+    else:
+        std_out = wk.submissions[0].jobscripts[0].direct_stdout_path.read_text()
+        std_err = wk.submissions[0].jobscripts[0].direct_stderr_path.read_text()
+        assert std_out.strip() == stdout_msg
+        assert std_err.strip() == stderr_msg
+
+
+@pytest.mark.integration
+@pytest.mark.skipif("hf.run_time_info.is_frozen")
+def test_IFG_pass_env_spec(null_config, tmp_path):
+    inp_file = hf.FileSpec(label="my_input_file", name="my_input_file.txt")
+
+    if os.name == "nt":
+        cmd = "Get-Content <<file:my_input_file>>"
+    else:
+        cmd = "cat <<file:my_input_file>>"
+
+    s1 = hf.TaskSchema(
+        objective="t1",
+        inputs=[hf.SchemaInput(parameter=hf.Parameter("p1"))],
+        actions=[
+            hf.Action(
+                commands=[hf.Command(cmd)],
+                input_file_generators=[
+                    hf.InputFileGenerator(
+                        input_file=inp_file,
+                        inputs=[hf.Parameter("p1")],
+                        script="<<script:env_specifier_test/input_file_generator_pass_env_spec.py>>",
+                        script_pass_env_spec=True,
+                    ),
+                ],
+                environments=[hf.ActionEnvironment(environment="python_env")],
+            )
+        ],
+    )
+    p1_val = 101
+    t1 = hf.Task(schema=s1, inputs={"p1": p1_val})
+    wk = hf.Workflow.from_template_data(
+        tasks=[t1],
+        template_name="input_file_generator_pass_env_spec",
+        path=tmp_path,
+    )
+    wk.submit(wait=True, add_to_known=False, status=False)
+
+    # check the command successfully printed the env spec and file contents to stdout:
+    std_out = wk.submissions[0].jobscripts[0].direct_stdout_path.read_text()
+    assert std_out.strip() == f"{{'name': 'python_env'}}\n{str(p1_val)}"
+
+
+@pytest.mark.integration
+@pytest.mark.skipif("hf.run_time_info.is_frozen")
+def test_env_specifier_in_input_file_generator_script_path(new_null_config, tmp_path):
+
+    py_env = hf.Environment(
+        name="python_env",
+        specifiers={"version": "v1"},
+        executables=[
+            hf.Executable(
+                label="python_script",
+                instances=[
+                    hf.ExecutableInstance(
+                        command="python <<script_path>> <<args>>",
+                        num_cores=1,
+                        parallel_mode=None,
+                    )
+                ],
+            )
+        ],
+    )
+    hf.envs.add_object(py_env, skip_duplicates=True)
+
+    inp_file = hf.FileSpec(label="my_input_file", name="my_input_file.txt")
+
+    if os.name == "nt":
+        cmd = "Get-Content <<file:my_input_file>>"
+    else:
+        cmd = "cat <<file:my_input_file>>"
+
+    s1 = hf.TaskSchema(
+        objective="t1",
+        inputs=[hf.SchemaInput(parameter=hf.Parameter("p1"))],
+        actions=[
+            hf.Action(
+                commands=[hf.Command(cmd)],
+                input_file_generators=[
+                    hf.InputFileGenerator(
+                        input_file=inp_file,
+                        inputs=[hf.Parameter("p1")],
+                        script="<<script:env_specifier_test/<<env:version>>/input_file_generator_basic.py>>",
+                    ),
+                ],
+                environments=[hf.ActionEnvironment(environment="python_env")],
+            )
+        ],
+    )
+    p1_val = 101
+    t1 = hf.Task(
+        schema=s1,
+        inputs={"p1": p1_val},
+        environments={"python_env": {"version": "v1"}},
+    )
+    wk = hf.Workflow.from_template_data(
+        tasks=[t1],
+        template_name="input_file_generator_test_env_specifier",
+        path=tmp_path,
+    )
+    wk.submit(wait=True, add_to_known=False, status=False)
+
+    # check the input file is written
+    run_0 = wk.get_all_EARs()[0]
+    exec_path = run_0.get_directory()
+    inp_file_path = exec_path.joinpath(inp_file.name.name)
+    inp_file_contents = inp_file_path.read_text()
+    assert inp_file_contents.strip() == str(p1_val)
+
+    # check the command successfully printed the file contents to stdout:
+    std_out = wk.submissions[0].jobscripts[0].direct_stdout_path.read_text()
+    assert std_out.strip() == str(p1_val)
+
+    hf.reload_template_components()  # remove extra envs