hpcflow 0.1.15__py3-none-any.whl → 0.2.0a271__py3-none-any.whl

hpcflow/data/scripts/bad_script.py

@@ -0,0 +1,2 @@
+def bad_script():
+    raise SyntaxError("oh no!")

hpcflow/data/scripts/demo_task_1_generate_t1_infile_1.py

@@ -0,0 +1,8 @@
+from random import randint
+from pathlib import Path
+
+
+def demo_task_1_generate_t1_infile_1(path, p1):
+    with Path(path).open("w") as fp:
+        fp.write(f"{randint(0, 1e6)}\n")
+        fp.write(f"p1: {p1}\n")

hpcflow/data/scripts/demo_task_1_generate_t1_infile_2.py

@@ -0,0 +1,8 @@
+from random import randint
+from pathlib import Path
+
+
+def demo_task_1_generate_t1_infile_2(path, p2):
+    with Path(path).open("w") as fp:
+        fp.write(f"{randint(0, 1e6)}\n")
+        fp.write(f"p2: {p2}\n")

hpcflow/data/scripts/demo_task_1_parse_p3.py

@@ -0,0 +1,7 @@
+from pathlib import Path
+
+
+def demo_task_1_parse_p3(t1_outfile_1):
+    with Path(t1_outfile_1).open("r") as fp:
+        p3 = int(fp.readline().strip())
+    return p3

hpcflow/data/scripts/do_nothing.py

@@ -0,0 +1,2 @@
+def do_nothing():
+    pass

hpcflow/data/scripts/env_specifier_test/input_file_generator_pass_env_spec.py

@@ -0,0 +1,4 @@
+def input_file_generator_pass_env_spec(path, p1, env_spec):
+    print(env_spec)
+    with path.open("w") as fp:
+        fp.write(f"{p1}\n")

hpcflow/data/scripts/env_specifier_test/main_script_test_pass_env_spec.py

@@ -0,0 +1,8 @@
+def main_script_test_pass_env_spec(p1, env_spec):
+    # process
+    p2 = p1 + 100
+
+    print(env_spec)
+
+    # return outputs
+    return {"p2": p2}

hpcflow/data/scripts/env_specifier_test/output_file_parser_pass_env_spec.py

@@ -0,0 +1,4 @@
+def output_file_parser_pass_env_spec(my_output_file, env_spec):
+    print(env_spec)
+    with my_output_file.open("r") as fp:
+        return int(fp.read().strip())

hpcflow/data/scripts/env_specifier_test/v1/input_file_generator_basic.py

@@ -0,0 +1,4 @@
+# version: v1
+def input_file_generator_basic(path, p1):
+    with path.open("w") as fp:
+        fp.write(f"{p1}\n")

hpcflow/data/scripts/env_specifier_test/v1/main_script_test_direct_in_direct_out.py

@@ -0,0 +1,7 @@
+# version: v1
+def main_script_test_direct_in_direct_out(p1):
+    # process
+    p2 = p1 + 100
+
+    # return outputs
+    return {"p2": p2}

hpcflow/data/scripts/env_specifier_test/v1/output_file_parser_basic.py

@@ -0,0 +1,4 @@
+# version: v1
+def output_file_parser_basic(my_output_file):
+    with my_output_file.open("r") as fp:
+        return int(fp.read().strip())

hpcflow/data/scripts/env_specifier_test/v2/main_script_test_direct_in_direct_out.py

@@ -0,0 +1,7 @@
+# version: v2
+def main_script_test_direct_in_direct_out(p1):
+    # process
+    p2 = p1 + 200
+
+    # return outputs
+    return {"p2": p2}

hpcflow/data/scripts/generate_t1_file_01.py

@@ -0,0 +1,7 @@
+from random import randint
+
+
+def generate_t1_file_01(path, p1):
+    with path.open("w") as fp:
+        fp.write(f"{randint(0, 1e6)}\n")
+        fp.write(f"p1: {p1}\n")

hpcflow/data/scripts/import_future_script.py

@@ -0,0 +1,7 @@
+from __future__ import annotations
+
+
+def import_future_script():
+    """Used to test the `__future__` import is moved to the top of the file in the
+    generated script in the case of `resources.combine_scripts == True`"""
+    pass

hpcflow/data/scripts/input_file_generator_basic.py

@@ -0,0 +1,3 @@
+def input_file_generator_basic(path, p1):
+    with path.open("w") as fp:
+        fp.write(f"{p1}\n")

hpcflow/data/scripts/input_file_generator_basic_FAIL.py

@@ -0,0 +1,3 @@
+def input_file_generator_basic_FAIL(path, p1):
+    # silently fail to generate the input file!
+    pass

hpcflow/data/scripts/input_file_generator_test_stdout_stderr.py

@@ -0,0 +1,8 @@
+import sys
+
+
+def input_file_generator_test_stdout_stderr(path, p1):
+    print(p1)
+    print(p1, file=sys.stderr)
+    with path.open("w") as fp:
+        fp.write(f"{p1}\n")

hpcflow/data/scripts/main_script_test_direct_in.py

@@ -0,0 +1,3 @@
+def main_script_test_direct_in(p1):
+    # process
+    print(p1 + 100)

hpcflow/data/scripts/main_script_test_direct_in_direct_out.py

@@ -0,0 +1,6 @@
+def main_script_test_direct_in_direct_out(p1):
+    # process
+    p2 = p1 + 100
+
+    # return outputs
+    return {"p2": p2}

hpcflow/data/scripts/main_script_test_direct_in_direct_out_2.py

@@ -0,0 +1,6 @@
+def main_script_test_direct_in_direct_out_2(p2):
+    # process
+    p3 = p2 + 100
+
+    # return outputs
+    return {"p3": p3}

hpcflow/data/scripts/main_script_test_direct_in_direct_out_2_fail_allowed.py

@@ -0,0 +1,6 @@
+def main_script_test_direct_in_direct_out_2_fail_allowed(p2):
+    # process, accounting for possible unset data:
+    p3 = (p2 if p2 is not None else 0) + 100
+
+    # return outputs
+    return {"p3": p3}

hpcflow/data/scripts/main_script_test_direct_in_direct_out_2_fail_allowed_group.py

@@ -0,0 +1,7 @@
+def main_script_test_direct_in_direct_out_2_fail_allowed_group(p2):
+    # process, accounting for possible unset data:
+    p2_sum = sum(i for i in p2 if i is not None)
+    p3 = p2_sum + 100
+
+    # return outputs
+    return {"p3": p3}

hpcflow/data/scripts/main_script_test_direct_in_direct_out_3.py

@@ -0,0 +1,6 @@
+def main_script_test_direct_in_direct_out_3(p3):
+    # process
+    p4 = p3 + 100
+
+    # return outputs
+    return {"p4": p4}

hpcflow/data/scripts/main_script_test_direct_in_direct_out_all_iters_test.py

@@ -0,0 +1,15 @@
+def main_script_test_direct_in_direct_out_all_iters_test(p1):
+    """Used for a simple loop test, where we pass all iterations to the script."""
+
+    # sum over all iterations:
+    all_p1_vals = []
+    for key, val in p1.items():
+        print(f"\nkey: {key}")
+        print(f"loop_idx: {val['loop_idx']}")
+        print(f"value: {val['value']}")
+        all_p1_vals.append(val["value"])
+
+    p1 = sum(all_p1_vals) + 1
+
+    # return outputs
+    return {"p1": p1}

hpcflow/data/scripts/main_script_test_direct_in_direct_out_env_spec.py

@@ -0,0 +1,7 @@
+def main_script_test_direct_in_direct_out_env_spec(p1, env_spec):
+
+    # process
+    p2 = p1 + 100
+
+    # return outputs
+    return {"p2": env_spec}

hpcflow/data/scripts/main_script_test_direct_in_direct_out_labels.py

@@ -0,0 +1,8 @@
+def main_script_test_direct_in_direct_out_labels(p1):
+    # process
+    p1_1 = p1["one"]
+    p1_2 = p1["two"]
+    p2 = p1_1 + p1_2
+
+    # return outputs
+    return {"p2": p2}

hpcflow/data/scripts/main_script_test_direct_in_group_direct_out_3.py

@@ -0,0 +1,6 @@
+def main_script_test_direct_in_group_direct_out_3(p3):
+    # process
+    p4 = sum(p3) + 100
+
+    # return outputs
+    return {"p4": p4}

hpcflow/data/scripts/main_script_test_direct_in_group_one_fail_direct_out_3.py

@@ -0,0 +1,6 @@
+def main_script_test_direct_in_group_one_fail_direct_out_3(p3):
+    # process, ignore some un-set inputs:
+    p4 = sum([i for i in p3 if i is not None]) + 100
+
+    # return outputs
+    return {"p4": p4}

hpcflow/data/scripts/main_script_test_direct_sub_param_in_direct_out.py

@@ -0,0 +1,6 @@
+def main_script_test_direct_sub_param_in_direct_out(a):
+    # process
+    p2 = a + 100
+
+    # return outputs
+    return {"p2": p2}

hpcflow/data/scripts/main_script_test_hdf5_in_obj.py

@@ -0,0 +1,12 @@
+import h5py  # type: ignore[import-untyped]
+
+
+def main_script_test_hdf5_in_obj(_input_files):
+    # read inputs
+    with h5py.File(_input_files["hdf5"], mode="r") as fh:
+        a = fh["p1c"].attrs["a"].item()
+
+    # process
+    p2 = a + 100
+
+    return {"p2": p2}

hpcflow/data/scripts/main_script_test_hdf5_in_obj_2.py

@@ -0,0 +1,12 @@
+import h5py  # type: ignore
+
+
+def main_script_test_hdf5_in_obj_2(p2, _input_files):
+    # read inputs
+    with h5py.File(_input_files["hdf5"], mode="r") as fh:
+        a = fh["p1c"].attrs["a"].item()
+
+    # process
+    p3 = a + 100
+
+    return {"p3": p3}

hpcflow/data/scripts/main_script_test_hdf5_in_obj_group.py

@@ -0,0 +1,12 @@
+import h5py  # type: ignore[import-untyped]
+
+
+def main_script_test_hdf5_in_obj_group(_input_files):
+    # read inputs
+    with h5py.File(_input_files["hdf5"], mode="r") as fh:
+        all_a = [p1c_dat.attrs["a"].item() for p1c_dat in fh["p1c"].values()]
+
+    # process
+    p2 = sum(all_a) + 100
+
+    return {"p2": p2}

hpcflow/data/scripts/main_script_test_hdf5_out_obj.py

@@ -0,0 +1,11 @@
+import h5py  # type: ignore[import-untyped]
+
+
+def main_script_test_hdf5_out_obj(p1, _output_files):
+    # process
+    p1c = {"a": p1 + 100}
+
+    # write outputs
+    with h5py.File(_output_files["hdf5"], mode="w") as fh:
+        p1c_grp = fh.create_group("p1c")
+        p1c_grp.attrs["a"] = p1c["a"]

hpcflow/data/scripts/main_script_test_json_and_direct_in_json_out.py

@@ -0,0 +1,14 @@
+import json
+
+
+def main_script_test_json_and_direct_in_json_out(p2, _input_files, _output_files):
+    # read inputs
+    with _input_files["json"].open("rt") as fh:
+        p1 = int(json.load(fh)["p1"])
+
+    # process
+    p3 = p1 + p2
+
+    # save outputs
+    with _output_files["json"].open("wt") as fh:
+        json.dump({"p3": p3}, fh)

hpcflow/data/scripts/main_script_test_json_in_json_and_direct_out.py

@@ -0,0 +1,17 @@
+import json
+
+
+def main_script_test_json_in_json_and_direct_out(_input_files, _output_files):
+    # read inputs
+    with _input_files["json"].open("rt") as fh:
+        p1 = int(json.load(fh)["p1"])
+
+    # process
+    p2 = p1 + 100
+    p3 = p1 + 200
+
+    # save outputs
+    with _output_files["json"].open("wt") as fh:
+        json.dump({"p2": p2}, fh)
+
+    return {"p3": p3}

hpcflow/data/scripts/main_script_test_json_in_json_out.py

@@ -0,0 +1,14 @@
+import json
+
+
+def main_script_test_json_in_json_out(_input_files, _output_files):
+    # read inputs
+    with _input_files["json"].open("rt") as fh:
+        p1 = int(json.load(fh)["p1"])
+
+    # process
+    p2 = p1 + 100
+
+    # save outputs
+    with _output_files["json"].open("wt") as fh:
+        json.dump({"p2": p2}, fh)

hpcflow/data/scripts/main_script_test_json_in_json_out_labels.py

@@ -0,0 +1,16 @@
+import json
+
+
+def main_script_test_json_in_json_out_labels(_input_files, _output_files):
+    # read inputs
+    with _input_files["json"].open("rt") as fh:
+        inp_dat = json.load(fh)
+        p1_1 = int(inp_dat["p1[one]"])
+        p1_2 = int(inp_dat["p1[two]"])
+
+    # process
+    p2 = p1_1 + p1_2
+
+    # save outputs
+    with _output_files["json"].open("wt") as fh:
+        json.dump({"p2": p2}, fh)

hpcflow/data/scripts/main_script_test_json_in_obj.py

@@ -0,0 +1,12 @@
+import json
+
+
+def main_script_test_json_in_obj(_input_files):
+    # read inputs
+    with _input_files["json"].open("rt") as fh:
+        p1c = json.load(fh)["p1c"]
+
+    # process
+    p2 = p1c["a"] + 100
+
+    return {"p2": p2}

hpcflow/data/scripts/main_script_test_json_out_FAIL.py

@@ -0,0 +1,3 @@
+def main_script_test_json_out_FAIL(_output_files):
+    pass
+    # don't generate a JSON output file!

hpcflow/data/scripts/main_script_test_json_out_obj.py

@@ -0,0 +1,10 @@
+import json
+
+
+def main_script_test_json_out_obj(p1, _output_files):
+    # process
+    p1c = {"a": p1 + 100}
+
+    # write outputs
+    with _output_files["json"].open("wt") as fh:
+        json.dump({"p1c": p1c}, fh)

hpcflow/data/scripts/main_script_test_json_sub_param_in_json_out_labels.py

@@ -0,0 +1,16 @@
+import json
+
+
+def main_script_test_json_sub_param_in_json_out_labels(_input_files, _output_files):
+    # read inputs
+    with _input_files["json"].open("rt") as fh:
+        inp_dat = json.load(fh)
+        a = int(inp_dat["a"])
+        p1_2 = int(inp_dat["p1[two]"])
+
+    # process
+    p2 = a + p1_2
+
+    # save outputs
+    with _output_files["json"].open("wt") as fh:
+        json.dump({"p2": p2}, fh)

hpcflow/data/scripts/main_script_test_shell_env_vars.py

@@ -0,0 +1,12 @@
+import json
+import os
+
+
+def main_script_test_shell_env_vars(p1):
+    with open("env_vars.json", "wt") as fp:
+        json.dump(
+            {k: v for k, v in os.environ.items() if k.startswith("HPCFLOW")},
+            fp,
+            indent=4,
+        )
+    return {"p1": p1 + 1}

hpcflow/data/scripts/main_script_test_std_out_std_err.py

@@ -0,0 +1,6 @@
+import sys
+
+
+def main_script_test_std_out_std_err(stdout_msg, stderr_msg):
+    print(stdout_msg)
+    print(stderr_msg, file=sys.stderr)

hpcflow/data/scripts/output_file_parser_basic.py

@@ -0,0 +1,3 @@
+def output_file_parser_basic(my_output_file):
+    with my_output_file.open("r") as fp:
+        return int(fp.read().strip())

hpcflow/data/scripts/output_file_parser_basic_FAIL.py

@@ -0,0 +1,7 @@
+def output_file_parser_basic_FAIL(my_output_file):
+    # parse the output file, but then delete it, so it cannot be saved (when run with
+    # `OFP.save_files=True`):
+    with my_output_file.open("r") as fp:
+        out = int(fp.read().strip())
+    my_output_file.unlink()
+    return out

hpcflow/data/scripts/output_file_parser_test_stdout_stderr.py

@@ -0,0 +1,8 @@
+import sys
+
+
+def output_file_parser_test_stdout_stderr(my_output_file, p1):
+    print(p1)
+    print(p1, file=sys.stderr)
+    with my_output_file.open("r") as fp:
+        return int(fp.read().strip())

hpcflow/data/scripts/parse_t1_file_01.py

@@ -0,0 +1,4 @@
+def parse_t1_file_01(t1_file_01):
+    with t1_file_01.open("r") as fp:
+        p4 = int(fp.readline().strip())
+    return p4

hpcflow/data/scripts/script_exit_test.py

@@ -0,0 +1,5 @@
+import sys
+
+
+def script_exit_test(exit_code):
+    sys.exit(exit_code)

hpcflow/data/template_components/__init__.py

@@ -0,0 +1 @@
+"""YAML files containing built-in template components, used for demonstrations."""

hpcflow/data/template_components/command_files.yaml

@@ -0,0 +1,26 @@
+- label: t1_infile_1
+  name:
+    name: t1_infile_1.txt
+  doc: Input file for demonstration task t1.
+- label: t1_infile_2
+  name:
+    name: t1_infile_2.txt
+  doc: Input file for demonstration task t1.
+- label: t1_outfile_1
+  name:
+    name: outfile.txt
+  doc: Output file for demonstration task t1.
+- label: lammps_atoms_file
+  name:
+    name: atoms.lammps
+  doc: The file of atoms processed by LAMMPS.
+- label: lammps_input_script
+  name:
+    name: in.lammps
+  doc: The LAMMPS control file.
+- label: lammps_dump_files
+  name:
+    name: (dump.\d+.txt)
+    is_regex: true
+  doc: >
+    A regular expression pattern that matches all the dump files generated by a LAMMPS run.

hpcflow/data/template_components/environments.yaml

@@ -0,0 +1,13 @@
+- name: null_env
+  doc: The default system environment.
+
+- name: python_env
+  executables:
+    - label: python_script
+      instances:
+        - command: python <<script_path>> <<args>>
+          num_cores: 1
+          parallel_mode: null
+  doc: |
+    An environment for running Python, typically with access to hpcflow and matflow modules and their dependencies.
+    May be a Python virtual environment.

hpcflow/data/template_components/parameters.yaml

@@ -0,0 +1,14 @@
+# These are only documented when their schemas are documented.
+- type: p1
+- type: p1c
+- type: p2
+- type: p3
+- type: p4
+- type: p5
+- type: orientations
+- type: number_atoms
+- type: box
+- type: atoms
+- type: empirical_potential
+- type: optimised_atoms
+- type: test_UPPER_case

hpcflow/data/template_components/task_schemas.yaml

@@ -0,0 +1,139 @@
+- objective: test_t1_bash
+  doc: >
+    Simple echo test using bash.
+    <code>p1</code> &rarr; <code>p2</code>
+  inputs:
+    - parameter: p1
+  outputs:
+    - parameter: p2
+  actions:
+    - environments:
+        any: null_env
+      commands:
+        - command: echo "$((<<parameter:p1>> + 100))"
+          stdout: <<parameter:p2>>
+
+- objective: test_t1_ps
+  doc: >
+    Simple echo test using powershell.
+    <code>p1</code> &rarr; <code>p2</code>
+  inputs:
+    - parameter: p1
+  outputs:
+    - parameter: p2
+  actions:
+    - environments:
+        any: null_env
+      commands:
+        - command: Write-Output ((<<parameter:p1>> + 100))
+          stdout: <<parameter:p2>>
+
+- objective: test_t1_conditional_OS
+  doc: >
+    Simple echo test showing conditional enactment schemes.
+    <code>p1</code> &rarr; <code>p2</code>
+  inputs:
+    - parameter: p1
+  outputs:
+    - parameter: p2
+  actions:
+    - rules:
+        - path: resources.os_name
+          condition: { value.equal_to: posix }
+      environments:
+        any: null_env
+      commands:
+        - command: echo "$((<<parameter:p1>> + 100))"
+          stdout: <<parameter:p2>>
+    - rules:
+        - path: resources.os_name
+          condition: { value.equal_to: nt }
+      environments:
+        any: null_env
+      commands:
+        - command: Write-Output ((<<parameter:p1>> + 100))
+          stdout: <<parameter:p2>>
+
+- objective: test_t2_bash
+  doc: >
+    Simple echo test using bash.
+    <code>p2</code> &rarr; <code>p3</code>
+  inputs:
+    - parameter: p2
+  outputs:
+    - parameter: p3
+  actions:
+    - environments:
+        any: null_env
+      commands:
+        - command: echo "$((<<parameter:p2>> + 100))"
+          stdout: <<parameter:p3>>
+
+- objective: test_t2_ps
+  doc: >
+    Simple echo test using powershell.
+    <code>p2</code> &rarr; <code>p3</code>
+  inputs:
+    - parameter: p2
+  outputs:
+    - parameter: p3
+  actions:
+    - environments:
+        any: null_env
+      commands:
+        - command: Write-Output ((<<parameter:p2>> + 100))
+          stdout: <<parameter:p3>>
+
+# - objective: demo_task_2
+#   inputs:
+#     - parameter: p3
+#     - parameter: orientations
+#   outputs:
+#     - parameter: p4
+#   actions:
+#     - environments:
+#         any: demo_env_1
+#       commands:
+#         - command: >
+#             echo "hello! p3=<<parameter:p3>> orientations=<<parameter:orientations>>"
+#       output_file_parsers:
+#         p4:
+#           from_files: [t1_outfile_1]
+
+- objective: dummy_task_1
+  doc: >
+    A dummy task showing simple output file parsing.
+    <code>(p1, p2, p5)</code> &rarr; <code>p3</code>
+  inputs:
+    - parameter: p1
+    - parameter: p2
+    - parameter: p5
+  outputs:
+    - parameter: p3
+  actions:
+    - environments:
+        any: null_env
+      commands:
+        - command: >-
+            Set-Content -Path .\test.txt -Value "hello from ${pwd}; p1=<<parameter:p1>> p2=<<parameter:p2>> p5=<<parameter:p5>>"
+      output_file_parsers:
+        p3:
+          from_files: [t1_outfile_1]
+
+- objective: dummy_task_2
+  doc: >
+    A dummy task showing simple output file parsing.
+    <code>(p3, orientations)</code> &rarr; <code>p4</code>
+  inputs:
+    - parameter: p3
+    - parameter: orientations
+  outputs:
+    - parameter: p4
+  actions:
+    - environments:
+        any: null_env
+      commands:
+        - command: Start-Sleep 1; echo "p3=<<parameter:p3>>; orientations=<<parameter:orientations>>"
+      output_file_parsers:
+        p4:
+          from_files: [t1_outfile_1]