hpcflow 0.1.0__py3-none-any.whl

hpcflow/__init__.py

@@ -0,0 +1,3 @@
+from hpcflow._version import __version__
+
+_app_name = "hpcFlow"

hpcflow/__pyinstaller/__init__.py

@@ -0,0 +1,5 @@
+from pathlib import Path
+
+
+def get_hook_dirs():
+    return [str(Path(__file__).parent.resolve())]

hpcflow/__pyinstaller/hook-hpcflow.py

@@ -0,0 +1,41 @@
+from PyInstaller.utils.hooks import collect_data_files
+
+from hpcflow.sdk import sdk_classes
+
+
+# most of the modules in `sdk_classes` are imported on-demand via the app object:
+hiddenimports = [
+    *sdk_classes.values(),
+    "hpcflow.sdk.data",
+    "hpcflow.data.data_manifests",
+    "hpcflow.data.scripts",
+    "hpcflow.data.jinja_templates",
+    "hpcflow.data.template_components",
+    "hpcflow.data.workflows",
+    "hpcflow.tests.data",
+    "hpcflow.sdk.core.test_utils",
+    "hpcflow.sdk.utils.patches",
+    "click.testing",
+    "requests",  # for GitHub fsspec file system
+    "fsspec.implementations.github",  # for GitHub fsspec file system
+    "hpcflow.pytest_plugin",
+]
+
+datas = (
+    collect_data_files("hpcflow.sdk.data")
+    + collect_data_files("hpcflow.data.data_manifests")
+    + collect_data_files(
+        "hpcflow.data.scripts", include_py_files=True, excludes=("**/__pycache__",)
+    )
+    + collect_data_files(
+        "hpcflow.data.jinja_templates",
+        include_py_files=True,
+        excludes=("**/__pycache__",),
+    )
+    + collect_data_files("hpcflow.data.template_components")
+    + collect_data_files("hpcflow.data.workflows")
+    + collect_data_files(
+        "hpcflow.tests", include_py_files=True, excludes=("**/__pycache__",)
+    )
+    + collect_data_files("hpcflow.tests.data")
+)

hpcflow/_version.py

@@ -0,0 +1 @@
+__version__ = "0.1.0"

hpcflow/app.py

@@ -0,0 +1,43 @@
+from __future__ import annotations
+from hpcflow import __version__, _app_name
+from hpcflow.sdk import app as sdk_app
+from hpcflow.sdk.config import ConfigOptions
+
+
+# provide access to app attributes:
+__getattr__ = sdk_app.get_app_attribute
+
+# ensure docs/help can see dynamically loaded attributes:
+__all__ = sdk_app.get_app_module_all()
+__dir__ = sdk_app.get_app_module_dir()
+
+# set app-level config options:
+config_options = ConfigOptions(
+    directory_env_var="HPCFLOW_CONFIG_DIR",
+    default_directory="~/.hpcflow",
+)
+
+# load built in template components (in this case, for demonstration purposes):
+template_components = sdk_app.BaseApp.load_builtin_template_component_data(
+    "hpcflow.data.template_components"
+)
+
+# initialise the App object:
+app: sdk_app.BaseApp = sdk_app.BaseApp(
+    name=_app_name,
+    version=__version__,
+    module=__name__,
+    docs_import_conv="hf",
+    description="Computational workflow management",
+    gh_org="hpcflow",
+    gh_repo="hpcflow",
+    config_options=config_options,
+    template_components=template_components,
+    scripts_dir="data.scripts",  # relative to root package
+    jinja_templates_dir="data.jinja_templates",  # relative to root package
+    workflows_dir="data.workflows",  # relative to root package
+    data_manifest_dir="hpcflow.data.data_manifests",
+    data_dir="github://hpcflow:hpcflow-data@main/data",
+    program_dir="github://hpcflow:hpcflow-data@main/programs",
+    docs_url="https://hpcflow.github.io/docs/stable",
+)  #: |app|

hpcflow/cli.py

@@ -0,0 +1,4 @@
+from hpcflow.app import cli
+
+if __name__ == "__main__":
+    cli()

hpcflow/data/data_manifests/__init__.py

@@ -0,0 +1,3 @@
+"""
+Manifest for demonstration data and builtin programs.
+"""

hpcflow/data/data_manifests/data.json

@@ -0,0 +1,15 @@
+{
+    "text_file_1.txt": {
+        "zip_file": "text_file_1.zip"
+    },
+    "text_file_2": {
+        "zip_contents": "text_file_2.zip"
+    },
+    "zip_file.zip": {},
+    "test_dataset_1": {
+        "zip_file": "test_dataset_1.zip"
+    },
+    "test_dataset_2": {
+        "zip_contents": "test_dataset_2.zip"
+    }
+}

hpcflow/data/data_manifests/programs.json

@@ -0,0 +1,25 @@
+{
+    "hello_world/linux_x86_64": {
+        "zip_contents": "hello_world/linux_x86_64/hello_world.zip",
+        "executable": "hello_world",
+        "set_executable": [
+            "hello_world"
+        ]
+    },
+    "hello_world/macos_x86_64": {
+        "executable": "hello_world",
+        "set_executable": [
+            "hello_world"
+        ]
+    },
+    "hello_world/macos_arm64": {
+        "zip_file": "hello_world/macos_arm64/hello_world.zip",
+        "executable": "hello_world",
+        "set_executable": [
+            "hello_world"
+        ]
+    },
+    "hello_world/win_AMD64": {
+        "executable": "hello_world.exe"
+    }
+}

hpcflow/data/jinja_templates/test/test_template.txt

@@ -0,0 +1,8 @@
+Hola, {{ name }}!
+
+This is a template, with a loop. Here are your specified fruits:
+
+{%- for fruit_i in fruits %}
+- {{ fruit_i -}}
+{% endfor %}
+

hpcflow/data/scripts/__init__.py

@@ -0,0 +1 @@
+"""This is required to allow the frozen app to find the scripts resources..."""

hpcflow/data/scripts/bad_script.py

@@ -0,0 +1,2 @@
+def bad_script():
+    raise SyntaxError("oh no!")

hpcflow/data/scripts/demo_task_1_generate_t1_infile_1.py

@@ -0,0 +1,8 @@
+from random import randint
+from pathlib import Path
+
+
+def demo_task_1_generate_t1_infile_1(path, p1):
+    with Path(path).open("w") as fp:
+        fp.write(f"{randint(0, 1e6)}\n")
+        fp.write(f"p1: {p1}\n")

hpcflow/data/scripts/demo_task_1_generate_t1_infile_2.py

@@ -0,0 +1,8 @@
+from random import randint
+from pathlib import Path
+
+
+def demo_task_1_generate_t1_infile_2(path, p2):
+    with Path(path).open("w") as fp:
+        fp.write(f"{randint(0, 1e6)}\n")
+        fp.write(f"p2: {p2}\n")

hpcflow/data/scripts/demo_task_1_parse_p3.py

@@ -0,0 +1,7 @@
+from pathlib import Path
+
+
+def demo_task_1_parse_p3(t1_outfile_1):
+    with Path(t1_outfile_1).open("r") as fp:
+        p3 = int(fp.readline().strip())
+    return p3

hpcflow/data/scripts/do_nothing.py

@@ -0,0 +1,2 @@
+def do_nothing():
+    pass

hpcflow/data/scripts/env_specifier_test/input_file_generator_pass_env_spec.py

@@ -0,0 +1,4 @@
+def input_file_generator_pass_env_spec(path, p1, env_spec):
+    print(env_spec)
+    with path.open("w") as fp:
+        fp.write(f"{p1}\n")

hpcflow/data/scripts/env_specifier_test/main_script_test_pass_env_spec.py

@@ -0,0 +1,8 @@
+def main_script_test_pass_env_spec(p1, env_spec):
+    # process
+    p2 = p1 + 100
+
+    print(env_spec)
+
+    # return outputs
+    return {"p2": p2}

hpcflow/data/scripts/env_specifier_test/output_file_parser_pass_env_spec.py

@@ -0,0 +1,4 @@
+def output_file_parser_pass_env_spec(my_output_file, env_spec):
+    print(env_spec)
+    with my_output_file.open("r") as fp:
+        return int(fp.read().strip())

hpcflow/data/scripts/env_specifier_test/v1/input_file_generator_basic.py

@@ -0,0 +1,4 @@
+# version: v1
+def input_file_generator_basic(path, p1):
+    with path.open("w") as fp:
+        fp.write(f"{p1}\n")

hpcflow/data/scripts/env_specifier_test/v1/main_script_test_direct_in_direct_out.py

@@ -0,0 +1,7 @@
+# version: v1
+def main_script_test_direct_in_direct_out(p1):
+    # process
+    p2 = p1 + 100
+
+    # return outputs
+    return {"p2": p2}

hpcflow/data/scripts/env_specifier_test/v1/output_file_parser_basic.py

@@ -0,0 +1,4 @@
+# version: v1
+def output_file_parser_basic(my_output_file):
+    with my_output_file.open("r") as fp:
+        return int(fp.read().strip())

hpcflow/data/scripts/env_specifier_test/v2/main_script_test_direct_in_direct_out.py

@@ -0,0 +1,7 @@
+# version: v2
+def main_script_test_direct_in_direct_out(p1):
+    # process
+    p2 = p1 + 200
+
+    # return outputs
+    return {"p2": p2}

hpcflow/data/scripts/generate_t1_file_01.py

@@ -0,0 +1,7 @@
+from random import randint
+
+
+def generate_t1_file_01(path, p1):
+    with path.open("w") as fp:
+        fp.write(f"{randint(0, 1e6)}\n")
+        fp.write(f"p1: {p1}\n")

hpcflow/data/scripts/import_future_script.py

@@ -0,0 +1,7 @@
+from __future__ import annotations
+
+
+def import_future_script():
+    """Used to test the `__future__` import is moved to the top of the file in the
+    generated script in the case of `resources.combine_scripts == True`"""
+    pass

hpcflow/data/scripts/input_file_generator_basic.py

@@ -0,0 +1,3 @@
+def input_file_generator_basic(path, p1):
+    with path.open("w") as fp:
+        fp.write(f"{p1}\n")

hpcflow/data/scripts/input_file_generator_basic_FAIL.py

@@ -0,0 +1,3 @@
+def input_file_generator_basic_FAIL(path, p1):
+    # silently fail to generate the input file!
+    pass

hpcflow/data/scripts/input_file_generator_test_stdout_stderr.py

@@ -0,0 +1,8 @@
+import sys
+
+
+def input_file_generator_test_stdout_stderr(path, p1):
+    print(p1)
+    print(p1, file=sys.stderr)
+    with path.open("w") as fp:
+        fp.write(f"{p1}\n")

hpcflow/data/scripts/main_script_test_direct_in.py

@@ -0,0 +1,3 @@
+def main_script_test_direct_in(p1):
+    # process
+    print(p1 + 100)

hpcflow/data/scripts/main_script_test_direct_in_direct_out.py

@@ -0,0 +1,6 @@
+def main_script_test_direct_in_direct_out(p1):
+    # process
+    p2 = p1 + 100
+
+    # return outputs
+    return {"p2": p2}

hpcflow/data/scripts/main_script_test_direct_in_direct_out_2.py

@@ -0,0 +1,6 @@
+def main_script_test_direct_in_direct_out_2(p2):
+    # process
+    p3 = p2 + 100
+
+    # return outputs
+    return {"p3": p3}

hpcflow/data/scripts/main_script_test_direct_in_direct_out_2_fail_allowed.py

@@ -0,0 +1,6 @@
+def main_script_test_direct_in_direct_out_2_fail_allowed(p2):
+    # process, accounting for possible unset data:
+    p3 = (p2 if p2 is not None else 0) + 100
+
+    # return outputs
+    return {"p3": p3}

hpcflow/data/scripts/main_script_test_direct_in_direct_out_2_fail_allowed_group.py

@@ -0,0 +1,7 @@
+def main_script_test_direct_in_direct_out_2_fail_allowed_group(p2):
+    # process, accounting for possible unset data:
+    p2_sum = sum(i for i in p2 if i is not None)
+    p3 = p2_sum + 100
+
+    # return outputs
+    return {"p3": p3}

hpcflow/data/scripts/main_script_test_direct_in_direct_out_3.py

@@ -0,0 +1,6 @@
+def main_script_test_direct_in_direct_out_3(p3):
+    # process
+    p4 = p3 + 100
+
+    # return outputs
+    return {"p4": p4}

hpcflow/data/scripts/main_script_test_direct_in_direct_out_all_iters_test.py

@@ -0,0 +1,15 @@
+def main_script_test_direct_in_direct_out_all_iters_test(p1):
+    """Used for a simple loop test, where we pass all iterations to the script."""
+
+    # sum over all iterations:
+    all_p1_vals = []
+    for key, val in p1.items():
+        print(f"\nkey: {key}")
+        print(f"loop_idx: {val['loop_idx']}")
+        print(f"value: {val['value']}")
+        all_p1_vals.append(val["value"])
+
+    p1 = sum(all_p1_vals) + 1
+
+    # return outputs
+    return {"p1": p1}

hpcflow/data/scripts/main_script_test_direct_in_direct_out_env_spec.py

@@ -0,0 +1,7 @@
+def main_script_test_direct_in_direct_out_env_spec(p1, env_spec):
+
+    # process
+    p2 = p1 + 100
+
+    # return outputs
+    return {"p2": env_spec}

hpcflow/data/scripts/main_script_test_direct_in_direct_out_labels.py

@@ -0,0 +1,8 @@
+def main_script_test_direct_in_direct_out_labels(p1):
+    # process
+    p1_1 = p1["one"]
+    p1_2 = p1["two"]
+    p2 = p1_1 + p1_2
+
+    # return outputs
+    return {"p2": p2}

hpcflow/data/scripts/main_script_test_direct_in_group_direct_out_3.py

@@ -0,0 +1,6 @@
+def main_script_test_direct_in_group_direct_out_3(p3):
+    # process
+    p4 = sum(p3) + 100
+
+    # return outputs
+    return {"p4": p4}

hpcflow/data/scripts/main_script_test_direct_in_group_one_fail_direct_out_3.py

@@ -0,0 +1,6 @@
+def main_script_test_direct_in_group_one_fail_direct_out_3(p3):
+    # process, ignore some un-set inputs:
+    p4 = sum([i for i in p3 if i is not None]) + 100
+
+    # return outputs
+    return {"p4": p4}

hpcflow/data/scripts/main_script_test_direct_sub_param_in_direct_out.py

@@ -0,0 +1,6 @@
+def main_script_test_direct_sub_param_in_direct_out(a):
+    # process
+    p2 = a + 100
+
+    # return outputs
+    return {"p2": p2}

hpcflow/data/scripts/main_script_test_hdf5_in_obj.py

@@ -0,0 +1,12 @@
+import h5py  # type: ignore[import-untyped]
+
+
+def main_script_test_hdf5_in_obj(_input_files):
+    # read inputs
+    with h5py.File(_input_files["hdf5"], mode="r") as fh:
+        a = fh["p1c"].attrs["a"].item()
+
+    # process
+    p2 = a + 100
+
+    return {"p2": p2}

hpcflow/data/scripts/main_script_test_hdf5_in_obj_2.py

@@ -0,0 +1,12 @@
+import h5py  # type: ignore
+
+
+def main_script_test_hdf5_in_obj_2(p2, _input_files):
+    # read inputs
+    with h5py.File(_input_files["hdf5"], mode="r") as fh:
+        a = fh["p1c"].attrs["a"].item()
+
+    # process
+    p3 = a + 100
+
+    return {"p3": p3}

hpcflow/data/scripts/main_script_test_hdf5_in_obj_group.py

@@ -0,0 +1,12 @@
+import h5py  # type: ignore[import-untyped]
+
+
+def main_script_test_hdf5_in_obj_group(_input_files):
+    # read inputs
+    with h5py.File(_input_files["hdf5"], mode="r") as fh:
+        all_a = [p1c_dat.attrs["a"].item() for p1c_dat in fh["p1c"].values()]
+
+    # process
+    p2 = sum(all_a) + 100
+
+    return {"p2": p2}

hpcflow/data/scripts/main_script_test_hdf5_out_obj.py

@@ -0,0 +1,11 @@
+import h5py  # type: ignore[import-untyped]
+
+
+def main_script_test_hdf5_out_obj(p1, _output_files):
+    # process
+    p1c = {"a": p1 + 100}
+
+    # write outputs
+    with h5py.File(_output_files["hdf5"], mode="w") as fh:
+        p1c_grp = fh.create_group("p1c")
+        p1c_grp.attrs["a"] = p1c["a"]

hpcflow/data/scripts/main_script_test_json_and_direct_in_json_out.py

@@ -0,0 +1,14 @@
+import json
+
+
+def main_script_test_json_and_direct_in_json_out(p2, _input_files, _output_files):
+    # read inputs
+    with _input_files["json"].open("rt") as fh:
+        p1 = int(json.load(fh)["p1"])
+
+    # process
+    p3 = p1 + p2
+
+    # save outputs
+    with _output_files["json"].open("wt") as fh:
+        json.dump({"p3": p3}, fh)

hpcflow/data/scripts/main_script_test_json_in_json_and_direct_out.py

@@ -0,0 +1,17 @@
+import json
+
+
+def main_script_test_json_in_json_and_direct_out(_input_files, _output_files):
+    # read inputs
+    with _input_files["json"].open("rt") as fh:
+        p1 = int(json.load(fh)["p1"])
+
+    # process
+    p2 = p1 + 100
+    p3 = p1 + 200
+
+    # save outputs
+    with _output_files["json"].open("wt") as fh:
+        json.dump({"p2": p2}, fh)
+
+    return {"p3": p3}

hpcflow/data/scripts/main_script_test_json_in_json_out.py

@@ -0,0 +1,14 @@
+import json
+
+
+def main_script_test_json_in_json_out(_input_files, _output_files):
+    # read inputs
+    with _input_files["json"].open("rt") as fh:
+        p1 = int(json.load(fh)["p1"])
+
+    # process
+    p2 = p1 + 100
+
+    # save outputs
+    with _output_files["json"].open("wt") as fh:
+        json.dump({"p2": p2}, fh)

hpcflow/data/scripts/main_script_test_json_in_json_out_labels.py

@@ -0,0 +1,16 @@
+import json
+
+
+def main_script_test_json_in_json_out_labels(_input_files, _output_files):
+    # read inputs
+    with _input_files["json"].open("rt") as fh:
+        inp_dat = json.load(fh)
+        p1_1 = int(inp_dat["p1[one]"])
+        p1_2 = int(inp_dat["p1[two]"])
+
+    # process
+    p2 = p1_1 + p1_2
+
+    # save outputs
+    with _output_files["json"].open("wt") as fh:
+        json.dump({"p2": p2}, fh)

hpcflow/data/scripts/main_script_test_json_in_obj.py

@@ -0,0 +1,12 @@
+import json
+
+
+def main_script_test_json_in_obj(_input_files):
+    # read inputs
+    with _input_files["json"].open("rt") as fh:
+        p1c = json.load(fh)["p1c"]
+
+    # process
+    p2 = p1c["a"] + 100
+
+    return {"p2": p2}

hpcflow/data/scripts/main_script_test_json_out_FAIL.py

@@ -0,0 +1,3 @@
+def main_script_test_json_out_FAIL(_output_files):
+    pass
+    # don't generate a JSON output file!

hpcflow/data/scripts/main_script_test_json_out_obj.py

@@ -0,0 +1,10 @@
+import json
+
+
+def main_script_test_json_out_obj(p1, _output_files):
+    # process
+    p1c = {"a": p1 + 100}
+
+    # write outputs
+    with _output_files["json"].open("wt") as fh:
+        json.dump({"p1c": p1c}, fh)

hpcflow/data/scripts/main_script_test_json_sub_param_in_json_out_labels.py

@@ -0,0 +1,16 @@
+import json
+
+
+def main_script_test_json_sub_param_in_json_out_labels(_input_files, _output_files):
+    # read inputs
+    with _input_files["json"].open("rt") as fh:
+        inp_dat = json.load(fh)
+        a = int(inp_dat["a"])
+        p1_2 = int(inp_dat["p1[two]"])
+
+    # process
+    p2 = a + p1_2
+
+    # save outputs
+    with _output_files["json"].open("wt") as fh:
+        json.dump({"p2": p2}, fh)

hpcflow/data/scripts/main_script_test_random_seed.py

@@ -0,0 +1,5 @@
+import os
+
+
+def main_script_test_random_seed():
+    return {"p2": int(os.environ["HPCFLOW_RUN_RANDOM_SEED"])}

hpcflow/data/scripts/main_script_test_shell_env_vars.py

@@ -0,0 +1,12 @@
+import json
+import os
+
+
+def main_script_test_shell_env_vars(p1):
+    with open("env_vars.json", "wt") as fp:
+        json.dump(
+            {k: v for k, v in os.environ.items() if k.startswith("HPCFLOW")},
+            fp,
+            indent=4,
+        )
+    return {"p1": p1 + 1}

hpcflow/data/scripts/main_script_test_std_out_std_err.py

@@ -0,0 +1,6 @@
+import sys
+
+
+def main_script_test_std_out_std_err(stdout_msg, stderr_msg):
+    print(stdout_msg)
+    print(stderr_msg, file=sys.stderr)

hpcflow/data/scripts/output_file_parser_basic.py

@@ -0,0 +1,3 @@
+def output_file_parser_basic(my_output_file):
+    with my_output_file.open("r") as fp:
+        return int(fp.read().strip())

hpcflow/data/scripts/output_file_parser_basic_FAIL.py

@@ -0,0 +1,7 @@
+def output_file_parser_basic_FAIL(my_output_file):
+    # parse the output file, but then delete it, so it cannot be saved (when run with
+    # `OFP.save_files=True`):
+    with my_output_file.open("r") as fp:
+        out = int(fp.read().strip())
+    my_output_file.unlink()
+    return out

hpcflow/data/scripts/output_file_parser_test_stdout_stderr.py

@@ -0,0 +1,8 @@
+import sys
+
+
+def output_file_parser_test_stdout_stderr(my_output_file, p1):
+    print(p1)
+    print(p1, file=sys.stderr)
+    with my_output_file.open("r") as fp:
+        return int(fp.read().strip())

hpcflow/data/scripts/parse_t1_file_01.py

@@ -0,0 +1,4 @@
+def parse_t1_file_01(t1_file_01):
+    with t1_file_01.open("r") as fp:
+        p4 = int(fp.readline().strip())
+    return p4

hpcflow/data/scripts/script_exit_test.py

@@ -0,0 +1,5 @@
+import sys
+
+
+def script_exit_test(exit_code):
+    sys.exit(exit_code)

hpcflow/data/template_components/__init__.py

@@ -0,0 +1 @@
+"""YAML files containing built-in template components, used for demonstrations."""