hpcflow-new2 0.2.0a190__py3-none-any.whl → 0.2.0a200__py3-none-any.whl

hpcflow/__pyinstaller/hook-hpcflow.py

@@ -13,6 +13,7 @@ hiddenimports = [
     "hpcflow.data.workflows",
     "hpcflow.tests.data",
     "hpcflow.sdk.core.test_utils",
+    "hpcflow.sdk.utils.patches",
     "click.testing",
     "requests",  # for GitHub fsspec file system
     "fsspec.implementations.github",  # for GitHub fsspec file system

hpcflow/_version.py

@@ -1 +1 @@
-__version__ = "0.2.0a190"
+__version__ = "0.2.0a200"

hpcflow/data/scripts/bad_script.py

@@ -0,0 +1,2 @@
+def bad_script():
+    raise SyntaxError("oh no!")

hpcflow/data/scripts/do_nothing.py

@@ -0,0 +1,2 @@
+def do_nothing():
+    pass

hpcflow/data/scripts/env_specifier_test/input_file_generator_pass_env_spec.py

@@ -0,0 +1,4 @@
+def input_file_generator_pass_env_spec(path, p1, env_spec):
+    print(env_spec)
+    with path.open("w") as fp:
+        fp.write(f"{p1}\n")

hpcflow/data/scripts/env_specifier_test/main_script_test_pass_env_spec.py

@@ -0,0 +1,8 @@
+def main_script_test_pass_env_spec(p1, env_spec):
+    # process
+    p2 = p1 + 100
+
+    print(env_spec)
+
+    # return outputs
+    return {"p2": p2}

hpcflow/data/scripts/env_specifier_test/output_file_parser_pass_env_spec.py

@@ -0,0 +1,4 @@
+def output_file_parser_pass_env_spec(my_output_file, env_spec):
+    print(env_spec)
+    with my_output_file.open("r") as fp:
+        return int(fp.read().strip())

hpcflow/data/scripts/env_specifier_test/v1/input_file_generator_basic.py

@@ -0,0 +1,4 @@
+# version: v1
+def input_file_generator_basic(path, p1):
+    with path.open("w") as fp:
+        fp.write(f"{p1}\n")

hpcflow/data/scripts/env_specifier_test/v1/main_script_test_direct_in_direct_out.py

@@ -0,0 +1,7 @@
+# version: v1
+def main_script_test_direct_in_direct_out(p1):
+    # process
+    p2 = p1 + 100
+
+    # return outputs
+    return {"p2": p2}

hpcflow/data/scripts/env_specifier_test/v1/output_file_parser_basic.py

@@ -0,0 +1,4 @@
+# version: v1
+def output_file_parser_basic(my_output_file):
+    with my_output_file.open("r") as fp:
+        return int(fp.read().strip())

hpcflow/data/scripts/env_specifier_test/v2/main_script_test_direct_in_direct_out.py

@@ -0,0 +1,7 @@
+# version: v2
+def main_script_test_direct_in_direct_out(p1):
+    # process
+    p2 = p1 + 200
+
+    # return outputs
+    return {"p2": p2}

hpcflow/data/scripts/input_file_generator_basic.py

@@ -0,0 +1,3 @@
+def input_file_generator_basic(path, p1):
+    with path.open("w") as fp:
+        fp.write(f"{p1}\n")

hpcflow/data/scripts/input_file_generator_basic_FAIL.py

@@ -0,0 +1,3 @@
+def input_file_generator_basic_FAIL(path, p1):
+    # silently fail to generate the input file!
+    pass

hpcflow/data/scripts/input_file_generator_test_stdout_stderr.py

@@ -0,0 +1,8 @@
+import sys
+
+
+def input_file_generator_test_stdout_stderr(path, p1):
+    print(p1)
+    print(p1, file=sys.stderr)
+    with path.open("w") as fp:
+        fp.write(f"{p1}\n")

hpcflow/data/scripts/main_script_test_direct_in.py

@@ -0,0 +1,3 @@
+def main_script_test_direct_in(p1):
+    # process
+    print(p1 + 100)

hpcflow/data/scripts/main_script_test_direct_in_direct_out_2.py

@@ -0,0 +1,6 @@
+def main_script_test_direct_in_direct_out_2(p2):
+    # process
+    p3 = p2 + 100
+
+    # return outputs
+    return {"p3": p3}

hpcflow/data/scripts/main_script_test_direct_in_direct_out_2_fail_allowed.py

@@ -0,0 +1,6 @@
+def main_script_test_direct_in_direct_out_2_fail_allowed(p2):
+    # process, accounting for possible unset data:
+    p3 = (p2 if p2 is not None else 0) + 100
+
+    # return outputs
+    return {"p3": p3}

hpcflow/data/scripts/main_script_test_direct_in_direct_out_2_fail_allowed_group.py

@@ -0,0 +1,7 @@
+def main_script_test_direct_in_direct_out_2_fail_allowed_group(p2):
+    # process, accounting for possible unset data:
+    p2_sum = sum(i for i in p2 if i is not None)
+    p3 = p2_sum + 100
+
+    # return outputs
+    return {"p3": p3}

hpcflow/data/scripts/main_script_test_direct_in_direct_out_3.py

@@ -0,0 +1,6 @@
+def main_script_test_direct_in_direct_out_3(p3):
+    # process
+    p4 = p3 + 100
+
+    # return outputs
+    return {"p4": p4}

hpcflow/data/scripts/main_script_test_direct_in_group_direct_out_3.py

@@ -0,0 +1,6 @@
+def main_script_test_direct_in_group_direct_out_3(p3):
+    # process
+    p4 = sum(p3) + 100
+
+    # return outputs
+    return {"p4": p4}

hpcflow/data/scripts/main_script_test_direct_in_group_one_fail_direct_out_3.py

@@ -0,0 +1,6 @@
+def main_script_test_direct_in_group_one_fail_direct_out_3(p3):
+    # process, ignore some un-set inputs:
+    p4 = sum([i for i in p3 if i is not None]) + 100
+
+    # return outputs
+    return {"p4": p4}

hpcflow/data/scripts/main_script_test_hdf5_in_obj_2.py

@@ -0,0 +1,12 @@
+import h5py  # type: ignore
+
+
+def main_script_test_hdf5_in_obj_2(p2, _input_files):
+    # read inputs
+    with h5py.File(_input_files["hdf5"], mode="r") as fh:
+        a = fh["p1c"].attrs["a"].item()
+
+    # process
+    p3 = a + 100
+
+    return {"p3": p3}

hpcflow/data/scripts/main_script_test_json_out_FAIL.py

@@ -0,0 +1,3 @@
+def main_script_test_json_out_FAIL(_output_files):
+    pass
+    # don't generate a JSON output file!

hpcflow/data/scripts/main_script_test_shell_env_vars.py

@@ -0,0 +1,12 @@
+import json
+import os
+
+
+def main_script_test_shell_env_vars(p1):
+    with open("env_vars.json", "wt") as fp:
+        json.dump(
+            {k: v for k, v in os.environ.items() if k.startswith("HPCFLOW")},
+            fp,
+            indent=4,
+        )
+    return {"p1": p1 + 1}

hpcflow/data/scripts/main_script_test_std_out_std_err.py

@@ -0,0 +1,6 @@
+import sys
+
+
+def main_script_test_std_out_std_err(stdout_msg, stderr_msg):
+    print(stdout_msg)
+    print(stderr_msg, file=sys.stderr)

hpcflow/data/scripts/output_file_parser_basic.py

@@ -0,0 +1,3 @@
+def output_file_parser_basic(my_output_file):
+    with my_output_file.open("r") as fp:
+        return int(fp.read().strip())

hpcflow/data/scripts/output_file_parser_basic_FAIL.py

@@ -0,0 +1,7 @@
+def output_file_parser_basic_FAIL(my_output_file):
+    # parse the output file, but then delete it, so it cannot be saved (when run with
+    # `OFP.save_files=True`):
+    with my_output_file.open("r") as fp:
+        out = int(fp.read().strip())
+    my_output_file.unlink()
+    return out

hpcflow/data/scripts/output_file_parser_test_stdout_stderr.py

@@ -0,0 +1,8 @@
+import sys
+
+
+def output_file_parser_test_stdout_stderr(my_output_file, p1):
+    print(p1)
+    print(p1, file=sys.stderr)
+    with my_output_file.open("r") as fp:
+        return int(fp.read().strip())

hpcflow/data/scripts/script_exit_test.py

@@ -0,0 +1,5 @@
+import sys
+
+
+def script_exit_test(exit_code):
+    sys.exit(exit_code)

hpcflow/data/template_components/environments.yaml

@@ -5,7 +5,7 @@
   executables:
     - label: python_script
       instances:
-        - command: python <<script_name>> <<args>>
+        - command: python <<script_path>> <<args>>
           num_cores: 1
           parallel_mode: null
   doc: |

hpcflow/sdk/__init__.py

@@ -12,6 +12,7 @@ from typing import Final
 sdk_classes: Final[Mapping[str, str]] = {
     "Workflow": "hpcflow.sdk.core.workflow",
     "Task": "hpcflow.sdk.core.task",
+    "MetaTask": "hpcflow.sdk.core.task",
     "ActionScopeType": "hpcflow.sdk.core.enums",
     "ActionScope": "hpcflow.sdk.core.actions",
     "ActionRule": "hpcflow.sdk.core.actions",
@@ -46,6 +47,7 @@ sdk_classes: Final[Mapping[str, str]] = {
     "NumCores": "hpcflow.sdk.core.environment",
     "ExecutableInstance": "hpcflow.sdk.core.environment",
     "Executable": "hpcflow.sdk.core.environment",
+    "Executor": "hpcflow.sdk.core.execute",
     "Environment": "hpcflow.sdk.core.environment",
     "Loop": "hpcflow.sdk.core.loop",
     "WorkflowLoop": "hpcflow.sdk.core.loop",
@@ -68,6 +70,7 @@ sdk_classes: Final[Mapping[str, str]] = {
     "SchemaInput": "hpcflow.sdk.core.parameters",
     "SchemaOutput": "hpcflow.sdk.core.parameters",
     "ValueSequence": "hpcflow.sdk.core.parameters",
+    "MultiPathSequence": "hpcflow.sdk.core.parameters",
     "ValuePerturbation": "hpcflow.sdk.core.parameters",
     "InputValue": "hpcflow.sdk.core.parameters",
     "ResourceSpec": "hpcflow.sdk.core.parameters",
@@ -77,6 +80,7 @@ sdk_classes: Final[Mapping[str, str]] = {
     "InputSource": "hpcflow.sdk.core.parameters",
     "TaskObjective": "hpcflow.sdk.core.task_schema",
     "TaskSchema": "hpcflow.sdk.core.task_schema",
+    "MetaTaskSchema": "hpcflow.sdk.core.task_schema",
     "ElementSet": "hpcflow.sdk.core.task",
     "Task": "hpcflow.sdk.core.task",
     "WorkflowTask": "hpcflow.sdk.core.task",
@@ -89,6 +93,7 @@ sdk_classes: Final[Mapping[str, str]] = {
     "Workflow": "hpcflow.sdk.core.workflow",
     "WorkflowBlueprint": "hpcflow.sdk.core.workflow",
     "Jobscript": "hpcflow.sdk.submission.jobscript",
+    "JobscriptBlock": "hpcflow.sdk.submission.jobscript",
     "Submission": "hpcflow.sdk.submission.submission",
     "QueuedScheduler": "hpcflow.sdk.submission.schedulers",
     "DirectWindows": "hpcflow.sdk.submission.schedulers.direct",