hpcflow-new2 0.2.0a190__tar.gz → 0.2.0a200__tar.gz

{hpcflow_new2-0.2.0a190 → hpcflow_new2-0.2.0a200}/PKG-INFO

@@ -1,12 +1,12 @@
 Metadata-Version: 2.1
 Name: hpcflow-new2
-Version: 0.2.0a190
+Version: 0.2.0a200
 Summary: Computational workflow management
-License: MIT
+License: MPL-2.0
 Author: aplowman
 Author-email: adam.plowman@manchester.ac.uk
 Requires-Python: >=3.9,<3.13
-Classifier: License :: OSI Approved :: MIT License
+Classifier: License :: OSI Approved
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
@@ -23,15 +23,18 @@ Requires-Dist: paramiko (>=3.2.0,<4.0.0)
 Requires-Dist: platformdirs (>=4.2.1,<5.0.0)
 Requires-Dist: psutil (>=5.9.4,<6.0.0)
 Requires-Dist: pytest (>=7.2.0,<8.0.0) ; extra == "test"
+Requires-Dist: pyzmq (>=26.0.3,<27.0.0)
 Requires-Dist: requests (>=2.31.0,<3.0.0)
 Requires-Dist: reretry (>=0.11.8,<0.12.0)
 Requires-Dist: rich (>=13.4.2,<14.0.0)
 Requires-Dist: ruamel-yaml (>=0.18.6,<0.19.0)
+Requires-Dist: scipy (>=1.13.1,<2.0.0) ; python_version < "3.10"
+Requires-Dist: scipy (>=1.15.2,<2.0.0) ; python_version >= "3.10"
 Requires-Dist: termcolor (>=1.1.0,<2.0.0)
 Requires-Dist: typing-extensions (>=4.12.2,<5.0.0)
 Requires-Dist: valida (>=0.7.5,<0.8.0)
 Requires-Dist: watchdog (>=2.1.9,<3.0.0)
-Requires-Dist: zarr (>=2.17.2,<3.0.0)
+Requires-Dist: zarr (==2.17.2)
 Description-Content-Type: text/markdown
 
 <div align="center">

{hpcflow_new2-0.2.0a190 → hpcflow_new2-0.2.0a200}/hpcflow/__pyinstaller/hook-hpcflow.py

@@ -13,6 +13,7 @@ hiddenimports = [
     "hpcflow.data.workflows",
     "hpcflow.tests.data",
     "hpcflow.sdk.core.test_utils",
+    "hpcflow.sdk.utils.patches",
     "click.testing",
     "requests",  # for GitHub fsspec file system
     "fsspec.implementations.github",  # for GitHub fsspec file system

hpcflow_new2-0.2.0a200/hpcflow/_version.py

@@ -0,0 +1 @@
+__version__ = "0.2.0a200"

hpcflow_new2-0.2.0a200/hpcflow/data/scripts/bad_script.py

@@ -0,0 +1,2 @@
+def bad_script():
+    raise SyntaxError("oh no!")

hpcflow_new2-0.2.0a200/hpcflow/data/scripts/do_nothing.py

@@ -0,0 +1,2 @@
+def do_nothing():
+    pass

hpcflow_new2-0.2.0a200/hpcflow/data/scripts/env_specifier_test/input_file_generator_pass_env_spec.py

@@ -0,0 +1,4 @@
+def input_file_generator_pass_env_spec(path, p1, env_spec):
+    print(env_spec)
+    with path.open("w") as fp:
+        fp.write(f"{p1}\n")

hpcflow_new2-0.2.0a200/hpcflow/data/scripts/env_specifier_test/main_script_test_pass_env_spec.py

@@ -0,0 +1,8 @@
+def main_script_test_pass_env_spec(p1, env_spec):
+    # process
+    p2 = p1 + 100
+
+    print(env_spec)
+
+    # return outputs
+    return {"p2": p2}

hpcflow_new2-0.2.0a200/hpcflow/data/scripts/env_specifier_test/output_file_parser_pass_env_spec.py

@@ -0,0 +1,4 @@
+def output_file_parser_pass_env_spec(my_output_file, env_spec):
+    print(env_spec)
+    with my_output_file.open("r") as fp:
+        return int(fp.read().strip())

hpcflow_new2-0.2.0a200/hpcflow/data/scripts/env_specifier_test/v1/input_file_generator_basic.py

@@ -0,0 +1,4 @@
+# version: v1
+def input_file_generator_basic(path, p1):
+    with path.open("w") as fp:
+        fp.write(f"{p1}\n")

hpcflow_new2-0.2.0a200/hpcflow/data/scripts/env_specifier_test/v1/main_script_test_direct_in_direct_out.py

@@ -0,0 +1,7 @@
+# version: v1
+def main_script_test_direct_in_direct_out(p1):
+    # process
+    p2 = p1 + 100
+
+    # return outputs
+    return {"p2": p2}

hpcflow_new2-0.2.0a200/hpcflow/data/scripts/env_specifier_test/v1/output_file_parser_basic.py

@@ -0,0 +1,4 @@
+# version: v1
+def output_file_parser_basic(my_output_file):
+    with my_output_file.open("r") as fp:
+        return int(fp.read().strip())

hpcflow_new2-0.2.0a200/hpcflow/data/scripts/env_specifier_test/v2/main_script_test_direct_in_direct_out.py

@@ -0,0 +1,7 @@
+# version: v2
+def main_script_test_direct_in_direct_out(p1):
+    # process
+    p2 = p1 + 200
+
+    # return outputs
+    return {"p2": p2}

hpcflow_new2-0.2.0a200/hpcflow/data/scripts/input_file_generator_basic.py

@@ -0,0 +1,3 @@
+def input_file_generator_basic(path, p1):
+    with path.open("w") as fp:
+        fp.write(f"{p1}\n")

hpcflow_new2-0.2.0a200/hpcflow/data/scripts/input_file_generator_basic_FAIL.py

@@ -0,0 +1,3 @@
+def input_file_generator_basic_FAIL(path, p1):
+    # silently fail to generate the input file!
+    pass

hpcflow_new2-0.2.0a200/hpcflow/data/scripts/input_file_generator_test_stdout_stderr.py

@@ -0,0 +1,8 @@
+import sys
+
+
+def input_file_generator_test_stdout_stderr(path, p1):
+    print(p1)
+    print(p1, file=sys.stderr)
+    with path.open("w") as fp:
+        fp.write(f"{p1}\n")

hpcflow_new2-0.2.0a200/hpcflow/data/scripts/main_script_test_direct_in.py

@@ -0,0 +1,3 @@
+def main_script_test_direct_in(p1):
+    # process
+    print(p1 + 100)

hpcflow_new2-0.2.0a200/hpcflow/data/scripts/main_script_test_direct_in_direct_out_2.py

@@ -0,0 +1,6 @@
+def main_script_test_direct_in_direct_out_2(p2):
+    # process
+    p3 = p2 + 100
+
+    # return outputs
+    return {"p3": p3}

hpcflow_new2-0.2.0a200/hpcflow/data/scripts/main_script_test_direct_in_direct_out_2_fail_allowed.py

@@ -0,0 +1,6 @@
+def main_script_test_direct_in_direct_out_2_fail_allowed(p2):
+    # process, accounting for possible unset data:
+    p3 = (p2 if p2 is not None else 0) + 100
+
+    # return outputs
+    return {"p3": p3}

hpcflow_new2-0.2.0a200/hpcflow/data/scripts/main_script_test_direct_in_direct_out_2_fail_allowed_group.py

@@ -0,0 +1,7 @@
+def main_script_test_direct_in_direct_out_2_fail_allowed_group(p2):
+    # process, accounting for possible unset data:
+    p2_sum = sum(i for i in p2 if i is not None)
+    p3 = p2_sum + 100
+
+    # return outputs
+    return {"p3": p3}

hpcflow_new2-0.2.0a200/hpcflow/data/scripts/main_script_test_direct_in_direct_out_3.py

@@ -0,0 +1,6 @@
+def main_script_test_direct_in_direct_out_3(p3):
+    # process
+    p4 = p3 + 100
+
+    # return outputs
+    return {"p4": p4}

hpcflow_new2-0.2.0a200/hpcflow/data/scripts/main_script_test_direct_in_group_direct_out_3.py

@@ -0,0 +1,6 @@
+def main_script_test_direct_in_group_direct_out_3(p3):
+    # process
+    p4 = sum(p3) + 100
+
+    # return outputs
+    return {"p4": p4}

hpcflow_new2-0.2.0a200/hpcflow/data/scripts/main_script_test_direct_in_group_one_fail_direct_out_3.py

@@ -0,0 +1,6 @@
+def main_script_test_direct_in_group_one_fail_direct_out_3(p3):
+    # process, ignore some un-set inputs:
+    p4 = sum([i for i in p3 if i is not None]) + 100
+
+    # return outputs
+    return {"p4": p4}

hpcflow_new2-0.2.0a200/hpcflow/data/scripts/main_script_test_hdf5_in_obj_2.py

@@ -0,0 +1,12 @@
+import h5py  # type: ignore
+
+
+def main_script_test_hdf5_in_obj_2(p2, _input_files):
+    # read inputs
+    with h5py.File(_input_files["hdf5"], mode="r") as fh:
+        a = fh["p1c"].attrs["a"].item()
+
+    # process
+    p3 = a + 100
+
+    return {"p3": p3}

hpcflow_new2-0.2.0a200/hpcflow/data/scripts/main_script_test_json_out_FAIL.py

@@ -0,0 +1,3 @@
+def main_script_test_json_out_FAIL(_output_files):
+    pass
+    # don't generate a JSON output file!

hpcflow_new2-0.2.0a200/hpcflow/data/scripts/main_script_test_shell_env_vars.py

@@ -0,0 +1,12 @@
+import json
+import os
+
+
+def main_script_test_shell_env_vars(p1):
+    with open("env_vars.json", "wt") as fp:
+        json.dump(
+            {k: v for k, v in os.environ.items() if k.startswith("HPCFLOW")},
+            fp,
+            indent=4,
+        )
+    return {"p1": p1 + 1}

hpcflow_new2-0.2.0a200/hpcflow/data/scripts/main_script_test_std_out_std_err.py

@@ -0,0 +1,6 @@
+import sys
+
+
+def main_script_test_std_out_std_err(stdout_msg, stderr_msg):
+    print(stdout_msg)
+    print(stderr_msg, file=sys.stderr)

hpcflow_new2-0.2.0a200/hpcflow/data/scripts/output_file_parser_basic.py

@@ -0,0 +1,3 @@
+def output_file_parser_basic(my_output_file):
+    with my_output_file.open("r") as fp:
+        return int(fp.read().strip())

hpcflow_new2-0.2.0a200/hpcflow/data/scripts/output_file_parser_basic_FAIL.py

@@ -0,0 +1,7 @@
+def output_file_parser_basic_FAIL(my_output_file):
+    # parse the output file, but then delete it, so it cannot be saved (when run with
+    # `OFP.save_files=True`):
+    with my_output_file.open("r") as fp:
+        out = int(fp.read().strip())
+    my_output_file.unlink()
+    return out

hpcflow_new2-0.2.0a200/hpcflow/data/scripts/output_file_parser_test_stdout_stderr.py

@@ -0,0 +1,8 @@
+import sys
+
+
+def output_file_parser_test_stdout_stderr(my_output_file, p1):
+    print(p1)
+    print(p1, file=sys.stderr)
+    with my_output_file.open("r") as fp:
+        return int(fp.read().strip())

hpcflow_new2-0.2.0a200/hpcflow/data/scripts/script_exit_test.py

@@ -0,0 +1,5 @@
+import sys
+
+
+def script_exit_test(exit_code):
+    sys.exit(exit_code)

{hpcflow_new2-0.2.0a190 → hpcflow_new2-0.2.0a200}/hpcflow/data/template_components/environments.yaml

@@ -5,7 +5,7 @@
   executables:
     - label: python_script
       instances:
-        - command: python <<script_name>> <<args>>
+        - command: python <<script_path>> <<args>>
           num_cores: 1
           parallel_mode: null
   doc: |

{hpcflow_new2-0.2.0a190 → hpcflow_new2-0.2.0a200}/hpcflow/sdk/__init__.py

@@ -12,6 +12,7 @@ from typing import Final
 sdk_classes: Final[Mapping[str, str]] = {
     "Workflow": "hpcflow.sdk.core.workflow",
     "Task": "hpcflow.sdk.core.task",
+    "MetaTask": "hpcflow.sdk.core.task",
     "ActionScopeType": "hpcflow.sdk.core.enums",
     "ActionScope": "hpcflow.sdk.core.actions",
     "ActionRule": "hpcflow.sdk.core.actions",
@@ -46,6 +47,7 @@ sdk_classes: Final[Mapping[str, str]] = {
     "NumCores": "hpcflow.sdk.core.environment",
     "ExecutableInstance": "hpcflow.sdk.core.environment",
     "Executable": "hpcflow.sdk.core.environment",
+    "Executor": "hpcflow.sdk.core.execute",
     "Environment": "hpcflow.sdk.core.environment",
     "Loop": "hpcflow.sdk.core.loop",
     "WorkflowLoop": "hpcflow.sdk.core.loop",
@@ -68,6 +70,7 @@ sdk_classes: Final[Mapping[str, str]] = {
     "SchemaInput": "hpcflow.sdk.core.parameters",
     "SchemaOutput": "hpcflow.sdk.core.parameters",
     "ValueSequence": "hpcflow.sdk.core.parameters",
+    "MultiPathSequence": "hpcflow.sdk.core.parameters",
     "ValuePerturbation": "hpcflow.sdk.core.parameters",
     "InputValue": "hpcflow.sdk.core.parameters",
     "ResourceSpec": "hpcflow.sdk.core.parameters",
@@ -77,6 +80,7 @@ sdk_classes: Final[Mapping[str, str]] = {
     "InputSource": "hpcflow.sdk.core.parameters",
     "TaskObjective": "hpcflow.sdk.core.task_schema",
     "TaskSchema": "hpcflow.sdk.core.task_schema",
+    "MetaTaskSchema": "hpcflow.sdk.core.task_schema",
     "ElementSet": "hpcflow.sdk.core.task",
     "Task": "hpcflow.sdk.core.task",
     "WorkflowTask": "hpcflow.sdk.core.task",
@@ -89,6 +93,7 @@ sdk_classes: Final[Mapping[str, str]] = {
     "Workflow": "hpcflow.sdk.core.workflow",
     "WorkflowBlueprint": "hpcflow.sdk.core.workflow",
     "Jobscript": "hpcflow.sdk.submission.jobscript",
+    "JobscriptBlock": "hpcflow.sdk.submission.jobscript",
     "Submission": "hpcflow.sdk.submission.submission",
     "QueuedScheduler": "hpcflow.sdk.submission.schedulers",
     "DirectWindows": "hpcflow.sdk.submission.schedulers.direct",