hpcflow-new2 0.2.0a189__tar.gz → 0.2.0a199__tar.gz

{hpcflow_new2-0.2.0a189 → hpcflow_new2-0.2.0a199}/PKG-INFO

@@ -1,12 +1,12 @@
 Metadata-Version: 2.1
 Name: hpcflow-new2
-Version: 0.2.0a189
+Version: 0.2.0a199
 Summary: Computational workflow management
-License: MIT
+License: MPL-2.0
 Author: aplowman
 Author-email: adam.plowman@manchester.ac.uk
 Requires-Python: >=3.9,<3.13
-Classifier: License :: OSI Approved :: MIT License
+Classifier: License :: OSI Approved
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
@@ -23,14 +23,18 @@ Requires-Dist: paramiko (>=3.2.0,<4.0.0)
 Requires-Dist: platformdirs (>=4.2.1,<5.0.0)
 Requires-Dist: psutil (>=5.9.4,<6.0.0)
 Requires-Dist: pytest (>=7.2.0,<8.0.0) ; extra == "test"
+Requires-Dist: pyzmq (>=26.0.3,<27.0.0)
 Requires-Dist: requests (>=2.31.0,<3.0.0)
 Requires-Dist: reretry (>=0.11.8,<0.12.0)
 Requires-Dist: rich (>=13.4.2,<14.0.0)
 Requires-Dist: ruamel-yaml (>=0.18.6,<0.19.0)
+Requires-Dist: scipy (>=1.13.1,<2.0.0) ; python_version < "3.10"
+Requires-Dist: scipy (>=1.15.2,<2.0.0) ; python_version >= "3.10"
 Requires-Dist: termcolor (>=1.1.0,<2.0.0)
+Requires-Dist: typing-extensions (>=4.12.2,<5.0.0)
 Requires-Dist: valida (>=0.7.5,<0.8.0)
 Requires-Dist: watchdog (>=2.1.9,<3.0.0)
-Requires-Dist: zarr (>=2.17.2,<3.0.0)
+Requires-Dist: zarr (==2.17.2)
 Description-Content-Type: text/markdown
 
 <div align="center">

{hpcflow_new2-0.2.0a189 → hpcflow_new2-0.2.0a199}/hpcflow/__pyinstaller/hook-hpcflow.py

@@ -4,9 +4,8 @@ from hpcflow.sdk import sdk_classes
 
 
 # most of the modules in `sdk_classes` are imported on-demand via the app object:
-hiddenimports = list(sdk_classes.values())
-
-hiddenimports += [
+hiddenimports = [
+    *sdk_classes.values(),
     "hpcflow.sdk.data",
     "hpcflow.data.demo_data_manifest",
     "hpcflow.data.scripts",
@@ -14,18 +13,22 @@ hiddenimports += [
     "hpcflow.data.workflows",
     "hpcflow.tests.data",
     "hpcflow.sdk.core.test_utils",
+    "hpcflow.sdk.utils.patches",
     "click.testing",
     "requests",  # for GitHub fsspec file system
     "fsspec.implementations.github",  # for GitHub fsspec file system
 ]
 
-py_include_kwargs = dict(include_py_files=True, excludes=("**/__pycache__",))
 datas = (
     collect_data_files("hpcflow.sdk.data")
     + collect_data_files("hpcflow.data.demo_data_manifest")
-    + collect_data_files("hpcflow.data.scripts", **py_include_kwargs)
+    + collect_data_files(
+        "hpcflow.data.scripts", include_py_files=True, excludes=("**/__pycache__",)
+    )
     + collect_data_files("hpcflow.data.template_components")
     + collect_data_files("hpcflow.data.workflows")
-    + collect_data_files("hpcflow.tests", **py_include_kwargs)
+    + collect_data_files(
+        "hpcflow.tests", include_py_files=True, excludes=("**/__pycache__",)
+    )
     + collect_data_files("hpcflow.tests.data")
 )

hpcflow_new2-0.2.0a199/hpcflow/_version.py

@@ -0,0 +1 @@
+__version__ = "0.2.0a199"

{hpcflow_new2-0.2.0a189 → hpcflow_new2-0.2.0a199}/hpcflow/app.py

@@ -1,3 +1,4 @@
+from __future__ import annotations
 from hpcflow import __version__, _app_name
 from hpcflow.sdk import app as sdk_app
 from hpcflow.sdk.config import ConfigOptions

hpcflow_new2-0.2.0a199/hpcflow/data/scripts/bad_script.py

@@ -0,0 +1,2 @@
+def bad_script():
+    raise SyntaxError("oh no!")

hpcflow_new2-0.2.0a199/hpcflow/data/scripts/do_nothing.py

@@ -0,0 +1,2 @@
+def do_nothing():
+    pass

hpcflow_new2-0.2.0a199/hpcflow/data/scripts/env_specifier_test/input_file_generator_pass_env_spec.py

@@ -0,0 +1,4 @@
+def input_file_generator_pass_env_spec(path, p1, env_spec):
+    print(env_spec)
+    with path.open("w") as fp:
+        fp.write(f"{p1}\n")

hpcflow_new2-0.2.0a199/hpcflow/data/scripts/env_specifier_test/main_script_test_pass_env_spec.py

@@ -0,0 +1,8 @@
+def main_script_test_pass_env_spec(p1, env_spec):
+    # process
+    p2 = p1 + 100
+
+    print(env_spec)
+
+    # return outputs
+    return {"p2": p2}

hpcflow_new2-0.2.0a199/hpcflow/data/scripts/env_specifier_test/output_file_parser_pass_env_spec.py

@@ -0,0 +1,4 @@
+def output_file_parser_pass_env_spec(my_output_file, env_spec):
+    print(env_spec)
+    with my_output_file.open("r") as fp:
+        return int(fp.read().strip())

hpcflow_new2-0.2.0a199/hpcflow/data/scripts/env_specifier_test/v1/input_file_generator_basic.py

@@ -0,0 +1,4 @@
+# version: v1
+def input_file_generator_basic(path, p1):
+    with path.open("w") as fp:
+        fp.write(f"{p1}\n")

hpcflow_new2-0.2.0a199/hpcflow/data/scripts/env_specifier_test/v1/main_script_test_direct_in_direct_out.py

@@ -0,0 +1,7 @@
+# version: v1
+def main_script_test_direct_in_direct_out(p1):
+    # process
+    p2 = p1 + 100
+
+    # return outputs
+    return {"p2": p2}

hpcflow_new2-0.2.0a199/hpcflow/data/scripts/env_specifier_test/v1/output_file_parser_basic.py

@@ -0,0 +1,4 @@
+# version: v1
+def output_file_parser_basic(my_output_file):
+    with my_output_file.open("r") as fp:
+        return int(fp.read().strip())

hpcflow_new2-0.2.0a199/hpcflow/data/scripts/env_specifier_test/v2/main_script_test_direct_in_direct_out.py

@@ -0,0 +1,7 @@
+# version: v2
+def main_script_test_direct_in_direct_out(p1):
+    # process
+    p2 = p1 + 200
+
+    # return outputs
+    return {"p2": p2}

hpcflow_new2-0.2.0a199/hpcflow/data/scripts/input_file_generator_basic.py

@@ -0,0 +1,3 @@
+def input_file_generator_basic(path, p1):
+    with path.open("w") as fp:
+        fp.write(f"{p1}\n")

hpcflow_new2-0.2.0a199/hpcflow/data/scripts/input_file_generator_basic_FAIL.py

@@ -0,0 +1,3 @@
+def input_file_generator_basic_FAIL(path, p1):
+    # silently fail to generate the input file!
+    pass

hpcflow_new2-0.2.0a199/hpcflow/data/scripts/input_file_generator_test_stdout_stderr.py

@@ -0,0 +1,8 @@
+import sys
+
+
+def input_file_generator_test_stdout_stderr(path, p1):
+    print(p1)
+    print(p1, file=sys.stderr)
+    with path.open("w") as fp:
+        fp.write(f"{p1}\n")

hpcflow_new2-0.2.0a199/hpcflow/data/scripts/main_script_test_direct_in.py

@@ -0,0 +1,3 @@
+def main_script_test_direct_in(p1):
+    # process
+    print(p1 + 100)

hpcflow_new2-0.2.0a199/hpcflow/data/scripts/main_script_test_direct_in_direct_out_2.py

@@ -0,0 +1,6 @@
+def main_script_test_direct_in_direct_out_2(p2):
+    # process
+    p3 = p2 + 100
+
+    # return outputs
+    return {"p3": p3}

hpcflow_new2-0.2.0a199/hpcflow/data/scripts/main_script_test_direct_in_direct_out_2_fail_allowed.py

@@ -0,0 +1,6 @@
+def main_script_test_direct_in_direct_out_2_fail_allowed(p2):
+    # process, accounting for possible unset data:
+    p3 = (p2 if p2 is not None else 0) + 100
+
+    # return outputs
+    return {"p3": p3}

hpcflow_new2-0.2.0a199/hpcflow/data/scripts/main_script_test_direct_in_direct_out_2_fail_allowed_group.py

@@ -0,0 +1,7 @@
+def main_script_test_direct_in_direct_out_2_fail_allowed_group(p2):
+    # process, accounting for possible unset data:
+    p2_sum = sum(i for i in p2 if i is not None)
+    p3 = p2_sum + 100
+
+    # return outputs
+    return {"p3": p3}

hpcflow_new2-0.2.0a199/hpcflow/data/scripts/main_script_test_direct_in_direct_out_3.py

@@ -0,0 +1,6 @@
+def main_script_test_direct_in_direct_out_3(p3):
+    # process
+    p4 = p3 + 100
+
+    # return outputs
+    return {"p4": p4}

hpcflow_new2-0.2.0a199/hpcflow/data/scripts/main_script_test_direct_in_group_direct_out_3.py

@@ -0,0 +1,6 @@
+def main_script_test_direct_in_group_direct_out_3(p3):
+    # process
+    p4 = sum(p3) + 100
+
+    # return outputs
+    return {"p4": p4}

hpcflow_new2-0.2.0a199/hpcflow/data/scripts/main_script_test_direct_in_group_one_fail_direct_out_3.py

@@ -0,0 +1,6 @@
+def main_script_test_direct_in_group_one_fail_direct_out_3(p3):
+    # process, ignore some un-set inputs:
+    p4 = sum([i for i in p3 if i is not None]) + 100
+
+    # return outputs
+    return {"p4": p4}

{hpcflow_new2-0.2.0a189 → hpcflow_new2-0.2.0a199}/hpcflow/data/scripts/main_script_test_hdf5_in_obj.py

@@ -1,4 +1,4 @@
-import h5py
+import h5py  # type: ignore[import-untyped]
 
 
 def main_script_test_hdf5_in_obj(_input_files):

hpcflow_new2-0.2.0a199/hpcflow/data/scripts/main_script_test_hdf5_in_obj_2.py

@@ -0,0 +1,12 @@
+import h5py  # type: ignore
+
+
+def main_script_test_hdf5_in_obj_2(p2, _input_files):
+    # read inputs
+    with h5py.File(_input_files["hdf5"], mode="r") as fh:
+        a = fh["p1c"].attrs["a"].item()
+
+    # process
+    p3 = a + 100
+
+    return {"p3": p3}

{hpcflow_new2-0.2.0a189 → hpcflow_new2-0.2.0a199}/hpcflow/data/scripts/main_script_test_hdf5_out_obj.py

@@ -1,4 +1,4 @@
-import h5py
+import h5py  # type: ignore[import-untyped]
 
 
 def main_script_test_hdf5_out_obj(p1, _output_files):

hpcflow_new2-0.2.0a199/hpcflow/data/scripts/main_script_test_json_out_FAIL.py

@@ -0,0 +1,3 @@
+def main_script_test_json_out_FAIL(_output_files):
+    pass
+    # don't generate a JSON output file!

hpcflow_new2-0.2.0a199/hpcflow/data/scripts/main_script_test_shell_env_vars.py

@@ -0,0 +1,12 @@
+import json
+import os
+
+
+def main_script_test_shell_env_vars(p1):
+    with open("env_vars.json", "wt") as fp:
+        json.dump(
+            {k: v for k, v in os.environ.items() if k.startswith("HPCFLOW")},
+            fp,
+            indent=4,
+        )
+    return {"p1": p1 + 1}

hpcflow_new2-0.2.0a199/hpcflow/data/scripts/main_script_test_std_out_std_err.py

@@ -0,0 +1,6 @@
+import sys
+
+
+def main_script_test_std_out_std_err(stdout_msg, stderr_msg):
+    print(stdout_msg)
+    print(stderr_msg, file=sys.stderr)

hpcflow_new2-0.2.0a199/hpcflow/data/scripts/output_file_parser_basic.py

@@ -0,0 +1,3 @@
+def output_file_parser_basic(my_output_file):
+    with my_output_file.open("r") as fp:
+        return int(fp.read().strip())

hpcflow_new2-0.2.0a199/hpcflow/data/scripts/output_file_parser_basic_FAIL.py

@@ -0,0 +1,7 @@
+def output_file_parser_basic_FAIL(my_output_file):
+    # parse the output file, but then delete it, so it cannot be saved (when run with
+    # `OFP.save_files=True`):
+    with my_output_file.open("r") as fp:
+        out = int(fp.read().strip())
+    my_output_file.unlink()
+    return out

hpcflow_new2-0.2.0a199/hpcflow/data/scripts/output_file_parser_test_stdout_stderr.py

@@ -0,0 +1,8 @@
+import sys
+
+
+def output_file_parser_test_stdout_stderr(my_output_file, p1):
+    print(p1)
+    print(p1, file=sys.stderr)
+    with my_output_file.open("r") as fp:
+        return int(fp.read().strip())

hpcflow_new2-0.2.0a199/hpcflow/data/scripts/script_exit_test.py

@@ -0,0 +1,5 @@
+import sys
+
+
+def script_exit_test(exit_code):
+    sys.exit(exit_code)

{hpcflow_new2-0.2.0a189 → hpcflow_new2-0.2.0a199}/hpcflow/data/template_components/environments.yaml

@@ -5,7 +5,7 @@
   executables:
     - label: python_script
       instances:
-        - command: python <<script_name>> <<args>>
+        - command: python <<script_path>> <<args>>
           num_cores: 1
           parallel_mode: null
   doc: |

{hpcflow_new2-0.2.0a189 → hpcflow_new2-0.2.0a199}/hpcflow/sdk/__init__.py

@@ -1,15 +1,19 @@
 """Sub-package to define an extensible hpcflow application."""
 
+from __future__ import annotations
+from collections.abc import Mapping
 import logging
 import os
 import sys
+from typing import Final
 
 #: Classes used in the construction of a workflow.
 #: :meta hide-value:
-sdk_classes = {
+sdk_classes: Final[Mapping[str, str]] = {
     "Workflow": "hpcflow.sdk.core.workflow",
     "Task": "hpcflow.sdk.core.task",
-    "ActionScopeType": "hpcflow.sdk.core.actions",
+    "MetaTask": "hpcflow.sdk.core.task",
+    "ActionScopeType": "hpcflow.sdk.core.enums",
     "ActionScope": "hpcflow.sdk.core.actions",
     "ActionRule": "hpcflow.sdk.core.actions",
     "Rule": "hpcflow.sdk.core.rule",
@@ -43,6 +47,7 @@ sdk_classes = {
     "NumCores": "hpcflow.sdk.core.environment",
     "ExecutableInstance": "hpcflow.sdk.core.environment",
     "Executable": "hpcflow.sdk.core.environment",
+    "Executor": "hpcflow.sdk.core.execute",
     "Environment": "hpcflow.sdk.core.environment",
     "Loop": "hpcflow.sdk.core.loop",
     "WorkflowLoop": "hpcflow.sdk.core.loop",
@@ -65,15 +70,17 @@ sdk_classes = {
     "SchemaInput": "hpcflow.sdk.core.parameters",
     "SchemaOutput": "hpcflow.sdk.core.parameters",
     "ValueSequence": "hpcflow.sdk.core.parameters",
+    "MultiPathSequence": "hpcflow.sdk.core.parameters",
     "ValuePerturbation": "hpcflow.sdk.core.parameters",
     "InputValue": "hpcflow.sdk.core.parameters",
     "ResourceSpec": "hpcflow.sdk.core.parameters",
-    "TaskSourceType": "hpcflow.sdk.core.parameters",
-    "InputSourceType": "hpcflow.sdk.core.parameters",
-    "ParameterPropagationMode": "hpcflow.sdk.core.parameters",
+    "TaskSourceType": "hpcflow.sdk.core.enums",
+    "InputSourceType": "hpcflow.sdk.core.enums",
+    "ParameterPropagationMode": "hpcflow.sdk.core.enums",
     "InputSource": "hpcflow.sdk.core.parameters",
     "TaskObjective": "hpcflow.sdk.core.task_schema",
     "TaskSchema": "hpcflow.sdk.core.task_schema",
+    "MetaTaskSchema": "hpcflow.sdk.core.task_schema",
     "ElementSet": "hpcflow.sdk.core.task",
     "Task": "hpcflow.sdk.core.task",
     "WorkflowTask": "hpcflow.sdk.core.task",
@@ -86,7 +93,9 @@ sdk_classes = {
     "Workflow": "hpcflow.sdk.core.workflow",
     "WorkflowBlueprint": "hpcflow.sdk.core.workflow",
     "Jobscript": "hpcflow.sdk.submission.jobscript",
+    "JobscriptBlock": "hpcflow.sdk.submission.jobscript",
     "Submission": "hpcflow.sdk.submission.submission",
+    "QueuedScheduler": "hpcflow.sdk.submission.schedulers",
     "DirectWindows": "hpcflow.sdk.submission.schedulers.direct",
     "DirectPosix": "hpcflow.sdk.submission.schedulers.direct",
     "SlurmPosix": "hpcflow.sdk.submission.schedulers.slurm",
@@ -98,7 +107,7 @@ sdk_classes = {
 # these are defined as `BaseApp` methods with an underscore prefix:
 #: Functions exported by the application.
 #: :meta hide-value:
-sdk_funcs = (
+sdk_funcs: Final[tuple[str, ...]] = (
     "make_workflow",
     "make_demo_workflow",
     "make_and_submit_workflow",
@@ -114,24 +123,26 @@ sdk_funcs = (
     "cancel",
 )
 
-_SDK_CONSOLE_LOG_LEVEL = os.environ.get("HPCFLOW_SDK_CONSOLE_LOG_LEVEL", "ERROR")
 
-
-def get_SDK_logger(name=None):
+def get_SDK_logger(name: str | None = None) -> logging.Logger:
     """Get a logger with prefix of "hpcflow_sdk" instead of "hpcflow.sdk" to ensure the
     handlers of the SDK logger and app logger are distinct."""
     name = ".".join(["hpcflow_sdk"] + (name or __name__).split(".")[2:])
     return logging.getLogger(name)
 
 
-_SDK_logger = get_SDK_logger()
-_SDK_logger.setLevel("DEBUG")
+def _init_logger() -> None:
+    level = os.environ.get("HPCFLOW_SDK_CONSOLE_LOG_LEVEL", "ERROR")
+    SDK_logger = get_SDK_logger()
+    SDK_logger.setLevel("DEBUG")
+
+    sh = logging.StreamHandler()
+    sh.setFormatter(logging.Formatter("%(levelname)s %(name)s: %(message)s"))
+    sh.setLevel(level)
+    SDK_logger.addHandler(sh)
 
-_sh = logging.StreamHandler()
-_sh.setFormatter(logging.Formatter("%(levelname)s %(name)s: %(message)s"))
-_sh.setLevel(_SDK_CONSOLE_LOG_LEVEL)
-_SDK_logger.addHandler(_sh)
 
+_init_logger()
 if getattr(sys, "frozen", False) and hasattr(sys, "_MEIPASS"):
     import multiprocessing