hpcflow-new2 0.2.0a218__tar.gz → 0.2.0a219__tar.gz

{hpcflow_new2-0.2.0a218 → hpcflow_new2-0.2.0a219}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hpcflow-new2
-Version: 0.2.0a218
+Version: 0.2.0a219
 Summary: Computational workflow management
 License: MPL-2.0
 Author: aplowman

hpcflow_new2-0.2.0a219/hpcflow/_version.py

@@ -0,0 +1 @@
+__version__ = "0.2.0a219"

hpcflow_new2-0.2.0a219/hpcflow/data/scripts/import_future_script.py

@@ -0,0 +1,7 @@
+from __future__ import annotations
+
+
+def import_future_script():
+    """Used to test the `__future__` import is moved to the top of the file in the
+    generated script in the case of `resources.combine_scripts == True`"""
+    pass

{hpcflow_new2-0.2.0a218 → hpcflow_new2-0.2.0a219}/hpcflow/sdk/app.py

@@ -1794,7 +1794,8 @@ class BaseApp(metaclass=Singleton):
         for comp_type in TEMPLATE_COMP_TYPES:
             with open_text_resource(package, f"{comp_type}.yaml") as fh:
                 SDK_logger.info(f"Parsing file as YAML: {fh.name!r}")
-                components[comp_type] = read_YAML_str(fh.read())
+                source = f"from {Path(fh.name)!r}"
+                components[comp_type] = read_YAML_str(fh.read(), source=source)
 
         return components
 

{hpcflow_new2-0.2.0a218 → hpcflow_new2-0.2.0a219}/hpcflow/sdk/core/errors.py

@@ -1020,3 +1020,9 @@ class MissingElementGroup(ValueError):
             f"Adding elements to task {task_name!r}: "
             f"no element group named {group_name!r} found for input {input_path!r}."
         )
+
+
+class YAMLError(ValueError):
+    """
+    A problem with parsing a YAML file.
+    """

{hpcflow_new2-0.2.0a218 → hpcflow_new2-0.2.0a219}/hpcflow/sdk/core/parameters.py

@@ -2788,31 +2788,31 @@ class InputSource(JSONLike):
 
         Examples
         --------
-            task.[task_ref].input
-            task.[task_ref].output
-            local
-            default
-            import.[import_ref]
+        For a local task input source, use:
+
+        >>> InputSource.from_string("local")
+
+        For a schema input default source, use:
+
+        >>> InputSource.from_string("default")
+
+        For task input sources, specify either the task insert ID (typically this is just
+        the task index within the workflow), or the task's unique name, which is usually
+        just the associated task schema's objective, but if multiple tasks use the same
+        schema, it will be suffixed by an index, starting from one.
+
+        >>> InputSource.from_string("task.0.input")
+        >>> InputSource.from_string("task.my_task.input")
         """
         return cls(**cls._parse_from_string(str_defn))
 
     @staticmethod
     def _parse_from_string(str_defn: str) -> dict[str, Any]:
-        """Parse a dot-delimited string definition of an InputSource.
-
-        Examples
-        --------
-            task.[task_ref].input
-            task.[task_ref].output
-            local
-            default
-            import.[import_ref]
-        """
+        """Parse a dot-delimited string definition of an InputSource."""
         parts = str_defn.split(".")
         source_type = get_enum_by_name_or_val(InputSourceType, parts[0])
-        task_ref: int | None = None
+        task_ref: int | str | None = None
         task_source_type: TaskSourceType | None = None
-        import_ref: int | None = None
         if (
             (
                 source_type in (InputSourceType.LOCAL, InputSourceType.DEFAULT)
@@ -2826,22 +2826,20 @@ class InputSource(JSONLike):
         if source_type is InputSourceType.TASK:
             # TODO: does this include element_iters?
             try:
+                # assume specified by task insert ID
                 task_ref = int(parts[1])
             except ValueError:
-                pass
+                # assume specified by task unique name
+                task_ref = parts[1]
             try:
                 task_source_type = get_enum_by_name_or_val(TaskSourceType, parts[2])
             except IndexError:
                 task_source_type = TaskSourceType.OUTPUT
         elif source_type is InputSourceType.IMPORT:
-            try:
-                import_ref = int(parts[1])
-            except ValueError:
-                pass
+            raise NotImplementedError("Import input sources are not yet supported.")
 
         return {
             "source_type": source_type,
-            "import_ref": import_ref,
             "task_ref": task_ref,
             "task_source_type": task_source_type,
         }

{hpcflow_new2-0.2.0a218 → hpcflow_new2-0.2.0a219}/hpcflow/sdk/core/utils.py

@@ -30,12 +30,14 @@ import fsspec  # type: ignore
 import numpy as np
 
 from ruamel.yaml import YAML
+from ruamel.yaml.error import MarkedYAMLError
 from watchdog.utils.dirsnapshot import DirectorySnapshot
 
 from hpcflow.sdk.core.errors import (
     ContainerKeyError,
     InvalidIdentifier,
     MissingVariableSubstitutionError,
+    YAMLError,
 )
 from hpcflow.sdk.log import TimeIt
 from hpcflow.sdk.utils.deferred_file import DeferredFileWriter
@@ -412,7 +414,10 @@ def substitute_string_vars(string: str, variables: dict[str, str]):
 
 @TimeIt.decorator
 def read_YAML_str(
-    yaml_str: str, typ="safe", variables: dict[str, str] | Literal[False] | None = None
+    yaml_str: str,
+    typ="safe",
+    variables: dict[str, str] | Literal[False] | None = None,
+    source: str | None = None,
 ) -> Any:
     """Load a YAML string. This will produce basic objects.
 
@@ -426,11 +431,21 @@ def read_YAML_str(
         String variables to substitute in `yaml_str`. Substitutions will be attempted if
         the file looks to contain variable references (like "<<var:name>>"). If set to
         `False`, no substitutions will occur.
+    source:
+        Used to document the source of the YAML string if raising a parsing error.
+        Typically, this should be a string that starts with "from ...", e.g.
+        "from the file path '/path/to/bad/file'".
     """
     if variables is not False and "<<var:" in yaml_str:
         yaml_str = substitute_string_vars(yaml_str, variables=variables or {})
     yaml = YAML(typ=typ)
-    return yaml.load(yaml_str)
+    try:
+        return yaml.load(yaml_str)
+    except MarkedYAMLError as err:  # includes `ScannerError` and `ParserError`
+        source_str = f"{source} " if source else ""
+        raise YAMLError(
+            f"The YAML string {source_str}is not formatted correctly."
+        ) from err
 
 
 @TimeIt.decorator
@@ -452,7 +467,7 @@ def read_YAML_file(
     """
     with fsspec.open(path, "rt") as f:
         yaml_str: str = f.read()
-    return read_YAML_str(yaml_str, typ=typ, variables=variables)
+    return read_YAML_str(yaml_str, typ=typ, variables=variables, source=f"from {path!r}")
 
 
 def write_YAML_file(obj, path: str | Path, typ: str = "safe") -> None:

{hpcflow_new2-0.2.0a218 → hpcflow_new2-0.2.0a219}/hpcflow/sdk/core/workflow.py

@@ -578,7 +578,13 @@ class WorkflowTemplate(JSONLike):
             set to `False`, no substitutions will occur, which may result in an invalid
             workflow template!
         """
-        return cls._from_data(read_YAML_str(string, variables=variables))
+        return cls._from_data(
+            read_YAML_str(
+                string,
+                variables=variables,
+                source="(from the inline workflow template definition)",
+            )
+        )
 
     @classmethod
     def _check_name(cls, data: dict[str, Any], path: PathLike) -> None:
@@ -962,9 +968,12 @@ class Workflow(AppAware):
                             f"({task.name!r})..."
                         )
                     wk._add_task(task)
-                if status:
-                    status.update(f"Preparing to add {len(template.loops)} loops...")
                 if template.loops:
+                    if status:
+                        status.update(
+                            f"Preparing to add {len(template.loops)} loops; building "
+                            f"cache..."
+                        )
                     # TODO: if loop with non-initialisable actions, will fail
                     cache = LoopCache.build(workflow=wk, loops=template.loops)
                     for idx, loop in enumerate(template.loops):
@@ -979,6 +988,8 @@ class Workflow(AppAware):
                             f"Added {len(template.loops)} loops. "
                             f"Committing to store..."
                         )
+                elif status:
+                    status.update("Committing to store...")
         except (Exception, NotImplementedError):
             if status:
                 status.stop()
@@ -4235,8 +4246,7 @@ class Workflow(AppAware):
                     input_source.task_ref = uniq_names_cur[input_source.task_ref]
                 except KeyError:
                     raise InvalidInputSourceTaskReference(
-                        f"Input source {input_source.to_string()!r} refers to a missing "
-                        f"or inaccessible task: {input_source.task_ref!r}."
+                        input_source, task_ref=input_source.task_ref
                     )
 
     @TimeIt.decorator

{hpcflow_new2-0.2.0a218 → hpcflow_new2-0.2.0a219}/hpcflow/sdk/persistence/zarr.py

@@ -124,7 +124,7 @@ def _encode_numpy_array(
     new_idx = (
         max((int(i.removeprefix("arr_")) for i in param_arr_group.keys()), default=-1) + 1
     )
-    param_arr_group.create_dataset(name=f"arr_{new_idx}", data=obj)
+    param_arr_group.create_dataset(name=f"arr_{new_idx}", data=obj, chunks=obj.shape)
     type_lookup["arrays"].append([path, new_idx])
 
     return len(type_lookup["arrays"]) - 1

{hpcflow_new2-0.2.0a218 → hpcflow_new2-0.2.0a219}/hpcflow/sdk/submission/jobscript.py

@@ -24,6 +24,7 @@ from hpcflow.sdk.core.errors import (
 from hpcflow.sdk.typing import hydrate
 from hpcflow.sdk.core.json_like import ChildObjectSpec, JSONLike
 from hpcflow.sdk.core.utils import nth_value, parse_timestamp, current_timestamp
+from hpcflow.sdk.utils.strings import extract_py_from_future_imports
 from hpcflow.sdk.log import TimeIt
 from hpcflow.sdk.submission.schedulers import QueuedScheduler
 from hpcflow.sdk.submission.schedulers.direct import DirectScheduler
@@ -1942,10 +1943,13 @@ class Jobscript(JSONLike):
         tab_indent = "    "
 
         script_funcs_lst: list[str] = []
+        future_imports: set[str] = set()
         for act_name, (_, snip_path) in script_data.items():
             main_func_name = snip_path.stem
             with snip_path.open("rt") as fp:
                 script_str = fp.read()
+            script_str, future_imports_i = extract_py_from_future_imports(script_str)
+            future_imports.update(future_imports_i)
             script_funcs_lst.append(
                 dedent(
                     """\
@@ -2325,13 +2329,22 @@ class Jobscript(JSONLike):
             func_invoc_lines=indent(func_invoc_lines, tab_indent * 4),
         )
 
+        future_imports_str = (
+            f"from __future__ import {', '.join(future_imports)}\n\n"
+            if future_imports
+            else ""
+        )
         script = dedent(
             """\
-            {script_funcs}
+            {future_imports_str}{script_funcs}
             if __name__ == "__main__":
             {main}
         """
-        ).format(script_funcs=script_funcs, main=indent(main, tab_indent))
+        ).format(
+            future_imports_str=future_imports_str,
+            script_funcs=script_funcs,
+            main=indent(main, tab_indent),
+        )
 
         num_elems = [i.num_elements for i in self.blocks]
         num_acts = [len(i) for i in action_scripts]

hpcflow_new2-0.2.0a219/hpcflow/sdk/utils/strings.py

@@ -0,0 +1,61 @@
+from typing import Iterable
+import re
+
+
+def shorten_list_str(
+    lst: Iterable, items: int = 10, end_num: int = 1, placeholder: str = "..."
+) -> str:
+    """Format a list as a string, including only some maximum number of items.
+
+    Parameters
+    ----------
+    lst:
+        The list to format in a shortened form.
+    items:
+        The total number of items to include in the formatted list.
+    end_num:
+        The number of items to include at the end of the formatted list.
+    placeholder
+        The placeholder to use to replace excess items in the formatted list.
+
+    Examples
+    --------
+    >>> shorten_list_str(list(range(20)), items=5)
+    '[0, 1, 2, 3, ..., 19]'
+
+    """
+    lst = list(lst)
+    if len(lst) <= items + 1:  # (don't replace only one item)
+        lst_short = lst
+    else:
+        start_num = items - end_num
+        lst_short = lst[:start_num] + ["..."] + lst[-end_num:]
+
+    return "[" + ", ".join(f"{i}" for i in lst_short) + "]"
+
+
+def extract_py_from_future_imports(py_str: str) -> tuple[str, set[str]]:
+    """
+    Remove any `from __future__ import <feature>` lines from a string of Python code, and
+    return the modified string, and a list of `<feature>`s that were imported.
+
+    Notes
+    -----
+    This is required when generated a combined-scripts jobscript that concatenates
+    multiple Python scripts into one script. If `__future__` statements are included in
+    these individual scripts, they must be moved to the top of the file [1].
+
+    References
+    ----------
+    [1] https://docs.python.org/3/reference/simple_stmts.html#future-statements
+
+    """
+
+    pattern = r"^from __future__ import (.*)\n"
+    if future_imports := (set(re.findall(pattern, py_str, flags=re.MULTILINE) or ())):
+        future_imports = {
+            j.strip() for i in future_imports for j in i.split(",") if j.strip()
+        }
+        py_str = re.sub(pattern, "", py_str, flags=re.MULTILINE)
+
+    return (py_str, future_imports)

{hpcflow_new2-0.2.0a218 → hpcflow_new2-0.2.0a219}/hpcflow/tests/scripts/test_main_scripts.py

@@ -6,6 +6,7 @@ import time
 import pytest
 
 from hpcflow.app import app as hf
+from hpcflow.sdk.core.enums import EARStatus
 from hpcflow.sdk.core.test_utils import P1_parameter_cls as P1
 
 # note: when testing the frozen app, we might not have MatFlow installed in the built in
@@ -1332,3 +1333,29 @@ def test_combine_scripts_script_data_multiple_input_file_formats(
     assert isinstance(t1_p3, hf.ElementParameter)
     assert t0_p2.value == p1_val + 100
     assert t1_p3.value == p1_val + 100
+
+
+@pytest.mark.integration
+@pytest.mark.skipif("hf.run_time_info.is_frozen")
+def test_combine_scripts_from_future_import(null_config, tmp_path: Path):
+    s1 = hf.TaskSchema(
+        objective="t1",
+        actions=[
+            hf.Action(
+                script="<<script:import_future_script.py>>",
+                script_exe="python_script",
+                environments=[hf.ActionEnvironment(environment="python_env")],
+            ),
+        ],
+    )
+
+    wk = hf.Workflow.from_template_data(
+        template_name="test_future_import",
+        tasks=[hf.Task(schema=s1)],
+        resources={"any": {"combine_scripts": True}},
+        path=tmp_path,
+    )
+    wk.submit(status=False, add_to_known=False, wait=True)
+
+    run = wk.get_EARs_from_IDs([0])[0]
+    assert run.status is EARStatus.success

{hpcflow_new2-0.2.0a218 → hpcflow_new2-0.2.0a219}/hpcflow/tests/unit/test_element_set.py

@@ -121,3 +121,28 @@ def test_nesting_order_paths_raise(null_config) -> None:
 
 def test_nesting_order_paths_no_raise(null_config) -> None:
     hf.ElementSet(nesting_order={"inputs.p1": 1, "resources.any": 2, "repeats": 3})
+
+
+def test_input_source_str_dict_list_str_list_dict_equivalence(null_config) -> None:
+    inp_source_dict: dict[str, str | int] = {
+        "source_type": "task",
+        "task_source_type": "output",
+        "task_ref": 0,
+    }
+    inp_source_str = "task.0.output"
+    inp_source_list_dict = [inp_source_dict]
+    inp_source_list_str = [inp_source_str]
+    assert (
+        hf.ElementSet.from_json_like(
+            {"input_sources": {"p1": inp_source_dict}}
+        ).input_sources
+        == hf.ElementSet.from_json_like(
+            {"input_sources": {"p1": inp_source_list_dict}}
+        ).input_sources
+        == hf.ElementSet.from_json_like(
+            {"input_sources": {"p1": inp_source_str}}
+        ).input_sources
+        == hf.ElementSet.from_json_like(
+            {"input_sources": {"p1": inp_source_list_str}}
+        ).input_sources
+    )

{hpcflow_new2-0.2.0a218 → hpcflow_new2-0.2.0a219}/hpcflow/tests/unit/test_input_source.py

@@ -1,4 +1,5 @@
 from __future__ import annotations
+from textwrap import dedent
 from typing import TYPE_CHECKING
 import numpy as np
 import pytest
@@ -183,6 +184,7 @@ def test_input_source_from_string_task_same_default_task_source() -> None:
     )
 
 
+@pytest.mark.skip(reason="Import not yet implemented.")
 def test_input_source_from_string_import() -> None:
     import_ref = 0
     assert hf.InputSource.from_string(f"import.{import_ref}") == hf.InputSource(
@@ -1283,3 +1285,72 @@ def test_input_source_inputs_from_multiple_element_sets_with_sub_parameter_seque
         {"a": 5, "b": 22},
         {"a": 5, "b": 22},
     ]
+
+
+def test_input_source_task_ref_equivalence(null_config, tmp_path):
+    yml = dedent(
+        """\
+    name: test
+    template_components:
+      task_schemas:
+        - objective: t1
+          inputs:
+            - parameter: p1
+    tasks:
+      - schema: t1
+        inputs:
+          p1: 100 # all subsequent tasks will source from this input
+    
+      - schema: t1 # t1_2
+        input_sources: # single source dict; by task insert ID
+          p1: 
+            source_type: task
+            task_source_type: input
+            task_ref: 0
+
+      - schema: t1 # t1_3
+        input_sources: # as a list of dicts; by task insert ID
+          p1: 
+            - source_type: task
+              task_source_type: input
+              task_ref: 0
+    
+      - schema: t1 # t1_4
+        input_sources: # as a single source dict; by task unique name
+          p1: 
+            source_type: task
+            task_source_type: input
+            task_ref: t1_1
+
+      - schema: t1 # t1_5
+        input_sources: # as a list of dicts; by task unique name
+          p1: 
+            - source_type: task
+              task_source_type: input
+              task_ref: t1_1
+    
+      - schema: t1 # t1_6
+        input_sources: # single source string; by task insert ID
+          p1: task.0.input
+
+      - schema: t1 # t1_7
+        input_sources: # as a list of strings; by task insert ID
+          p1: 
+            - task.0.input
+
+      - schema: t1 # t1_8
+        input_sources: # single source string; by task unique name
+          p1: task.t1_1.input
+
+      - schema: t1 # t1_9
+        input_sources: # as a list of strings; by task unique name
+          p1: 
+            - task.t1_1.input
+        
+    """
+    )
+    wk = hf.Workflow.from_YAML_string(YAML_str=yml, path=tmp_path)
+
+    all_sources = (task.elements[0].input_sources["inputs.p1"] for task in wk.tasks[1:])
+    all_task_refs = (src.task_ref for src in all_sources)
+    assert all(task_ref == 0 for task_ref in all_task_refs)

{hpcflow_new2-0.2.0a218 → hpcflow_new2-0.2.0a219}/hpcflow/tests/unit/test_utils.py

@@ -1,9 +1,14 @@
 from pathlib import Path
+from textwrap import dedent
 import pytest
 import zarr  # type: ignore
 import numpy as np
 from numpy.typing import NDArray
-from hpcflow.sdk.core.errors import InvalidIdentifier, MissingVariableSubstitutionError
+from hpcflow.sdk.core.errors import (
+    InvalidIdentifier,
+    MissingVariableSubstitutionError,
+    YAMLError,
+)
 
 from hpcflow.sdk.core.utils import (
     JSONLikeDirSnapShot,
@@ -16,6 +21,7 @@ from hpcflow.sdk.core.utils import (
     nth_key,
     nth_value,
     process_string_nodes,
+    read_YAML_str,
     replace_items,
     check_valid_py_identifier,
     reshape,
@@ -575,3 +581,36 @@ def test_nth_key_raises():
 
     with pytest.raises(Exception):
         nth_key(dct, -1)
+
+
+def test_read_YAML_str():
+    good_yaml = dedent(
+        """\
+        a: 1
+        b: 2
+        """
+    )
+    assert read_YAML_str(good_yaml) == {"a": 1, "b": 2}
+
+
+def test_read_YAML_str_raise_on_bad_indent():
+    bad_yaml = dedent(
+        """\
+        a: 1
+          b: 2
+        """
+    )
+    with pytest.raises(YAMLError):
+        read_YAML_str(bad_yaml)
+
+
+def test_read_YAML_str_raise_on_mixed_tabs_spaces():
+    bad_yaml = dedent(
+        """\
+        a:
+          a1: 2 # this has a space indent
+        	a2: 3 # this has a tab indent
+        """
+    )
+    with pytest.raises(YAMLError):
+        read_YAML_str(bad_yaml)

hpcflow_new2-0.2.0a219/hpcflow/tests/unit/utils/test_strings.py

@@ -0,0 +1,97 @@
+from textwrap import dedent
+
+from hpcflow.sdk.utils.strings import extract_py_from_future_imports
+
+
+def test_extract_py_from_future_imports_none():
+    py_str = dedent(
+        """\
+
+    def my_function():
+        print("blah!")
+    """
+    )
+    new_str, imports = extract_py_from_future_imports(py_str)
+    assert imports == set()
+    assert new_str == py_str
+
+
+def test_extract_py_from_future_imports_single():
+    py_str = dedent(
+        """\
+    from __future__ import annotations
+
+    def my_function():
+        print("blah!")
+    """
+    )
+    new_str, imports = extract_py_from_future_imports(py_str)
+    assert imports == {"annotations"}
+    assert new_str == dedent(
+        """\
+
+    def my_function():
+        print("blah!")
+    """
+    )
+
+
+def test_extract_py_from_future_imports_multi():
+    py_str = dedent(
+        """\
+    from __future__ import annotations, feature_2
+
+    def my_function():
+        print("blah!")
+    """
+    )
+    new_str, imports = extract_py_from_future_imports(py_str)
+    assert imports == {"annotations", "feature_2"}
+    assert new_str == dedent(
+        """\
+
+    def my_function():
+        print("blah!")
+    """
+    )
+
+
+def test_extract_py_from_future_imports_trailing_comma():
+    py_str = dedent(
+        """\
+    from __future__ import annotations,
+
+    def my_function():
+        print("blah!")
+    """
+    )
+    new_str, imports = extract_py_from_future_imports(py_str)
+    assert imports == {"annotations"}
+    assert new_str == dedent(
+        """\
+
+    def my_function():
+        print("blah!")
+    """
+    )
+
+
+def test_extract_py_from_future_imports_multi_lines():
+    py_str = dedent(
+        """\
+    from __future__ import annotations, feature_2
+    from __future__ import feature_2, feature_3,
+
+    def my_function():
+        print("blah!")
+    """
+    )
+    new_str, imports = extract_py_from_future_imports(py_str)
+    assert imports == {"annotations", "feature_2", "feature_3"}
+    assert new_str == dedent(
+        """\
+
+    def my_function():
+        print("blah!")
+    """
+    )

{hpcflow_new2-0.2.0a218 → hpcflow_new2-0.2.0a219}/pyproject.toml

@@ -1,7 +1,7 @@
 
 [tool.poetry]
 name = "hpcflow-new2"
-version = "0.2.0a218"
+version = "0.2.0a219"
 
 description = "Computational workflow management"
 authors = ["aplowman <adam.plowman@manchester.ac.uk>"]
@@ -100,7 +100,7 @@ hook-dirs = "hpcflow.__pyinstaller:get_hook_dirs"
 
 [tool.commitizen]
 name = "cz_conventional_commits"
-version = "0.2.0a218"
+version = "0.2.0a219"
 tag_format = "v$version"
 version_files = [ 
     "pyproject.toml:version",

hpcflow_new2-0.2.0a218/hpcflow/_version.py

@@ -1 +0,0 @@
-__version__ = "0.2.0a218"