hpcflow-new2 0.2.0a190__py3-none-any.whl → 0.2.0a199__py3-none-any.whl

hpcflow/sdk/core/cache.py

@@ -14,41 +14,50 @@ if TYPE_CHECKING:
     from collections.abc import Sequence
     from typing_extensions import Self
     from .element import Element, ElementIteration
+    from .actions import ElementActionRun
     from .workflow import Workflow
     from ..persistence.base import StoreEAR, StoreElement, StoreElementIter
 
 
 @dataclass
-class DependencyCache:
-    """
-    Class to bulk-retrieve dependencies between elements, iterations, and runs.
-    """
+class ObjectCache:
+    """Class to bulk-retrieve and store elements, iterations, runs and their various
+    dependencies."""
+
+    #: The elements of the workflow that this cache was built from.
+    elements: list[Element] | None = None
+    #: The iterations of the workflow that this cache was built from.
+    iterations: list[ElementIteration] | None = None
+    #: The runs of the workflow that this cache was built from.
+    runs: list[ElementActionRun] | None = None
 
     #: What EARs (by ID) a given EAR depends on.
-    run_dependencies: dict[int, set[int]]
+    run_dependencies: dict[int, set[int]] | None = None
     #: What EARs (by ID) are depending on a given EAR.
-    run_dependents: dict[int, set[int]]
+    run_dependents: dict[int, set[int]] | None = None
     #: What EARs (by ID) a given iteration depends on.
-    iter_run_dependencies: dict[int, set[int]]
+    iter_run_dependencies: dict[int, set[int]] | None = None
     #: What iterations (by ID) a given iteration depends on.
-    iter_iter_dependencies: dict[int, set[int]]
+    iter_iter_dependencies: dict[int, set[int]] | None = None
     #: What iterations (by ID) a given element depends on.
-    elem_iter_dependencies: dict[int, set[int]]
+    elem_iter_dependencies: dict[int, set[int]] | None = None
     #: What elements (by ID) a given element depends on.
-    elem_elem_dependencies: dict[int, set[int]]
+    elem_elem_dependencies: dict[int, set[int]] | None = None
     #: What elements (by ID) are depending on a given element.
-    elem_elem_dependents: dict[int, set[int]]
+    elem_elem_dependents: dict[int, set[int]] | None = None
     #: Transitive closure of :py:attr:`elem_elem_dependents`.
-    elem_elem_dependents_rec: dict[int, set[int]]
-
-    #: The elements of the workflow that this cache was built from.
-    elements: list[Element]
-    #: The iterations of the workflow that this cache was built from.
-    iterations: list[ElementIteration]
+    elem_elem_dependents_rec: dict[int, set[int]] | None = None
 
     @classmethod
     @TimeIt.decorator
-    def build(cls, workflow: Workflow) -> Self:
+    def build(
+        cls,
+        workflow: Workflow,
+        dependencies: bool = False,
+        elements: bool = False,
+        iterations: bool = False,
+        runs: bool = False,
+    ):
         """
         Build a cache instance.
 
@@ -56,7 +65,51 @@ class DependencyCache:
         ----------
         workflow: ~hpcflow.app.Workflow
             The workflow to build the cache from.
+        dependencies
+            If True, calculate dependencies.
+        elements
+            If True, include elements in the cache.
+        iterations
+            If True, include iterations in the cache.
+        runs
+            If True, include runs in the cache.
+
         """
+        kwargs = {}
+        if dependencies:
+            kwargs.update(cls._get_dependencies(workflow))
+
+        if elements:
+            kwargs["elements"] = workflow.get_all_elements()
+
+        if iterations:
+            kwargs["iterations"] = workflow.get_all_element_iterations()
+
+        if runs:
+            kwargs["runs"] = workflow.get_all_EARs()
+
+        return cls(**kwargs)
+
+    @classmethod
+    @TimeIt.decorator
+    def _get_dependencies(cls, workflow: Workflow):
+        def _get_recursive_deps(elem_id: int, seen_ids: list[int] | None = None):
+            if seen_ids is None:
+                seen_ids = [elem_id]
+            elif elem_id in seen_ids:
+                # stop recursion
+                return set()
+            else:
+                seen_ids.append(elem_id)
+            return set(elem_elem_dependents[elem_id]).union(
+                [
+                    j
+                    for i in elem_elem_dependents[elem_id]
+                    for j in _get_recursive_deps(i, seen_ids)
+                    if j != elem_id
+                ]
+            )
+
         num_iters = workflow.num_element_iterations
         num_elems = workflow.num_elements
         num_runs = workflow.num_EARs
@@ -84,8 +137,8 @@ class DependencyCache:
         for idx, dict_i in enumerate(all_data_idx):
             run_i_sources = set(
                 run_k
-                for idx in chain.from_iterable(dict_i.values())
-                if (run_k := all_param_sources[idx].get("EAR_ID")) is not None
+                for dat_idx_k in chain.from_iterable(dict_i.values())
+                if (run_k := all_param_sources[dat_idx_k].get("EAR_ID")) is not None
                 and run_k != idx
             )
             run_dependencies[idx] = run_i_sources
@@ -149,14 +202,8 @@ class DependencyCache:
 
         # for each element, which elements depend on it (recursively)?
         elem_elem_dependents_rec: defaultdict[int, set[int]] = defaultdict(set)
-        for k in tuple(elem_elem_dependents):
-            # NB: code below modifies elem_elem_dependents during this loop;
-            # copy above is mandatory!
-            for i in elem_elem_dependents[k]:
-                elem_elem_dependents_rec[k].add(i)
-                elem_elem_dependents_rec[k].update(
-                    m for m in elem_elem_dependents[i] if m != k
-                )
+        for i in list(elem_elem_dependents):
+            elem_elem_dependents_rec[i] = _get_recursive_deps(i)
 
         # add missing keys and downgrade to dict:
         for elem_idx in range(num_elems):
@@ -165,10 +212,7 @@ class DependencyCache:
         elem_elem_dependents.default_factory = None
         elem_elem_dependents_rec.default_factory = None
 
-        elements = workflow.get_all_elements()
-        iterations = workflow.get_all_element_iterations()
-
-        return cls(
+        return dict(
             run_dependencies=run_dependencies,
             run_dependents=run_dependents,
             iter_run_dependencies=iter_run_dependencies,
@@ -177,6 +221,4 @@ class DependencyCache:
             elem_elem_dependencies=elem_elem_dependencies,
             elem_elem_dependents=elem_elem_dependents,
             elem_elem_dependents_rec=elem_elem_dependents_rec,
-            elements=elements,
-            iterations=iterations,
         )

hpcflow/sdk/core/command_files.py

@@ -10,13 +10,14 @@ from textwrap import dedent
 from typing import Protocol, cast, overload, TYPE_CHECKING
 from typing_extensions import Final, override
 
-from hpcflow.sdk.typing import hydrate, ParamSource
+from hpcflow.sdk.typing import PathLike, hydrate, ParamSource
 from hpcflow.sdk.core.json_like import ChildObjectSpec, JSONLike
 from hpcflow.sdk.core.utils import search_dir_files_by_regex
 from hpcflow.sdk.core.zarr_io import zarr_decode
 from hpcflow.sdk.core.parameters import _process_demo_data_strings
 
 if TYPE_CHECKING:
+    import os
     from collections.abc import Mapping
     from typing import Any, ClassVar
     from typing_extensions import Self
@@ -33,7 +34,7 @@ class FileNamePart(Protocol):
     A filename or piece of filename that can be expanded.
     """
 
-    def value(self, directory: str = ".") -> str | list[str]:
+    def value(self, directory: str | os.PathLike = ".") -> str | list[str]:
         """
         Get the part of the file, possibly with directory specified.
         Implementations of this may ignore the directory.
@@ -74,7 +75,7 @@ class FileSpec(JSONLike):
         self._hash_value = _hash_value
         self.__hash = hash((label, self.name))
 
-    def value(self, directory: str = ".") -> str:
+    def value(self, directory: str | os.PathLike = ".") -> str:
         """
         The path to a file, optionally resolved with respect to a particular directory.
         """
@@ -178,17 +179,17 @@ class FileNameSpec(JSONLike):
         """
         return self._app.FileNameExt(self)
 
-    def value(self, directory: str = ".") -> list[str] | str:
+    def value(self, directory: str | os.PathLike = ".") -> list[str] | str:
         """
         Get the template-resolved name of the file
         (or files matched if the name is a regex pattern).
 
         Parameters
         ----------
-        directory: str
+        directory: PathLike
             Where to resolve values with respect to.
         """
-        format_args = [arg.value(directory) for arg in self.args]
+        format_args = [arg.value(Path(directory)) for arg in self.args]
         value = self.name.format(*format_args)
         if self.is_regex:
             return search_dir_files_by_regex(value, directory=directory)
@@ -207,7 +208,7 @@ class FileNameStem(JSONLike):
     #: The file specification this is derived from.
     file_name: FileNameSpec
 
-    def value(self, directory: str = ".") -> str:
+    def value(self, directory: str | os.PathLike = ".") -> str:
         """
         Get the stem, possibly with directory specified.
         """
@@ -227,7 +228,7 @@ class FileNameExt(JSONLike):
     #: The file specification this is derived from.
     file_name: FileNameSpec
 
-    def value(self, directory: str = ".") -> str:
+    def value(self, directory: str | os.PathLike = ".") -> str:
         """
         Get the extension.
         """
@@ -301,6 +302,8 @@ class InputFileGenerator(JSONLike):
     abortable: bool = False
     #: User-specified rules for whether to run the generator.
     rules: list[ActionRule] = field(default_factory=list)
+    #: Whether the generator requires a working directory.
+    requires_dir: bool = True
 
     def get_action_rules(self) -> list[ActionRule]:
         """
@@ -311,58 +314,6 @@ class InputFileGenerator(JSONLike):
             self._app.ActionRule.check_missing(f"input_files.{self.input_file.label}")
         ] + self.rules
 
-    def compose_source(self, snip_path: Path) -> str:
-        """Generate the file contents of this input file generator source."""
-
-        script_main_func = snip_path.stem
-        with snip_path.open("rt") as fp:
-            script_str = fp.read()
-
-        main_block = dedent(
-            """\
-            if __name__ == "__main__":
-                import sys
-                from pathlib import Path
-                import {app_module} as app
-                app.load_config(
-                    log_file_path=Path("{run_log_file}").resolve(),
-                    config_dir=r"{cfg_dir}",
-                    config_key=r"{cfg_invoc_key}",
-                )
-                wk_path, EAR_ID = sys.argv[1:]
-                EAR_ID = int(EAR_ID)
-                wk = app.Workflow(wk_path)
-                EAR = wk.get_EARs_from_IDs([EAR_ID])[0]
-                {script_main_func}(path=Path({file_path!r}), **EAR.get_IFG_input_values())
-        """
-        )
-        main_block = main_block.format(
-            run_log_file=self._app.RunDirAppFiles.get_log_file_name(),
-            app_module=self._app.module,
-            cfg_dir=self._app.config.config_directory,
-            cfg_invoc_key=self._app.config.config_key,
-            script_main_func=script_main_func,
-            file_path=self.input_file.name.value(),
-        )
-
-        out = dedent(
-            """\
-            {script_str}
-            {main_block}
-        """
-        )
-
-        return out.format(script_str=script_str, main_block=main_block)
-
-    def write_source(self, action: Action, env_spec: Mapping[str, Any]) -> None:
-        """
-        Write the script if it is specified as a snippet script, otherwise we assume
-        the script already exists in the working directory.
-        """
-        if snip_path := action.get_snippet_script_path(self.script, env_spec):
-            with Path(snip_path.name).open("wt", newline="\n") as fp:
-                fp.write(self.compose_source(snip_path))
-
 
 @dataclass
 @hydrate
@@ -449,7 +400,7 @@ class OutputFileParser(JSONLike):
     inputs: list[str] | None = None
     #: Optional multiple outputs from the upstream actions of the schema that are
     #: required to parametrise this parser.
-    #: Not to be confused with :py:attr:`output` (plural).
+    #: Not to be confused with :py:attr:`output` (singular).
     outputs: list[str] | None = None
     #: Miscellaneous options.
     options: dict[str, Any] | None = None
@@ -464,6 +415,8 @@ class OutputFileParser(JSONLike):
     clean_up: list[str] = field(default_factory=list)
     #: Rules for whether to enable this parser.
     rules: list[ActionRule] = field(default_factory=list)
+    #: Whether the parser requires a working directory.
+    requires_dir: bool = True
 
     def __post_init__(self, save_files: list[FileSpec] | bool) -> None:
         if not save_files:
@@ -501,73 +454,6 @@ class OutputFileParser(JSONLike):
             for out_f in self.output_files
         ] + self.rules
 
-    def compose_source(self, snip_path: Path) -> str:
-        """Generate the file contents of this output file parser source."""
-
-        if self.output is None:
-            # might be used just for saving files:
-            return ""
-
-        script_main_func = snip_path.stem
-        with snip_path.open("rt") as fp:
-            script_str = fp.read()
-
-        main_block = dedent(
-            """\
-            if __name__ == "__main__":
-                import sys
-                from pathlib import Path
-                import {app_module} as app
-                app.load_config(
-                    log_file_path=Path("{run_log_file}").resolve(),
-                    config_dir=r"{cfg_dir}",
-                    config_key=r"{cfg_invoc_key}",
-                )
-                wk_path, EAR_ID = sys.argv[1:]
-                EAR_ID = int(EAR_ID)
-                wk = app.Workflow(wk_path)
-                EAR = wk.get_EARs_from_IDs([EAR_ID])[0]
-                value = {script_main_func}(
-                    **EAR.get_OFP_output_files(),
-                    **EAR.get_OFP_inputs(),
-                    **EAR.get_OFP_outputs(),
-                )
-                wk.save_parameter(name="{param_name}", value=value, EAR_ID=EAR_ID)
-
-        """
-        )
-        main_block = main_block.format(
-            run_log_file=self._app.RunDirAppFiles.get_log_file_name(),
-            app_module=self._app.module,
-            cfg_dir=self._app.config.config_directory,
-            cfg_invoc_key=self._app.config.config_key,
-            script_main_func=script_main_func,
-            param_name=f"outputs.{self.output.typ}",
-        )
-
-        out = dedent(
-            """\
-            {script_str}
-            {main_block}
-        """
-        )
-
-        return out.format(script_str=script_str, main_block=main_block)
-
-    def write_source(self, action: Action, env_spec: Mapping[str, Any]) -> None:
-        """
-        Write the actual output parser to a file so it can be enacted.
-        """
-        if self.output is None:
-            # might be used just for saving files:
-            return
-
-        # write the script if it is specified as a snippet script, otherwise we assume
-        # the script already exists in the working directory:
-        if snip_path := action.get_snippet_script_path(self.script, env_spec):
-            with Path(snip_path.name).open("wt", newline="\n") as fp:
-                fp.write(self.compose_source(snip_path))
-
 
 @hydrate
 class _FileContentsSpecifier(JSONLike):

hpcflow/sdk/core/commands.py

@@ -20,7 +20,7 @@ from hpcflow.sdk.core.parameters import ParameterValue
 if TYPE_CHECKING:
     from collections.abc import Callable, Iterable, Mapping, Sequence
     from re import Pattern
-    from .actions import ActionRule
+    from .actions import ActionRule, Action
     from .element import ElementActionRun
     from .environment import Environment
     from ..submission.shells import Shell
@@ -81,6 +81,11 @@ class Command(JSONLike):
     #: Rules that state whether this command is eligible to run.
     rules: list[ActionRule] = field(default_factory=list)
 
+    action: Action | None = None  # assigned by parent Action
+
+    def __post_init__(self):
+        self._set_parent_refs()
+
     def __repr__(self) -> str:
         out = []
         if self.command:
@@ -102,6 +107,20 @@ class Command(JSONLike):
 
         return f"{self.__class__.__name__}({', '.join(out)})"
 
+    def __eq__(self, other):
+        if not isinstance(other, self.__class__):
+            return False
+        return (
+            self.command == other.command
+            and self.executable == other.executable
+            and self.arguments == other.arguments
+            and self.variables == other.variables
+            and self.stdout == other.stdout
+            and self.stderr == other.stderr
+            and self.stdin == other.stdin
+            and self.rules == other.rules
+        )
+
     def __get_initial_command_line(self) -> str:
         if self.command:
             return self.command
@@ -142,6 +161,7 @@ class Command(JSONLike):
                 exec_cmd = executable.filter_instances(**filter_exec)[0].command
                 return exec_cmd.replace("<<num_cores>>", str(EAR.resources.num_cores))
             elif typ == "script":
+                # TODO: is this needed? we have <<script_name>> <<script_path>> etc as command variables
                 return EAR.action.get_script_name(val)
             else:
                 raise ValueError("impossible match occurred")
@@ -191,7 +211,11 @@ class Command(JSONLike):
                     cmd_str = cmd_str.rstrip()
 
         # remove any left over "<<args>>" and "<<script_name>>"s:
-        cmd_str = cmd_str.replace("<<args>>", "").replace("<<script_name>>", "")
+        cmd_str = (
+            cmd_str.replace("<<args>>", "")
+            .replace("<<script_name>>", "")
+            .replace("<<script_path>>", "")
+        )
 
         # substitute input parameters in command:
         types_pattern = "|".join(parse_types)
@@ -208,7 +232,10 @@ class Command(JSONLike):
                 cmd_inp = ".".join(cmd_inp_parts[:-1])
             else:
                 cmd_inp = cmd_inp_full
-            inp_val = EAR.get(f"inputs.{cmd_inp}")  # TODO: what if schema output?
+            inp_val = EAR.get(
+                f"inputs.{cmd_inp}",
+                raise_on_unset=True,
+            )  # TODO: what if schema output?
             pattern_i = pattern.format(
                 types_pattern=types_pattern,
                 name=re.escape(cmd_inp),
@@ -219,9 +246,11 @@ class Command(JSONLike):
                 string=cmd_str,
             )
 
-        # substitute input files in command:
-        for cmd_file in EAR.action.get_command_input_file_labels():
-            file_path = EAR.get(f"input_files.{cmd_file}")  # TODO: what if out file?
+        # substitute input/output files in command:
+        for cmd_file in EAR.action.get_command_file_labels():
+            file_path = EAR.get(
+                f"input_files.{cmd_file}", raise_on_unset=True
+            ) or EAR.get(f"output_files.{cmd_file}", raise_on_unset=True)
             # assuming we have copied this file to the EAR directory, then we just
             # need the file name:
             file_name = Path(file_path).name