hpcflow-new2 0.2.0a199__tar.gz → 0.2.0a200__tar.gz

{hpcflow_new2-0.2.0a199 → hpcflow_new2-0.2.0a200}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hpcflow-new2
-Version: 0.2.0a199
+Version: 0.2.0a200
 Summary: Computational workflow management
 License: MPL-2.0
 Author: aplowman

hpcflow_new2-0.2.0a200/hpcflow/_version.py

@@ -0,0 +1 @@
+__version__ = "0.2.0a200"

{hpcflow_new2-0.2.0a199 → hpcflow_new2-0.2.0a200}/hpcflow/sdk/app.py

@@ -2370,10 +2370,23 @@ class BaseApp(metaclass=Singleton):
             else:
                 print(contents)
 
-    def load_demo_workflow(self, name: str) -> _WorkflowTemplate:
-        """Load a WorkflowTemplate object from a builtin demo template file."""
+    def load_demo_workflow(
+        self, name: str, variables: dict[str, str] | Literal[False] | None = None
+    ) -> _WorkflowTemplate:
+        """Load a WorkflowTemplate object from a builtin demo template file.
+
+        Parameters
+        ----------
+        name:
+            Name of the demo workflow to load.
+        variables:
+            String variables to substitute in the demo workflow. Substitutions will be
+            attempted if the file looks to contain variable references (like
+            "<<var:name>>"). If set to `False`, no substitutions will occur, which may
+            result in an invalid workflow template!
+        """
         with self.get_demo_workflow_template_file(name) as path:
-            return self.WorkflowTemplate.from_file(path)
+            return self.WorkflowTemplate.from_file(path, variables=variables)
 
     def template_components_from_json_like(
         self, json_like: dict[str, dict]

{hpcflow_new2-0.2.0a199 → hpcflow_new2-0.2.0a200}/hpcflow/sdk/core/utils.py

@@ -414,20 +414,44 @@ def substitute_string_vars(string: str, variables: dict[str, str]):
 
 @TimeIt.decorator
 def read_YAML_str(
-    yaml_str: str, typ="safe", variables: dict[str, str] | None = None
+    yaml_str: str, typ="safe", variables: dict[str, str] | Literal[False] | None = None
 ) -> Any:
-    """Load a YAML string. This will produce basic objects."""
-    if variables is not None and "<<var:" in yaml_str:
-        yaml_str = substitute_string_vars(yaml_str, variables=variables)
+    """Load a YAML string. This will produce basic objects.
+
+    Parameters
+    ----------
+    yaml_str:
+        The YAML string to parse.
+    typ:
+        Load type passed to the YAML library.
+    variables:
+        String variables to substitute in `yaml_str`. Substitutions will be attempted if
+        the file looks to contain variable references (like "<<var:name>>"). If set to
+        `False`, no substitutions will occur.
+    """
+    if variables is not False and "<<var:" in yaml_str:
+        yaml_str = substitute_string_vars(yaml_str, variables=variables or {})
     yaml = YAML(typ=typ)
     return yaml.load(yaml_str)
 
 
 @TimeIt.decorator
 def read_YAML_file(
-    path: PathLike, typ="safe", variables: dict[str, str] | None = None
+    path: PathLike, typ="safe", variables: dict[str, str] | Literal[False] | None = None
 ) -> Any:
-    """Load a YAML file. This will produce basic objects."""
+    """Load a YAML file. This will produce basic objects.
+
+    Parameters
+    ----------
+    path:
+        Path to the YAML file to parse.
+    typ:
+        Load type passed to the YAML library.
+    variables:
+        String variables to substitute in the file given by `path`. Substitutions will be
+        attempted if the file looks to contain variable references (like "<<var:name>>").
+        If set to `False`, no substitutions will occur.
+    """
     with fsspec.open(path, "rt") as f:
         yaml_str: str = f.read()
     return read_YAML_str(yaml_str, typ=typ, variables=variables)
@@ -440,15 +464,37 @@ def write_YAML_file(obj, path: str | Path, typ: str = "safe") -> None:
         yaml.dump(obj, fp)
 
 
-def read_JSON_string(json_str: str, variables: dict[str, str] | None = None) -> Any:
-    """Load a JSON string. This will produce basic objects."""
-    if variables is not None and "<<var:" in json_str:
-        json_str = substitute_string_vars(json_str, variables=variables)
+def read_JSON_string(
+    json_str: str, variables: dict[str, str] | Literal[False] | None = None
+) -> Any:
+    """Load a JSON string. This will produce basic objects.
+
+    Parameters
+    ----------
+    json_str:
+        The JSON string to parse.
+    variables:
+        String variables to substitute in `json_str`. Substitutions will be attempted if
+        the file looks to contain variable references (like "<<var:name>>"). If set to
+        `False`, no substitutions will occur.
+    """
+    if variables is not False and "<<var:" in json_str:
+        json_str = substitute_string_vars(json_str, variables=variables or {})
     return json.loads(json_str)
 
 
-def read_JSON_file(path, variables: dict[str, str] | None = None) -> Any:
-    """Load a JSON file. This will produce basic objects."""
+def read_JSON_file(path, variables: dict[str, str] | Literal[False] | None = None) -> Any:
+    """Load a JSON file. This will produce basic objects.
+
+    Parameters
+    ----------
+    path:
+        Path to the JSON file to parse.
+    variables:
+        String variables to substitute in the file given by `path`. Substitutions will be
+        attempted if the file looks to contain variable references (like "<<var:name>>").
+        If set to `False`, no substitutions will occur.
+    """
     with fsspec.open(path, "rt") as f:
         json_str: str = f.read()
     return read_JSON_string(json_str, variables=variables)

{hpcflow_new2-0.2.0a199 → hpcflow_new2-0.2.0a200}/hpcflow/sdk/core/workflow.py

@@ -564,7 +564,7 @@ class WorkflowTemplate(JSONLike):
     def from_YAML_string(
         cls,
         string: str,
-        variables: dict[str, str] | None = None,
+        variables: dict[str, str] | Literal[False] | None = None,
     ) -> WorkflowTemplate:
         """Load from a YAML string.
 
@@ -573,7 +573,10 @@ class WorkflowTemplate(JSONLike):
         string
             The YAML string containing the workflow template parametrisation.
         variables
-            String variables to substitute in `string`.
+            String variables to substitute in `string`. Substitutions will be attempted if
+            the YAML string looks to contain variable references (like "<<var:name>>"). If
+            set to `False`, no substitutions will occur, which may result in an invalid
+            workflow template!
         """
         return cls._from_data(read_YAML_str(string, variables=variables))
 
@@ -597,7 +600,7 @@ class WorkflowTemplate(JSONLike):
     def from_YAML_file(
         cls,
         path: PathLike,
-        variables: dict[str, str] | None = None,
+        variables: dict[str, str] | Literal[False] | None = None,
     ) -> WorkflowTemplate:
         """Load from a YAML file.
 
@@ -606,7 +609,10 @@ class WorkflowTemplate(JSONLike):
         path
             The path to the YAML file containing the workflow template parametrisation.
         variables
-            String variables to substitute in the file given by `path`.
+            String variables to substitute in the file given by `path`. Substitutions will
+            be attempted if the YAML file looks to contain variable references (like
+            "<<var:name>>"). If set to `False`, no substitutions will occur, which may
+            result in an invalid workflow template!
 
         """
         cls._app.logger.debug("parsing workflow template from a YAML file")
@@ -620,7 +626,7 @@ class WorkflowTemplate(JSONLike):
     def from_JSON_string(
         cls,
         string: str,
-        variables: dict[str, str] | None = None,
+        variables: dict[str, str] | Literal[False] | None = None,
     ) -> WorkflowTemplate:
         """Load from a JSON string.
 
@@ -629,7 +635,10 @@ class WorkflowTemplate(JSONLike):
         string
             The JSON string containing the workflow template parametrisation.
         variables
-            String variables to substitute in `string`.
+            String variables to substitute in `string`. Substitutions will be attempted if
+            the JSON string looks to contain variable references (like "<<var:name>>"). If
+            set to `False`, no substitutions will occur, which may result in an invalid
+            workflow template!
         """
         return cls._from_data(read_JSON_string(string, variables=variables))
 
@@ -638,7 +647,7 @@ class WorkflowTemplate(JSONLike):
     def from_JSON_file(
         cls,
         path: PathLike,
-        variables: dict[str, str] | None = None,
+        variables: dict[str, str] | Literal[False] | None = None,
     ) -> WorkflowTemplate:
         """Load from a JSON file.
 
@@ -647,7 +656,10 @@ class WorkflowTemplate(JSONLike):
         path
             The path to the JSON file containing the workflow template parametrisation.
         variables
-            String variables to substitute in the file given by `path`.
+            String variables to substitute in the file given by `path`. Substitutions will
+            be attempted if the JSON file looks to contain variable references (like
+            "<<var:name>>"). If set to `False`, no substitutions will occur, which may
+            result in an invalid workflow template!
         """
         cls._app.logger.debug("parsing workflow template from a JSON file")
         data = read_JSON_file(path, variables=variables)
@@ -661,7 +673,7 @@ class WorkflowTemplate(JSONLike):
         cls,
         path: PathLike,
         template_format: Literal["yaml", "json"] | None = None,
-        variables: dict[str, str] | None = None,
+        variables: dict[str, str] | Literal[False] | None = None,
     ) -> WorkflowTemplate:
         """Load from either a YAML or JSON file, depending on the file extension.
 
@@ -673,8 +685,10 @@ class WorkflowTemplate(JSONLike):
             The file format to expect at `path`. One of "json" or "yaml", if specified. By
             default, "yaml".
         variables
-            String variables to substitute in the file given by `path`.
-
+            String variables to substitute in the file given by `path`. Substitutions will
+            be attempted if the file looks to contain variable references (like
+            "<<var:name>>"). If set to `False`, no substitutions will occur, which may
+            result in an invalid workflow template!
         """
         path_ = Path(path or ".")
         fmt = template_format.lower() if template_format else None
@@ -983,7 +997,7 @@ class Workflow(AppAware):
         ts_fmt: str | None = None,
         ts_name_fmt: str | None = None,
         store_kwargs: dict[str, Any] | None = None,
-        variables: dict[str, str] | None = None,
+        variables: dict[str, str] | Literal[False] | None = None,
     ) -> Workflow:
         """Generate from a YAML file.
 
@@ -1013,7 +1027,10 @@ class Workflow(AppAware):
         store_kwargs:
             Keyword arguments to pass to the store's `write_empty_workflow` method.
         variables:
-            String variables to substitute in the file given by `YAML_path`.
+            String variables to substitute in the file given by `YAML_path`. Substitutions
+            will be attempted if the YAML file looks to contain variable references (like
+            "<<var:name>>"). If set to `False`, no substitutions will occur, which may
+            result in an invalid workflow template!
         """
         template = cls._app.WorkflowTemplate.from_YAML_file(
             path=YAML_path,
@@ -1041,7 +1058,7 @@ class Workflow(AppAware):
         ts_fmt: str | None = None,
         ts_name_fmt: str | None = None,
         store_kwargs: dict[str, Any] | None = None,
-        variables: dict[str, str] | None = None,
+        variables: dict[str, str] | Literal[False] | None = None,
         status: Status | None = None,
     ) -> Workflow:
         """Generate from a YAML string.
@@ -1072,7 +1089,10 @@ class Workflow(AppAware):
         store_kwargs:
             Keyword arguments to pass to the store's `write_empty_workflow` method.
         variables:
-            String variables to substitute in the string `YAML_str`.
+            String variables to substitute in the string `YAML_str`. Substitutions will be
+            attempted if the YAML string looks to contain variable references (like
+            "<<var:name>>"). If set to `False`, no substitutions will occur, which may
+            result in an invalid workflow template!
         """
         template = cls._app.WorkflowTemplate.from_YAML_string(
             string=YAML_str,
@@ -1101,7 +1121,7 @@ class Workflow(AppAware):
         ts_fmt: str | None = None,
         ts_name_fmt: str | None = None,
         store_kwargs: dict[str, Any] | None = None,
-        variables: dict[str, str] | None = None,
+        variables: dict[str, str] | Literal[False] | None = None,
         status: Status | None = None,
     ) -> Workflow:
         """Generate from a JSON file.
@@ -1132,7 +1152,10 @@ class Workflow(AppAware):
         store_kwargs:
             Keyword arguments to pass to the store's `write_empty_workflow` method.
         variables:
-            String variables to substitute in the file given by `JSON_path`.
+            String variables to substitute in the file given by `JSON_path`. Substitutions
+            will be attempted if the JSON file looks to contain variable references (like
+            "<<var:name>>"). If set to `False`, no substitutions will occur, which may
+            result in an invalid workflow template!
         """
         template = cls._app.WorkflowTemplate.from_JSON_file(
             path=JSON_path,
@@ -1161,7 +1184,7 @@ class Workflow(AppAware):
         ts_fmt: str | None = None,
         ts_name_fmt: str | None = None,
         store_kwargs: dict[str, Any] | None = None,
-        variables: dict[str, str] | None = None,
+        variables: dict[str, str] | Literal[False] | None = None,
         status: Status | None = None,
     ) -> Workflow:
         """Generate from a JSON string.
@@ -1192,7 +1215,10 @@ class Workflow(AppAware):
         store_kwargs:
             Keyword arguments to pass to the store's `write_empty_workflow` method.
         variables:
-            String variables to substitute in the string `JSON_str`.
+            String variables to substitute in the string `JSON_str`. Substitutions will be
+            attempted if the JSON string looks to contain variable references (like
+            "<<var:name>>"). If set to `False`, no substitutions will occur, which may
+            result in an invalid workflow template!
         """
         template = cls._app.WorkflowTemplate.from_JSON_string(
             string=JSON_str,
@@ -1223,7 +1249,7 @@ class Workflow(AppAware):
         ts_fmt: str | None = None,
         ts_name_fmt: str | None = None,
         store_kwargs: dict[str, Any] | None = None,
-        variables: dict[str, str] | None = None,
+        variables: dict[str, str] | Literal[False] | None = None,
         status: Status | None = None,
     ) -> Workflow:
         """Generate from either a YAML or JSON file, depending on the file extension.
@@ -1259,6 +1285,9 @@ class Workflow(AppAware):
             Keyword arguments to pass to the store's `write_empty_workflow` method.
         variables:
             String variables to substitute in the file given by `template_path`.
+            Substitutions will be attempted if the file looks to contain variable
+            references (like "<<var:name>>"). If set to `False`, no substitutions will
+            occur, which may result in an invalid workflow template!
         """
         try:
             template = cls._app.WorkflowTemplate.from_file(

hpcflow_new2-0.2.0a200/hpcflow/tests/data/benchmark_script_runner.yaml

@@ -0,0 +1,26 @@
+doc: |
+  A workflow for benchmarking the overhead introduced by hpcflow in running a Python
+  script `N` times.
+
+template_components:
+  task_schemas:
+    - objective: run_script
+      inputs:
+        - parameter: p1
+      outputs:
+        - parameter: p2
+      actions:
+        - environments:
+            - scope:
+                type: any
+              environment: python_env
+          script: <<script:main_script_test_direct_in_direct_out.py>>
+          script_exe: python_script
+          script_data_in: direct
+          script_data_out: direct
+
+tasks:
+  - schema: run_script
+    inputs:
+      p1: 101
+    repeats: <<var:N[default=1]>>

{hpcflow_new2-0.2.0a199 → hpcflow_new2-0.2.0a200}/hpcflow/tests/unit/test_workflow_template.py

@@ -1,5 +1,7 @@
+from textwrap import dedent
 import pytest
 from hpcflow.app import app as hf
+from hpcflow.sdk.core.errors import MissingVariableSubstitutionError
 from hpcflow.sdk.core.test_utils import (
     make_test_data_YAML_workflow_template,
 )
@@ -38,6 +40,35 @@ def test_workflow_template_vars(tmp_path, new_null_config):
     assert wkt.tasks[0].element_sets[0].repeats[0]["number"] == num_repeats
 
 
+def test_workflow_template_vars_raise_no_vars(tmp_path, new_null_config):
+    # no default value for the variable is provided in `benchmark_N_elements`, so should
+    # raise if the variables dict is not passed:
+    with pytest.raises(MissingVariableSubstitutionError):
+        make_test_data_YAML_workflow_template("benchmark_N_elements.yaml")
+
+
+def test_workflow_template_vars_defaults_used(tmp_path, new_null_config):
+    # `benchmark_script_runner` contains a default value for the variable `N`, so that
+    # should be used, since we don't pass any variables:
+    wkt = make_test_data_YAML_workflow_template("benchmark_script_runner.yaml")
+    assert wkt.tasks[0].element_sets[0].repeats[0]["number"] == 1
+
+
+def test_workflow_template_vars_False_no_substitution(tmp_path, new_null_config):
+    # read a yaml template, check variables are not substituted, when `variables=False`:
+    wkt_yaml = dedent(
+        """\
+        name: workflow_1
+        tasks:
+          - schema: test_t1_conditional_OS            
+            inputs:
+              p1: <<var:my_var>>
+    """
+    )
+    wkt = hf.WorkflowTemplate.from_YAML_string(wkt_yaml, variables=False)
+    assert wkt.tasks[0].element_sets[0].inputs[0].value == "<<var:my_var>>"
+
+
 def test_env_preset_merge_simple(null_config):
     s1 = hf.TaskSchema(
         objective="s1",

{hpcflow_new2-0.2.0a199 → hpcflow_new2-0.2.0a200}/pyproject.toml

@@ -1,7 +1,7 @@
 
 [tool.poetry]
 name = "hpcflow-new2"
-version = "0.2.0a199"
+version = "0.2.0a200"
 
 description = "Computational workflow management"
 authors = ["aplowman <adam.plowman@manchester.ac.uk>"]
@@ -55,6 +55,10 @@ sphinx-design = "^0.4.1"
 sphinx-togglebutton = "^0.3.2"
 pydata-sphinx-theme = "^0.13.3"
 ipykernel = "^6.25.1"
+myst-nb = [
+    {version = "0.17.2", python = "<3.9"},
+    {version = "1.1.1", python = ">=3.9"}
+]
 
 [tool.poetry.group.test.dependencies]
 pytest = "^7.2.0"
@@ -89,7 +93,7 @@ hook-dirs = "hpcflow.__pyinstaller:get_hook_dirs"
 
 [tool.commitizen]
 name = "cz_conventional_commits"
-version = "0.2.0a199"
+version = "0.2.0a200"
 tag_format = "v$version"
 version_files = [ 
     "pyproject.toml:version",

hpcflow_new2-0.2.0a199/hpcflow/_version.py

@@ -1 +0,0 @@
-__version__ = "0.2.0a199"