hpcflow-new2 0.2.0a149__tar.gz → 0.2.0a153__tar.gz

{hpcflow_new2-0.2.0a149 → hpcflow_new2-0.2.0a153}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hpcflow-new2
-Version: 0.2.0a149
+Version: 0.2.0a153
 Summary: Computational workflow management
 License: MIT
 Author: aplowman

hpcflow_new2-0.2.0a153/hpcflow/_version.py

@@ -0,0 +1 @@
+__version__ = "0.2.0a153"

{hpcflow_new2-0.2.0a149 → hpcflow_new2-0.2.0a153}/hpcflow/sdk/app.py

@@ -1,4 +1,5 @@
 """An hpcflow application."""
+
 from __future__ import annotations
 
 from collections import defaultdict
@@ -37,7 +38,7 @@ from hpcflow.sdk.core.actions import EARStatus
 from hpcflow.sdk.core.errors import WorkflowNotFoundError
 from hpcflow.sdk.core.object_list import ObjectList
 from hpcflow.sdk.core.utils import (
-    read_YAML,
+    read_YAML_str,
     read_YAML_file,
     read_JSON_file,
     write_YAML_file,
@@ -46,7 +47,7 @@ from hpcflow.sdk.core.utils import (
 from hpcflow.sdk import sdk_classes, sdk_funcs, get_SDK_logger
 from hpcflow.sdk.config import Config, ConfigFile
 from hpcflow.sdk.core import ALL_TEMPLATE_FORMATS
-from hpcflow.sdk.log import AppLog
+from hpcflow.sdk.log import AppLog, TimeIt
 from hpcflow.sdk.persistence import DEFAULT_STORE_FORMAT
 from hpcflow.sdk.persistence.base import TEMPLATE_COMP_TYPES
 from hpcflow.sdk.runtime import RunTimeInfo
@@ -95,7 +96,10 @@ class Singleton(type):
     _instances = {}
 
     def __call__(cls, *args, **kwargs):
-        SDK_logger.info(f"App metaclass __call__ with {args=} {kwargs=}")
+        SDK_logger.info(
+            f"App metaclass __call__: "
+            f"name={kwargs['name']!r}, version={kwargs['version']!r}."
+        )
         if cls not in cls._instances:
             SDK_logger.info(f"App metaclass initialising new object {kwargs['name']!r}.")
             cls._instances[cls] = super(Singleton, cls).__call__(*args, **kwargs)
@@ -255,6 +259,14 @@ class BaseApp(metaclass=Singleton):
     def log(self) -> AppLog:
         return self._log
 
+    @property
+    def timeit(self) -> bool:
+        return TimeIt.active
+
+    @timeit.setter
+    def timeit(self, value: bool):
+        TimeIt.active = bool(value)
+
     @property
     def template_components(self) -> Dict[str, ObjectList]:
         if not self.is_template_components_loaded:
@@ -281,6 +293,7 @@ class BaseApp(metaclass=Singleton):
             warnings.warn("Template components not loaded; loading now.")
         self._load_template_components()
 
+    @TimeIt.decorator
     def _load_template_components(self, *include) -> None:
         """Combine any builtin template components with user-defined template components
         and initialise list objects."""
@@ -366,7 +379,7 @@ class BaseApp(metaclass=Singleton):
                 fh = resources.open_text(package, resource)
             SDK_logger.info(f"Parsing file as YAML: {fh.name!r}")
             comp_dat = fh.read()
-            components[comp_type] = read_YAML(comp_dat)
+            components[comp_type] = read_YAML_str(comp_dat)
             fh.close()
 
         return components
@@ -625,6 +638,7 @@ class BaseApp(metaclass=Singleton):
             shutil.rmtree(self.user_cache_hostname_dir)
             self._ensure_user_cache_hostname_dir()
 
+    @TimeIt.decorator
     def _load_config(self, config_dir, config_key, **overrides) -> None:
         self.logger.info("Loading configuration.")
         self._ensure_user_data_dir()
@@ -714,6 +728,7 @@ class BaseApp(metaclass=Singleton):
         self._config_files = {}
         self._load_config(config_dir, config_key, **overrides)
 
+    @TimeIt.decorator
     def _load_scripts(self):
         # TODO: load custom directories / custom functions (via decorator)
 
@@ -1126,6 +1141,7 @@ class BaseApp(metaclass=Singleton):
         ts_fmt: Optional[str] = None,
         ts_name_fmt: Optional[str] = None,
         store_kwargs: Optional[Dict] = None,
+        variables: Optional[Dict[str, str]] = None,
     ) -> get_app_attribute("Workflow"):
         """Generate a new {app_name} workflow from a file or string containing a workflow
         template parametrisation.
@@ -1160,6 +1176,8 @@ class BaseApp(metaclass=Singleton):
             includes a timestamp.
         store_kwargs
             Keyword arguments to pass to the store's `write_empty_workflow` method.
+        variables
+            String variables to substitute in `template_file_or_str`.
         """
 
         self.API_logger.info("make_workflow called")
@@ -1172,6 +1190,7 @@ class BaseApp(metaclass=Singleton):
             "ts_fmt": ts_fmt,
             "ts_name_fmt": ts_name_fmt,
             "store_kwargs": store_kwargs,
+            "variables": variables,
         }
 
         if not is_string:
@@ -1212,6 +1231,7 @@ class BaseApp(metaclass=Singleton):
         ts_fmt: Optional[str] = None,
         ts_name_fmt: Optional[str] = None,
         store_kwargs: Optional[Dict] = None,
+        variables: Optional[Dict[str, str]] = None,
         JS_parallelism: Optional[bool] = None,
         wait: Optional[bool] = False,
         add_to_known: Optional[bool] = True,
@@ -1252,6 +1272,8 @@ class BaseApp(metaclass=Singleton):
             includes a timestamp.
         store_kwargs
             Keyword arguments to pass to the store's `write_empty_workflow` method.
+        variables
+            String variables to substitute in `template_file_or_str`.
         JS_parallelism
             If True, allow multiple jobscripts to execute simultaneously. Raises if set to
             True but the store type does not support the `jobscript_parallelism` feature. If
@@ -1282,6 +1304,7 @@ class BaseApp(metaclass=Singleton):
             ts_fmt=ts_fmt,
             ts_name_fmt=ts_name_fmt,
             store_kwargs=store_kwargs,
+            variables=variables,
         )
         return wk.submit(
             JS_parallelism=JS_parallelism,
@@ -1302,6 +1325,7 @@ class BaseApp(metaclass=Singleton):
         ts_fmt: Optional[str] = None,
         ts_name_fmt: Optional[str] = None,
         store_kwargs: Optional[Dict] = None,
+        variables: Optional[Dict[str, str]] = None,
     ) -> get_app_attribute("Workflow"):
         """Generate a new {app_name} workflow from a builtin demo workflow template.
 
@@ -1333,6 +1357,8 @@ class BaseApp(metaclass=Singleton):
             includes a timestamp.
         store_kwargs
             Keyword arguments to pass to the store's `write_empty_workflow` method.
+        variables
+            String variables to substitute in the demo workflow template file.
         """
 
         self.API_logger.info("make_demo_workflow called")
@@ -1348,6 +1374,7 @@ class BaseApp(metaclass=Singleton):
                 ts_fmt=ts_fmt,
                 ts_name_fmt=ts_name_fmt,
                 store_kwargs=store_kwargs,
+                variables=variables,
             )
         return wk
 
@@ -1362,6 +1389,7 @@ class BaseApp(metaclass=Singleton):
         ts_fmt: Optional[str] = None,
         ts_name_fmt: Optional[str] = None,
         store_kwargs: Optional[Dict] = None,
+        variables: Optional[Dict[str, str]] = None,
         JS_parallelism: Optional[bool] = None,
         wait: Optional[bool] = False,
         add_to_known: Optional[bool] = True,
@@ -1399,6 +1427,8 @@ class BaseApp(metaclass=Singleton):
             includes a timestamp.
         store_kwargs
             Keyword arguments to pass to the store's `write_empty_workflow` method.
+        variables
+            String variables to substitute in the demo workflow template file.
         JS_parallelism
             If True, allow multiple jobscripts to execute simultaneously. Raises if set to
             True but the store type does not support the `jobscript_parallelism` feature. If
@@ -1428,6 +1458,7 @@ class BaseApp(metaclass=Singleton):
             ts_fmt=ts_fmt,
             ts_name_fmt=ts_name_fmt,
             store_kwargs=store_kwargs,
+            variables=variables,
         )
         return wk.submit(
             JS_parallelism=JS_parallelism,

{hpcflow_new2-0.2.0a149 → hpcflow_new2-0.2.0a153}/hpcflow/sdk/cli.py

@@ -19,6 +19,7 @@ from hpcflow.sdk.cli_common import (
     store_option,
     ts_fmt_option,
     ts_name_fmt_option,
+    variables_option,
     js_parallelism_option,
     wait_option,
     add_to_known_opt,
@@ -31,6 +32,7 @@ from hpcflow.sdk.cli_common import (
     unzip_log_opt,
 )
 from hpcflow.sdk.helper.cli import get_helper_CLI
+from hpcflow.sdk.log import TimeIt
 from hpcflow.sdk.submission.shells import ALL_SHELLS
 
 string_option = click.option(
@@ -68,6 +70,7 @@ def _make_API_CLI(app):
     @store_option
     @ts_fmt_option
     @ts_name_fmt_option
+    @variables_option
     def make_workflow(
         template_file_or_str,
         string,
@@ -78,6 +81,7 @@ def _make_API_CLI(app):
         store,
         ts_fmt=None,
         ts_name_fmt=None,
+        variables=None,
     ):
         """Generate a new {app_name} workflow.
 
@@ -95,6 +99,7 @@ def _make_API_CLI(app):
             store=store,
             ts_fmt=ts_fmt,
             ts_name_fmt=ts_name_fmt,
+            variables=dict(variables),
         )
         click.echo(wk.path)
 
@@ -108,6 +113,7 @@ def _make_API_CLI(app):
     @store_option
     @ts_fmt_option
     @ts_name_fmt_option
+    @variables_option
     @js_parallelism_option
     @wait_option
     @add_to_known_opt
@@ -123,6 +129,7 @@ def _make_API_CLI(app):
         store,
         ts_fmt=None,
         ts_name_fmt=None,
+        variables=None,
         js_parallelism=None,
         wait=False,
         add_to_known=True,
@@ -146,6 +153,7 @@ def _make_API_CLI(app):
             store=store,
             ts_fmt=ts_fmt,
             ts_name_fmt=ts_name_fmt,
+            variables=dict(variables),
             JS_parallelism=js_parallelism,
             wait=wait,
             add_to_known=add_to_known,
@@ -1032,9 +1040,27 @@ def make_cli(app):
         nargs=2,
         multiple=True,
     )
+    @click.option(
+        "--timeit",
+        help=(
+            "Time function pathways as the code executes and write out a summary at the "
+            "end. Only functions decorated by `TimeIt.decorator` are included."
+        ),
+        is_flag=True,
+    )
+    @click.option(
+        "--timeit-file",
+        help=(
+            "Time function pathways as the code executes and write out a summary at the "
+            "end to a text file given by this file path. Only functions decorated by "
+            "`TimeIt.decorator` are included."
+        ),
+    )
     @click.pass_context
-    def new_CLI(ctx, config_dir, config_key, with_config):
+    def new_CLI(ctx, config_dir, config_key, with_config, timeit, timeit_file):
         app.run_time_info.from_CLI = True
+        TimeIt.active = timeit or timeit_file
+        TimeIt.file_path = timeit_file
         if ctx.invoked_subcommand != "manage":
             # load the config
             overrides = {kv[0]: kv[1] for kv in with_config}
@@ -1048,6 +1074,11 @@ def make_cli(app):
                 click.echo(f"{colored(err.__class__.__name__, 'red')}: {err}")
                 ctx.exit(1)
 
+    @new_CLI.result_callback()
+    def post_execution(*args, **kwargs):
+        if TimeIt.active:
+            TimeIt.summarise_string()
+
     @new_CLI.command()
     @click.argument("name")
     @click.option("--use-current-env", is_flag=True, default=False)

{hpcflow_new2-0.2.0a149 → hpcflow_new2-0.2.0a153}/hpcflow/sdk/cli_common.py

@@ -63,6 +63,17 @@ ts_name_fmt_option = click.option(
         "includes a timestamp."
     ),
 )
+variables_option = click.option(
+    "-v",
+    "--var",
+    "variables",
+    type=(str, str),
+    multiple=True,
+    help=(
+        "Workflow template variable value to be substituted in to the template file or "
+        "string. Multiple variable values can be specified."
+    ),
+)
 js_parallelism_option = click.option(
     "--js-parallelism",
     help=(

{hpcflow_new2-0.2.0a149 → hpcflow_new2-0.2.0a153}/hpcflow/sdk/core/actions.py

@@ -29,6 +29,7 @@ from hpcflow.sdk.core.utils import (
     split_param_label,
     swap_nested_dict_keys,
 )
+from hpcflow.sdk.log import TimeIt
 
 
 ACTION_SCOPE_REGEX = r"(\w*)(?:\[(.*)\])?"
@@ -1720,6 +1721,7 @@ class Action(JSONLike):
     def get_output_file_labels(self):
         return tuple(i.label for i in self.output_files)
 
+    @TimeIt.decorator
     def generate_data_index(
         self,
         act_idx,

{hpcflow_new2-0.2.0a149 → hpcflow_new2-0.2.0a153}/hpcflow/sdk/core/errors.py

@@ -387,3 +387,7 @@ class UnknownScriptDataParameter(ValueError):
 
 class UnknownScriptDataKey(ValueError):
     pass
+
+
+class MissingVariableSubstitutionError(KeyError):
+    pass

{hpcflow_new2-0.2.0a149 → hpcflow_new2-0.2.0a153}/hpcflow/sdk/core/task.py

@@ -10,6 +10,7 @@ from valida.rules import Rule
 
 from hpcflow.sdk import app
 from hpcflow.sdk.core.task_schema import TaskSchema
+from hpcflow.sdk.log import TimeIt
 from hpcflow.sdk.submission.shells import DEFAULT_SHELL_NAMES
 from .json_like import ChildObjectSpec, JSONLike
 from .element import ElementGroup
@@ -719,6 +720,7 @@ class Task(JSONLike):
         """Find the nesting order for a task sequence."""
         return self.nesting_order[seq.normalised_path] if len(seq.values) > 1 else -1
 
+    @TimeIt.decorator
     def _prepare_persistent_outputs(self, workflow, local_element_idx_range):
         # TODO: check that schema is present when adding task? (should this be here?)
 
@@ -1790,6 +1792,7 @@ class WorkflowTask:
 
         return element_dat_idx
 
+    @TimeIt.decorator
     def initialise_EARs(self) -> List[int]:
         """Try to initialise any uninitialised EARs of this task."""
         initialised = []
@@ -1815,6 +1818,7 @@ class WorkflowTask:
                         self.workflow.set_EARs_initialised(iter_i.id_)
         return initialised
 
+    @TimeIt.decorator
     def _initialise_element_iter_EARs(self, element_iter: app.ElementIteration) -> None:
         # keys are (act_idx, EAR_idx):
         all_data_idx = {}
@@ -1878,6 +1882,7 @@ class WorkflowTask:
         for pid, src in param_src_updates.items():
             self.workflow._store.update_param_source(pid, src)
 
+    @TimeIt.decorator
     def _add_element_set(self, element_set):
         """
         Returns
@@ -1982,6 +1987,7 @@ class WorkflowTask:
                 return_indices=return_indices,
             )
 
+    @TimeIt.decorator
     def _add_elements(
         self,
         base_element=None,
@@ -2153,6 +2159,7 @@ class WorkflowTask:
 
         return deps
 
+    @TimeIt.decorator
     def get_dependent_tasks(
         self,
         as_objects: bool = False,
@@ -2251,6 +2258,7 @@ class WorkflowTask:
 
         return params
 
+    @TimeIt.decorator
     def _get_merged_parameter_data(
         self,
         data_index,

{hpcflow_new2-0.2.0a149 → hpcflow_new2-0.2.0a153}/hpcflow/sdk/core/utils.py

@@ -15,7 +15,7 @@ import string
 import subprocess
 from datetime import datetime, timezone
 import sys
-from typing import Optional, Tuple, Type, Union, List
+from typing import Dict, Optional, Tuple, Type, Union, List
 import fsspec
 import numpy as np
 
@@ -27,7 +27,9 @@ from hpcflow.sdk.core.errors import (
     ContainerKeyError,
     FromSpecMissingObjectError,
     InvalidIdentifier,
+    MissingVariableSubstitutionError,
 )
+from hpcflow.sdk.log import TimeIt
 from hpcflow.sdk.typing import PathLike
 
 
@@ -385,19 +387,42 @@ def check_in_object_list(spec_name, spec_pos=1, obj_list_pos=2):
     return decorator
 
 
-def read_YAML(loadable_yaml, typ="safe"):
+def substitute_string_vars(string, variables: Dict[str, str] = None):
+    variables = variables or {}
+
+    def var_repl(match_obj):
+        var_name = match_obj.group(1)
+        try:
+            out = str(variables[var_name])
+        except KeyError:
+            raise MissingVariableSubstitutionError(
+                f"The variable {var_name!r} referenced in the string does not match any "
+                f"of the provided variables: {list(variables)!r}."
+            )
+        return out
+
+    new_str = re.sub(
+        pattern=r"\<\<var:(.*?)\>\>",
+        repl=var_repl,
+        string=string,
+    )
+    return new_str
+
+
+@TimeIt.decorator
+def read_YAML_str(yaml_str, typ="safe", variables: Dict[str, str] = None):
+    """Load a YAML string."""
+    if variables:
+        yaml_str = substitute_string_vars(yaml_str, variables=variables)
     yaml = YAML(typ=typ)
-    return yaml.load(loadable_yaml)
+    return yaml.load(yaml_str)
 
 
-def read_YAML_file(path: PathLike, typ="safe"):
-    if is_fsspec_url(str(path)):
-        with fsspec.open(path, "rt") as f:
-            data = f.read()
-        loadable_yaml = data
-    else:
-        loadable_yaml = Path(path)
-    return read_YAML(loadable_yaml, typ=typ)
+@TimeIt.decorator
+def read_YAML_file(path: PathLike, typ="safe", variables: Dict[str, str] = None):
+    with fsspec.open(path, "rt") as f:
+        yaml_str = f.read()
+    return read_YAML_str(yaml_str, typ=typ, variables=variables)
 
 
 def write_YAML_file(obj, path: PathLike, typ="safe"):
@@ -406,13 +431,16 @@ def write_YAML_file(obj, path: PathLike, typ="safe"):
         yaml.dump(obj, fp)
 
 
-def read_JSON_string(string: str):
-    return json.loads(string)
+def read_JSON_string(json_str: str, variables: Dict[str, str] = None):
+    if variables:
+        json_str = substitute_string_vars(json_str, variables=variables)
+    return json.loads(json_str)
 
 
-def read_JSON_file(path):
-    with Path(path).open("rt") as fh:
-        return json.load(fh)
+def read_JSON_file(path, variables: Dict[str, str] = None):
+    with fsspec.open(path, "rt") as f:
+        json_str = f.read()
+    return read_JSON_string(json_str, variables=variables)
 
 
 def write_JSON_file(obj, path: PathLike):