hpcflow-new2 0.2.0a50__py3-none-any.whl → 0.2.0a52__py3-none-any.whl

hpcflow/_version.py

@@ -1 +1 @@
-__version__ = "0.2.0a50"
+__version__ = "0.2.0a52"

hpcflow/sdk/__init__.py

@@ -98,7 +98,7 @@ sdk_funcs = {
 
 sdk_objs = {**sdk_classes, **sdk_funcs}
 
-_SDK_CONSOLE_LOG_LEVEL = os.environ.get("HPCFLOW_SDK_CONSOLE_LOG_LEVEL", "WARNING")
+_SDK_CONSOLE_LOG_LEVEL = os.environ.get("HPCFLOW_SDK_CONSOLE_LOG_LEVEL", "ERROR")
 
 
 def get_SDK_logger(name=None):

hpcflow/sdk/api.py

@@ -4,7 +4,7 @@ from __future__ import annotations
 import importlib
 import os
 from typing import Optional, TypeVar, Union
-from hpcflow.sdk.core.workflow import ALL_TEMPLATE_FORMATS, DEFAULT_TEMPLATE_FORMAT
+from hpcflow.sdk.core import ALL_TEMPLATE_FORMATS, DEFAULT_TEMPLATE_FORMAT
 from hpcflow.sdk.persistence import DEFAULT_STORE_FORMAT
 
 from hpcflow.sdk.submission.shells import get_shell

hpcflow/sdk/app.py

@@ -2,14 +2,14 @@
 from __future__ import annotations
 
 import enum
-from functools import partial, wraps
+from functools import wraps
 from importlib import resources, import_module
-import importlib
 from logging import Logger
 import os
 from pathlib import Path
 from typing import Any, Callable, Dict, List, Optional, Type, Union
 import warnings
+from reretry import retry
 
 from setuptools import find_packages
 
@@ -18,9 +18,10 @@ from hpcflow.sdk.core.object_list import ObjectList
 from hpcflow.sdk.core.utils import read_YAML, read_YAML_file
 from hpcflow.sdk import sdk_objs, sdk_classes, sdk_funcs, get_SDK_logger
 from hpcflow.sdk.config import Config
-from hpcflow.sdk.core.workflow import ALL_TEMPLATE_FORMATS, DEFAULT_TEMPLATE_FORMAT
+from hpcflow.sdk.core import ALL_TEMPLATE_FORMATS, DEFAULT_TEMPLATE_FORMAT
 from hpcflow.sdk.log import AppLog
 from hpcflow.sdk.persistence import DEFAULT_STORE_FORMAT
+from hpcflow.sdk.persistence.base import TEMPLATE_COMP_TYPES
 from hpcflow.sdk.runtime import RunTimeInfo
 from hpcflow.sdk.cli import make_cli
 from hpcflow.sdk.submission.shells import get_shell
@@ -95,13 +96,6 @@ class BaseApp(metaclass=Singleton):
 
     """
 
-    _template_component_types = (
-        "parameters",
-        "command_files",
-        "environments",
-        "task_schemas",
-    )
-
     def __init__(
         self,
         name,
@@ -281,7 +275,7 @@ class BaseApp(metaclass=Singleton):
             f"Loading built-in template component data for package: {package!r}."
         )
         components = {}
-        for comp_type in cls._template_component_types:
+        for comp_type in TEMPLATE_COMP_TYPES:
             resource = f"{comp_type}.yaml"
             try:
                 fh = resources.files(package).joinpath(resource).open("rt")
@@ -336,6 +330,10 @@ class BaseApp(metaclass=Singleton):
     def config_logger(self) -> Logger:
         return self.logger.getChild("config")
 
+    @property
+    def persistence_logger(self) -> Logger:
+        return self.logger.getChild("persistence")
+
     @property
     def runtime_info_logger(self) -> Logger:
         return self.logger.getChild("runtime")
@@ -354,6 +352,17 @@ class BaseApp(metaclass=Singleton):
             self.load_config()
         return self._config
 
+    def perm_error_retry(self):
+        """Return a decorator for retrying functions on permission and OS errors that
+        might be associated with cloud-storage desktop sync. engine operations."""
+        return retry(
+            (PermissionError, OSError),
+            tries=10,
+            delay=1,
+            backoff=2,
+            logger=self.persistence_logger,
+        )
+
     def _load_config(self, config_dir, config_invocation_key, **overrides) -> None:
         self.logger.info("Loading configuration.")
         self._config = Config(

hpcflow/sdk/cli.py

@@ -5,7 +5,7 @@ from termcolor import colored
 from hpcflow import __version__, _app_name
 from hpcflow.sdk.config.cli import get_config_CLI
 from hpcflow.sdk.config.errors import ConfigError
-from hpcflow.sdk.core.workflow import ALL_TEMPLATE_FORMATS, DEFAULT_TEMPLATE_FORMAT
+from hpcflow.sdk.core import ALL_TEMPLATE_FORMATS, DEFAULT_TEMPLATE_FORMAT
 from hpcflow.sdk.demo.cli import get_demo_software_CLI
 from hpcflow.sdk.helper.cli import get_helper_CLI
 from hpcflow.sdk.persistence import ALL_STORE_FORMATS, DEFAULT_STORE_FORMAT
@@ -410,21 +410,22 @@ def _make_internal_CLI(app):
     @click.pass_context
     @click.argument("submission_idx", type=click.INT)
     @click.argument("jobscript_idx", type=click.INT)
-    @click.argument("js_element_idx", type=click.INT)
     @click.argument("js_action_idx", type=click.INT)
+    @click.argument("ear_id", type=click.INT)
     def write_commands(
         ctx,
         submission_idx: int,
         jobscript_idx: int,
-        js_element_idx: int,
         js_action_idx: int,
+        ear_id: int,
     ):
+        app.CLI_logger.info(f"write commands for EAR ID {ear_id!r}.")
         ctx.exit(
             ctx.obj["workflow"].write_commands(
                 submission_idx,
                 jobscript_idx,
-                js_element_idx,
                 js_action_idx,
+                ear_id,
             )
         )
 
@@ -432,73 +433,47 @@ def _make_internal_CLI(app):
     @click.pass_context
     @click.argument("name")
     @click.argument("value")
-    @click.argument("submission_idx", type=click.INT)
-    @click.argument("jobscript_idx", type=click.INT)
-    @click.argument("js_element_idx", type=click.INT)
-    @click.argument("js_action_idx", type=click.INT)
+    @click.argument("ear_id", type=click.INT)
     def save_parameter(
         ctx,
         name: str,
         value: str,
-        submission_idx: int,
-        jobscript_idx: int,
-        js_element_idx: int,
-        js_action_idx: int,
+        ear_id: int,
     ):
-        ctx.exit(
-            ctx.obj["workflow"].save_parameter(
-                name,
-                value,
-                submission_idx,
-                jobscript_idx,
-                js_element_idx,
-                js_action_idx,
-            )
-        )
+        app.CLI_logger.info(f"save parameter {name!r} for EAR ID {ear_id!r}.")
+        ctx.exit(ctx.obj["workflow"].save_parameter(name, value, ear_id))
 
     @workflow.command()
     @click.pass_context
-    @click.argument("submission_idx", type=click.INT)
-    @click.argument("jobscript_idx", type=click.INT)
-    @click.argument("js_element_idx", type=click.INT)
-    @click.argument("js_action_idx", type=click.INT)
-    def set_EAR_start(
-        ctx,
-        submission_idx: int,
-        jobscript_idx: int,
-        js_element_idx: int,
-        js_action_idx: int,
-    ):
-        ctx.exit(
-            ctx.obj["workflow"].set_EAR_start(
-                submission_idx,
-                jobscript_idx,
-                js_element_idx,
-                js_action_idx,
-            )
-        )
+    @click.argument("ear_id", type=click.INT)
+    def set_EAR_start(ctx, ear_id: int):
+        app.CLI_logger.info(f"set EAR start for EAR ID {ear_id!r}.")
+        ctx.exit(ctx.obj["workflow"].set_EAR_start(ear_id))
 
     @workflow.command()
     @click.pass_context
-    @click.argument("submission_idx", type=click.INT)
-    @click.argument("jobscript_idx", type=click.INT)
-    @click.argument("js_element_idx", type=click.INT)
-    @click.argument("js_action_idx", type=click.INT)
-    def set_EAR_end(
-        ctx,
-        submission_idx: int,
-        jobscript_idx: int,
-        js_element_idx: int,
-        js_action_idx: int,
-    ):
-        ctx.exit(
-            ctx.obj["workflow"].set_EAR_end(
-                submission_idx,
-                jobscript_idx,
-                js_element_idx,
-                js_action_idx,
-            )
+    @click.argument("ear_id", type=click.INT)
+    @click.argument("exit_code", type=click.INT)
+    def set_EAR_end(ctx, ear_id: int, exit_code: int):
+        app.CLI_logger.info(
+            f"set EAR end for EAR ID {ear_id!r} with exit code {exit_code!r}."
         )
+        ctx.exit(ctx.obj["workflow"].set_EAR_end(ear_id, exit_code))
+
+    @workflow.command()
+    @click.pass_context
+    @click.argument("ear_id", type=click.INT)
+    def set_EAR_skip(ctx, ear_id: int):
+        app.CLI_logger.info(f"set EAR skip for EAR ID {ear_id!r}.")
+        ctx.exit(ctx.obj["workflow"].set_EAR_skip(ear_id))
+
+    @workflow.command()
+    @click.pass_context
+    @click.argument("ear_id", type=click.INT)
+    def get_EAR_skipped(ctx, ear_id: int):
+        """Return 1 if the given EAR is to be skipped, else return 0."""
+        app.CLI_logger.info(f"get EAR skip for EAR ID {ear_id!r}.")
+        click.echo(int(ctx.obj["workflow"].get_EAR_skipped(ear_id)))
 
     # TODO: in general, maybe the workflow command group can expose the simple Workflow
     # properties; maybe use a decorator on the Workflow property object to signify

hpcflow/sdk/core/__init__.py

@@ -1 +1,13 @@
-"""Core programmatic models for hpcflow."""
+"""Core programmatic models for hpcflow.
+
+EAR abort exit code is set to 64 [1].
+
+References
+----------
+https://tldp.org/LDP/abs/html/exitcodes.html
+
+"""
+
+DEFAULT_TEMPLATE_FORMAT = "yaml"
+ALL_TEMPLATE_FORMATS = ("yaml", "json")
+ABORT_EXIT_CODE = 64