hpcflow-new2 0.2.0a37__tar.gz → 0.2.0a38__tar.gz

{hpcflow_new2-0.2.0a37 → hpcflow_new2-0.2.0a38}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hpcflow-new2
-Version: 0.2.0a37
+Version: 0.2.0a38
 Summary: Computational workflow management
 License: MIT
 Author: aplowman

hpcflow_new2-0.2.0a38/hpcflow/_version.py

@@ -0,0 +1 @@
+__version__ = "0.2.0a38"

{hpcflow_new2-0.2.0a37 → hpcflow_new2-0.2.0a38}/hpcflow/sdk/__init__.py

@@ -3,15 +3,23 @@ import logging
 import os
 import sys
 
-_SDK_CONSOLE_LOG_LEVEL = os.environ.get("HPCFLOW_SDK_CONSOLE_LOG_LEVEL", "ERROR")
+_SDK_CONSOLE_LOG_LEVEL = os.environ.get("HPCFLOW_SDK_CONSOLE_LOG_LEVEL", "WARNING")
 
-SDK_logger = logging.getLogger()
-SDK_logger.setLevel("DEBUG")
+
+def get_SDK_logger(name=None):
+    """Get a logger with prefix of "hpcflow_sdk" instead of "hpcflow.sdk" to ensure the
+    handlers of the SDK logger and app logger are distinct."""
+    name = ".".join(["hpcflow_sdk"] + (name or __name__).split(".")[2:])
+    return logging.getLogger(name)
+
+
+_SDK_logger = get_SDK_logger()
+_SDK_logger.setLevel("DEBUG")
 
 _sh = logging.StreamHandler()
 _sh.setFormatter(logging.Formatter("%(levelname)s %(name)s: %(message)s"))
 _sh.setLevel(_SDK_CONSOLE_LOG_LEVEL)
-SDK_logger.addHandler(_sh)
+_SDK_logger.addHandler(_sh)
 
 if getattr(sys, "frozen", False) and hasattr(sys, "_MEIPASS"):
     import multiprocessing

{hpcflow_new2-0.2.0a37 → hpcflow_new2-0.2.0a38}/hpcflow/sdk/app.py

@@ -3,7 +3,6 @@
 from functools import wraps
 from importlib import resources, import_module
 from pathlib import Path
-import time
 from typing import Dict
 import warnings
 
@@ -17,7 +16,7 @@ from hpcflow.sdk.persistence import ALL_STORE_FORMATS, DEFAULT_STORE_FORMAT
 from hpcflow.sdk.submission.shells import ALL_SHELLS
 from .core.json_like import JSONLike
 from .core.utils import read_YAML, read_YAML_file
-from . import api, SDK_logger
+from . import api, get_SDK_logger
 from .config import Config
 from .config.cli import get_config_CLI
 from .config.errors import ConfigError
@@ -62,7 +61,7 @@ from .helper.cli import get_helper_CLI
 from .log import AppLog
 from .runtime import RunTimeInfo
 
-SDK_logger = SDK_logger.getChild(__name__)
+SDK_logger = get_SDK_logger(__name__)
 
 
 class BaseApp:
@@ -139,6 +138,9 @@ class BaseApp:
                 api_method.__doc__ = func.__doc__.format(app_name=name)
                 setattr(self, func.__name__, api_method)
 
+    def __repr__(self):
+        return f"{self.__class__.__name__}(name={self.name}, version={self.version})"
+
     @property
     def template_components(self):
         if not self.is_template_components_loaded:
@@ -339,15 +341,21 @@ class BaseApp:
         return self._config
 
     def _load_config(self, config_dir, config_invocation_key, **overrides):
-        self.logger.debug("Loading configuration.")
+        self.logger.info("Loading configuration.")
         self._config = Config(
             app=self,
             options=self.config_options,
             config_dir=config_dir,
             config_invocation_key=config_invocation_key,
             logger=self.config_logger,
+            variables={"app_name": self.name, "app_version": self.version},
             **overrides,
         )
+        self.log.update_console_level(self.config.get("log_console_level"))
+        self.log.add_file_logger(
+            path=self.config.get("log_file_path"),
+            level=self.config.get("log_file_level"),
+        )
         self.logger.info(f"Configuration loaded from: {self.config.config_file_path}")
 
     def load_config(self, config_dir=None, config_invocation_key=None, **overrides):

{hpcflow_new2-0.2.0a37 → hpcflow_new2-0.2.0a38}/hpcflow/sdk/config/callbacks.py

@@ -1,9 +1,27 @@
 """Module that defines built-in callback functions for configuration item values."""
 
 
+import re
 import fsspec
 
 
+def callback_vars(config, value):
+    """Substitute configuration variables."""
+
+    def vars_repl(match_obj):
+        var_name = match_obj.groups()[0]
+        return config._variables[var_name]
+
+    vars_join = "|".join(list(config._variables.keys()))
+    vars_regex = f"\<\<({vars_join})\>\>"
+    value = re.sub(
+        pattern=vars_regex,
+        repl=vars_repl,
+        string=value,
+    )
+    return value
+
+
 def callback_file_paths(config, file_path):
     if isinstance(file_path, list):
         return [config._resolve_path(i) for i in file_path]

{hpcflow_new2-0.2.0a37 → hpcflow_new2-0.2.0a38}/hpcflow/sdk/config/config.py

@@ -9,7 +9,7 @@ import uuid
 from dataclasses import dataclass, field
 from hashlib import new
 from pathlib import Path
-from typing import Any, Dict, List, Optional, Union
+from typing import Dict, List, Optional, Tuple, Union
 
 from fsspec.registry import known_implementations as fsspec_protocols
 from platformdirs import user_data_dir
@@ -20,6 +20,7 @@ from hpcflow.sdk.log import AppLog
 
 from .callbacks import (
     callback_bool,
+    callback_vars,
     callback_file_paths,
     set_callback_file_paths,
     check_load_data_files,
@@ -47,7 +48,7 @@ DEFAULT_CONFIG_FILE = {
             "config": {
                 "machine": socket.gethostname(),
                 "telemetry": True,
-                "log_file_path": "logs/app.log",
+                "log_file_path": "logs/<<app_name>>_v<<app_version>>.log",
                 "environment_sources": [],
                 "task_schema_sources": [],
                 "command_file_sources": [],
@@ -104,13 +105,14 @@ class Config:
         logger,
         uid=None,
         callbacks=None,
+        variables=None,
         **overrides,
     ):
-
         self._app = app
         self._options = options
         self._overrides = overrides
         self._logger = logger
+        self._variables = variables or {}
 
         # Callbacks are run on get:
         self._get_callbacks = {
@@ -118,7 +120,10 @@ class Config:
             "environment_sources": (callback_file_paths,),
             "parameter_sources": (callback_file_paths,),
             "command_file_sources": (callback_file_paths,),
-            "log_file_path": (callback_file_paths,),
+            "log_file_path": (
+                callback_vars,
+                callback_file_paths,
+            ),
             "telemetry": (callback_bool,),
             **(callbacks or {}),
         }
@@ -318,7 +323,10 @@ class Config:
                 self._logger.debug(
                     f"Invoking `config.get` callback ({cb.__name__!r}) for item {name!r}={value!r}"
                 )
-                value = cb(self, value)
+                try:
+                    value = cb(self, value)
+                except Exception as err:
+                    raise ConfigItemCallbackError(name, cb, err)
         return value
 
     def get(
@@ -361,10 +369,7 @@ class Config:
             )
 
         if callback:
-            try:
-                val = self._get_callback_value(name, val)
-            except Exception as err:
-                raise ConfigItemCallbackError(name, err)
+            val = self._get_callback_value(name, val)
 
         if as_str:
             if isinstance(val, (list, tuple, set)):
@@ -382,7 +387,6 @@ class Config:
         return value
 
     def set(self, name, value, is_json=False, callback=True):
-
         self._logger.debug(f"Attempting to set config item {name!r} to {value!r}.")
 
         if name not in self._configurable_keys:
@@ -396,7 +400,6 @@ class Config:
             file_val = self._get_callback_value(name, file_val_raw)
 
             if callback_val != current_val:
-
                 was_in_modified = False
                 was_in_unset = False
                 prev_modified_val = None
@@ -426,7 +429,6 @@ class Config:
                             cb(self, callback_val)
 
                 except ConfigValidationError as err:
-
                     # revert:
                     if modified_updated:
                         if was_in_modified:
@@ -445,7 +447,6 @@ class Config:
                 print(f"value is already: {callback_val!r}")
 
     def unset(self, name):
-
         if name not in self._configurable_keys:
             raise ConfigNonConfigurableError(name=name)
         if name in self._unset_keys or not self._file.is_item_set(name):

{hpcflow_new2-0.2.0a37 → hpcflow_new2-0.2.0a38}/hpcflow/sdk/config/config_file.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import copy
+import fnmatch
 import io
 import logging
 import os
@@ -45,7 +46,9 @@ class ConfigFile:
             for c_name_i, c_dat_i in self.data["configs"].items():
                 is_match = True
                 for match_k, match_v in c_dat_i["invocation"]["match"].items():
-                    if getattr(self.config._app.run_time_info, match_k) != match_v:
+                    # test for a matching glob pattern:
+                    k_value = getattr(self.config._app.run_time_info, match_k)
+                    if not fnmatch.filter(names=[k_value], pat=match_v):
                         is_match = False
                         break
                 if is_match:
@@ -180,7 +183,7 @@ class ConfigFile:
             raise ConfigDefaultValidationError(err) from None
 
         yaml = YAML(typ="rt")
-        with config_path.open("w") as handle:
+        with config_path.open("w", newline="\n") as handle:
             yaml.dump(default_config, handle)
 
     def _load_file_data(self):

{hpcflow_new2-0.2.0a37 → hpcflow_new2-0.2.0a38}/hpcflow/sdk/config/errors.py

@@ -39,9 +39,10 @@ class ConfigFileValidationError(ConfigError):
 
 
 class ConfigItemCallbackError(ConfigError):
-    def __init__(self, name, err, message=None):
+    def __init__(self, name, callback, err, message=None):
         self.message = message or (
-            f"Callback function for configuration item {name!r} failed: \n\n{err!r}"
+            f"Callback function {callback.__name__!r} for configuration item {name!r} "
+            f"failed with exception: \n\n{err!r}"
         )
         super().__init__(self.message)
 

{hpcflow_new2-0.2.0a37 → hpcflow_new2-0.2.0a38}/hpcflow/sdk/core/actions.py

@@ -411,7 +411,7 @@ class ElementActionRun:
             app_package_name=self.app.package_name,
             app_name=self.app.name,
             cfg_dir=self.app.config.config_directory,
-            cfg_invoc_key=self.app.config._file.invoc_key,
+            cfg_invoc_key=self.app.config.config_invocation_key,
             script_main_func=script_main_func,
         )
 

{hpcflow_new2-0.2.0a37 → hpcflow_new2-0.2.0a38}/hpcflow/sdk/core/command_files.py

@@ -167,7 +167,7 @@ class InputFileGenerator(JSONLike):
             app_package_name=self.app.package_name,
             app_name=self.app.name,
             cfg_dir=self.app.config.config_directory,
-            cfg_invoc_key=self.app.config._file.invoc_key,
+            cfg_invoc_key=self.app.config.config_invocation_key,
             script_main_func=script_main_func,
             file_path=self.input_file.name.value(),
         )
@@ -259,7 +259,7 @@ class OutputFileParser(JSONLike):
             app_package_name=self.app.package_name,
             app_name=self.app.name,
             cfg_dir=self.app.config.config_directory,
-            cfg_invoc_key=self.app.config._file.invoc_key,
+            cfg_invoc_key=self.app.config.config_invocation_key,
             script_main_func=script_main_func,
             param_name=f"outputs.{self.output.typ}",
         )

{hpcflow_new2-0.2.0a37 → hpcflow_new2-0.2.0a38}/hpcflow/sdk/core/utils.py

@@ -10,10 +10,8 @@ import re
 import socket
 import string
 from datetime import datetime, timezone
-from typing import Mapping, List, Union
+from typing import Mapping
 
-import zarr
-import numpy.typing as npt
 from ruamel.yaml import YAML
 import sentry_sdk
 
@@ -116,7 +114,6 @@ def group_by_dict_key_values(lst, *keys):
     grouped = [[lst[0]]]
     for lst_item in lst[1:]:
         for group_idx, group in enumerate(grouped):
-
             try:
                 is_vals_equal = all(lst_item[k] == group[0][k] for k in keys)
 
@@ -162,7 +159,6 @@ def get_in_container(cont, path, cast_indices=False):
 
 
 def set_in_container(cont, path, value, ensure_path=False):
-
     if ensure_path:
         num_path = len(path)
         for idx in range(1, num_path):
@@ -291,7 +287,6 @@ def check_in_object_list(spec_name, spec_pos=1, obj_list_pos=2):
     def decorator(func):
         @wraps(func)
         def wrap(*args, **kwargs):
-
             spec = args[spec_pos]
             obj_list = args[obj_list_pos]
             if spec[spec_name] not in obj_list:

hpcflow_new2-0.2.0a38/hpcflow/sdk/log.py

@@ -0,0 +1,41 @@
+import logging
+
+
+class AppLog:
+    DEFAULT_LOG_CONSOLE_LEVEL = "WARNING"
+    DEFAULT_LOG_FILE_LEVEL = "INFO"
+
+    def __init__(self, app, log_console_level=None):
+        self.app = app
+        self.logger = logging.getLogger(app.package_name)
+        self.logger.setLevel(logging.DEBUG)
+        self.console_handler = self._add_console_logger(
+            level=log_console_level or AppLog.DEFAULT_LOG_CONSOLE_LEVEL
+        )
+
+    def _add_console_logger(self, level, fmt=None):
+        fmt = fmt or "%(levelname)s %(name)s: %(message)s"
+        handler = logging.StreamHandler()
+        handler.setFormatter(logging.Formatter(fmt))
+        handler.setLevel(level)
+        self.logger.addHandler(handler)
+        return handler
+
+    def update_console_level(self, new_level):
+        if new_level:
+            self.console_handler.setLevel(new_level.upper())
+
+    def add_file_logger(self, path, level=None, fmt=None, max_bytes=None):
+        fmt = fmt or f"%(asctime)s %(levelname)s %(name)s: %(message)s"
+        level = level or AppLog.DEFAULT_LOG_FILE_LEVEL
+        max_bytes = max_bytes or int(10e6)
+
+        if not path.parent.is_dir():
+            self.logger.info(f"Generating log file parent directory: {path.parent!r}")
+            path.parent.mkdir(exist_ok=True, parents=True)
+
+        handler = logging.handlers.RotatingFileHandler(filename=path, maxBytes=max_bytes)
+        handler.setFormatter(logging.Formatter(fmt))
+        handler.setLevel(level.upper())
+        self.logger.addHandler(handler)
+        return handler

{hpcflow_new2-0.2.0a37 → hpcflow_new2-0.2.0a38}/hpcflow/sdk/runtime.py

@@ -41,11 +41,13 @@ class RunTimeInfo(PrettyPrinter):
         self.is_frozen = is_frozen
         self.working_dir = os.getcwd()
         self.logger = logger
-        self.machine = socket.gethostname()
+        self.hostname = socket.gethostname()
 
         path_exec = Path(sys.executable)
         path_argv = Path(sys.argv[0])
 
+        self.in_ipython = False
+
         if self.is_frozen:
             self.bundle_dir = Path(bundle_dir)
             self.executable_path = path_argv
@@ -57,6 +59,12 @@ class RunTimeInfo(PrettyPrinter):
             self.resolved_script_path = path_argv.absolute()
             self.python_executable_path = path_exec
 
+            try:
+                get_ipython
+                self.in_ipython = True
+            except NameError:
+                pass
+
         self.python_version = platform.python_version()
         self.is_venv = hasattr(sys, "real_prefix") or sys.base_prefix != sys.prefix
         self.is_conda_venv = "CONDA_PREFIX" in os.environ
@@ -101,6 +109,7 @@ class RunTimeInfo(PrettyPrinter):
                 "is_conda_venv",
                 "executable_name",
                 "python_version",
+                "in_ipython",
             ):
                 sentry_sdk.set_tag(f"rti.{k}", v)
 
@@ -108,9 +117,10 @@ class RunTimeInfo(PrettyPrinter):
         out = {
             "is_frozen": self.is_frozen,
             "python_version": self.python_version,
-            "machine": self.machine,
+            "hostname": self.hostname,
             "working_dir": self.working_dir,
             "invocation_command": self.get_invocation_command(),
+            "in_ipython": self.in_ipython,
         }
         if self.is_frozen:
             out.update(
@@ -140,8 +150,7 @@ class RunTimeInfo(PrettyPrinter):
 
     def __repr__(self):
         out = f"{self.__class__.__name__}("
-        for k, v in self._get_members().items():
-            out += f"{k}={v!r}"
+        out += ", ".join(f"{k}={v!r}" for k, v in self._get_members().items())
         return out
 
     def _set_venv_path(self):
@@ -168,16 +177,8 @@ class RunTimeInfo(PrettyPrinter):
         if self.is_frozen:
             return [str(self.resolved_executable_path)]
         else:
-            in_ipython = False
-            try:
-                get_ipython
-                in_ipython = True
-            except NameError:
-                pass
-
-            if in_ipython:
+            if self.in_ipython:
                 app_module = import_module(self.package_name)
-
                 CLI_path = Path(*app_module.__path__, "cli.py")
                 command = [str(self.python_executable_path), str(CLI_path)]
 

{hpcflow_new2-0.2.0a37 → hpcflow_new2-0.2.0a38}/hpcflow/sdk/submission/jobscript.py

@@ -682,7 +682,7 @@ class Jobscript(JSONLike):
                 "env_setup": env_setup,
                 "app_invoc": app_invoc,
                 "config_dir": str(self.app.config.config_directory),
-                "config_invoc_key": self.app.config._file.invoc_key,
+                "config_invoc_key": self.app.config.config_invocation_key,
                 "workflow_path": self.workflow.path,
                 "sub_idx": self.submission.index,
                 "js_idx": self.index,

{hpcflow_new2-0.2.0a37 → hpcflow_new2-0.2.0a38}/pyproject.toml

@@ -1,7 +1,7 @@
 
 [tool.poetry]
 name = "hpcflow-new2"
-version = "0.2.0a37"
+version = "0.2.0a38"
 
 description = "Computational workflow management"
 authors = ["aplowman <adam.plowman@manchester.ac.uk>"]
@@ -57,7 +57,7 @@ hook-dirs = "hpcflow.__pyinstaller:get_hook_dirs"
 
 [tool.commitizen]
 name = "cz_conventional_commits"
-version = "0.2.0a37"
+version = "0.2.0a38"
 tag_format = "v$version"
 version_files = [ 
     "pyproject.toml:version",

hpcflow_new2-0.2.0a37/hpcflow/_version.py

@@ -1 +0,0 @@
-__version__ = "0.2.0a37"

hpcflow_new2-0.2.0a37/hpcflow/api/config.py

@@ -1,17 +0,0 @@
-from hpcflow.config import Config
-
-
-def get_config_file_contents():
-    return Config.config_file_contents
-
-
-def show_config_file_contents():
-    print(get_config_file_contents())
-
-
-def show_config():
-    print(Config.to_string(exclude=["config_file_contents"]))
-
-
-def check():
-    Config.check_data_files()

hpcflow_new2-0.2.0a37/hpcflow/sdk/log.py

@@ -1,26 +0,0 @@
-import logging
-
-
-class AppLog:
-
-    DEFAULT_LOG_LEVEL_CONSOLE = "WARNING"
-    DEFAULT_LOG_LEVEL_FILE = "INFO"
-    DEFAULT_LOG_FILE_PATH = "app.log"
-
-    def __init__(self, app, log_console_level=None):
-        self.app = app
-        self.logger = logging.getLogger(app.package_name)
-        self.logger.setLevel(logging.DEBUG)
-        self.console_handler = self._add_console_logger(
-            level=log_console_level or AppLog.DEFAULT_LOG_LEVEL_CONSOLE
-        )
-        # print(f"log_console_level: {log_console_level}")
-
-    def _add_console_logger(self, level, fmt=None):
-        if not fmt:
-            fmt = "%(levelname)s %(name)s: %(message)s"
-        handler = logging.StreamHandler()
-        handler.setFormatter(logging.Formatter(fmt))
-        handler.setLevel(level)
-        self.logger.addHandler(handler)
-        return handler