hpcflow 0.1.9__py3-none-any.whl → 0.2.0a271__py3-none-any.whl

hpcflow/sdk/helper/watcher.py

@@ -0,0 +1,192 @@
+"""
+File-system watcher classes.
+"""
+
+from __future__ import annotations
+from collections.abc import Callable
+from datetime import timedelta
+from logging import Logger
+from pathlib import Path
+from typing import cast
+from watchdog.observers.polling import PollingObserver
+from watchdog.events import (
+    FileSystemEvent,
+    FileSystemEventHandler,
+    PatternMatchingEventHandler,
+)
+
+
+class _PMEHDelegate(PatternMatchingEventHandler):
+    def __init__(self, pattern: str, on_modified: Callable[[FileSystemEvent], None]):
+        super().__init__(patterns=[pattern])
+        self.__on_modified = on_modified
+
+    def on_modified(self, event: FileSystemEvent) -> None:
+        self.__on_modified(event)
+
+
+class _FSEHDelegate(FileSystemEventHandler):
+    def __init__(self, on_modified: Callable[[FileSystemEvent], None]) -> None:
+        self.__on_modified = on_modified
+
+    def on_modified(self, event: FileSystemEvent) -> None:
+        self.__on_modified(event)
+
+
+class MonitorController:
+    """
+    Controller for tracking watch files.
+    """
+
+    def __init__(
+        self,
+        workflow_dirs_file_path: str | Path,
+        watch_interval: float | timedelta,
+        logger: Logger,
+    ):
+        if isinstance(watch_interval, timedelta):
+            self.watch_interval = int(watch_interval.total_seconds())
+        else:
+            self.watch_interval = int(watch_interval)
+        self.workflow_dirs_file_path = Path(workflow_dirs_file_path).absolute()
+        self.logger = logger
+
+        if not self.workflow_dirs_file_path.exists():
+            self.logger.info(
+                f"Watch file does not exist; creating {str(self.workflow_dirs_file_path)}."
+            )
+            with self.workflow_dirs_file_path.open("wt") as fp:
+                fp.write("\n")
+
+        self.logger.info(f"Watching file: {str(self.workflow_dirs_file_path)}")
+
+        self.event_handler = _PMEHDelegate("watch_workflows.txt", self.on_modified)
+
+        self.observer = PollingObserver(timeout=self.watch_interval)
+        self.observer.schedule(
+            self.event_handler,
+            path=cast("str", self.workflow_dirs_file_path.parent),
+            recursive=False,
+        )
+
+        self.observer.start()
+
+        workflow_paths = self.parse_watch_workflows_file(
+            self.workflow_dirs_file_path, logger=self.logger
+        )
+        self.workflow_monitor = WorkflowMonitor(
+            workflow_paths,
+            watch_interval=self.watch_interval,
+            logger=self.logger,
+        )
+
+    @staticmethod
+    def parse_watch_workflows_file(
+        path: str | Path, logger: Logger
+    ) -> list[dict[str, Path]]:
+        """
+        Parse the file describing what workflows to watch.
+        """
+        # TODO: and parse element IDs as well; and record which are set/unset.
+        with Path(path).open("rt") as fp:
+            lns = fp.readlines()
+
+        wks: list[dict[str, Path]] = []
+        for ln in lns:
+            ln_s = ln.strip()
+            if not ln_s:
+                continue
+            wk_path = Path(ln_s).absolute()
+            if not wk_path.is_dir():
+                logger.warning(f"{str(wk_path)} is not a workflow")
+                continue
+
+            wks.append(
+                {
+                    "path": wk_path,
+                }
+            )
+
+        return wks
+
+    def on_modified(self, event: FileSystemEvent):
+        """
+        Callback when files are modified.
+        """
+        self.logger.info(f"Watch file modified: {event.src_path!r}")
+        wks = self.parse_watch_workflows_file(
+            cast("str", event.src_path), logger=self.logger
+        )
+        self.workflow_monitor.update_workflow_paths(wks)
+
+    def join(self) -> None:
+        """
+        Join the worker thread.
+        """
+        self.observer.join()
+
+    def stop(self) -> None:
+        """
+        Stop this monitor.
+        """
+        self.observer.stop()
+        self.observer.join()  # wait for it to stop!
+        self.workflow_monitor.stop()
+
+
+class WorkflowMonitor:
+    """
+    Workflow monitor.
+    """
+
+    def __init__(
+        self,
+        workflow_paths: list[dict[str, Path]],
+        watch_interval: float | timedelta,
+        logger: Logger,
+    ):
+        if isinstance(watch_interval, timedelta):
+            self.watch_interval = int(watch_interval.total_seconds())
+        else:
+            self.watch_interval = int(watch_interval)
+
+        self.event_handler = _FSEHDelegate(self.on_modified)
+        self.workflow_paths = workflow_paths
+        self.logger = logger
+
+        self._monitor_workflow_paths()
+
+    def _monitor_workflow_paths(self) -> None:
+        observer = PollingObserver(timeout=self.watch_interval)
+        self.observer: PollingObserver | None = observer
+        for i in self.workflow_paths:
+            observer.schedule(
+                self.event_handler, path=cast("str", i["path"]), recursive=False
+            )
+            self.logger.info(f"Watching workflow: {i['path'].name}")
+
+        observer.start()
+
+    def on_modified(self, event: FileSystemEvent):
+        """
+        Triggered on a workflow being modified.
+        """
+        self.logger.info(f"Workflow modified: {event.src_path!r}")
+
+    def update_workflow_paths(self, new_paths: list[dict[str, Path]]):
+        """
+        Change the set of paths to monitored workflows.
+        """
+        self.logger.info("Updating watched workflows.")
+        self.stop()
+        self.workflow_paths = new_paths
+        self._monitor_workflow_paths()
+
+    def stop(self) -> None:
+        """
+        Stop this monitor.
+        """
+        if self.observer:
+            self.observer.stop()
+            self.observer.join()  # wait for it to stop!
+            self.observer = None

hpcflow/sdk/log.py

@@ -0,0 +1,288 @@
+"""
+Interface to the standard logger, and performance logging utility.
+"""
+
+from __future__ import annotations
+from functools import wraps
+import logging
+import logging.handlers
+from pathlib import Path
+import time
+from collections import defaultdict
+from collections.abc import Callable, Sequence
+import statistics
+from dataclasses import dataclass
+from typing import ClassVar, TypeVar, TYPE_CHECKING
+from typing_extensions import ParamSpec
+
+if TYPE_CHECKING:
+    from .app import BaseApp
+
+
+P = ParamSpec("P")
+T = TypeVar("T")
+
+
+@dataclass
+class _Summary:
+    """
+    Summary of a particular node's execution time.
+    """
+
+    number: int
+    mean: float
+    stddev: float
+    min: float
+    max: float
+    sum: float
+    children: dict[tuple[str, ...], _Summary]
+
+
+class TimeIt:
+    """
+    Method execution time instrumentation.
+    """
+
+    #: Whether the instrumentation is active.
+    active: ClassVar = False
+    #: Where to log to.
+    file_path: ClassVar[str | None] = None
+    #: The details be tracked.
+    timers: ClassVar[dict[tuple[str, ...], list[float]]] = defaultdict(list)
+    #: Traces of the stack.
+    trace: ClassVar[list[str]] = []
+    #: Trace indices.
+    trace_idx: ClassVar[list[int]] = []
+    #: Preceding traces.
+    trace_prev: ClassVar[list[str]] = []
+    #: Preceding trace indices.
+    trace_idx_prev: ClassVar[list[int]] = []
+
+    def __enter__(self):
+        self.__class__.active = True
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        try:
+            self.__class__.summarise_string()
+        finally:
+            self.__class__.reset()
+            self.__class__.active = False
+
+    @classmethod
+    def decorator(cls, func: Callable[P, T]) -> Callable[P, T]:
+        """
+        Decorator for a method that is to have its execution time monitored.
+        """
+
+        @wraps(func)
+        def wrapper(*args, **kwargs) -> T:
+            if not cls.active:
+                return func(*args, **kwargs)
+
+            cls.trace.append(func.__qualname__)
+
+            if cls.trace_prev == cls.trace:
+                new_trace_idx = cls.trace_idx_prev[-1] + 1
+            else:
+                new_trace_idx = 0
+            cls.trace_idx.append(new_trace_idx)
+
+            tic = time.perf_counter()
+            out = func(*args, **kwargs)
+            toc = time.perf_counter()
+            elapsed = toc - tic
+
+            cls.timers[tuple(cls.trace)].append(elapsed)
+
+            cls.trace_prev = list(cls.trace)
+            cls.trace_idx_prev = list(cls.trace_idx)
+
+            cls.trace.pop()
+            cls.trace_idx.pop()
+
+            return out
+
+        return wrapper
+
+    @classmethod
+    def _summarise(cls) -> dict[tuple[str, ...], _Summary]:
+        """
+        Produce a machine-readable summary of method execution time statistics.
+        """
+        stats = {
+            k: _Summary(
+                len(v),
+                statistics.mean(v),
+                statistics.pstdev(v),
+                min(v),
+                max(v),
+                sum(v),
+                {},
+            )
+            for k, v in cls.timers.items()
+        }
+
+        # make a graph
+        for key in sorted(stats, key=lambda x: len(x), reverse=True):
+            if len(key) == 1:
+                continue
+            value = stats.pop(key)
+            parent_key = key[:-1]
+            if parent_key in stats:
+                stats[parent_key].children[key] = value
+
+        return stats
+
+    @classmethod
+    def summarise_string(cls) -> None:
+        """
+        Produce a human-readable summary of method execution time statistics.
+        """
+
+        def _format_nodes(
+            node: dict[tuple[str, ...], _Summary],
+            depth: int = 0,
+            depth_final: Sequence[bool] = (),
+        ):
+            for idx, (k, v) in enumerate(node.items()):
+                is_final_child = idx == len(node) - 1
+                angle = "└ " if is_final_child else "├ "
+                bars = ""
+                if depth > 0:
+                    bars = "".join(f"{'│ ' if not i else '  '}" for i in depth_final)
+                k_str = bars + (angle if depth > 0 else "") + f"{k[depth]}"
+                min_str = f"{v.min:10.6f}" if v.number > 1 else f"{f'-':^12s}"
+                max_str = f"{v.max:10.6f}" if v.number > 1 else f"{f'-':^12s}"
+                stddev_str = f"({v.stddev:8.6f})" if v.number > 1 else f"{f' ':^10s}"
+                out.append(
+                    f"{k_str:.<80s} {v.sum:12.6f} "
+                    f"{v.mean:10.6f} {stddev_str} {v.number:8d} "
+                    f"{min_str} {max_str} "
+                )
+                depth_final_next = list(depth_final)
+                if depth > 0:
+                    depth_final_next.append(is_final_child)
+                _format_nodes(v.children, depth + 1, depth_final_next)
+
+        summary = cls._summarise()
+
+        out = [
+            f"{'function':^80s} {'sum /s':^12s} {'mean (stddev) /s':^20s} {'N':^8s} "
+            f"{'min /s':^12s} {'max /s':^12s}"
+        ]
+        _format_nodes(summary)
+        out_str = "\n".join(out)
+        if cls.file_path:
+            Path(cls.file_path).write_text(out_str, encoding="utf-8")
+        else:
+            print(out_str)
+
+    @classmethod
+    def reset(cls):
+        cls.timers = defaultdict(list)
+        cls.trace = []
+        cls.trace_idx = []
+        cls.trace_prev = []
+        cls.trace_idx_prev = []
+
+
+class AppLog:
+    """
+    Application log control.
+    """
+
+    #: Default logging level for the console.
+    DEFAULT_LOG_CONSOLE_LEVEL: ClassVar = "WARNING"
+    #: Default logging level for log files.
+    DEFAULT_LOG_FILE_LEVEL: ClassVar = "WARNING"
+
+    def __init__(self, app: BaseApp, log_console_level: str | None = None) -> None:
+        #: The application context.
+        self._app = app
+        #: The base logger for the application.
+        self.logger = logging.getLogger(app.package_name)
+        self.logger.setLevel(logging.WARNING)
+        #: The handler for directing logging messages to the console.
+        self.console_handler = self.__add_console_logger(
+            level=log_console_level or AppLog.DEFAULT_LOG_CONSOLE_LEVEL
+        )
+        self.file_handler: logging.FileHandler | None = None
+
+    def _ensure_logger_level(self):
+        """Ensure the logger's level is set to a level that triggers the handlers.
+
+        Notes
+        -----
+        Previously, we fixed the logger to DEBUG, but we found other Python packages
+        could then trigger debug logs in hpcflow even though the handlers were set to e.g.
+        ERROR.
+
+        """
+        min_level = min((handler.level for handler in self.logger.handlers), default=0)
+        if self.logger.level != min_level:
+            self.logger.setLevel(min_level)
+
+    def __add_console_logger(self, level: str, fmt: str | None = None) -> logging.Handler:
+        fmt = fmt or "%(levelname)s %(name)s: %(message)s"
+        handler = logging.StreamHandler()
+        handler.setFormatter(logging.Formatter(fmt))
+        handler.setLevel(level)
+        self.logger.addHandler(handler)
+        self._ensure_logger_level()
+        return handler
+
+    def update_console_level(self, new_level: str | None = None) -> None:
+        """
+        Set the logging level for console messages.
+        """
+        new_level = new_level or AppLog.DEFAULT_LOG_CONSOLE_LEVEL
+        self.console_handler.setLevel(new_level.upper())
+        self._ensure_logger_level()
+
+    def update_file_level(self, new_level: str | None = None) -> None:
+        if self.file_handler:
+            new_level = new_level or AppLog.DEFAULT_LOG_FILE_LEVEL
+            self.file_handler.setLevel(new_level.upper())
+            self._ensure_logger_level()
+
+    def add_file_logger(
+        self,
+        path: str | Path,
+        level: str | None = None,
+        fmt: str | None = None,
+        max_bytes: int | None = None,
+        backup_count: int = 4,
+    ) -> None:
+        """
+        Add a log file.
+        """
+        path = Path(path)
+        fmt = fmt or "%(asctime)s %(levelname)s %(name)s: %(message)s"
+        level = level or AppLog.DEFAULT_LOG_FILE_LEVEL
+        max_bytes = max_bytes or int(50e6)
+
+        if not path.parent.is_dir():
+            self.logger.info(f"Generating log file parent directory: {path.parent!r}")
+            path.parent.mkdir(exist_ok=True, parents=True)
+
+        handler = logging.handlers.RotatingFileHandler(
+            filename=path,
+            maxBytes=max_bytes,
+            backupCount=backup_count,
+        )
+        handler.setFormatter(logging.Formatter(fmt))
+        handler.setLevel(level.upper())
+        self.logger.addHandler(handler)
+        self.file_handler = handler
+        self._ensure_logger_level()
+
+    def remove_file_handler(self) -> None:
+        """Remove the file handler."""
+        if self.file_handler:
+            self.logger.debug(
+                f"Removing file handler from the AppLog: {self.file_handler!r}."
+            )
+            self.logger.removeHandler(self.file_handler)
+            self.file_handler = None
+            self._ensure_logger_level()

hpcflow/sdk/persistence/__init__.py

@@ -0,0 +1,18 @@
+"""
+Workflow persistence subsystem.
+"""
+
+from __future__ import annotations
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from .base import PersistentStore
+
+
+def store_cls_from_str(store_format: str) -> type[PersistentStore]:
+    """
+    Get the class that implements the persistence store from its name.
+    """
+    from .discovery import store_cls_from_str as impl
+
+    return impl(store_format)