runem 0.0.29__py3-none-any.whl → 0.0.31__py3-none-any.whl

runem/hook_manager.py

@@ -0,0 +1,116 @@
+import typing
+from collections import defaultdict
+
+from runem.config_metadata import ConfigMetadata
+from runem.job import Job
+from runem.job_execute import job_execute
+from runem.log import log
+from runem.types import (
+    FilePathListLookup,
+    HookConfig,
+    HookName,
+    Hooks,
+    HooksStore,
+    JobConfig,
+)
+
+
+class HookManager:
+    hooks_store: HooksStore
+
+    def __init__(self, hooks: Hooks, verbose: bool) -> None:
+        self.hooks_store: HooksStore = defaultdict(list)
+        self.initialise_hooks(hooks, verbose)
+
+    @staticmethod
+    def is_valid_hook_name(hook_name: typing.Union[HookName, str]) -> bool:
+        """Returns True/False depending on hook-name validity."""
+        if isinstance(hook_name, str):
+            try:
+                HookName(hook_name)  # lookup by value
+                return True
+            except ValueError:
+                return False
+        # the type is a HookName
+        if not isinstance(hook_name, HookName):
+            return False
+        return True
+
+    def register_hook(
+        self, hook_name: HookName, hook_config: HookConfig, verbose: bool
+    ) -> None:
+        """Registers a hook_config to a specific hook-type."""
+        if not self.is_valid_hook_name(hook_name):
+            raise ValueError(f"Hook {hook_name} does not exist.")
+        self.hooks_store[hook_name].append(hook_config)
+        if verbose:
+            log(
+                f"hooks: registered hook for '{hook_name}', "
+                f"have {len(self.hooks_store[hook_name])}: "
+                f"{Job.get_job_name(hook_config)}"  # type: ignore[arg-type]
+            )
+
+    def deregister_hook(
+        self, hook_name: HookName, hook_config: HookConfig, verbose: bool
+    ) -> None:
+        """Deregisters a hook_config from a specific hook-type."""
+        if not (
+            hook_name in self.hooks_store and hook_config in self.hooks_store[hook_name]
+        ):
+            raise ValueError(f"Function not found in hook {hook_name}.")
+        self.hooks_store[hook_name].remove(hook_config)
+        if verbose:
+            log(
+                f"hooks: deregistered hooks for '{hook_name}', "
+                f"have {len(self.hooks_store[hook_name])}"
+            )
+
+    def invoke_hooks(
+        self,
+        hook_name: HookName,
+        config_metadata: ConfigMetadata,
+        **kwargs: typing.Any,
+    ) -> None:
+        """Invokes all functions registered to a specific hook."""
+        hooks: typing.List[HookConfig] = self.hooks_store.get(hook_name, [])
+        if config_metadata.args.verbose:
+            log(f"hooks: invoking {len(hooks)} hooks for '{hook_name}'")
+
+        hook_config: HookConfig
+        for hook_config in hooks:
+            job_config: JobConfig = {
+                "label": str(hook_name),
+                "ctx": None,
+                "when": {"phase": str(hook_name), "tags": {str(hook_name)}},
+            }
+            if "addr" in hook_config:
+                job_config["addr"] = hook_config["addr"]
+            if "command" in hook_config:
+                job_config["command"] = hook_config["command"]
+            file_lists: FilePathListLookup = defaultdict(list)
+            file_lists[str(hook_name)] = [__file__]
+            job_execute(
+                job_config,
+                running_jobs={},
+                config_metadata=config_metadata,
+                file_lists=file_lists,
+                **kwargs,
+            )
+
+        if config_metadata.args.verbose:
+            log(f"hooks: done invoking '{hook_name}'")
+
+    def initialise_hooks(self, hooks: Hooks, verbose: bool) -> None:
+        """Initialised the hook with the configured data."""
+        if verbose:
+            num_hooks: int = sum(len(hooks_list) for hooks_list in hooks.values())
+            if num_hooks:
+                log(f"hooks: initialising {num_hooks} hooks")
+        for hook_name in hooks:
+            hook: HookConfig
+            if verbose:
+                log(
+                    f"hooks:\tinitialising {len(hooks[hook_name])} hooks for '{hook_name}'"
+                )
+            for hook in hooks[hook_name]:
+                self.register_hook(hook_name, hook, verbose)

runem/job_execute.py

@@ -1,4 +1,3 @@
-import inspect
 import os
 import pathlib
 import typing
@@ -10,7 +9,7 @@ from runem.config_metadata import ConfigMetadata
 from runem.informative_dict import ReadOnlyInformativeDict
 from runem.job import Job
 from runem.job_wrapper import get_job_wrapper
-from runem.log import log
+from runem.log import error, log
 from runem.types import (
     FilePathListLookup,
     JobConfig,
@@ -27,6 +26,7 @@ def job_execute_inner(
     job_config: JobConfig,
     config_metadata: ConfigMetadata,
     file_lists: FilePathListLookup,
+    **kwargs: typing.Any,
 ) -> typing.Tuple[JobTiming, JobReturn]:
     """Wrapper for running a job inside a sub-process.
 
@@ -73,31 +73,26 @@ def job_execute_inner(
         os.chdir(root_path)
 
     start = timer()
-    func_signature = inspect.signature(function)
     if config_metadata.args.verbose:
         log(f"job: running: '{Job.get_job_name(job_config)}'")
     reports: JobReturn
     try:
-        if "args" in func_signature.parameters:
-            reports = function(  # type: ignore  # FIXME: which function do we have?
-                config_metadata.args, config_metadata.options, file_list
-            )
-        else:
-            reports = function(
-                options=ReadOnlyInformativeDict(config_metadata.options),  # type: ignore
-                file_list=file_list,
-                procs=config_metadata.args.procs,
-                root_path=root_path,
-                verbose=config_metadata.args.verbose,
-                # unpack useful data points from the job_config
-                label=Job.get_job_name(job_config),
-                job=job_config,
-                record_sub_job_time=_record_sub_job_time,
-            )
+        reports = function(
+            options=ReadOnlyInformativeDict(config_metadata.options),  # type: ignore
+            file_list=file_list,
+            procs=config_metadata.args.procs,
+            root_path=root_path,
+            verbose=config_metadata.args.verbose,
+            # unpack useful data points from the job_config
+            label=Job.get_job_name(job_config),
+            job=job_config,
+            record_sub_job_time=_record_sub_job_time,
+            **kwargs,
+        )
     except BaseException:  # pylint: disable=broad-exception-caught
         # log that we hit an error on this job and re-raise
         log(decorate=False)
-        log(f"job: ERROR: job '{Job.get_job_name(job_config)}' failed to complete!")
+        error(f"job: job '{Job.get_job_name(job_config)}' failed to complete!")
         # re-raise
         raise
 
@@ -114,6 +109,7 @@ def job_execute(
     running_jobs: typing.Dict[str, str],
     config_metadata: ConfigMetadata,
     file_lists: FilePathListLookup,
+    **kwargs: typing.Any,
 ) -> typing.Tuple[JobTiming, JobReturn]:
     """Thin-wrapper around job_execute_inner needed for mocking in tests.
 
@@ -121,6 +117,11 @@ def job_execute(
     """
     this_id: str = str(uuid.uuid4())
     running_jobs[this_id] = Job.get_job_name(job_config)
-    results = job_execute_inner(job_config, config_metadata, file_lists)
+    results = job_execute_inner(
+        job_config,
+        config_metadata,
+        file_lists,
+        **kwargs,
+    )
     del running_jobs[this_id]
     return results

runem/job_filter.py

@@ -35,7 +35,7 @@ def _should_filter_out_by_tags(
         if verbose:
             log(
                 (
-                    f"not running job '{job['label']}' because it doesn't have "
+                    f"not running job '{Job.get_job_name(job)}' because it doesn't have "
                     f"any of the following tags: {printable_set(tags)}"
                 )
             )
@@ -46,7 +46,7 @@ def _should_filter_out_by_tags(
         if verbose:
             log(
                 (
-                    f"not running job '{job['label']}' because it contains the "
+                    f"not running job '{Job.get_job_name(job)}' because it contains the "
                     f"following tags: {printable_set(has_tags_to_avoid)}"
                 )
             )

runem/job_runner_simple_command.py

@@ -5,6 +5,12 @@ from runem.run_command import run_command
 from runem.types import JobConfig
 
 
+def validate_simple_command(command_string: str) -> typing.List[str]:
+    # use shlex to handle parsing of the command string, a non-trivial problem.
+    split_command: typing.List[str] = shlex.split(command_string)
+    return split_command
+
+
 def job_runner_simple_command(
     **kwargs: typing.Any,
 ) -> None:
@@ -17,7 +23,7 @@ def job_runner_simple_command(
     command_string: str = job_config["command"]
 
     # use shlex to handle parsing of the command string, a non-trivial problem.
-    result = shlex.split(command_string)
+    result = validate_simple_command(command_string)
 
     # preserve quotes for consistent handling of strings and avoid the "word
     # splitting" problem for unix-like shells.

runem/job_wrapper.py

@@ -1,19 +1,25 @@
 import pathlib
 
-from runem.job_runner_simple_command import job_runner_simple_command
+from runem.job_runner_simple_command import (
+    job_runner_simple_command,
+    validate_simple_command,
+)
 from runem.job_wrapper_python import get_job_wrapper_py_func
-from runem.types import JobConfig, JobFunction
+from runem.types import JobFunction, JobWrapper
 
 
-def get_job_wrapper(job_config: JobConfig, cfg_filepath: pathlib.Path) -> JobFunction:
+def get_job_wrapper(job_wrapper: JobWrapper, cfg_filepath: pathlib.Path) -> JobFunction:
     """Given a job-description determines the job-runner, returning it as a function.
 
     NOTE: Side-effects: also re-addressed the job-config in the case of functions see
           get_job_function.
     """
-    if "command" in job_config:
+    if "command" in job_wrapper:
+        # validate that the command is "understandable" and usable.
+        command_string: str = job_wrapper["command"]
+        validate_simple_command(command_string)
         return job_runner_simple_command  # type: ignore # NO_COMMIT
 
     # if we do not have a simple command address assume we have just an addressed
     # function
-    return get_job_wrapper_py_func(job_config, cfg_filepath)
+    return get_job_wrapper_py_func(job_wrapper, cfg_filepath)

runem/job_wrapper_python.py

@@ -3,7 +3,7 @@ import sys
 from importlib.util import module_from_spec
 from importlib.util import spec_from_file_location as module_spec_from_file_location
 
-from runem.types import FunctionNotFound, JobConfig, JobFunction
+from runem.types import FunctionNotFound, JobFunction, JobWrapper
 
 
 def _load_python_function_from_module(
@@ -86,22 +86,22 @@ def _find_job_module(cfg_filepath: pathlib.Path, module_file_path: str) -> pathl
 
 
 def get_job_wrapper_py_func(
-    job_config: JobConfig, cfg_filepath: pathlib.Path
+    job_wrapper: JobWrapper, cfg_filepath: pathlib.Path
 ) -> JobFunction:
     """For a job, dynamically loads the associated python job-function.
 
     Side-effects: also re-addressed the job-config.
     """
-    function_to_load: str = job_config["addr"]["function"]
+    function_to_load: str = job_wrapper["addr"]["function"]
     try:
         module_file_path: pathlib.Path = _find_job_module(
-            cfg_filepath, job_config["addr"]["file"]
+            cfg_filepath, job_wrapper["addr"]["file"]
         )
     except FunctionNotFound as err:
         raise FunctionNotFound(
             (
-                f"Whilst loading job '{job_config['label']}' runem failed to find "
-                f"job.addr.file '{job_config['addr']['file']}' looking for "
+                "runem failed to find "
+                f"job.addr.file '{job_wrapper['addr']['file']}' looking for "
                 f"job.addr.function '{function_to_load}'"
             )
         ) from err
@@ -118,5 +118,5 @@ def get_job_wrapper_py_func(
     )
 
     # re-write the job-config file-path for the module with the one that worked
-    job_config["addr"]["file"] = str(module_file_path)
+    job_wrapper["addr"]["file"] = str(module_file_path)
     return function

runem/log.py

@@ -14,3 +14,11 @@ def log(msg: str = "", decorate: bool = True, end: typing.Optional[str] = None)
     # print in a blocking manner, waiting for system resources to free up if a
     # runem job is contending on stdout or similar.
     blocking_print(msg, end=end)
+
+
+def warn(msg: str) -> None:
+    log(f"WARNING: {msg}")
+
+
+def error(msg: str) -> None:
+    log(f"ERROR: {msg}")

runem/report.py

@@ -43,7 +43,7 @@ def _align_bar_graphs_workaround(original_text: str) -> str:
     return formatted_text
 
 
-def _replace_bar_characters(text: str, end_str: str, replace_char: str) -> str:
+def replace_bar_graph_characters(text: str, end_str: str, replace_char: str) -> str:
     """Replaces block characters in lines containing `end_str` with give char.
 
     Args:
@@ -56,16 +56,19 @@ def _replace_bar_characters(text: str, end_str: str, replace_char: str) -> str:
     """
     # Define the block character and its light shade replacement
     block_chars = (
-        "▏▎▋▊█▌▐▄▀─"  # Extend this string with any additional block characters you use
+        "▏▎▍▋▊▉█▌▐▄▀─"  # Extend this string with any additional block characters you use
+        "░·"  # also include the chars we might replace with for special bars
     )
 
     text_lines: typing.List[str] = text.split("\n")
 
     # Process each line, replacing block characters if `end_str` is present
     modified_lines = [
-        line.translate(str.maketrans(block_chars, replace_char * len(block_chars)))
-        if end_str in line
-        else line
+        (
+            line.translate(str.maketrans(block_chars, replace_char * len(block_chars)))
+            if end_str in line
+            else line
+        )
         for line in text_lines
     ]
 
@@ -74,12 +77,12 @@ def _replace_bar_characters(text: str, end_str: str, replace_char: str) -> str:
 
 def _semi_shade_phase_totals(text: str) -> str:
     light_shade_char = "░"
-    return _replace_bar_characters(text, "(user-time)", light_shade_char)
+    return replace_bar_graph_characters(text, "(user-time)", light_shade_char)
 
 
 def _dot_jobs(text: str) -> str:
     dot_char = "·"
-    return _replace_bar_characters(text, "(+)", dot_char)
+    return replace_bar_graph_characters(text, "(+)", dot_char)
 
 
 def _plot_times(
@@ -103,8 +106,8 @@ def _plot_times(
 
     for idx, phase in enumerate(phase_run_oder):
         not_last_phase: bool = idx < len(phase_run_oder) - 1
-        utf8_phase = "├" if not_last_phase else "└"
-        utf8_phase_group = "│" if not_last_phase else " "
+        utf8_phase = " ├" if not_last_phase else " └"
+        utf8_phase_group = " │" if not_last_phase else "  "
         # log(f"Phase '{phase}' jobs took:")
         phase_start_idx = len(labels)
 
@@ -121,9 +124,11 @@ def _plot_times(
 
     runem_app_timing: typing.List[JobTiming] = timing_data["_app"]
     job_metadata: JobTiming
-    for job_metadata in reversed(runem_app_timing):
+    for idx, job_metadata in enumerate(reversed(runem_app_timing)):
+        last_group: bool = idx == 0  # reverse sorted
+        utf8_group = "├" if not last_group else "└"
         job_label, job_time_total = job_metadata["job"]
-        labels.insert(0, f"├runem.{job_label}")
+        labels.insert(0, f"{utf8_group}runem.{job_label}")
         times.insert(0, job_time_total.total_seconds())
     labels.insert(0, "runem (total wall-clock)")
     times.insert(0, wall_clock_for_runem_main.total_seconds())
@@ -175,7 +180,7 @@ def _gen_jobs_report(
         utf8_job = "├" if not_last else "└"
         utf8_sub_jobs = "│" if not_last else " "
         job_label, job_time_total = job_timing["job"]
-        job_bar_label: str = f"{phase}.{job_label}"
+        job_bar_label: str = f"{job_label}"
         labels.append(f"{utf8_phase_group}{utf8_job}{job_bar_label}")
         times.append(job_time_total.total_seconds())
         job_time_sum += job_time_total
@@ -191,8 +196,7 @@ def _gen_jobs_report(
             if idx == len(sub_command_times) - 1:
                 sub_utf8 = "└"
             labels.append(
-                f"{utf8_phase_group}{utf8_sub_jobs}{sub_utf8}{job_bar_label}"
-                f".{sub_job_label} (+)"
+                f"{utf8_phase_group}{utf8_sub_jobs}{sub_utf8}{sub_job_label} (+)"
             )
             times.append(sub_job_time.total_seconds())
     return job_time_sum

runem/runem.py

@@ -34,18 +34,19 @@ from timeit import default_timer as timer
 from halo import Halo
 
 from runem.command_line import parse_args
-from runem.config import load_config
+from runem.config import load_project_config, load_user_configs
 from runem.config_metadata import ConfigMetadata
-from runem.config_parse import parse_config
+from runem.config_parse import load_config_metadata
 from runem.files import find_files
 from runem.job import Job
 from runem.job_execute import job_execute
 from runem.job_filter import filter_jobs
-from runem.log import log
+from runem.log import error, log, warn
 from runem.report import report_on_run
 from runem.types import (
     Config,
     FilePathListLookup,
+    HookName,
     JobReturn,
     JobRunMetadata,
     JobRunMetadatasByPhase,
@@ -68,8 +69,11 @@ def _determine_run_parameters(argv: typing.List[str]) -> ConfigMetadata:
     """
     config: Config
     cfg_filepath: pathlib.Path
-    config, cfg_filepath = load_config()
-    config_metadata: ConfigMetadata = parse_config(config, cfg_filepath)
+    config, cfg_filepath = load_project_config()
+    user_configs: typing.List[typing.Tuple[Config, pathlib.Path]] = load_user_configs()
+    config_metadata: ConfigMetadata = load_config_metadata(
+        config, cfg_filepath, user_configs, verbose=("--verbose" in argv)
+    )
 
     # Now we parse the cli arguments extending them with information from the
     # .runem.yml config.
@@ -263,18 +267,21 @@ def _process_jobs_by_phase(
         )
         if failure_exception is not None:
             if config_metadata.args.verbose:
-                log(f"ERROR: running phase {phase}: aborting run")
+                error(f"running phase {phase}: aborting run")
             return failure_exception
 
     # ALl phases completed aok.
     return None
 
 
+MainReturnType = typing.Tuple[
+    ConfigMetadata, JobRunMetadatasByPhase, typing.Optional[BaseException]
+]
+
+
 def _main(
     argv: typing.List[str],
-) -> typing.Tuple[
-    OrderedPhases, JobRunMetadatasByPhase, typing.Optional[BaseException]
-]:
+) -> MainReturnType:
     start = timer()
 
     config_metadata: ConfigMetadata = _determine_run_parameters(argv)
@@ -283,7 +290,10 @@ def _main(
     os.chdir(config_metadata.cfg_filepath.parent)
 
     file_lists: FilePathListLookup = find_files(config_metadata)
-    assert file_lists
+    if not file_lists:
+        warn("no files found")
+        return (config_metadata, {}, None)
+
     if config_metadata.args.verbose:
         log(f"found {len(file_lists)} batches, ", end="")
         for tag in sorted(file_lists.keys()):
@@ -322,7 +332,7 @@ def _main(
     phase_run_report: JobReturn = None
     phase_run_metadata: JobRunMetadata = (phase_run_timing, phase_run_report)
     job_run_metadatas["_app"].append(phase_run_metadata)
-    return config_metadata.phases, job_run_metadatas, failure_exception
+    return config_metadata, job_run_metadatas, failure_exception
 
 
 def timed_main(argv: typing.List[str]) -> None:
@@ -332,10 +342,11 @@ def timed_main(argv: typing.List[str]) -> None:
                are representative.
     """
     start = timer()
-    phase_run_oder: OrderedPhases
+    config_metadata: ConfigMetadata
     job_run_metadatas: JobRunMetadatasByPhase
     failure_exception: typing.Optional[BaseException]
-    phase_run_oder, job_run_metadatas, failure_exception = _main(argv)
+    config_metadata, job_run_metadatas, failure_exception = _main(argv)
+    phase_run_oder: OrderedPhases = config_metadata.phases
     end = timer()
     time_taken: timedelta = timedelta(seconds=end - start)
     wall_clock_time_saved: timedelta
@@ -354,6 +365,12 @@ def timed_main(argv: typing.List[str]) -> None:
         )
     )
 
+    config_metadata.hook_manager.invoke_hooks(
+        hook_name=HookName.ON_EXIT,
+        config_metadata=config_metadata,
+        wall_clock_time_saved=wall_clock_time_saved,
+    )
+
     if failure_exception is not None:
         # we got a failure somewhere, now that we've reported the timings we
         # re-raise.

runem/types.py

@@ -2,6 +2,7 @@ import argparse
 import pathlib
 import typing
 from datetime import timedelta
+from enum import Enum
 
 from runem.informative_dict import InformativeDict, ReadOnlyInformativeDict
 
@@ -26,6 +27,15 @@ ReportUrlInfo = typing.Tuple[ReportName, ReportUrl]
 ReportUrls = typing.List[ReportUrlInfo]
 
 
+class HookName(Enum):
+    # at exit
+    ON_EXIT = "on-exit"
+    # before all tasks are run, after config is read
+    # BEFORE_ALL = "before-all"
+    # after all tasks are done, before reporting
+    # AFTER_ALL = "after-all"
+
+
 class JobReturnData(typing.TypedDict, total=False):
     """A dict that defines job result to be reported to the user."""
 
@@ -125,7 +135,14 @@ class JobWhen(typing.TypedDict, total=False):
     phase: PhaseName  # the phase when the job should be run
 
 
-class JobConfig(typing.TypedDict, total=False):
+class JobWrapper(typing.TypedDict, total=False):
+    """A base-type for jobs, hooks, and things that can be invoked."""
+
+    addr: JobAddressConfig  # which callable to call
+    command: str  # a one-liner command to be run
+
+
+class JobConfig(JobWrapper, total=False):
     """A dict that defines a job to be run.
 
     It consists of the label, address, context and filter information
@@ -134,8 +151,6 @@ class JobConfig(typing.TypedDict, total=False):
     """
 
     label: JobName  # the name of the job
-    addr: JobAddressConfig  # which callable to call
-    command: str  # a one-liner command to be run
     ctx: typing.Optional[JobContextConfig]  # how to call the callable
     when: JobWhen  # when to call the job
 
@@ -193,6 +208,30 @@ class GlobalSerialisedConfig(typing.TypedDict):
     config: GlobalConfig
 
 
+class HookConfig(JobWrapper, total=False):
+    """Specification for hooks.
+
+    Like JobConfig with use addr or command to specify what to execute.
+    """
+
+    hook_name: HookName  # the hook for when this is called
+
+
+Hooks = typing.DefaultDict[HookName, typing.List[HookConfig]]
+
+# A dictionary to hold hooks, with hook names as keys
+HooksStore = typing.Dict[HookName, typing.List[HookConfig]]
+
+
+class HookSerialisedConfig(typing.TypedDict):
+    """Intended to make reading a config file easier.
+
+    Also, unlike JobSerialisedConfig, this type may not actually help readability.
+    """
+
+    hook: HookConfig
+
+
 class JobSerialisedConfig(typing.TypedDict):
     """Makes serialised configs easier to read.
 
@@ -204,6 +243,10 @@ class JobSerialisedConfig(typing.TypedDict):
     job: JobConfig
 
 
-ConfigNodes = typing.Union[GlobalSerialisedConfig, JobSerialisedConfig]
+ConfigNodes = typing.Union[
+    GlobalSerialisedConfig, JobSerialisedConfig, HookSerialisedConfig
+]
 # The config format as it is serialised to/from disk
 Config = typing.List[ConfigNodes]
+
+UserConfigMetadata = typing.List[typing.Tuple[Config, pathlib.Path]]