runem 0.0.28__py3-none-any.whl → 0.0.30__py3-none-any.whl

runem/VERSION

@@ -1 +1 @@
-0.0.28
+0.0.30

runem/cli.py

@@ -1,4 +1,5 @@
 """CLI interface for runem project."""
+
 import sys
 
 from runem.runem import timed_main

runem/command_line.py

@@ -6,7 +6,7 @@ import typing
 
 from runem.config_metadata import ConfigMetadata
 from runem.informative_dict import InformativeDict
-from runem.log import log
+from runem.log import error, log
 from runem.runem_version import get_runem_version
 from runem.types import JobNames, OptionConfig, OptionsWritable
 from runem.utils import printable_set
@@ -22,7 +22,12 @@ def parse_args(
 
     Returns the parsed args, the jobs_names_to_run, job_phases_to_run, job_tags_to_run
     """
-    parser = argparse.ArgumentParser(description="Runs the Lursight Lang test-suite")
+    parser = argparse.ArgumentParser(
+        add_help=False, description="Runs the Lursight Lang test-suite"
+    )
+    parser.add_argument(
+        "-H", "--help", action="help", help="show this help message and exit"
+    )
 
     job_group = parser.add_argument_group("jobs")
     all_job_names: JobNames = set(name for name in config_metadata.all_job_names)
@@ -112,6 +117,26 @@ def parse_args(
         required=False,
     )
 
+    parser.add_argument(
+        "-f",
+        "--modified-files-only",
+        dest="check_modified_files_only",
+        help="only use files that have changed",
+        action=argparse.BooleanOptionalAction,
+        default=False,
+        required=False,
+    )
+
+    parser.add_argument(
+        "-h",
+        "--git-head-files-only",
+        dest="check_head_files_only",
+        help="fast run of files",
+        action=argparse.BooleanOptionalAction,
+        default=False,
+        required=False,
+    )
+
     parser.add_argument(
         "--procs",
         "-j",
@@ -206,9 +231,9 @@ def _validate_filters(
     for name, name_list in (("--jobs", args.jobs), ("--not-jobs", args.jobs_excluded)):
         for job_name in name_list:
             if job_name not in config_metadata.all_job_names:
-                log(
+                error(
                     (
-                        f"ERROR: invalid job-name '{job_name}' for {name}, "
+                        f"invalid job-name '{job_name}' for {name}, "
                         f"choose from one of {printable_set(config_metadata.all_job_names)}"
                     )
                 )
@@ -218,9 +243,9 @@ def _validate_filters(
     for name, tag_list in (("--tags", args.tags), ("--not-tags", args.tags_excluded)):
         for tag in tag_list:
             if tag not in config_metadata.all_job_tags:
-                log(
+                error(
                     (
-                        f"ERROR: invalid tag '{tag}' for {name}, "
+                        f"invalid tag '{tag}' for {name}, "
                         f"choose from one of {printable_set(config_metadata.all_job_tags)}"
                     )
                 )
@@ -233,9 +258,9 @@ def _validate_filters(
     ):
         for phase in phase_list:
             if phase not in config_metadata.all_job_phases:
-                log(
+                error(
                     (
-                        f"ERROR: invalid phase '{phase}' for {name}, "
+                        f"invalid phase '{phase}' for {name}, "
                         f"choose from one of {printable_set(config_metadata.all_job_phases)}"
                     )
                 )

runem/config.py

@@ -5,9 +5,9 @@ import typing
 import yaml
 from packaging.version import Version
 
-from runem.log import log
+from runem.log import error, log
 from runem.runem_version import get_runem_version
-from runem.types import Config, GlobalConfig, GlobalSerialisedConfig
+from runem.types import Config, GlobalConfig, GlobalSerialisedConfig, UserConfigMetadata
 
 CFG_FILE_YAML = pathlib.Path(".runem.yml")
 
@@ -40,20 +40,53 @@ def _search_up_multiple_dirs_for_file(
     return None
 
 
-def _find_cfg() -> pathlib.Path:
-    """Searches up from the cwd for a .runem.yml config file."""
+def _find_config_file(
+    config_filename: typing.Union[str, pathlib.Path]
+) -> typing.Tuple[typing.Optional[pathlib.Path], typing.Tuple[pathlib.Path, ...]]:
+    """Searches up from the cwd for the given config file-name."""
     start_dirs = (pathlib.Path(".").absolute(),)
     cfg_candidate: typing.Optional[pathlib.Path] = _search_up_multiple_dirs_for_file(
-        start_dirs, CFG_FILE_YAML
+        start_dirs, config_filename
     )
+    return cfg_candidate, start_dirs
+
+
+def _find_project_cfg() -> pathlib.Path:
+    """Searches up from the cwd for the project .runem.yml config file."""
+    cfg_candidate: typing.Optional[pathlib.Path]
+    start_dirs: typing.Tuple[pathlib.Path, ...]
+    cfg_candidate, start_dirs = _find_config_file(config_filename=CFG_FILE_YAML)
+
     if cfg_candidate:
         return cfg_candidate
 
     # error out and exit as we currently require the cfg file as it lists jobs.
-    log(f"ERROR: Config not found! Looked from {start_dirs}")
+    error(f"Config not found! Looked from {start_dirs}")
     sys.exit(1)
 
 
+def _find_local_configs() -> typing.List[pathlib.Path]:
+    """Searches for all user-configs and returns the found ones.
+
+    TODO: add some priorities to the files, such that
+            - .runem.local.yml has lowest priority
+            - $HOME/.runem.user.yml is applied after .local
+            - .runem.user.yml overloads all others
+    """
+    local_configs: typing.List[pathlib.Path] = []
+    for config_filename in (".runem.local.yml", ".runem.user.yml"):
+        cfg_candidate: typing.Optional[pathlib.Path]
+        cfg_candidate, _ = _find_config_file(config_filename)
+        if cfg_candidate:
+            local_configs.append(cfg_candidate)
+
+    user_home_config: pathlib.Path = pathlib.Path("~/.runem.user.yml")
+    if user_home_config.exists():
+        local_configs.append(user_home_config)
+
+    return local_configs
+
+
 def _conform_global_config_types(
     all_config: Config,
 ) -> typing.Tuple[Config, typing.Optional[GlobalConfig]]:
@@ -77,9 +110,8 @@ def _conform_global_config_types(
     return all_config, global_config
 
 
-def load_config() -> typing.Tuple[Config, pathlib.Path]:
-    """Finds and loads the .runem.yml file."""
-    cfg_filepath: pathlib.Path = _find_cfg()
+def load_and_parse_config(cfg_filepath: pathlib.Path) -> Config:
+    """For the given config file pass, project or user, load it & parse/conform it."""
     with cfg_filepath.open("r+", encoding="utf-8") as config_file_handle:
         all_config = yaml.full_load(config_file_handle)
 
@@ -104,5 +136,22 @@ def load_config() -> typing.Tuple[Config, pathlib.Path]:
                 )
             )
             sys.exit(1)
+    return conformed_config
+
+
+def load_project_config() -> typing.Tuple[Config, pathlib.Path]:
+    """Finds and loads the .runem.yml file for the current project."""
+    cfg_filepath: pathlib.Path = _find_project_cfg()
+    conformed_config: Config = load_and_parse_config(cfg_filepath)
 
     return conformed_config, cfg_filepath
+
+
+def load_user_configs() -> UserConfigMetadata:
+    """Returns the user-local configs, that extend/override runem behaviour."""
+    user_configs: typing.List[typing.Tuple[Config, pathlib.Path]] = []
+    user_config_paths: typing.List[pathlib.Path] = _find_local_configs()
+    for config_path in user_config_paths:
+        user_config: Config = load_and_parse_config(config_path)
+        user_configs.append((user_config, config_path))
+    return user_configs

runem/config_metadata.py

@@ -1,5 +1,6 @@
 import argparse
 import pathlib
+import typing
 
 from runem.informative_dict import InformativeDict
 from runem.types import (
@@ -13,6 +14,9 @@ from runem.types import (
     TagFileFilters,
 )
 
+if typing.TYPE_CHECKING:  # pragma: no cover
+    from runem.hook_manager import HookManager
+
 
 class ConfigMetadata:
     """Full metadata about what can and should be run."""
@@ -20,6 +24,7 @@ class ConfigMetadata:
     phases: OrderedPhases  # the phases and orders to run them in
     options_config: OptionConfigs  # the options to add to the cli and pass to jobs
     file_filters: TagFileFilters  # which files to get for which tag
+    hook_manager: "HookManager"  # the hooks register for the run
     jobs: PhaseGroupedJobs  # the jobs to be run ordered by phase
     all_job_names: JobNames  # the set of job-names
     all_job_phases: JobPhases  # the set of job-phases (should be subset of 'phases')
@@ -39,6 +44,7 @@ class ConfigMetadata:
         phases: OrderedPhases,
         options_config: OptionConfigs,
         file_filters: TagFileFilters,
+        hook_manager: "HookManager",
         jobs: PhaseGroupedJobs,
         all_job_names: JobNames,
         all_job_phases: JobPhases,
@@ -48,6 +54,8 @@ class ConfigMetadata:
         self.phases = phases
         self.options_config = options_config
         self.file_filters = file_filters
+        # initialise all the registered call-back hooks
+        self.hook_manager = hook_manager
         self.jobs = jobs
         self.all_job_names = all_job_names
         self.all_job_phases = all_job_phases

runem/config_parse.py

@@ -6,14 +6,20 @@ from collections import defaultdict
 from collections.abc import Iterable
 
 from runem.config_metadata import ConfigMetadata
+from runem.hook_manager import HookManager
 from runem.job import Job
 from runem.job_wrapper import get_job_wrapper
-from runem.log import log
+from runem.log import error, log, warn
 from runem.types import (
     Config,
     ConfigNodes,
+    FunctionNotFound,
     GlobalConfig,
     GlobalSerialisedConfig,
+    HookConfig,
+    HookName,
+    Hooks,
+    HookSerialisedConfig,
     JobConfig,
     JobNames,
     JobPhases,
@@ -61,7 +67,31 @@ def _parse_global_config(
     return phases, options, file_filters
 
 
-def _parse_job(
+def parse_hook_config(
+    hook: HookConfig,
+    cfg_filepath: pathlib.Path,
+) -> None:
+    """Get the hook information, verifying validity."""
+    try:
+        if not HookManager.is_valid_hook_name(hook["hook_name"]):
+            raise ValueError(
+                f"invalid hook-name '{str(hook['hook_name'])}'. "
+                f"Valid hook names are: {[hook.value for hook in HookName]}"
+            )
+        # cast the hook-name to a HookName type
+        hook["hook_name"] = HookName(hook["hook_name"])
+        get_job_wrapper(hook, cfg_filepath)
+    except KeyError as err:
+        raise ValueError(
+            f"hook config entry is missing '{err.args[0]}' key. Have {tuple(hook.keys())}"
+        ) from err
+    except FunctionNotFound as err:
+        raise FunctionNotFound(
+            f"Whilst loading job '{str(hook['hook_name'])}'. {str(err)}"
+        ) from err
+
+
+def _parse_job(  # noqa: C901
     cfg_filepath: pathlib.Path,
     job: JobConfig,
     in_out_tags: JobTags,
@@ -69,22 +99,43 @@ def _parse_job(
     in_out_job_names: JobNames,
     in_out_phases: JobPhases,
     phase_order: OrderedPhases,
+    warn_missing_phase: bool = True,
 ) -> None:
     """Parse an individual job."""
     job_name: str = Job.get_job_name(job)
     job_names_used = job_name in in_out_job_names
     if job_names_used:
-        log("ERROR: duplicate job label!")
-        log(f"\t'{job['label']}' is used twice or more in {str(cfg_filepath)}")
+        error(
+            "duplicate job label!"
+            f"\t'{job['label']}' is used twice or more in {str(cfg_filepath)}"
+        )
         sys.exit(1)
 
-    # try and load the function _before_ we schedule it's execution
-    get_job_wrapper(job, cfg_filepath)
+    try:
+        # try and load the function _before_ we schedule it's execution
+        get_job_wrapper(job, cfg_filepath)
+    except FunctionNotFound as err:
+        raise FunctionNotFound(
+            f"Whilst loading job '{job['label']}'. {str(err)}"
+        ) from err
+
     try:
         phase_id: PhaseName = job["when"]["phase"]
     except KeyError:
-        fallback_phase = phase_order[0]
-        log(f"WARNING: no phase found for '{job_name}', using '{fallback_phase}'")
+        try:
+            fallback_phase = phase_order[0]
+            if warn_missing_phase:
+                warn(f"no phase found for '{job_name}', using '{fallback_phase}'")
+        except IndexError:
+            fallback_phase = "<NO PHASES FOUND>"
+            if warn_missing_phase:
+                warn(
+                    (
+                        f"no phases found for '{job_name}', "
+                        f"or in '{str(cfg_filepath)}', "
+                        f"using '{fallback_phase}'"
+                    )
+                )
         phase_id = fallback_phase
     in_out_jobs_by_phase[phase_id].append(job)
 
@@ -169,7 +220,18 @@ def parse_job_config(
         ) from err
 
 
-def parse_config(config: Config, cfg_filepath: pathlib.Path) -> ConfigMetadata:
+def parse_config(
+    config: Config, cfg_filepath: pathlib.Path, hooks_only: bool = False
+) -> typing.Tuple[
+    Hooks,  # hooks:
+    OrderedPhases,  # phase_order:
+    OptionConfigs,  # options:
+    TagFileFilters,  # file_filters:
+    PhaseGroupedJobs,  # jobs_by_phase:
+    JobNames,  # job_names:
+    JobPhases,  # job_phases:
+    JobTags,  # tags:
+]:
     """Validates and restructure the config to make it more convenient to use."""
     jobs_by_phase: PhaseGroupedJobs = defaultdict(list)
     job_names: JobNames = set()
@@ -180,6 +242,7 @@ def parse_config(config: Config, cfg_filepath: pathlib.Path) -> ConfigMetadata:
     phase_order: OrderedPhases = ()
     options: OptionConfigs = ()
     file_filters: TagFileFilters = {}
+    hooks: Hooks = defaultdict(list)
 
     # first search for the global config
     for entry in config:
@@ -204,12 +267,28 @@ def parse_config(config: Config, cfg_filepath: pathlib.Path) -> ConfigMetadata:
             phase_order, options, file_filters = _parse_global_config(global_config)
             continue
 
+        # we apply a type-ignore here as we know (for now) that jobs have "job"
+        # keys and global configs have "global" keys
+        isinstance_hooks: bool = "hook" in entry
+        if isinstance_hooks:
+            hook_entry: HookSerialisedConfig = entry  # type: ignore  # see above
+            hook: HookConfig = hook_entry["hook"]
+            parse_hook_config(hook, cfg_filepath)
+
+            # if we get here we have validated the hook, add it to the hooks list
+            hook_name: HookName = hook["hook_name"]
+            hooks[hook_name].append(hook)
+
+            # continue to the next element and do NOT error
+            continue
+
         # not a global or a job entry, what is it
-        raise RuntimeError(f"invalid 'job' or 'global' config entry, {entry}")
+        raise RuntimeError(f"invalid 'job', 'hook, or 'global' config entry, {entry}")
 
     if not phase_order:
-        log("WARNING: phase ordering not configured! Runs will be non-deterministic!")
-        phase_order = tuple(job_phases)
+        if not hooks_only:
+            warn("phase ordering not configured! Runs will be non-deterministic!")
+            phase_order = tuple(job_phases)
 
     # now parse out the job_configs
     for entry in config:
@@ -228,13 +307,109 @@ def parse_config(config: Config, cfg_filepath: pathlib.Path) -> ConfigMetadata:
             in_out_phases=job_phases,
             phase_order=phase_order,
         )
+    return (
+        hooks,
+        phase_order,
+        options,
+        file_filters,
+        jobs_by_phase,
+        job_names,
+        job_phases,
+        tags,
+    )
+
 
-    # tags = tags.union(("python", "es", "firebase_funcs"))
+def generate_config(
+    cfg_filepath: pathlib.Path,
+    hooks: Hooks,
+    phase_order: OrderedPhases,
+    verbose: bool,
+    options: OptionConfigs,
+    file_filters: TagFileFilters,
+    jobs_by_phase: PhaseGroupedJobs,
+    job_names: JobNames,
+    job_phases: JobPhases,
+    tags: JobTags,
+) -> ConfigMetadata:
+    """Constructs the ConfigMetadata from parsed config parts."""
     return ConfigMetadata(
         cfg_filepath,
         phase_order,
         options,
         file_filters,
+        HookManager(hooks, verbose),
+        jobs_by_phase,
+        job_names,
+        job_phases,
+        tags,
+    )
+
+
+def _load_user_hooks_from_config(
+    user_config: Config, cfg_filepath: pathlib.Path
+) -> Hooks:
+    hooks: Hooks
+    (
+        hooks,
+        _,
+        _,
+        _,
+        _,
+        _,
+        _,
+        _,
+    ) = parse_config(user_config, cfg_filepath, hooks_only=True)
+    return hooks
+
+
+def load_config_metadata(
+    config: Config,
+    cfg_filepath: pathlib.Path,
+    user_configs: typing.List[typing.Tuple[Config, pathlib.Path]],
+    verbose: bool = False,
+) -> ConfigMetadata:
+    hooks: Hooks
+    phase_order: OrderedPhases
+    options: OptionConfigs
+    file_filters: TagFileFilters
+    jobs_by_phase: PhaseGroupedJobs
+    job_names: JobNames
+    job_phases: JobPhases
+    tags: JobTags
+    (
+        hooks,
+        phase_order,
+        options,
+        file_filters,
+        jobs_by_phase,
+        job_names,
+        job_phases,
+        tags,
+    ) = parse_config(config, cfg_filepath)
+
+    user_config: Config
+    user_config_path: pathlib.Path
+    for user_config, user_config_path in user_configs:
+        user_hooks: Hooks = _load_user_hooks_from_config(user_config, user_config_path)
+        if user_hooks:
+            if verbose:
+                log(f"hooks: loading user hooks from '{str(user_config_path)}'")
+        hook_name: HookName
+        hooks_for_name: typing.List[HookConfig]
+        for hook_name, hooks_for_name in user_hooks.items():
+            hooks[hook_name].extend(hooks_for_name)
+            if verbose:
+                log(
+                    f"hooks:\tadded {len(hooks_for_name)} user hooks for '{str(hook_name)}'"
+                )
+
+    return generate_config(
+        cfg_filepath,
+        hooks,
+        phase_order,
+        verbose,
+        options,
+        file_filters,
         jobs_by_phase,
         job_names,
         job_phases,

runem/files.py

@@ -1,6 +1,7 @@
 import re
 import typing
 from collections import defaultdict
+from pathlib import Path
 from subprocess import check_output as subprocess_check_output
 
 from runem.config_metadata import ConfigMetadata
@@ -23,14 +24,38 @@ def find_files(config_metadata: ConfigMetadata) -> FilePathListLookup:
     """
     file_lists: FilePathListLookup = defaultdict(list)
 
-    file_paths: typing.List[str] = (
-        subprocess_check_output(
-            "git ls-files",
-            shell=True,
+    file_paths: typing.List[str] = []
+
+    if config_metadata.args.check_modified_files_only:
+        file_paths = (
+            subprocess_check_output(
+                "git diff --name-only",
+                shell=True,
+            )
+            .decode("utf-8")
+            .splitlines()
+        )
+    elif config_metadata.args.check_head_files_only:
+        # Fetching modified and added files from the HEAD commit that are still on disk
+        file_paths = (
+            subprocess_check_output(
+                "git diff-tree --no-commit-id --name-only -r HEAD",
+                shell=True,
+            )
+            .decode("utf-8")
+            .splitlines()
+        )
+        file_paths = [file_path for file_path in file_paths if Path(file_path).exists()]
+    else:
+        # fall-back to all files
+        file_paths = (
+            subprocess_check_output(
+                "git ls-files",
+                shell=True,
+            )
+            .decode("utf-8")
+            .splitlines()
         )
-        .decode("utf-8")
-        .splitlines()
-    )
     _bucket_file_by_tag(
         file_paths,
         config_metadata,

runem/hook_manager.py

@@ -0,0 +1,116 @@
+import typing
+from collections import defaultdict
+
+from runem.config_metadata import ConfigMetadata
+from runem.job import Job
+from runem.job_execute import job_execute
+from runem.log import log
+from runem.types import (
+    FilePathListLookup,
+    HookConfig,
+    HookName,
+    Hooks,
+    HooksStore,
+    JobConfig,
+)
+
+
+class HookManager:
+    hooks_store: HooksStore
+
+    def __init__(self, hooks: Hooks, verbose: bool) -> None:
+        self.hooks_store: HooksStore = defaultdict(list)
+        self.initialise_hooks(hooks, verbose)
+
+    @staticmethod
+    def is_valid_hook_name(hook_name: typing.Union[HookName, str]) -> bool:
+        """Returns True/False depending on hook-name validity."""
+        if isinstance(hook_name, str):
+            try:
+                HookName(hook_name)  # lookup by value
+                return True
+            except ValueError:
+                return False
+        # the type is a HookName
+        if not isinstance(hook_name, HookName):
+            return False
+        return True
+
+    def register_hook(
+        self, hook_name: HookName, hook_config: HookConfig, verbose: bool
+    ) -> None:
+        """Registers a hook_config to a specific hook-type."""
+        if not self.is_valid_hook_name(hook_name):
+            raise ValueError(f"Hook {hook_name} does not exist.")
+        self.hooks_store[hook_name].append(hook_config)
+        if verbose:
+            log(
+                f"hooks: registered hook for '{hook_name}', "
+                f"have {len(self.hooks_store[hook_name])}: "
+                f"{Job.get_job_name(hook_config)}"  # type: ignore[arg-type]
+            )
+
+    def deregister_hook(
+        self, hook_name: HookName, hook_config: HookConfig, verbose: bool
+    ) -> None:
+        """Deregisters a hook_config from a specific hook-type."""
+        if not (
+            hook_name in self.hooks_store and hook_config in self.hooks_store[hook_name]
+        ):
+            raise ValueError(f"Function not found in hook {hook_name}.")
+        self.hooks_store[hook_name].remove(hook_config)
+        if verbose:
+            log(
+                f"hooks: deregistered hooks for '{hook_name}', "
+                f"have {len(self.hooks_store[hook_name])}"
+            )
+
+    def invoke_hooks(
+        self,
+        hook_name: HookName,
+        config_metadata: ConfigMetadata,
+        **kwargs: typing.Any,
+    ) -> None:
+        """Invokes all functions registered to a specific hook."""
+        hooks: typing.List[HookConfig] = self.hooks_store.get(hook_name, [])
+        if config_metadata.args.verbose:
+            log(f"hooks: invoking {len(hooks)} hooks for '{hook_name}'")
+
+        hook_config: HookConfig
+        for hook_config in hooks:
+            job_config: JobConfig = {
+                "label": str(hook_name),
+                "ctx": None,
+                "when": {"phase": str(hook_name), "tags": {str(hook_name)}},
+            }
+            if "addr" in hook_config:
+                job_config["addr"] = hook_config["addr"]
+            if "command" in hook_config:
+                job_config["command"] = hook_config["command"]
+            file_lists: FilePathListLookup = defaultdict(list)
+            file_lists[str(hook_name)] = [__file__]
+            job_execute(
+                job_config,
+                running_jobs={},
+                config_metadata=config_metadata,
+                file_lists=file_lists,
+                **kwargs,
+            )
+
+        if config_metadata.args.verbose:
+            log(f"hooks: done invoking '{hook_name}'")
+
+    def initialise_hooks(self, hooks: Hooks, verbose: bool) -> None:
+        """Initialised the hook with the configured data."""
+        if verbose:
+            num_hooks: int = sum(len(hooks_list) for hooks_list in hooks.values())
+            if num_hooks:
+                log(f"hooks: initialising {num_hooks} hooks")
+        for hook_name in hooks:
+            hook: HookConfig
+            if verbose:
+                log(
+                    f"hooks:\tinitialising {len(hooks[hook_name])} hooks for '{hook_name}'"
+                )
+            for hook in hooks[hook_name]:
+                self.register_hook(hook_name, hook, verbose)