runem 0.0.31__py3-none-any.whl → 0.1.0__py3-none-any.whl

runem/VERSION

@@ -1 +1 @@
-0.0.31
+0.1.0

runem/command_line.py

@@ -12,6 +12,45 @@ from runem.types import JobNames, OptionConfig, OptionsWritable
 from runem.utils import printable_set
 
 
+class HelpFormatterFixedWidth(argparse.HelpFormatter):
+    """This works around test issues via constant width helo output.
+
+    This ensures that we get more constant for help-text by fixing the width to
+    something reasonable.
+    """
+
+    def __init__(self, prog: typing.Any) -> None:
+        # Override the default width with a fixed width, for tests.
+        super().__init__(
+            prog,
+            # Pretty wide so we do not get wrapping on directories
+            # or process-count output.
+            width=1000,
+        )
+
+
+def _get_argparse_help_formatter() -> typing.Any:
+    """Returns a help-formatter for argparse.
+
+    This is for tests only to fake terminals of a constant with when rendering help
+    output.
+    """
+    # Check environment variable to see if we're in tests and need a fixed width
+    # help output.
+    use_fixed_width = os.getenv("RUNEM_FIXED_HELP_WIDTH", None)
+
+    if use_fixed_width:
+        # Use custom formatter with the width specified in the environment variable
+        return (
+            lambda prog: HelpFormatterFixedWidth(  # pylint: disable=unnecessary-lambda
+                prog
+            )
+        )
+
+    # Use default formatter
+    return argparse.HelpFormatter
+
+
 def parse_args(
     config_metadata: ConfigMetadata, argv: typing.List[str]
 ) -> ConfigMetadata:
@@ -23,7 +62,9 @@ def parse_args(
     Returns the parsed args, the jobs_names_to_run, job_phases_to_run, job_tags_to_run
     """
     parser = argparse.ArgumentParser(
-        add_help=False, description="Runs the Lursight Lang test-suite"
+        add_help=False,
+        description="Runs the Lursight Lang test-suite",
+        formatter_class=_get_argparse_help_formatter(),
     )
     parser.add_argument(
         "-H", "--help", action="help", help="show this help message and exit"
@@ -136,6 +177,29 @@ def parse_args(
         default=False,
         required=False,
     )
+    parser.add_argument(
+        "--always-files",
+        dest="always_files",
+        help=(
+            "list of paths/files to always check (overriding -f/-h), if the path "
+            "matches the filter regex and if file-paths exist"
+        ),
+        nargs="+",
+        default=None,
+        required=False,
+    )
+
+    parser.add_argument(
+        "--git-files-since-branch",
+        dest="git_since_branch",
+        help=(
+            "Get the list of paths/files changed between a branch, e.g., since "
+            "'origin/main'. Useful for checking files changed before pushing."
+        ),
+        default=None,  # Default to None if no branch is specified
+        required=False,  # Not required, users may not want to specify a branch
+        type=str,  # Accepts a string input representing the branch name
+    )
 
     parser.add_argument(
         "--procs",
@@ -151,7 +215,7 @@ def parse_args(
         type=int,
     )
 
-    config_dir: pathlib.Path = config_metadata.cfg_filepath.parent
+    config_dir: pathlib.Path = _get_config_dir(config_metadata)
     parser.add_argument(
         "--root",
         dest="root_dir",
@@ -163,6 +227,15 @@ def parse_args(
         required=False,
     )
 
+    parser.add_argument(
+        "--root-show",
+        dest="show_root_path_and_exit",
+        help="show the root-path of runem and exit",
+        action=argparse.BooleanOptionalAction,
+        default=False,
+        required=False,
+    )
+
     parser.add_argument(
         "--spinner",
         dest="show_spinner",
@@ -196,6 +269,11 @@ def parse_args(
 
     args = parser.parse_args(argv[1:])
 
+    if args.show_root_path_and_exit:
+        log(str(config_metadata.cfg_filepath.parent), decorate=False)
+        # cleanly exit
+        sys.exit(0)
+
     if args.show_version_and_exit:
         log(str(get_runem_version()), decorate=False)
         # cleanly exit
@@ -219,6 +297,11 @@ def parse_args(
     return config_metadata
 
 
+def _get_config_dir(config_metadata: ConfigMetadata) -> pathlib.Path:
+    """A function to get the path, that we can mock in tests."""
+    return config_metadata.cfg_filepath.parent
+
+
 def _validate_filters(
     config_metadata: ConfigMetadata,
     args: argparse.Namespace,
@@ -338,7 +421,7 @@ def _define_option_args(
 
 
 def _alias_to_switch(switch_name_alias: str, negatise: bool = False) -> str:
-    """Util function to generate a alias switch for argsparse."""
+    """Util function to generate a alias switch for argparse."""
     single_letter_variant = not negatise and len(switch_name_alias) == 1
     if single_letter_variant:
         return f"-{switch_name_alias}"

runem/files.py

@@ -29,6 +29,7 @@ def find_files(config_metadata: ConfigMetadata) -> FilePathListLookup:
     if (
         config_metadata.args.check_modified_files
         or config_metadata.args.check_head_files
+        or (config_metadata.args.git_since_branch is not None)
     ):
         if config_metadata.args.check_modified_files:
             # get modified, un-staged files first
@@ -60,10 +61,25 @@ def find_files(config_metadata: ConfigMetadata) -> FilePathListLookup:
                 .decode("utf-8")
                 .splitlines()
             )
+
+        if config_metadata.args.git_since_branch is not None:
+            # Add all files changed since a particular branch e..g `origin/main`
+            # Useful for quickly checking branches before pushing.
+            # NOTE: without dependency checking this might report false-positives.
+            target_branch: str = config_metadata.args.git_since_branch
+            file_paths.extend(
+                subprocess_check_output(
+                    f"git diff --name-only {target_branch}...HEAD",
+                    shell=True,
+                )
+                .decode("utf-8")
+                .splitlines()
+            )
         # ensure files are unique, and still on disk i.e. filter-out deleted files
         file_paths = list(
             {file_path for file_path in file_paths if Path(file_path).exists()}
         )
+
     else:
         # fall-back to all files
         file_paths = (
@@ -74,6 +90,16 @@ def find_files(config_metadata: ConfigMetadata) -> FilePathListLookup:
             .decode("utf-8")
             .splitlines()
         )
+
+    if config_metadata.args.always_files is not None:
+        # a poor-man's version of adding path-regex's
+        existent_files = [
+            filepath
+            for filepath in config_metadata.args.always_files
+            if Path(filepath).exists()
+        ]
+        file_paths.extend(existent_files)
+
     _bucket_file_by_tag(
         file_paths,
         config_metadata,

runem/hook_manager.py

@@ -1,6 +1,8 @@
 import typing
 from collections import defaultdict
 
+from typing_extensions import Unpack
+
 from runem.config_metadata import ConfigMetadata
 from runem.job import Job
 from runem.job_execute import job_execute
@@ -10,6 +12,7 @@ from runem.types import (
     HookConfig,
     HookName,
     Hooks,
+    HookSpecificKwargs,
     HooksStore,
     JobConfig,
 )
@@ -69,7 +72,7 @@ class HookManager:
         self,
         hook_name: HookName,
         config_metadata: ConfigMetadata,
-        **kwargs: typing.Any,
+        **kwargs: Unpack[HookSpecificKwargs],
     ) -> None:
         """Invokes all functions registered to a specific hook."""
         hooks: typing.List[HookConfig] = self.hooks_store.get(hook_name, [])

runem/job_execute.py

@@ -5,13 +5,17 @@ import uuid
 from datetime import timedelta
 from timeit import default_timer as timer
 
+from typing_extensions import Unpack
+
 from runem.config_metadata import ConfigMetadata
 from runem.informative_dict import ReadOnlyInformativeDict
 from runem.job import Job
 from runem.job_wrapper import get_job_wrapper
 from runem.log import error, log
 from runem.types import (
+    FilePathList,
     FilePathListLookup,
+    HookSpecificKwargs,
     JobConfig,
     JobFunction,
     JobReturn,
@@ -26,7 +30,7 @@ def job_execute_inner(
     job_config: JobConfig,
     config_metadata: ConfigMetadata,
     file_lists: FilePathListLookup,
-    **kwargs: typing.Any,
+    **kwargs: Unpack[HookSpecificKwargs],
 ) -> typing.Tuple[JobTiming, JobReturn]:
     """Wrapper for running a job inside a sub-process.
 
@@ -36,13 +40,12 @@ def job_execute_inner(
     if config_metadata.args.verbose:
         log(f"START: '{label}'")
     root_path: pathlib.Path = config_metadata.cfg_filepath.parent
-    function: JobFunction
     job_tags: typing.Optional[JobTags] = Job.get_job_tags(job_config)
     os.chdir(root_path)
-    function = get_job_wrapper(job_config, config_metadata.cfg_filepath)
+    function: JobFunction = get_job_wrapper(job_config, config_metadata.cfg_filepath)
 
     # get the files for all files found for this job's tags
-    file_list = Job.get_job_files(file_lists, job_tags)
+    file_list: FilePathList = Job.get_job_files(file_lists, job_tags)
 
     if not file_list:
         # no files to work on
@@ -77,8 +80,9 @@ def job_execute_inner(
         log(f"job: running: '{Job.get_job_name(job_config)}'")
     reports: JobReturn
     try:
+        assert isinstance(function, JobFunction)
         reports = function(
-            options=ReadOnlyInformativeDict(config_metadata.options),  # type: ignore
+            options=ReadOnlyInformativeDict(config_metadata.options),
             file_list=file_list,
             procs=config_metadata.args.procs,
             root_path=root_path,
@@ -109,7 +113,7 @@ def job_execute(
     running_jobs: typing.Dict[str, str],
     config_metadata: ConfigMetadata,
     file_lists: FilePathListLookup,
-    **kwargs: typing.Any,
+    **kwargs: Unpack[HookSpecificKwargs],
 ) -> typing.Tuple[JobTiming, JobReturn]:
     """Thin-wrapper around job_execute_inner needed for mocking in tests.
 

runem/job_wrapper.py

@@ -18,7 +18,7 @@ def get_job_wrapper(job_wrapper: JobWrapper, cfg_filepath: pathlib.Path) -> JobF
         # validate that the command is "understandable" and usable.
         command_string: str = job_wrapper["command"]
         validate_simple_command(command_string)
-        return job_runner_simple_command  # type: ignore # NO_COMMIT
+        return job_runner_simple_command
 
     # if we do not have a simple command address assume we have just an addressed
     # function

runem/runem.py

@@ -30,8 +30,11 @@ from datetime import timedelta
 from itertools import repeat
 from multiprocessing.managers import DictProxy, ListProxy, ValueProxy
 from timeit import default_timer as timer
+from types import TracebackType
 
-from halo import Halo
+from rich.console import Console, ConsoleOptions, ConsoleRenderable, RenderResult
+from rich.spinner import Spinner
+from rich.text import Text
 
 from runem.command_line import parse_args
 from runem.config import load_project_config, load_user_configs
@@ -58,6 +61,8 @@ from runem.types import (
 )
 from runem.utils import printable_set
 
+rich_console = Console()
+
 
 def _determine_run_parameters(argv: typing.List[str]) -> ConfigMetadata:
     """Loads config, parsing cli input and produces the run config.
@@ -85,6 +90,36 @@ def _determine_run_parameters(argv: typing.List[str]) -> ConfigMetadata:
     return config_metadata
 
 
+class DummySpinner(ConsoleRenderable):  # pragma: no cover
+    """A dummy spinner for when spinners are disabled."""
+
+    def __init__(self) -> None:
+        self.text = ""
+
+    def __rich__(self) -> Text:
+        """Return a rich Text object for rendering."""
+        return Text(self.text)
+
+    def __rich_console__(
+        self, console: Console, options: ConsoleOptions
+    ) -> RenderResult:
+        """Yield an empty string or placeholder text."""
+        yield Text(self.text)
+
+    def __enter__(self) -> None:
+        """Support for context manager."""
+        pass
+
+    def __exit__(
+        self,
+        exc_type: typing.Optional[typing.Type[BaseException]],
+        exc_value: typing.Optional[BaseException],
+        traceback: typing.Optional[TracebackType],
+    ) -> None:
+        """Support for context manager."""
+        pass
+
+
 def _update_progress(
     label: str,
     running_jobs: typing.Dict[str, str],
@@ -104,52 +139,55 @@ def _update_progress(
         is_running (ValueProxy[bool]): Flag indicating if jobs are still running.
         num_workers (int): Indicates the number of workers performing the jobs.
     """
-    # Using Halo library to show a loading spinner on console
+    # Using the `rich` module to show a loading spinner on console
+    spinner: typing.Union[Spinner, DummySpinner]
     if show_spinner:
-        spinner = Halo(text="", spinner="dots")
-        spinner.start()
+        spinner = Spinner("dots", text="Starting tasks...")
+    else:
+        spinner = DummySpinner()
 
-    # The set of all job labels, and the set of completed jobs
-    all_job_names: typing.Set[str] = {Job.get_job_name(job) for job in all_jobs}
-    completed_jobs: typing.Set[str] = set()
+    with rich_console.status(spinner):
 
-    # This dataset is used to track changes between iterations
-    last_running_jobs_set: typing.Set[str] = set()
+        # The set of all job labels, and the set of completed jobs
+        all_job_names: typing.Set[str] = {Job.get_job_name(job) for job in all_jobs}
+        completed_jobs: typing.Set[str] = set()
 
-    while is_running.value:
-        running_jobs_set: typing.Set[str] = set(running_jobs.values())
-        seen_jobs = list(running_jobs_set.union(seen_jobs))  # Update the seen jobs
+        # This dataset is used to track changes between iterations
+        last_running_jobs_set: typing.Set[str] = set()
 
-        # Jobs that have disappeared since last check
-        disappeared_jobs: typing.Set[str] = last_running_jobs_set - running_jobs_set
+        while is_running.value:
+            running_jobs_set: typing.Set[str] = set(running_jobs.values())
+            seen_jobs = list(running_jobs_set.union(seen_jobs))  # Update the seen jobs
 
-        # Jobs that have not yet completed
-        remaining_jobs: typing.Set[str] = all_job_names - completed_jobs
+            # Jobs that have disappeared since last check
+            disappeared_jobs: typing.Set[str] = last_running_jobs_set - running_jobs_set
 
-        # Check if we're closing to completion
-        workers_retiring: bool = len(remaining_jobs) <= num_workers
+            # Jobs that have not yet completed
+            remaining_jobs: typing.Set[str] = all_job_names - completed_jobs
 
-        if workers_retiring:
-            # Handle edge case: a task may have disappeared whilst process was sleeping
-            all_completed_jobs: typing.Set[str] = all_job_names - remaining_jobs
-            disappeared_jobs.update(all_completed_jobs - running_jobs_set)
+            # Check if we're closing to completion
+            workers_retiring: bool = len(remaining_jobs) <= num_workers
 
-        completed_jobs.update(disappeared_jobs)
+            if workers_retiring:
+                # Handle edge case: a task may have disappeared whilst process was sleeping
+                all_completed_jobs: typing.Set[str] = all_job_names - remaining_jobs
+                disappeared_jobs.update(all_completed_jobs - running_jobs_set)
 
-        # Prepare progress report
-        progress: str = f"{len(completed_jobs)}/{len(all_jobs)}"
-        running_jobs_list = printable_set(running_jobs_set)
-        if show_spinner:
-            spinner.text = f"{label}: {progress}({num_workers}): {running_jobs_list}"
+            completed_jobs.update(disappeared_jobs)
 
-        # Update the tracked dataset for the next iteration
-        last_running_jobs_set = running_jobs_set
+            # Prepare progress report
+            progress: str = f"{len(completed_jobs)}/{len(all_jobs)}"
+            running_jobs_list = printable_set(running_jobs_set)
+            if show_spinner:
+                spinner.text = (
+                    f"{label}: {progress}({num_workers}): {running_jobs_list}"
+                )
 
-        # Sleep to decrease frequency of updates and reduce CPU usage
-        time.sleep(0.1)
+            # Update the tracked dataset for the next iteration
+            last_running_jobs_set = running_jobs_set
 
-    if show_spinner:
-        spinner.stop()
+            # Sleep to decrease frequency of updates and reduce CPU usage
+            time.sleep(0.1)
 
 
 def _process_jobs(
@@ -209,7 +247,7 @@ def _process_jobs(
             with multiprocessing.Pool(processes=num_concurrent_procs) as pool:
                 # use starmap so we can pass down the job-configs and the args and the files
                 in_out_job_run_metadatas[phase] = pool.starmap(
-                    job_execute,
+                    job_execute,  # no kwargs passed for jobs here
                     zip(
                         jobs,
                         repeat(running_jobs),

runem/types.py

@@ -1,9 +1,37 @@
-import argparse
+"""
+
+Some note on Unpack and kwargs:
+    We *try* to strongly type `**kwargs` for clarity.
+    We have tried several ways to define a Generic type that encapsulates
+        `**kwargs: SingleType`
+    ... but none of the solutions worked with python 3.9 -> 3.12 and mypy 1.9.0,
+    so we have to recommend instead using:
+        `**kwargs: Unpack[KwArgsType]`
+
+    For this to work across versions of python where support for Unpack changes;
+    for example `Unpack` is a python 3.12 feature, but available in the
+    `typing_extensions` module.
+
+    So, for now, it looks like we get away with importing `Unpack` from the
+    `typing_extensions` module, even in python 3.12, so we will use, and
+    recommend using, the `typing_extensions` of `Unpack`, until it becomes
+    obsolete.
+
+    Alternatively, we can use the following, but it's unnecessarily verbose.
+
+    if sys.version_info >= (3, 12):  # pragma: no coverage
+        from typing import Unpack
+    else:  # pragma: no coverage
+        from typing_extensions import Unpack
+"""
+
 import pathlib
 import typing
 from datetime import timedelta
 from enum import Enum
 
+from typing_extensions import Unpack
+
 from runem.informative_dict import InformativeDict, ReadOnlyInformativeDict
 
 
@@ -96,12 +124,6 @@ FilePathSerialise = str
 FilePathList = typing.List[FilePathSerialise]
 FilePathListLookup = typing.DefaultDict[JobTag, FilePathList]
 
-# FIXME: this type is no-longer the actual spec of the test-functions
-JobFunction = typing.Union[
-    typing.Callable[[argparse.Namespace, OptionsWritable, FilePathList], None],
-    typing.Callable[[typing.Any], None],
-]
-
 
 class JobParamConfig(typing.TypedDict):
     """Configures what parameters are passed to the test-callable.
@@ -250,3 +272,86 @@ ConfigNodes = typing.Union[
 Config = typing.List[ConfigNodes]
 
 UserConfigMetadata = typing.List[typing.Tuple[Config, pathlib.Path]]
+
+
+class CommonKwargs(
+    typing.TypedDict,
+    total=True,  # each of these are guaranteed to exist in jobs and hooks
+):
+    """Defines the base args that are passed to all jobs.
+
+    As we call hooks and job-task in the same manner, this defines the variables that we
+    can access from both hooks and job-tasks.
+    """
+
+    root_path: pathlib.Path  # the path where the .runem.yml file is
+    job: JobConfig  # the job or hook task spec ¢ TODO: rename this
+    label: str  # the name of the hook or the job-label
+    options: Options  # options passed in on the command line
+    procs: int  # the max number of concurrent procs to run
+    verbose: bool  # control log verbosity
+
+
+class HookSpecificKwargs(typing.TypedDict, total=False):
+    """Defines the args that are passed down to the hooks.
+
+    NOTE: that although these however
+     outside of the *hook* context, the data will not be present. Such is the
+     difficulty in dynamic programming.
+    """
+
+    wall_clock_time_saved: timedelta  # only on `HookName.ON_EXIT`
+
+
+class JobTaskKwargs(
+    typing.TypedDict,
+    total=False,  # for now, we don't enforce these types for job-context, but we should.
+):
+    """Defines the task-specific args for job-task functions."""
+
+    file_list: FilePathList
+    record_sub_job_time: typing.Optional[typing.Callable[[str, timedelta], None]]
+
+
+class HookKwargs(CommonKwargs, HookSpecificKwargs):
+    """A merged set of kwargs for runem-hooks."""
+
+    pass
+
+
+class JobKwargs(CommonKwargs, JobTaskKwargs):
+    """A merged set of kwargs for job-tasks."""
+
+    pass
+
+
+class AllKwargs(CommonKwargs, JobTaskKwargs, HookSpecificKwargs):
+    """A merged set of kwargs for al job-functions."""
+
+    pass
+
+
+@typing.runtime_checkable
+class JobFunction(typing.Protocol):
+    def __call__(self, **kwargs: Unpack[AllKwargs]) -> JobReturn:  # pragma: no cover
+        """Defines the call() protocol's abstract pattern for job-tasks."""
+
+    @property
+    def __name__(self) -> str:  # pragma: no cover
+        """Defines the name protocol for job-task functions.
+
+        This is primarily used for internal tests but can be useful for introspection.
+        """
+
+
+def _hook_example(
+    wall_clock_time_saved: timedelta,
+    **kwargs: typing.Any,
+) -> None:
+    """An example hook."""
+
+
+def _job_task_example(
+    **kwargs: Unpack[JobKwargs],
+) -> None:
+    """An example job-task function."""