runem 0.5.0__py3-none-any.whl → 0.7.0__py3-none-any.whl

runem/runem.py

@@ -19,6 +19,8 @@ We do:
 - time tests and tell you what used the most time, and how much time run-tests saved
   you
 """
+
+import contextlib
 import multiprocessing
 import os
 import pathlib
@@ -30,10 +32,9 @@ from datetime import timedelta
 from itertools import repeat
 from multiprocessing.managers import DictProxy, ValueProxy
 from timeit import default_timer as timer
-from types import TracebackType
 
-from rich.console import Console, ConsoleOptions, ConsoleRenderable, RenderResult
 from rich.spinner import Spinner
+from rich.status import Status
 from rich.text import Text
 
 from runem.blocking_print import RICH_CONSOLE
@@ -46,7 +47,9 @@ from runem.job_execute import job_execute
 from runem.job_filter import filter_jobs
 from runem.log import error, log, warn
 from runem.report import report_on_run
+from runem.run_command import RunemJobError
 from runem.types.common import OrderedPhases, PhaseName
+from runem.types.errors import SystemExitBad
 from runem.types.filters import FilePathListLookup
 from runem.types.hooks import HookName
 from runem.types.runem_config import Config, Jobs, PhaseGroupedJobs
@@ -56,7 +59,7 @@ from runem.types.types_jobs import (
     JobRunMetadatasByPhase,
     JobTiming,
 )
-from runem.utils import printable_set
+from runem.utils import printable_set_coloured
 
 
 def _determine_run_parameters(argv: typing.List[str]) -> ConfigMetadata:
@@ -67,7 +70,6 @@ def _determine_run_parameters(argv: typing.List[str]) -> ConfigMetadata:
 
     Return a ConfigMetadata object with all the required information.
     """
-
     # Because we want to be able to show logging whilst parsing .runem.yml config, we
     # need to check the state of the logging-verbosity switches here, manually, as well.
     verbose = "--verbose" in argv
@@ -92,38 +94,8 @@ def _determine_run_parameters(argv: typing.List[str]) -> ConfigMetadata:
     return config_metadata
 
 
-class DummySpinner(ConsoleRenderable):  # pragma: no cover
-    """A dummy spinner for when spinners are disabled."""
-
-    def __init__(self) -> None:
-        self.text = ""
-
-    def __rich__(self) -> Text:
-        """Return a rich Text object for rendering."""
-        return Text(self.text)
-
-    def __rich_console__(
-        self, console: Console, options: ConsoleOptions
-    ) -> RenderResult:
-        """Yield an empty string or placeholder text."""
-        yield Text(self.text)
-
-    def __enter__(self) -> None:
-        """Support for context manager."""
-        pass
-
-    def __exit__(
-        self,
-        exc_type: typing.Optional[typing.Type[BaseException]],
-        exc_value: typing.Optional[BaseException],
-        traceback: typing.Optional[TracebackType],
-    ) -> None:
-        """Support for context manager."""
-        pass
-
-
 def _update_progress(
-    label: str,
+    phase: str,
     running_jobs: typing.Dict[str, str],
     completed_jobs: typing.Dict[str, str],
     all_jobs: Jobs,
@@ -134,36 +106,44 @@ def _update_progress(
     """Updates progress report periodically for running tasks.
 
     Args:
-        label (str): The identifier.
+        phase (str): The currently running phase.
         running_jobs (Dict[str, str]): The currently running jobs.
+        completed_jobs (Dict[str, str]): The jobs that have finished work.
         all_jobs (Jobs): All jobs, encompassing both completed and running jobs.
         is_running (ValueProxy[bool]): Flag indicating if jobs are still running.
         num_workers (int): Indicates the number of workers performing the jobs.
+        show_spinner (bool): Whether to show the animated spinner or not.
     """
-    # Using the `rich` module to show a loading spinner on console
-    spinner: typing.Union[Spinner, DummySpinner]
-    if show_spinner:
-        spinner = Spinner("dots", text="Starting tasks...")
-    else:
-        spinner = DummySpinner()
-
     last_running_jobs_set: typing.Set[str] = set()
 
-    with RICH_CONSOLE.status(spinner):
+    # Using the `rich` module to show a loading spinner on console
+    spinner_ctx: typing.Union[Status, typing.ContextManager[None]] = (
+        RICH_CONSOLE.status(Spinner("dots", text="Starting tasks..."))
+        if show_spinner
+        else contextlib.nullcontext()
+    )
+
+    with spinner_ctx:
         while is_running.value:
             running_jobs_set: typing.Set[str] = set(running_jobs.values())
 
             # Progress report
             progress: str = f"{len(completed_jobs)}/{len(all_jobs)}"
-            running_jobs_list = printable_set(
-                running_jobs_set
+            running_jobs_list = printable_set_coloured(
+                running_jobs_set,
+                "blue",
             )  # Reflect current running jobs accurately
-            report: str = f"{label}: {progress}({num_workers}): {running_jobs_list}"
+            report: str = (
+                f"[green]{phase}[/green]: {progress}({num_workers}): "
+                f"{running_jobs_list}"
+            )
             if show_spinner:
-                spinner.text = report
+                assert isinstance(spinner_ctx, Status)
+                spinner_ctx.update(Text.from_markup(report))
             else:
                 if last_running_jobs_set != running_jobs_set:
                     RICH_CONSOLE.log(report)
+                    last_running_jobs_set = running_jobs_set
 
             # Sleep for reduced CPU usage
             time.sleep(0.1)
@@ -176,7 +156,7 @@ def _process_jobs(
     phase: PhaseName,
     jobs: Jobs,
     show_spinner: bool,
-) -> typing.Optional[BaseException]:
+) -> typing.Optional[RunemJobError]:
     """Execute each given job asynchronously.
 
     This is where the major real-world time savings happen, and it could be
@@ -196,12 +176,12 @@ def _process_jobs(
     num_concurrent_procs: int = min(max_num_concurrent_procs, len(jobs))
     log(
         (
-            f"Running '{phase}' with {num_concurrent_procs} workers (of "
+            f"Running '[green]{phase}[/green]' with {num_concurrent_procs} workers (of "
             f"{max_num_concurrent_procs} max) processing {len(jobs)} jobs"
         )
     )
 
-    subprocess_error: typing.Optional[BaseException] = None
+    subprocess_error: typing.Optional[RunemJobError] = None
 
     with multiprocessing.Manager() as manager:
         running_jobs: DictProxy[typing.Any, typing.Any] = manager.dict()
@@ -235,7 +215,7 @@ def _process_jobs(
                         repeat(file_lists),
                     ),
                 )
-        except BaseException as err:  # pylint: disable=broad-exception-caught
+        except RunemJobError as err:  # pylint: disable=broad-exception-caught
             subprocess_error = err
         finally:
             # Signal the terminal_writer process to exit
@@ -251,7 +231,7 @@ def _process_jobs_by_phase(
     filtered_jobs_by_phase: PhaseGroupedJobs,
     in_out_job_run_metadatas: JobRunMetadatasByPhase,
     show_spinner: bool,
-) -> typing.Optional[BaseException]:
+) -> typing.Optional[RunemJobError]:
     """Execute each job asynchronously, grouped by phase.
 
     Whilst it is conceptually useful to group jobs by 'phase', Phases are
@@ -275,7 +255,7 @@ def _process_jobs_by_phase(
         if config_metadata.args.verbose:
             log(f"Running Phase {phase}")
 
-        failure_exception: typing.Optional[BaseException] = _process_jobs(
+        failure_exception: typing.Optional[RunemJobError] = _process_jobs(
             config_metadata,
             file_lists,
             in_out_job_run_metadatas,
@@ -293,7 +273,7 @@ def _process_jobs_by_phase(
 
 
 MainReturnType = typing.Tuple[
-    ConfigMetadata, JobRunMetadatasByPhase, typing.Optional[BaseException]
+    ConfigMetadata, JobRunMetadatasByPhase, typing.Optional[RunemJobError]
 ]
 
 
@@ -316,8 +296,8 @@ def _main(
         log(f"found {len(file_lists)} batches, ", end="")
         for tag in sorted(file_lists.keys()):
             file_list = file_lists[tag]
-            log(f"{len(file_list)} '{tag}' files, ", decorate=False, end="")
-        log(decorate=False)  # new line
+            log(f"{len(file_list)} '{tag}' files, ", prefix=False, end="")
+        log(prefix=False)  # new line
 
     filtered_jobs_by_phase: PhaseGroupedJobs = filter_jobs(
         config_metadata=config_metadata,
@@ -333,7 +313,7 @@ def _main(
 
     start = timer()
 
-    failure_exception: typing.Optional[BaseException] = _process_jobs_by_phase(
+    failure_exception: typing.Optional[RunemJobError] = _process_jobs_by_phase(
         config_metadata,
         file_lists,
         filtered_jobs_by_phase,
@@ -362,7 +342,7 @@ def timed_main(argv: typing.List[str]) -> None:
     start = timer()
     config_metadata: ConfigMetadata
     job_run_metadatas: JobRunMetadatasByPhase
-    failure_exception: typing.Optional[BaseException]
+    failure_exception: typing.Optional[RunemJobError]
     config_metadata, job_run_metadatas, failure_exception = _main(argv)
     phase_run_oder: OrderedPhases = config_metadata.phases
     end = timer()
@@ -372,14 +352,15 @@ def timed_main(argv: typing.List[str]) -> None:
     system_time_spent, wall_clock_time_saved = report_on_run(
         phase_run_oder, job_run_metadatas, time_taken
     )
-    message: str = "DONE: runem took"
+    message: str = "[green bold]DONE[/green bold]: runem took"
     if failure_exception:
-        message = "FAILED: your jobs failed after"
+        message = "[red bold]FAILED[/red bold]: your jobs failed after"
     log(
         (
             f"{message}: {time_taken.total_seconds()}s, "
-            f"saving you {wall_clock_time_saved.total_seconds()}s, "
-            f"without runem you would have waited {system_time_spent.total_seconds()}s"
+            f"saving you [green]{wall_clock_time_saved.total_seconds()}s[/green], "
+            "without runem you would have waited "
+            f"[red]{system_time_spent.total_seconds()}s[/red]"
         )
     )
 
@@ -392,7 +373,8 @@ def timed_main(argv: typing.List[str]) -> None:
     if failure_exception is not None:
         # we got a failure somewhere, now that we've reported the timings we
         # re-raise.
-        raise failure_exception
+        error(failure_exception.stdout)
+        raise SystemExitBad(1) from failure_exception
 
 
 if __name__ == "__main__":

runem/schema.yml

@@ -0,0 +1,137 @@
+#%RAML 1.0  (← just a comment so VS Code picks up YAML)
+$schema: "https://json-schema.org/draft/2020-12/schema"
+$title: Runem pipeline definition
+$defs:
+  # ----- common pieces -------------------------------------------------------
+  phase:
+    type: string
+
+  addr:
+    type: object
+    required: [file, function]
+    additionalProperties: false
+    properties:
+      file:     { type: string, minLength: 1 }
+      function: { type: string, minLength: 1 }
+
+  ctx:
+    type: object
+    additionalProperties: false
+    properties:
+      cwd:
+        oneOf:
+          - type: string
+          - type: array
+            minItems: 1
+            items: { type: string, minLength: 1 }
+      params:
+        type: object      # free‑form kv‑pairs for hooks
+        additionalProperties: true
+
+  when:
+    type: object
+    required: [phase]
+    additionalProperties: false
+    properties:
+      phase: { $ref: "#/$defs/phase" }
+      tags:
+        type: array
+        items: { type: string, minLength: 1 }
+        uniqueItems: true
+
+  # ----- top‑level entity types ---------------------------------------------
+  config:
+    type: object
+    required: []
+    additionalProperties: false
+    properties:
+      min_version:
+        type: string
+
+      phases:
+        type: array
+        minItems: 1
+        items: { $ref: "#/$defs/phase" }
+        uniqueItems: true
+
+      files:
+        type: [array, 'null']
+        minItems: 0
+        items:
+          type: object
+          required: [filter]
+          additionalProperties: false
+          properties:
+            filter:
+              type: object
+              required: [tag, regex]
+              additionalProperties: false
+              properties:
+                tag:   { type: string, minLength: 1 }
+                regex: { type: string, minLength: 1 } # leave pattern‑checking to the engine
+
+      options:
+        type: [array, 'null']
+        minItems: 0
+        items:
+          type: object
+          required: [option]
+          additionalProperties: false
+          properties:
+            option:
+              type: object
+              required: [name, type, default, desc]
+              additionalProperties: false
+              properties:
+                name:    { type: string, minLength: 1 }
+                alias:   { type: string, minLength: 1 }
+                desc:    { type: string, minLength: 1 }
+                type:
+                  const: bool           # always "bool" per sample
+                default: { type: boolean }
+
+  hook:
+    type: object
+    required: [hook_name]
+    oneOf:
+      - required: [command]
+      - required: [addr]
+    additionalProperties: false
+    properties:
+      hook_name: { type: string, minLength: 1 }
+      addr:      { $ref: "#/$defs/addr" }
+      command: { type: string, minLength: 1 }
+
+  job:
+    type: object
+    oneOf:
+      - required: [command]
+      - required: [addr]
+    additionalProperties: false
+    properties:
+      label:   { type: string, minLength: 1 }
+      addr:    { $ref: "#/$defs/addr" }
+      command: { type: string, minLength: 1 }
+      ctx:    { $ref: "#/$defs/ctx" }
+      when:    { $ref: "#/$defs/when" }
+    oneOf:
+      - required: [addr]    # either addr
+      - required: [command] # or command, but not both
+    not:
+      anyOf:
+        - required: [addr, command]   # forbid both together
+
+# ---------- ROOT -------------------------------------------------------------
+type: array
+minItems: 1
+items:
+  type: object
+  additionalProperties: false
+  oneOf:
+    - required: [config]
+    - required: [hook]
+    - required: [job]
+  properties:
+    config: { $ref: "#/$defs/config" }
+    hook:   { $ref: "#/$defs/hook" }
+    job:    { $ref: "#/$defs/job" }

runem/types/__init__.py

@@ -1,11 +1,12 @@
 from runem.types.common import FilePathList, JobName
 from runem.types.options import Options
-from runem.types.types_jobs import HookKwargs, JobKwargs, JobReturnData
+from runem.types.types_jobs import HookKwargs, JobKwargs, JobReturn, JobReturnData
 
 __all__ = [
     "FilePathList",
     "HookKwargs",
     "JobName",
+    "JobReturn",
     "JobReturnData",
     "Options",
     "JobKwargs",

runem/types/errors.py

@@ -1,4 +1,14 @@
+from typing import Optional
+
+
 class FunctionNotFound(ValueError):
     """Thrown when the test-function cannot be found."""
 
     pass
+
+
+class SystemExitBad(SystemExit):
+    def __init__(self, code: Optional[int] = None) -> None:
+        super().__init__()
+        self.code = 1 if code is None else code  # non-zero bad exit code
+        assert self.code > 0, "A bad exit code should be non-zero and >0"

runem/types/hooks.py

@@ -4,7 +4,7 @@ import enum
 class HookName(enum.Enum):
     """List supported hooks.
 
-    TODO:
+    Todo:
     - before all tasks are run, after config is read
     - BEFORE_ALL = "before-all"
     - after all tasks are done, before reporting

runem/types/types_jobs.py

@@ -1,28 +1,25 @@
-"""
+"""Job‑typing helpers.
+
+Cross‑version advice
+--------------------
+* Type variadic keyword arguments as **kwargs: Unpack[KwArgsT] for clarity.
+* Always import Unpack from ``typing_extensions``.
+  - Std‑lib Unpack appears only in Py 3.12+.
+  - ``typing_extensions`` works on 3.9‑3.12, so one import path keeps
+    mypy/pyright happy without conditional logic.
+
+Example:
+~~~~~~~
+from typing_extensions import TypedDict, Unpack
+
+
+class SaveKwArgs(TypedDict):
+    path: str
+    overwrite: bool
+
 
-Some note on Unpack and kwargs:
-    We *try* to strongly type `**kwargs` for clarity.
-    We have tried several ways to define a Generic type that encapsulates
-        `**kwargs: SingleType`
-    ... but none of the solutions worked with python 3.9 -> 3.12 and mypy 1.9.0,
-    so we have to recommend instead using:
-        `**kwargs: Unpack[KwArgsType]`
-
-    For this to work across versions of python where support for Unpack changes;
-    for example `Unpack` is a python 3.12 feature, but available in the
-    `typing_extensions` module.
-
-    So, for now, it looks like we get away with importing `Unpack` from the
-    `typing_extensions` module, even in python 3.12, so we will use, and
-    recommend using, the `typing_extensions` of `Unpack`, until it becomes
-    obsolete.
-
-    Alternatively, we can use the following, but it's unnecessarily verbose.
-
-    if sys.version_info >= (3, 12):  # pragma: no coverage
-        from typing import Unpack
-    else:  # pragma: no coverage
-        from typing_extensions import Unpack
+def save_job(**kwargs: Unpack[SaveKwArgs]) -> None:
+    ...
 """
 
 import pathlib

runem/utils.py

@@ -4,3 +4,15 @@ import typing
 def printable_set(some_set: typing.Set[typing.Any]) -> str:
     """Get a printable, deterministic string version of a set."""
     return ", ".join([f"'{set_item}'" for set_item in sorted(list(some_set))])
+
+
+def printable_set_coloured(some_set: typing.Set[typing.Any], colour: str) -> str:
+    """`printable_set` but elements are surrounded with colour mark-up.
+
+    Parameters:
+        some_set: a set of anything
+        colour: a `rich` Console supported colour
+    """
+    return ", ".join(
+        [f"'[{colour}]{set_item}[/{colour}]'" for set_item in sorted(list(some_set))]
+    )

runem/yaml_utils.py

@@ -0,0 +1,19 @@
+import pathlib
+import typing
+
+import yaml
+
+
+def load_yaml_object(yaml_file: pathlib.Path) -> typing.Any:
+    """Loads using full_load, a yaml file.
+
+    This is likely to have safety concerns in non-trusted projects.
+
+    Returns:
+        YAML Loader object: the full PyYAML loader object.
+    """
+    # Do a full, untrusted load of the runem config
+    # TODO: work out safety concerns of this
+    with yaml_file.open("r+", encoding="utf-8") as file_handle:
+        full_yaml_object: typing.Any = yaml.full_load(file_handle)
+    return full_yaml_object

runem/yaml_validation.py

@@ -0,0 +1,28 @@
+from typing import Any, List
+
+from jsonschema import Draft202012Validator, ValidationError
+
+# For now just return the raw ValidationErrors as a list
+ValidationErrors = List[ValidationError]
+
+
+def validate_yaml(yaml_data: Any, schema: Any) -> ValidationErrors:
+    """Validates the give yaml data against the given schema, returning any errors.
+
+    We use more future-looking validation so that we can have richer and more
+    descriptive schema.
+
+    Params:
+        instance: JSON data loaded via `load_json` or similar
+        schema: schema object compatible with a Draft202012Validator
+
+    Returns:
+        ValidationErrors: a sorted list of errors in the file, empty if none found
+    """
+    validator = Draft202012Validator(schema)
+    errors: ValidationErrors = sorted(
+        validator.iter_errors(yaml_data),
+        key=lambda e: e.path,
+    )
+
+    return errors