runem 0.6.0__py3-none-any.whl → 0.7.1__py3-none-any.whl

runem/VERSION

@@ -1 +1 @@
-0.6.0
+0.7.1

runem/blocking_print.py

@@ -21,6 +21,9 @@ def _reset_console() -> Console:
         # `highlight` is what colourises string and number in print() calls.
         # We do not want this to be auto-magic.
         highlight=False,
+        # `soft_wrap=True` disables word-wrap & cropping by default:
+        #  - `soft_wrap` reads like a misnomer to me
+        soft_wrap=True,
     )
     return RICH_CONSOLE
 

runem/cli/initialise_options.py

@@ -13,7 +13,6 @@ def initialise_options(
 
     Returns the options dictionary
     """
-
     options: OptionsWritable = InformativeDict(
         {option["name"]: option["default"] for option in config_metadata.options_config}
     )

runem/command_line.py

@@ -43,10 +43,8 @@ def _get_argparse_help_formatter() -> typing.Any:
 
     if use_fixed_width:
         # Use custom formatter with the width specified in the environment variable
-        return (
-            lambda prog: HelpFormatterFixedWidth(  # pylint: disable=unnecessary-lambda
-                prog
-            )
+        return lambda prog: HelpFormatterFixedWidth(  # pylint: disable=unnecessary-lambda
+            prog
         )
 
     # Use default formatter
@@ -294,12 +292,12 @@ def parse_args(
     error_on_log_logic(args.verbose, args.silent)
 
     if args.show_root_path_and_exit:
-        log(str(config_metadata.cfg_filepath.parent), decorate=False)
+        log(str(config_metadata.cfg_filepath.parent), prefix=False)
         # cleanly exit
         sys.exit(0)
 
     if args.show_version_and_exit:
-        log(str(get_runem_version()), decorate=False)
+        log(str(get_runem_version()), prefix=False)
         # cleanly exit
         sys.exit(0)
 
@@ -383,7 +381,6 @@ def initialise_options(
 
     Returns the options dictionary
     """
-
     options: OptionsWritable = InformativeDict(
         {option["name"]: option["default"] for option in config_metadata.options_config}
     )

runem/config.py

@@ -2,9 +2,9 @@ import pathlib
 import sys
 import typing
 
-import yaml
 from packaging.version import Version
 
+from runem.config_validate import validate_runem_file
 from runem.log import error, log
 from runem.runem_version import get_runem_version
 from runem.types.runem_config import (
@@ -13,6 +13,7 @@ from runem.types.runem_config import (
     GlobalSerialisedConfig,
     UserConfigMetadata,
 )
+from runem.yaml_utils import load_yaml_object
 
 CFG_FILE_YAML = pathlib.Path(".runem.yml")
 
@@ -46,7 +47,7 @@ def _search_up_multiple_dirs_for_file(
 
 
 def _find_config_file(
-    config_filename: typing.Union[str, pathlib.Path]
+    config_filename: typing.Union[str, pathlib.Path],
 ) -> typing.Tuple[typing.Optional[pathlib.Path], typing.Tuple[pathlib.Path, ...]]:
     """Searches up from the cwd for the given config file-name."""
     start_dirs = (pathlib.Path(".").absolute(),)
@@ -117,8 +118,11 @@ def _conform_global_config_types(
 
 def load_and_parse_config(cfg_filepath: pathlib.Path) -> Config:
     """For the given config file pass, project or user, load it & parse/conform it."""
-    with cfg_filepath.open("r+", encoding="utf-8") as config_file_handle:
-        all_config = yaml.full_load(config_file_handle)
+    all_config = load_yaml_object(cfg_filepath)
+    validate_runem_file(
+        cfg_filepath,
+        all_config,
+    )
 
     conformed_config: Config
     global_config: typing.Optional[GlobalConfig]

runem/config_parse.py

@@ -11,7 +11,7 @@ from runem.job import Job
 from runem.job_wrapper import get_job_wrapper
 from runem.log import error, log, warn
 from runem.types.common import JobNames, JobPhases, JobTags, OrderedPhases, PhaseName
-from runem.types.errors import FunctionNotFound
+from runem.types.errors import FunctionNotFound, SystemExitBad
 from runem.types.filters import TagFileFilter, TagFileFilters
 from runem.types.hooks import HookName
 from runem.types.runem_config import (
@@ -81,9 +81,8 @@ def parse_hook_config(
             f"hook config entry is missing '{err.args[0]}' key. Have {tuple(hook.keys())}"
         ) from err
     except FunctionNotFound as err:
-        raise FunctionNotFound(
-            f"Whilst loading job '{str(hook['hook_name'])}'. {str(err)}"
-        ) from err
+        error(f"Whilst loading hook '{str(hook['hook_name'])}'. {str(err)}")
+        raise SystemExitBad(2) from err
 
 
 def _parse_job(  # noqa: C901
@@ -110,9 +109,8 @@ def _parse_job(  # noqa: C901
         # try and load the function _before_ we schedule it's execution
         get_job_wrapper(job, cfg_filepath)
     except FunctionNotFound as err:
-        raise FunctionNotFound(
-            f"Whilst loading job '{job['label']}'. {str(err)}"
-        ) from err
+        error(f"Whilst loading job '{job['label']}'. {str(err)}")
+        raise SystemExitBad(2) from err
 
     try:
         phase_id: PhaseName = job["when"]["phase"]
@@ -162,7 +160,8 @@ def parse_job_config(
             ("cwd" in job["ctx"]) and (job["ctx"]["cwd"] is not None)
         )
         if (not have_ctw_cwd) or isinstance(
-            job["ctx"]["cwd"], str  # type: ignore # handled above
+            job["ctx"]["cwd"],  # type: ignore # handled above
+            str,
         ):
             # if
             # - we don't have a cwd, ctx

runem/config_validate.py

@@ -0,0 +1,47 @@
+import pathlib
+import typing
+
+from runem.log import error, log
+from runem.types.errors import SystemExitBad
+from runem.yaml_utils import load_yaml_object
+from runem.yaml_validation import ValidationErrors, validate_yaml
+
+
+def _load_runem_schema() -> typing.Any:
+    """Loads and returns the yaml schema for runem.
+
+    Returns:
+        Any: the Draft202012Validator conformant schema.
+    """
+    schema_path: pathlib.Path = pathlib.Path(__file__).with_name("schema.yml")
+    if not schema_path.exists():
+        error(
+            (
+                "runem schema file not found, cannot continue! "
+                f"Is the install corrupt? {schema_path}"
+            )
+        )
+        raise SystemExitBad(1)
+    schema: typing.Any = load_yaml_object(schema_path)
+    return schema
+
+
+def validate_runem_file(
+    cfg_filepath: pathlib.Path,
+    all_config: typing.Any,
+) -> None:
+    """Validates the config Loader object against the runem schema.
+
+    Exits if the files does not validate.
+    """
+    schema: typing.Any = _load_runem_schema()
+    errors: ValidationErrors = validate_yaml(all_config, schema)
+    if not errors:
+        # aok
+        return
+
+    error(f"failed to validate runem config [yellow]{cfg_filepath}[/yellow]")
+    for err in errors:
+        path = ".".join(map(str, err.path)) or "<root>"
+        log(f" [yellow]{path}[/yellow]: {err.message}")
+    raise SystemExit("Config validation failed.")

runem/informative_dict.py

@@ -9,8 +9,7 @@ class InformativeDict(typing.Dict[K, V], typing.Generic[K, V]):
     """A dictionary type that prints out the available keys."""
 
     def __getitem__(self, key: K) -> V:
-        """Attempt to retrieve an item, raising a detailed exception if the key is not
-        found."""
+        """Attempt to get item, raising a detailed exception if the key is not found."""
         try:
             return super().__getitem__(key)
         except KeyError:
@@ -24,19 +23,25 @@ class ReadOnlyInformativeDict(InformativeDict[K, V], typing.Generic[K, V]):
     """A read-only variant of the above."""
 
     def __setitem__(self, key: K, value: V) -> None:
+        """Readonly object, setitem disallowed."""
         raise NotImplementedError("This dictionary is read-only")
 
     def __delitem__(self, key: K) -> None:
+        """Readonly object, delitem disallowed."""
         raise NotImplementedError("This dictionary is read-only")
 
     def pop(self, *args: typing.Any, **kwargs: typing.Any) -> V:
+        """Readonly object, pop disallowed."""
         raise NotImplementedError("This dictionary is read-only")
 
     def popitem(self) -> typing.Tuple[K, V]:
+        """Readonly object, popitem disallowed."""
         raise NotImplementedError("This dictionary is read-only")
 
     def clear(self) -> None:
+        """Readonly object, clear disallowed."""
         raise NotImplementedError("This dictionary is read-only")
 
     def update(self, *args: typing.Any, **kwargs: typing.Any) -> None:
+        """Readonly object, update disallowed."""
         raise NotImplementedError("This dictionary is read-only")

runem/job.py

@@ -71,7 +71,6 @@ class Job:
 
         TODO: make a non-static member function
         """
-
         # default to all file-tags
         tags_for_files: typing.Iterable[str] = file_lists.keys()
         use_default_tags: bool = job_tags is None
@@ -91,7 +90,6 @@ class Job:
 
         TODO: make a non-static member function
         """
-
         # First try one of the following keys.
         valid_name_keys = ("label", "command")
         for candidate in valid_name_keys:
@@ -101,6 +99,6 @@ class Job:
 
         # The try the python-wrapper address
         try:
-            return f'{job["addr"]["file"]}.{job["addr"]["function"]}'
+            return f"{job['addr']['file']}.{job['addr']['function']}"
         except KeyError:
             raise NoJobName()  # pylint: disable=raise-missing-from

runem/job_execute.py

@@ -105,7 +105,7 @@ def job_execute_inner(
         reports = function(**all_k_args)
     except BaseException:  # pylint: disable=broad-exception-caught
         # log that we hit an error on this job and re-raise
-        log(decorate=False)
+        log(prefix=False)
         error(f"job: job '{Job.get_job_name(job_config)}' failed to complete!")
         # re-raise
         raise
@@ -132,12 +132,15 @@ def job_execute(
     """
     this_id: str = str(uuid.uuid4())
     running_jobs[this_id] = Job.get_job_name(job_config)
-    results = job_execute_inner(
-        job_config,
-        config_metadata,
-        file_lists,
-        **kwargs,
-    )
-    completed_jobs[this_id] = running_jobs[this_id]
-    del running_jobs[this_id]
+    try:
+        results = job_execute_inner(
+            job_config,
+            config_metadata,
+            file_lists,
+            **kwargs,
+        )
+    finally:
+        # Always tidy-up job statuses
+        completed_jobs[this_id] = running_jobs[this_id]
+        del running_jobs[this_id]
     return results

runem/job_filter.py

@@ -101,21 +101,21 @@ def filter_jobs(  # noqa: C901
         if tags_to_run:
             log(
                 f"filtering for tags {printable_set(tags_to_run)}",
-                decorate=True,
+                prefix=True,
                 end="",
             )
         if tags_to_avoid:
             if tags_to_run:
-                log(", ", decorate=False, end="")
+                log(", ", prefix=False, end="")
             else:
-                log(decorate=True, end="")
+                log(prefix=True, end="")
             log(
                 f"excluding jobs with tags {printable_set(tags_to_avoid)}",
-                decorate=False,
+                prefix=False,
                 end="",
             )
         if tags_to_run or tags_to_avoid:
-            log(decorate=False)
+            log(prefix=False)
     filtered_jobs: PhaseGroupedJobs = defaultdict(list)
     for phase in config_metadata.phases:
         if phase not in phases_to_run:

runem/job_wrapper_python.py

@@ -15,7 +15,6 @@ def _load_python_function_from_module(
     function_to_load: str,
 ) -> JobFunction:
     """Given a job-description dynamically loads the test-function so we can call it."""
-
     # first locate the module relative to the config file
     abs_module_file_path: pathlib.Path = (
         cfg_filepath.parent / module_file_path
@@ -56,9 +55,9 @@ def _load_python_function_from_module(
     except AttributeError as err:
         raise FunctionNotFound(
             (
-                f"ERROR! Check that function '{function_to_load}' "
-                f"exists in '{str(module_file_path)}' as expected in "
-                f"your config at '{str(cfg_filepath)}"
+                f"Check that function '[blue]{function_to_load}[/blue]' "
+                f"exists in '[blue]{str(module_file_path)}[/blue]' as expected in "
+                f"your config at '[blue]{str(cfg_filepath)}[/blue]'"
             )
         ) from err
     return function
@@ -109,9 +108,9 @@ def get_job_wrapper_py_func(
         ) from err
 
     anchored_file_path = cfg_filepath.parent / module_file_path
-    assert (
-        anchored_file_path.exists()
-    ), f"{module_file_path} not found at {anchored_file_path}!"
+    assert anchored_file_path.exists(), (
+        f"{module_file_path} not found at {anchored_file_path}!"
+    )
 
     module_name = module_file_path.stem.replace(" ", "_").replace("-", "_")
 

runem/log.py

@@ -5,7 +5,7 @@ from runem.blocking_print import blocking_print
 
 def log(
     msg: str = "",
-    decorate: bool = True,
+    prefix: typing.Optional[bool] = None,
     end: typing.Optional[str] = None,
 ) -> None:
     """Thin wrapper around 'print', change the 'msg' & handles system-errors.
@@ -26,7 +26,10 @@ def log(
     # Remove any markup as it will probably error, if unsanitised.
     # msg = escape(msg)
 
-    if decorate:
+    if prefix is None:
+        prefix = True
+
+    if prefix:
         # Make it clear that the message comes from `runem` internals.
         msg = f"[light_slate_grey]runem[/light_slate_grey]: {msg}"
 

runem/report.py

@@ -46,12 +46,13 @@ def replace_bar_graph_characters(text: str, end_str: str, replace_char: str) ->
     """Replaces block characters in lines containing `end_str` with give char.
 
     Args:
-        text_lines (List[str]): A list of strings, each representing a line of text.
+        text (str): Text containing lines of bar-graphs (perhaps)
+        end_str (str): If contained by a line, the bar-graph shapes are replaced.
         replace_char (str): The character to replace all bocks with
 
     Returns:
-        List[str]: The modified list of strings with block characters replaced
-                   on specified lines.
+        str: The modified `text` with block characters replaced on specific
+             lines.
     """
     # Define the block character and its light shade replacement
     block_chars = (

runem/run_command.py

@@ -41,9 +41,7 @@ RecordSubJobTimeType = typing.Callable[[str, timedelta], None]
 
 
 def parse_stdout(stdout: str, prefix: str) -> str:
-    """Prefixes each line of the output with a given label, except trailing new
-    lines."""
-
+    """Prefixes each line of output with a given label, except trailing new lines."""
     # Edge case: Return the prefix immediately for an empty string
     if not stdout:
         return prefix
@@ -93,13 +91,13 @@ def _log_command_execution(
     if verbose:
         log(
             f"running: start: [blue]{label}[/blue]: [yellow]{cmd_string}[yellow]",
-            decorate=decorate_logs,
+            prefix=decorate_logs,
         )
         if valid_exit_ids is not None:
             valid_exit_strs = ",".join(str(exit_code) for exit_code in valid_exit_ids)
             log(
                 f"\tallowed return ids are: [green]{valid_exit_strs}[/green]",
-                decorate=decorate_logs,
+                prefix=decorate_logs,
             )
 
         if env_overrides:
@@ -108,11 +106,11 @@ def _log_command_execution(
             )
             log(
                 f"ENV OVERRIDES: [yellow]{env_overrides_as_string} {cmd_string}[/yellow]",
-                decorate=decorate_logs,
+                prefix=decorate_logs,
             )
 
         if cwd:
-            log(f"cwd: {str(cwd)}", decorate=decorate_logs)
+            log(f"cwd: {str(cwd)}", prefix=decorate_logs)
 
 
 def run_command(  # noqa: C901
@@ -175,7 +173,7 @@ def run_command(  # noqa: C901
                         parse_stdout(
                             line, prefix=f"[green]| [/green][blue]{label}[/blue]: "
                         ),
-                        decorate=False,
+                        prefix=False,
                     )
 
             # Wait for the subprocess to finish and get the exit code
@@ -206,7 +204,7 @@ def run_command(  # noqa: C901
         error_string = (
             f"runem: [red bold]FATAL[/red bold]: command failed: [blue]{label}[/blue]"
             f"\n\t[yellow]{env_overrides_as_string}{cmd_string}[/yellow]"
-            f"\n[red underline]| ERROR[/red underline]"
+            f"\n[red underline]| ERROR[/red underline]: [blue]{label}[/blue]"
             f"\n{str(parsed_stdout)}"
             f"\n[red underline]| ERROR END[/red underline]"
         )
@@ -219,7 +217,7 @@ def run_command(  # noqa: C901
     if verbose:
         log(
             f"running: done: [blue]{label}[/blue]: [yellow]{cmd_string}[/yellow]",
-            decorate=decorate_logs,
+            prefix=decorate_logs,
         )
 
     if record_sub_job_time is not None:

runem/runem.py

@@ -19,6 +19,7 @@ We do:
 - time tests and tell you what used the most time, and how much time run-tests saved
   you
 """
+
 import contextlib
 import multiprocessing
 import os
@@ -48,6 +49,7 @@ from runem.log import error, log, warn
 from runem.report import report_on_run
 from runem.run_command import RunemJobError
 from runem.types.common import OrderedPhases, PhaseName
+from runem.types.errors import SystemExitBad
 from runem.types.filters import FilePathListLookup
 from runem.types.hooks import HookName
 from runem.types.runem_config import Config, Jobs, PhaseGroupedJobs
@@ -68,7 +70,6 @@ def _determine_run_parameters(argv: typing.List[str]) -> ConfigMetadata:
 
     Return a ConfigMetadata object with all the required information.
     """
-
     # Because we want to be able to show logging whilst parsing .runem.yml config, we
     # need to check the state of the logging-verbosity switches here, manually, as well.
     verbose = "--verbose" in argv
@@ -105,13 +106,14 @@ def _update_progress(
     """Updates progress report periodically for running tasks.
 
     Args:
-        label (str): The identifier.
+        phase (str): The currently running phase.
         running_jobs (Dict[str, str]): The currently running jobs.
+        completed_jobs (Dict[str, str]): The jobs that have finished work.
         all_jobs (Jobs): All jobs, encompassing both completed and running jobs.
         is_running (ValueProxy[bool]): Flag indicating if jobs are still running.
         num_workers (int): Indicates the number of workers performing the jobs.
+        show_spinner (bool): Whether to show the animated spinner or not.
     """
-
     last_running_jobs_set: typing.Set[str] = set()
 
     # Using the `rich` module to show a loading spinner on console
@@ -132,7 +134,8 @@ def _update_progress(
                 "blue",
             )  # Reflect current running jobs accurately
             report: str = (
-                f"[green]{phase}[/green]: {progress}({num_workers}): {running_jobs_list}"
+                f"[green]{phase}[/green]: {progress}({num_workers}): "
+                f"{running_jobs_list}"
             )
             if show_spinner:
                 assert isinstance(spinner_ctx, Status)
@@ -293,8 +296,8 @@ def _main(
         log(f"found {len(file_lists)} batches, ", end="")
         for tag in sorted(file_lists.keys()):
             file_list = file_lists[tag]
-            log(f"{len(file_list)} '{tag}' files, ", decorate=False, end="")
-        log(decorate=False)  # new line
+            log(f"{len(file_list)} '{tag}' files, ", prefix=False, end="")
+        log(prefix=False)  # new line
 
     filtered_jobs_by_phase: PhaseGroupedJobs = filter_jobs(
         config_metadata=config_metadata,
@@ -371,7 +374,7 @@ def timed_main(argv: typing.List[str]) -> None:
         # we got a failure somewhere, now that we've reported the timings we
         # re-raise.
         error(failure_exception.stdout)
-        raise failure_exception
+        raise SystemExitBad(1) from failure_exception
 
 
 if __name__ == "__main__":

runem/schema.yml

@@ -0,0 +1,137 @@
+#%RAML 1.0  (← just a comment so VS Code picks up YAML)
+$schema: "https://json-schema.org/draft/2020-12/schema"
+$title: Runem pipeline definition
+$defs:
+  # ----- common pieces -------------------------------------------------------
+  phase:
+    type: string
+
+  addr:
+    type: object
+    required: [file, function]
+    additionalProperties: false
+    properties:
+      file:     { type: string, minLength: 1 }
+      function: { type: string, minLength: 1 }
+
+  ctx:
+    type: object
+    additionalProperties: false
+    properties:
+      cwd:
+        oneOf:
+          - type: string
+          - type: array
+            minItems: 1
+            items: { type: string, minLength: 1 }
+      params:
+        type: object      # free‑form kv‑pairs for hooks
+        additionalProperties: true
+
+  when:
+    type: object
+    required: [phase]
+    additionalProperties: false
+    properties:
+      phase: { $ref: "#/$defs/phase" }
+      tags:
+        type: array
+        items: { type: string, minLength: 1 }
+        uniqueItems: true
+
+  # ----- top‑level entity types ---------------------------------------------
+  config:
+    type: object
+    required: []
+    additionalProperties: false
+    properties:
+      min_version:
+        type: string
+
+      phases:
+        type: array
+        minItems: 1
+        items: { $ref: "#/$defs/phase" }
+        uniqueItems: true
+
+      files:
+        type: [array, 'null']
+        minItems: 0
+        items:
+          type: object
+          required: [filter]
+          additionalProperties: false
+          properties:
+            filter:
+              type: object
+              required: [tag, regex]
+              additionalProperties: false
+              properties:
+                tag:   { type: string, minLength: 1 }
+                regex: { type: string, minLength: 1 } # leave pattern‑checking to the engine
+
+      options:
+        type: [array, 'null']
+        minItems: 0
+        items:
+          type: object
+          required: [option]
+          additionalProperties: false
+          properties:
+            option:
+              type: object
+              required: [name, type, default, desc]
+              additionalProperties: false
+              properties:
+                name:    { type: string, minLength: 1 }
+                alias:   { type: string, minLength: 1 }
+                desc:    { type: string, minLength: 1 }
+                type:
+                  const: bool           # always "bool" per sample
+                default: { type: boolean }
+
+  hook:
+    type: object
+    required: [hook_name]
+    oneOf:
+      - required: [command]
+      - required: [addr]
+    additionalProperties: false
+    properties:
+      hook_name: { type: string, minLength: 1 }
+      addr:      { $ref: "#/$defs/addr" }
+      command: { type: string, minLength: 1 }
+
+  job:
+    type: object
+    oneOf:
+      - required: [command]
+      - required: [addr]
+    additionalProperties: false
+    properties:
+      label:   { type: string, minLength: 1 }
+      addr:    { $ref: "#/$defs/addr" }
+      command: { type: string, minLength: 1 }
+      ctx:    { $ref: "#/$defs/ctx" }
+      when:    { $ref: "#/$defs/when" }
+    oneOf:
+      - required: [addr]    # either addr
+      - required: [command] # or command, but not both
+    not:
+      anyOf:
+        - required: [addr, command]   # forbid both together
+
+# ---------- ROOT -------------------------------------------------------------
+type: array
+minItems: 1
+items:
+  type: object
+  additionalProperties: false
+  oneOf:
+    - required: [config]
+    - required: [hook]
+    - required: [job]
+  properties:
+    config: { $ref: "#/$defs/config" }
+    hook:   { $ref: "#/$defs/hook" }
+    job:    { $ref: "#/$defs/job" }

runem/types/errors.py

@@ -1,4 +1,14 @@
+from typing import Optional
+
+
 class FunctionNotFound(ValueError):
     """Thrown when the test-function cannot be found."""
 
     pass
+
+
+class SystemExitBad(SystemExit):
+    def __init__(self, code: Optional[int] = None) -> None:
+        super().__init__()
+        self.code = 1 if code is None else code  # non-zero bad exit code
+        assert self.code > 0, "A bad exit code should be non-zero and >0"

runem/types/hooks.py

@@ -4,7 +4,7 @@ import enum
 class HookName(enum.Enum):
     """List supported hooks.
 
-    TODO:
+    Todo:
     - before all tasks are run, after config is read
     - BEFORE_ALL = "before-all"
     - after all tasks are done, before reporting

runem/types/types_jobs.py

@@ -1,28 +1,25 @@
-"""
+"""Job‑typing helpers.
+
+Cross‑version advice
+--------------------
+* Type variadic keyword arguments as **kwargs: Unpack[KwArgsT] for clarity.
+* Always import Unpack from ``typing_extensions``.
+  - Std‑lib Unpack appears only in Py 3.12+.
+  - ``typing_extensions`` works on 3.9‑3.12, so one import path keeps
+    mypy/pyright happy without conditional logic.
+
+Example:
+~~~~~~~
+from typing_extensions import TypedDict, Unpack
+
+
+class SaveKwArgs(TypedDict):
+    path: str
+    overwrite: bool
+
 
-Some note on Unpack and kwargs:
-    We *try* to strongly type `**kwargs` for clarity.
-    We have tried several ways to define a Generic type that encapsulates
-        `**kwargs: SingleType`
-    ... but none of the solutions worked with python 3.9 -> 3.12 and mypy 1.9.0,
-    so we have to recommend instead using:
-        `**kwargs: Unpack[KwArgsType]`
-
-    For this to work across versions of python where support for Unpack changes;
-    for example `Unpack` is a python 3.12 feature, but available in the
-    `typing_extensions` module.
-
-    So, for now, it looks like we get away with importing `Unpack` from the
-    `typing_extensions` module, even in python 3.12, so we will use, and
-    recommend using, the `typing_extensions` of `Unpack`, until it becomes
-    obsolete.
-
-    Alternatively, we can use the following, but it's unnecessarily verbose.
-
-    if sys.version_info >= (3, 12):  # pragma: no coverage
-        from typing import Unpack
-    else:  # pragma: no coverage
-        from typing_extensions import Unpack
+def save_job(**kwargs: Unpack[SaveKwArgs]) -> None:
+    ...
 """
 
 import pathlib

runem/yaml_utils.py

@@ -0,0 +1,19 @@
+import pathlib
+import typing
+
+import yaml
+
+
+def load_yaml_object(yaml_file: pathlib.Path) -> typing.Any:
+    """Loads using full_load, a yaml file.
+
+    This is likely to have safety concerns in non-trusted projects.
+
+    Returns:
+        YAML Loader object: the full PyYAML loader object.
+    """
+    # Do a full, untrusted load of the runem config
+    # TODO: work out safety concerns of this
+    with yaml_file.open("r+", encoding="utf-8") as file_handle:
+        full_yaml_object: typing.Any = yaml.full_load(file_handle)
+    return full_yaml_object

runem/yaml_validation.py

@@ -0,0 +1,28 @@
+from typing import Any, List
+
+from jsonschema import Draft202012Validator, ValidationError
+
+# For now just return the raw ValidationErrors as a list
+ValidationErrors = List[ValidationError]
+
+
+def validate_yaml(yaml_data: Any, schema: Any) -> ValidationErrors:
+    """Validates the give yaml data against the given schema, returning any errors.
+
+    We use more future-looking validation so that we can have richer and more
+    descriptive schema.
+
+    Params:
+        instance: JSON data loaded via `load_json` or similar
+        schema: schema object compatible with a Draft202012Validator
+
+    Returns:
+        ValidationErrors: a sorted list of errors in the file, empty if none found
+    """
+    validator = Draft202012Validator(schema)
+    errors: ValidationErrors = sorted(
+        validator.iter_errors(yaml_data),
+        key=lambda e: e.path,
+    )
+
+    return errors

{runem-0.6.0.dist-info → runem-0.7.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: runem
-Version: 0.6.0
+Version: 0.7.1
 Summary: Awesome runem created by lursight
 Author: lursight
 License: Specify your license here
@@ -16,31 +16,32 @@ Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: packaging>=22.0
 Requires-Dist: PyYAML>=5.0.0
+Requires-Dist: jsonschema>=4.22
 Requires-Dist: rich>10.0.0
 Requires-Dist: typing_extensions>3.0.0
 Provides-Extra: tests
-Requires-Dist: black==24.10.0; extra == "tests"
 Requires-Dist: coverage==7.5; extra == "tests"
-Requires-Dist: docformatter==1.7.5; extra == "tests"
 Requires-Dist: flake8-bugbear==24.2.6; extra == "tests"
 Requires-Dist: flake8==7.0.0; extra == "tests"
 Requires-Dist: gitchangelog==3.0.4; extra == "tests"
-Requires-Dist: isort==5.13.2; extra == "tests"
 Requires-Dist: mkdocs==1.5.3; extra == "tests"
 Requires-Dist: mypy==1.9.0; extra == "tests"
 Requires-Dist: pydocstyle==6.3.0; extra == "tests"
-Requires-Dist: pylint==3.1.0; extra == "tests"
+Requires-Dist: pylint==3.3.6; extra == "tests"
 Requires-Dist: pylama==8.4.1; extra == "tests"
-Requires-Dist: pytest-cov==6.0.0; extra == "tests"
+Requires-Dist: pytest-cov==6.1.1; extra == "tests"
 Requires-Dist: pytest-profiling==1.7.0; extra == "tests"
 Requires-Dist: pytest-xdist==3.6.1; extra == "tests"
-Requires-Dist: pytest==8.3.3; extra == "tests"
+Requires-Dist: pytest==8.3.5; extra == "tests"
+Requires-Dist: ruff==0.11.6; extra == "tests"
 Requires-Dist: setuptools; extra == "tests"
 Requires-Dist: termplotlib==0.3.9; extra == "tests"
 Requires-Dist: tox; extra == "tests"
 Requires-Dist: types-PyYAML==6.0.12.20240311; extra == "tests"
 Requires-Dist: requests-mock==1.11.0; extra == "tests"
+Requires-Dist: types-jsonschema; extra == "tests"
 Requires-Dist: types-setuptools; extra == "tests"
+Dynamic: license-file
 
 <!-- [![codecov](https://codecov.io/gh/lursight/runem/branch/main/graph/badge.svg?token=run-test_token_here)](https://codecov.io/gh/lursight/runem) -->
 [![CI](https://github.com/lursight/runem/actions/workflows/main.yml/badge.svg)](https://github.com/lursight/runem/actions/workflows/main.yml)

{runem-0.6.0.dist-info → runem-0.7.1.dist-info}/RECORD

@@ -1,41 +1,46 @@
-runem/VERSION,sha256=l6XW5UCmEg0Jw53bZn4Ojiusf8wv_vgTuC4I_WA2W84,6
+runem/VERSION,sha256=kCMRx7s4-hZY_0A972FY7nN4_p1uLihPocOHNcMb0ws,6
 runem/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 runem/__main__.py,sha256=dsOiVZegpfK9JOs5n7UmbX5iwwbj7iFkEbLoVeEgAn4,136
 runem/base.py,sha256=EZfR7FIlwEdU9Vfe47Wk2DOO8GQqpKxxLNKp6YHueZ4,316
-runem/blocking_print.py,sha256=2nCvc10zXl1DRJldkruKthKjfmZKErcmxQzt3pjmN-c,2289
+runem/blocking_print.py,sha256=UKU_BM7wzPvn6RKw_tFPw4Lzpjdtbwq5MIaAdL1_zN8,2435
 runem/cli.py,sha256=wEt_Jnumhl8SiOdKdSJzLkJpWv6n3_Odhi_HeIixr1k,134
-runem/command_line.py,sha256=qkZFCCq9hUl6RO398SJzoigv8di5jGw2sdNwgTVBdd8,14474
-runem/config.py,sha256=UiEU0Jyg5qjrNStvasWYjMOABQHhpZjbPiX3-sH_CMg,5969
+runem/command_line.py,sha256=Q5xH7kGzc3YpIbGpU43B_pwKvp-LchSW7IgfGnf2Ke0,14437
+runem/config.py,sha256=PWj4wj90WrRiGuXOUy6mpevJXQM57epDWG5NYDrGK2w,6049
 runem/config_metadata.py,sha256=krDomUcADsAeUQrxwNmOS58eeaNIlqmhWIKWv8mUH4A,3300
-runem/config_parse.py,sha256=zXQ4rpj-igQufB5JtTsI1mOE_gBTdBcI2hI6HWU28gg,13830
+runem/config_parse.py,sha256=A1pLfEL5CEaexH-Kn_PYthAa9mwmOS5UrD0kEbDcqPo,13843
+runem/config_validate.py,sha256=gtatObD1qBkEK-0CJ2rJPT5A7EBTWE_9V_AiGZZ1FQI,1424
 runem/files.py,sha256=59boeFvUANYOS-PllIjeKIht6lNINZ43WxahDg90oAc,4392
 runem/hook_manager.py,sha256=H0TL3HCqU2mgKm_-dgCD7TsK5T1bLT4g7x6kpytMPhU,4350
-runem/informative_dict.py,sha256=U7p9z78UwOT4TAfng1iDXCEyeYz6C-XZlx9Z1pWNVrI,1548
-runem/job.py,sha256=NOdRQnGePPyYdmIR_6JKVFzp9nbgNGetpE13bHEHaf4,3442
-runem/job_execute.py,sha256=-76IJI0PDU_XdQiDxTKUfOHEno9pixxQb_zi58rFumo,4702
-runem/job_filter.py,sha256=7vgG4YWJ9gyGBFjV7QbSojG5ofYoszAmxXx9HnMLkHo,5384
+runem/informative_dict.py,sha256=4UUE_RU6zEX1JFFlVUK5EMdmPdJ5ZOo5scU93fYi0iU,1831
+runem/job.py,sha256=SX_uHaFxocFm2QT2hRTfAv7mADnMj2th85L_JzYwS-4,3440
+runem/job_execute.py,sha256=Sn5v7KoyCOw2FH-bJaur_zYChwmvCXhmdZ692B3SZA8,4792
+runem/job_filter.py,sha256=4KMjsI5-tiK0b90RTlivxm5Xdt0gWdRz30voqqL_ijI,5374
 runem/job_runner_simple_command.py,sha256=iP5an6yixW8o4C0ZBtu6csb-oVK3Q62ZZgtHBmxlXaU,2428
 runem/job_wrapper.py,sha256=q5GtopZ5vhSJ581rwU4-lF9KnbL3ZYgOC8fqaCnXD_g,983
-runem/job_wrapper_python.py,sha256=rx7J_N-JXs8GgMq7Sla7B9s_ZAfofKUhEnzgMcq_bts,4303
-runem/log.py,sha256=6r6HIJyvp19P6PZNo93qdIxE0SpTAsY4ELrETBl1dC4,1363
+runem/job_wrapper_python.py,sha256=9rM6OXs0rNRVNwZ7OVB6L0K7RocEpiG0N-qIMuprjRA,4335
+runem/log.py,sha256=MEGWEBeFs0YVkr_U9UXnl4Isqc5MvGbEn_I3fo-V35Q,1422
 runem/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-runem/report.py,sha256=beye95AV9Lop_K7eOoLY40vKt0VGkIdg-ig-SmJ_5MY,9083
-runem/run_command.py,sha256=R77jqAtrXPBkFtT7QXJfnQkivU4h01M8G0Q2sjWY6Gs,7691
-runem/runem.py,sha256=6XZqf59_ZMyrBaK1FlaLzMXYNoDQsTUcbFqTbpZL1ik,13192
+runem/report.py,sha256=qiGu0PPcbeFh9KJHtFKjH_LQHDfYuDgLVbUA81fNsLI,9124
+runem/run_command.py,sha256=jz-vP9yw3WDM_0mc5KrBpqFcASSQUFB9zz9u_le-m8A,7692
+runem/runem.py,sha256=Lm0mg8oTe7pCvQVZob6KR6zq7G5xoMTjrCfhBlCQwZQ,13436
 runem/runem_version.py,sha256=MbETwZO2Tb1Y3hX_OYZjKepEMKA1cjNvr-7Cqhz6e3s,271
+runem/schema.yml,sha256=LmQVtVbziaH_Z3aaE5-njR7qPYTP8qqruNQt8rVfv5M,3720
 runem/utils.py,sha256=MEYfox09rLvb6xmay_3rV1cWmdqMbhaAjOomYGNk15k,602
-runem/cli/initialise_options.py,sha256=zx_EduWQk7yGBr3XUNrffHCSInPv05edFItHLnlo9dk,918
+runem/yaml_utils.py,sha256=RyAvEp679JvavE0Kbs263Ofh8_tTXXHUdtblSeHovfU,554
+runem/yaml_validation.py,sha256=j8vnufeV8FRwg15wbitL_QLP4nh25Rx1L_14Oc2puwU,877
+runem/cli/initialise_options.py,sha256=91QjAHdfhUNI9nWVDnHAqNshxCXytXQfbsEaQo3FMLc,917
 runem/types/__init__.py,sha256=0bWG7hE7VeqJ2oIu-xhrqQud8hcNp6WNbF3uMfT_n9g,314
 runem/types/common.py,sha256=gPMSoJ3yRUYjHnoviRrpSg0gRwsGLFGWGpbTWkq4jX0,279
-runem/types/errors.py,sha256=rbM5BA6UhY1X7Q0OZLUNsG7JXAjgNFTG5KQuqPNuZm8,103
+runem/types/errors.py,sha256=9A4V5qT3ofolqRxrqBI9i9jX3r60x8nQnQnt9w1Y3co,403
 runem/types/filters.py,sha256=8R5fyMssN0ISGBilJhEtbdHFl6OP7uI51WKkB5SH6EA,255
-runem/types/hooks.py,sha256=lgrv5QAuHCEzr5dXDj4-azNcs63addY9zdrGWj5zv_s,292
+runem/types/hooks.py,sha256=9Q5THuBEH0Asdx5cj0caNmO54RckwR0tT3AgRXFROhs,292
 runem/types/options.py,sha256=y8_hyWYvhalC9-kZbvoDtxm0trZgyyGcswQqfuQy_pM,265
 runem/types/runem_config.py,sha256=qG_bghm5Nr-ZTbaZbf1v8Fx447V-hgEvvRy5NZ3t-Io,5141
-runem/types/types_jobs.py,sha256=wqiiBmRIJDbGlKcfOqewHGKx350w0p4_7pysMm7xGmo,4906
+runem/types/types_jobs.py,sha256=99b2TLwBiqjOurxRWWU7E9BlphR0KketUUSqUD_Kh5c,4432
+runem-0.7.1.dist-info/licenses/LICENSE,sha256=awOCsWJ58m_2kBQwBUGWejVqZm6wuRtCL2hi9rfa0X4,1211
 scripts/test_hooks/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 scripts/test_hooks/json_validators.py,sha256=N2FyWcpjWzfFGycXLo-ecNLJkxTFPPbqPfVBcJLBlb4,967
-scripts/test_hooks/py.py,sha256=YUbwNny7NPmv2bY7k7YcbJ-jRcnNfjQajE9Hn1MLaBc,8821
+scripts/test_hooks/py.py,sha256=Nku44p6ZqJb5d9uo2bPfpeqf8g1LDHRDqL4ou0Y0G_k,10520
 scripts/test_hooks/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 scripts/test_hooks/runem_hooks.py,sha256=FJMuDBEOz3dr9gBW3WW6yKbUJs_LFXb3klpqSzCAZRk,628
 scripts/test_hooks/yarn.py,sha256=1QsG1rKAclpZoqp86ntkuvzYaYN4UkEvO0JhO2Kf5C8,1082
@@ -44,9 +49,8 @@ tests/data/help_output.3.10.txt,sha256=5TUpNITVL6pD5BpFAl-Orh3vkOpStveijZzvgJuI_
 tests/data/help_output.3.11.txt,sha256=ycrF-xKgdQ8qrWzkkR-vbHe7NulUTsCsS0_Gda8xYDs,4162
 tests/test_types/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/test_types/test_public_api.py,sha256=QHiwt7CetQur65JSbFRnOzQxhCJkX5MVLymHHVd_6yc,160
-runem-0.6.0.dist-info/LICENSE,sha256=awOCsWJ58m_2kBQwBUGWejVqZm6wuRtCL2hi9rfa0X4,1211
-runem-0.6.0.dist-info/METADATA,sha256=00_gsiXfiHPCwcZumnmOU2KjaUTvR6kF9c9nr4p2YHc,5892
-runem-0.6.0.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-runem-0.6.0.dist-info/entry_points.txt,sha256=nu0g_vBeuPihYtimbtlNusxWovylMppvJ8UxdJlJfvM,46
-runem-0.6.0.dist-info/top_level.txt,sha256=NkdxkwLKNNhxItveR2KqNqTshTZ268m5D7SjJEmG4-Y,20
-runem-0.6.0.dist-info/RECORD,,
+runem-0.7.1.dist-info/METADATA,sha256=wX6dRVH8E905TMINRtxhwtGoVeyaokkjrQcg-2h1TgY,5894
+runem-0.7.1.dist-info/WHEEL,sha256=pxyMxgL8-pra_rKaQ4drOZAegBVuX-G_4nRHjjgWbmo,91
+runem-0.7.1.dist-info/entry_points.txt,sha256=nu0g_vBeuPihYtimbtlNusxWovylMppvJ8UxdJlJfvM,46
+runem-0.7.1.dist-info/top_level.txt,sha256=NkdxkwLKNNhxItveR2KqNqTshTZ268m5D7SjJEmG4-Y,20
+runem-0.7.1.dist-info/RECORD,,

{runem-0.6.0.dist-info → runem-0.7.1.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (75.8.0)
+Generator: setuptools (79.0.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

scripts/test_hooks/py.py

@@ -9,7 +9,69 @@ from runem.run_command import RunCommandUnhandledError, run_command
 from runem.types import FilePathList, JobKwargs, JobName, JobReturnData, Options
 
 
-def _job_py_code_reformat(
+def _job_py_code_ruff_reformat(
+    **kwargs: typing.Any,
+) -> None:
+    """Runs python formatting code in serial order as one influences the other."""
+    label: JobName = kwargs["label"]
+    options: Options = kwargs["options"]
+    python_files: FilePathList = kwargs["file_list"]
+
+    # put into 'check' mode if requested on the command line
+    extra_args = []
+    if options["check-only"]:
+        extra_args.append("--check")
+
+    if not options["ruff"]:
+        # Do not run `ruff` if opted-out
+        return
+
+    # If ruff is enabled we do NOT run black etc. because ruff does that
+    # for us, faster and better.
+    ruff_format_cmd = [
+        "python3",
+        "-m",
+        "ruff",
+        "format",
+        *extra_args,
+        *python_files,
+    ]
+    kwargs["label"] = f"{label} ruff"
+    run_command(cmd=ruff_format_cmd, **kwargs)
+
+
+def _job_py_ruff_lint(
+    **kwargs: typing.Any,
+) -> None:
+    """Runs python formatting code in serial order as one influences the other."""
+    label: JobName = kwargs["label"]
+    options: Options = kwargs["options"]
+    python_files: FilePathList = kwargs["file_list"]
+
+    # try to auto-fix issues (one benefit of ruff over flake8 etc.)
+    extra_args = []
+    if options["fix"]:
+        extra_args.append("--fix")
+
+    if not options["ruff"]:
+        # Do not run `ruff` if opted-out
+        return
+
+    # If ruff is enabled we do NOT run black etc. because ruff does that
+    # for us, faster and better.
+    ruff_lint_cmd = [
+        "python3",
+        "-m",
+        "ruff",
+        "check",
+        *extra_args,
+        *python_files,
+    ]
+    kwargs["label"] = f"{label} ruff"
+    run_command(cmd=ruff_lint_cmd, **kwargs)
+
+
+def _job_py_code_reformat_deprecated(
     **kwargs: Unpack[JobKwargs],
 ) -> None:
     """Runs python formatting code in serial order as one influences the other."""