runem 0.5.0__py3-none-any.whl → 0.7.0__py3-none-any.whl

runem/VERSION

@@ -1 +1 @@
-0.5.0
+0.7.0

runem/blocking_print.py

@@ -13,7 +13,14 @@ def _reset_console() -> Console:
 
     RICH_CONSOLE = Console(
         log_path=False,  # Do NOT print the source path.
-        markup=False,  # Do NOT print markup e.g. `[blink]Don't Panic![/blink]`.
+        # We allow markup here, BUT stdout/stderr from other procs should have
+        # `escape()` called on them so they don't error here.
+        # This means 'rich' effects/colors can be judiciously applied:
+        # e.g. `[blink]Don't Panic![/blink]`.
+        markup=True,
+        # `highlight` is what colourises string and number in print() calls.
+        # We do not want this to be auto-magic.
+        highlight=False,
     )
     return RICH_CONSOLE
 
@@ -31,7 +38,8 @@ def _reset_console_for_tests() -> None:
     RICH_CONSOLE = Console(
         log_path=False,  # Do NOT print the source path.
         log_time=False,  # Do not prefix with log time e.g. `[time] log message`.
-        markup=False,  # Do NOT print markup e.g. `[blink]Don't Panic![/blink]`.
+        markup=True,  # Allow some markup e.g. `[blink]Don't Panic![/blink]`.
+        highlight=False,
         width=999,  # A very wide width.
     )
 

runem/cli/initialise_options.py

@@ -13,7 +13,6 @@ def initialise_options(
 
     Returns the options dictionary
     """
-
     options: OptionsWritable = InformativeDict(
         {option["name"]: option["default"] for option in config_metadata.options_config}
     )

runem/command_line.py

@@ -43,10 +43,8 @@ def _get_argparse_help_formatter() -> typing.Any:
 
     if use_fixed_width:
         # Use custom formatter with the width specified in the environment variable
-        return (
-            lambda prog: HelpFormatterFixedWidth(  # pylint: disable=unnecessary-lambda
-                prog
-            )
+        return lambda prog: HelpFormatterFixedWidth(  # pylint: disable=unnecessary-lambda
+            prog
         )
 
     # Use default formatter
@@ -294,12 +292,12 @@ def parse_args(
     error_on_log_logic(args.verbose, args.silent)
 
     if args.show_root_path_and_exit:
-        log(str(config_metadata.cfg_filepath.parent), decorate=False)
+        log(str(config_metadata.cfg_filepath.parent), prefix=False)
         # cleanly exit
         sys.exit(0)
 
     if args.show_version_and_exit:
-        log(str(get_runem_version()), decorate=False)
+        log(str(get_runem_version()), prefix=False)
         # cleanly exit
         sys.exit(0)
 
@@ -383,7 +381,6 @@ def initialise_options(
 
     Returns the options dictionary
     """
-
     options: OptionsWritable = InformativeDict(
         {option["name"]: option["default"] for option in config_metadata.options_config}
     )

runem/config.py

@@ -2,9 +2,9 @@ import pathlib
 import sys
 import typing
 
-import yaml
 from packaging.version import Version
 
+from runem.config_validate import validate_runem_file
 from runem.log import error, log
 from runem.runem_version import get_runem_version
 from runem.types.runem_config import (
@@ -13,6 +13,7 @@ from runem.types.runem_config import (
     GlobalSerialisedConfig,
     UserConfigMetadata,
 )
+from runem.yaml_utils import load_yaml_object
 
 CFG_FILE_YAML = pathlib.Path(".runem.yml")
 
@@ -46,7 +47,7 @@ def _search_up_multiple_dirs_for_file(
 
 
 def _find_config_file(
-    config_filename: typing.Union[str, pathlib.Path]
+    config_filename: typing.Union[str, pathlib.Path],
 ) -> typing.Tuple[typing.Optional[pathlib.Path], typing.Tuple[pathlib.Path, ...]]:
     """Searches up from the cwd for the given config file-name."""
     start_dirs = (pathlib.Path(".").absolute(),)
@@ -117,8 +118,11 @@ def _conform_global_config_types(
 
 def load_and_parse_config(cfg_filepath: pathlib.Path) -> Config:
     """For the given config file pass, project or user, load it & parse/conform it."""
-    with cfg_filepath.open("r+", encoding="utf-8") as config_file_handle:
-        all_config = yaml.full_load(config_file_handle)
+    all_config = load_yaml_object(cfg_filepath)
+    validate_runem_file(
+        cfg_filepath,
+        all_config,
+    )
 
     conformed_config: Config
     global_config: typing.Optional[GlobalConfig]

runem/config_parse.py

@@ -162,7 +162,8 @@ def parse_job_config(
             ("cwd" in job["ctx"]) and (job["ctx"]["cwd"] is not None)
         )
         if (not have_ctw_cwd) or isinstance(
-            job["ctx"]["cwd"], str  # type: ignore # handled above
+            job["ctx"]["cwd"],  # type: ignore # handled above
+            str,
         ):
             # if
             # - we don't have a cwd, ctx

runem/config_validate.py

@@ -0,0 +1,47 @@
+import pathlib
+import typing
+
+from runem.log import error, log
+from runem.types.errors import SystemExitBad
+from runem.yaml_utils import load_yaml_object
+from runem.yaml_validation import ValidationErrors, validate_yaml
+
+
+def _load_runem_schema() -> typing.Any:
+    """Loads and returns the yaml schema for runem.
+
+    Returns:
+        Any: the Draft202012Validator conformant schema.
+    """
+    schema_path: pathlib.Path = pathlib.Path(__file__).with_name("schema.yml")
+    if not schema_path.exists():
+        error(
+            (
+                "runem schema file not found, cannot continue! "
+                f"Is the install corrupt? {schema_path}"
+            )
+        )
+        raise SystemExitBad(1)
+    schema: typing.Any = load_yaml_object(schema_path)
+    return schema
+
+
+def validate_runem_file(
+    cfg_filepath: pathlib.Path,
+    all_config: typing.Any,
+) -> None:
+    """Validates the config Loader object against the runem schema.
+
+    Exits if the files does not validate.
+    """
+    schema: typing.Any = _load_runem_schema()
+    errors: ValidationErrors = validate_yaml(all_config, schema)
+    if not errors:
+        # aok
+        return
+
+    error(f"failed to validate runem config [yellow]{cfg_filepath}[/yellow]")
+    for err in errors:
+        path = ".".join(map(str, err.path)) or "<root>"
+        log(f" [yellow]{path}[/yellow]: {err.message}")
+    raise SystemExit("Config validation failed.")

runem/informative_dict.py

@@ -9,8 +9,7 @@ class InformativeDict(typing.Dict[K, V], typing.Generic[K, V]):
     """A dictionary type that prints out the available keys."""
 
     def __getitem__(self, key: K) -> V:
-        """Attempt to retrieve an item, raising a detailed exception if the key is not
-        found."""
+        """Attempt to get item, raising a detailed exception if the key is not found."""
         try:
             return super().__getitem__(key)
         except KeyError:
@@ -24,19 +23,25 @@ class ReadOnlyInformativeDict(InformativeDict[K, V], typing.Generic[K, V]):
     """A read-only variant of the above."""
 
     def __setitem__(self, key: K, value: V) -> None:
+        """Readonly object, setitem disallowed."""
         raise NotImplementedError("This dictionary is read-only")
 
     def __delitem__(self, key: K) -> None:
+        """Readonly object, delitem disallowed."""
         raise NotImplementedError("This dictionary is read-only")
 
     def pop(self, *args: typing.Any, **kwargs: typing.Any) -> V:
+        """Readonly object, pop disallowed."""
         raise NotImplementedError("This dictionary is read-only")
 
     def popitem(self) -> typing.Tuple[K, V]:
+        """Readonly object, popitem disallowed."""
         raise NotImplementedError("This dictionary is read-only")
 
     def clear(self) -> None:
+        """Readonly object, clear disallowed."""
         raise NotImplementedError("This dictionary is read-only")
 
     def update(self, *args: typing.Any, **kwargs: typing.Any) -> None:
+        """Readonly object, update disallowed."""
         raise NotImplementedError("This dictionary is read-only")

runem/job.py

@@ -71,7 +71,6 @@ class Job:
 
         TODO: make a non-static member function
         """
-
         # default to all file-tags
         tags_for_files: typing.Iterable[str] = file_lists.keys()
         use_default_tags: bool = job_tags is None
@@ -91,7 +90,6 @@ class Job:
 
         TODO: make a non-static member function
         """
-
         # First try one of the following keys.
         valid_name_keys = ("label", "command")
         for candidate in valid_name_keys:
@@ -101,6 +99,6 @@ class Job:
 
         # The try the python-wrapper address
         try:
-            return f'{job["addr"]["file"]}.{job["addr"]["function"]}'
+            return f"{job['addr']['file']}.{job['addr']['function']}"
         except KeyError:
             raise NoJobName()  # pylint: disable=raise-missing-from

runem/job_execute.py

@@ -105,7 +105,7 @@ def job_execute_inner(
         reports = function(**all_k_args)
     except BaseException:  # pylint: disable=broad-exception-caught
         # log that we hit an error on this job and re-raise
-        log(decorate=False)
+        log(prefix=False)
         error(f"job: job '{Job.get_job_name(job_config)}' failed to complete!")
         # re-raise
         raise
@@ -132,12 +132,15 @@ def job_execute(
     """
     this_id: str = str(uuid.uuid4())
     running_jobs[this_id] = Job.get_job_name(job_config)
-    results = job_execute_inner(
-        job_config,
-        config_metadata,
-        file_lists,
-        **kwargs,
-    )
-    completed_jobs[this_id] = running_jobs[this_id]
-    del running_jobs[this_id]
+    try:
+        results = job_execute_inner(
+            job_config,
+            config_metadata,
+            file_lists,
+            **kwargs,
+        )
+    finally:
+        # Always tidy-up job statuses
+        completed_jobs[this_id] = running_jobs[this_id]
+        del running_jobs[this_id]
     return results

runem/job_filter.py

@@ -101,21 +101,21 @@ def filter_jobs(  # noqa: C901
         if tags_to_run:
             log(
                 f"filtering for tags {printable_set(tags_to_run)}",
-                decorate=True,
+                prefix=True,
                 end="",
             )
         if tags_to_avoid:
             if tags_to_run:
-                log(", ", decorate=False, end="")
+                log(", ", prefix=False, end="")
             else:
-                log(decorate=True, end="")
+                log(prefix=True, end="")
             log(
                 f"excluding jobs with tags {printable_set(tags_to_avoid)}",
-                decorate=False,
+                prefix=False,
                 end="",
             )
         if tags_to_run or tags_to_avoid:
-            log(decorate=False)
+            log(prefix=False)
     filtered_jobs: PhaseGroupedJobs = defaultdict(list)
     for phase in config_metadata.phases:
         if phase not in phases_to_run:

runem/job_wrapper_python.py

@@ -15,7 +15,6 @@ def _load_python_function_from_module(
     function_to_load: str,
 ) -> JobFunction:
     """Given a job-description dynamically loads the test-function so we can call it."""
-
     # first locate the module relative to the config file
     abs_module_file_path: pathlib.Path = (
         cfg_filepath.parent / module_file_path
@@ -109,9 +108,9 @@ def get_job_wrapper_py_func(
         ) from err
 
     anchored_file_path = cfg_filepath.parent / module_file_path
-    assert (
-        anchored_file_path.exists()
-    ), f"{module_file_path} not found at {anchored_file_path}!"
+    assert anchored_file_path.exists(), (
+        f"{module_file_path} not found at {anchored_file_path}!"
+    )
 
     module_name = module_file_path.stem.replace(" ", "_").replace("-", "_")
 

runem/log.py

@@ -3,13 +3,35 @@ import typing
 from runem.blocking_print import blocking_print
 
 
-def log(msg: str = "", decorate: bool = True, end: typing.Optional[str] = None) -> None:
+def log(
+    msg: str = "",
+    prefix: typing.Optional[bool] = None,
+    end: typing.Optional[str] = None,
+) -> None:
     """Thin wrapper around 'print', change the 'msg' & handles system-errors.
 
     One way we change it is to decorate the output with 'runem'
+
+    Parameters:
+        msg: str - the message to log out. Any `rich` markup will be escaped
+                   and not applied.
+        decorate: str - whether to add runem-specific prefix text. We do this
+                        to identify text that comes from the app vs text that
+                        comes from hooks or other third-parties.
+        end: Optional[str] - same as the end option used by `print()` and
+                             `rich`
+    Returns:
+        Nothing
     """
-    if decorate:
-        msg = f"runem: {msg}"
+    # Remove any markup as it will probably error, if unsanitised.
+    # msg = escape(msg)
+
+    if prefix is None:
+        prefix = True
+
+    if prefix:
+        # Make it clear that the message comes from `runem` internals.
+        msg = f"[light_slate_grey]runem[/light_slate_grey]: {msg}"
 
     # print in a blocking manner, waiting for system resources to free up if a
     # runem job is contending on stdout or similar.
@@ -17,8 +39,8 @@ def log(msg: str = "", decorate: bool = True, end: typing.Optional[str] = None)
 
 
 def warn(msg: str) -> None:
-    log(f"WARNING: {msg}")
+    log(f"[yellow]WARNING[/yellow]: {msg}")
 
 
 def error(msg: str) -> None:
-    log(f"ERROR: {msg}")
+    log(f"[red]ERROR[/red]: {msg}")

runem/report.py

@@ -46,12 +46,13 @@ def replace_bar_graph_characters(text: str, end_str: str, replace_char: str) ->
     """Replaces block characters in lines containing `end_str` with give char.
 
     Args:
-        text_lines (List[str]): A list of strings, each representing a line of text.
+        text (str): Text containing lines of bar-graphs (perhaps)
+        end_str (str): If contained by a line, the bar-graph shapes are replaced.
         replace_char (str): The character to replace all bocks with
 
     Returns:
-        List[str]: The modified list of strings with block characters replaced
-                   on specified lines.
+        str: The modified `text` with block characters replaced on specific
+             lines.
     """
     # Define the block character and its light shade replacement
     block_chars = (
@@ -211,7 +212,10 @@ def _print_reports_by_phase(
         for job_report_url_info in report_urls:
             if not job_report_url_info:
                 continue
-            log(f"report: {str(job_report_url_info[0])}: {str(job_report_url_info[1])}")
+            log(
+                f"report: [blue]{str(job_report_url_info[0])}[/blue]: "
+                f"{str(job_report_url_info[1])}"
+            )
 
 
 def report_on_run(

runem/run_command.py

@@ -7,17 +7,33 @@ from subprocess import STDOUT as SUBPROCESS_STDOUT
 from subprocess import Popen
 from timeit import default_timer as timer
 
+from rich.markup import escape
+
 from runem.log import log
 
 TERMINAL_WIDTH = 86
 
 
-class RunCommandBadExitCode(RuntimeError):
-    pass
+class RunemJobError(RuntimeError):
+    """An exception type that stores the stdout/stderr.
+
+    Designed so that we do not print the full stdout via the exception stack, instead,
+    allows an opportunity to parse the markup in it.
+    """
+
+    def __init__(self, friendly_message: str, stdout: str):
+        self.stdout = stdout
+        super().__init__(friendly_message)
 
 
-class RunCommandUnhandledError(RuntimeError):
-    pass
+class RunCommandBadExitCode(RunemJobError):
+    def __init__(self, stdout: str):
+        super().__init__(friendly_message="Bad exit-code", stdout=stdout)
+
+
+class RunCommandUnhandledError(RunemJobError):
+    def __init__(self, stdout: str):
+        super().__init__(friendly_message="Unhandled job error", stdout=stdout)
 
 
 # A function type for recording timing information.
@@ -25,27 +41,25 @@ RecordSubJobTimeType = typing.Callable[[str, timedelta], None]
 
 
 def parse_stdout(stdout: str, prefix: str) -> str:
-    """Prefixes each line of the output with a given label, except trailing new
-    lines."""
+    """Prefixes each line of output with a given label, except trailing new lines."""
     # Edge case: Return the prefix immediately for an empty string
     if not stdout:
         return prefix
 
-    # Split stdout into lines, noting if it ends with a newline
-    ends_with_newline = stdout.endswith("\n")
+    # Stop errors in `rich` by parsing out anything that might look like
+    # rich-markup.
+    stdout = escape(stdout)
+
+    # Split stdout into lines
     lines = stdout.split("\n")
 
     # Apply prefix to all lines except the last if it's empty (due to a trailing newline)
-    modified_lines = [f"{prefix}{line}" for line in lines[:-1]] + (
-        [lines[-1]]
-        if lines[-1] == "" and ends_with_newline
-        else [f"{prefix}{lines[-1]}"]
+    modified_lines = [f"{prefix}{escape(line)}" for line in lines[:-1]] + (
+        [f"{prefix}{escape(lines[-1])}"]
     )
 
     # Join the lines back together, appropriately handling the final newline
-    modified_stdout = "\n".join(modified_lines)
-    # if ends_with_newline:
-    #     modified_stdout += "\n"
+    modified_stdout: str = "\n".join(modified_lines)
 
     return modified_stdout
 
@@ -69,24 +83,34 @@ def _log_command_execution(
     label: str,
     env_overrides: typing.Optional[typing.Dict[str, str]],
     valid_exit_ids: typing.Optional[typing.Tuple[int, ...]],
+    decorate_logs: bool,
     verbose: bool,
     cwd: typing.Optional[pathlib.Path] = None,
 ) -> None:
     """Logs out useful debug information on '--verbose'."""
     if verbose:
-        log(f"running: start: {label}: {cmd_string}")
+        log(
+            f"running: start: [blue]{label}[/blue]: [yellow]{cmd_string}[yellow]",
+            prefix=decorate_logs,
+        )
         if valid_exit_ids is not None:
             valid_exit_strs = ",".join(str(exit_code) for exit_code in valid_exit_ids)
-            log(f"\tallowed return ids are: {valid_exit_strs}")
+            log(
+                f"\tallowed return ids are: [green]{valid_exit_strs}[/green]",
+                prefix=decorate_logs,
+            )
 
         if env_overrides:
             env_overrides_as_string = " ".join(
                 [f"{key}='{value}'" for key, value in env_overrides.items()]
             )
-            log(f"ENV OVERRIDES: {env_overrides_as_string} {cmd_string}")
+            log(
+                f"ENV OVERRIDES: [yellow]{env_overrides_as_string} {cmd_string}[/yellow]",
+                prefix=decorate_logs,
+            )
 
         if cwd:
-            log(f"cwd: {str(cwd)}")
+            log(f"cwd: {str(cwd)}", prefix=decorate_logs)
 
 
 def run_command(  # noqa: C901
@@ -98,6 +122,7 @@ def run_command(  # noqa: C901
     valid_exit_ids: typing.Optional[typing.Tuple[int, ...]] = None,
     cwd: typing.Optional[pathlib.Path] = None,
     record_sub_job_time: typing.Optional[RecordSubJobTimeType] = None,
+    decorate_logs: bool = True,
     **kwargs: typing.Any,
 ) -> str:
     """Runs the given command, returning stdout or throwing on any error."""
@@ -115,6 +140,7 @@ def run_command(  # noqa: C901
         label,
         env_overrides,
         valid_exit_ids,
+        decorate_logs,
         verbose,
         cwd,
     )
@@ -143,7 +169,12 @@ def run_command(  # noqa: C901
                 stdout += line
                 if verbose:
                     # print each line of output, assuming that each has a newline
-                    log(parse_stdout(line, prefix=f"{label}: "))
+                    log(
+                        parse_stdout(
+                            line, prefix=f"[green]| [/green][blue]{label}[/blue]: "
+                        ),
+                        prefix=False,
+                    )
 
             # Wait for the subprocess to finish and get the exit code
             process.wait()
@@ -154,15 +185,15 @@ def run_command(  # noqa: C901
                 )
                 raise RunCommandBadExitCode(
                     (
-                        f"non-zero exit {process.returncode} (allowed are "
-                        f"{valid_exit_strs}) from {cmd_string}"
+                        f"non-zero exit [red]{process.returncode}[/red] (allowed are "
+                        f"[green]{valid_exit_strs}[/green]) from {cmd_string}"
                     )
                 )
     except BaseException as err:
         if ignore_fails:
             return ""
         parsed_stdout: str = (
-            parse_stdout(stdout, prefix=f"{label}: ERROR: ") if process else ""
+            parse_stdout(stdout, prefix="[red]| [/red]") if process else ""
         )
         env_overrides_as_string = ""
         if env_overrides:
@@ -171,11 +202,11 @@ def run_command(  # noqa: C901
             )
             env_overrides_as_string = f"{env_overrides_as_string} "
         error_string = (
-            f"runem: test: FATAL: command failed: {label}"
-            f"\n\t{env_overrides_as_string}{cmd_string}"
-            f"\nERROR"
+            f"runem: [red bold]FATAL[/red bold]: command failed: [blue]{label}[/blue]"
+            f"\n\t[yellow]{env_overrides_as_string}{cmd_string}[/yellow]"
+            f"\n[red underline]| ERROR[/red underline]: [blue]{label}[/blue]"
             f"\n{str(parsed_stdout)}"
-            f"\nERROR END"
+            f"\n[red underline]| ERROR END[/red underline]"
         )
 
         if isinstance(err, RunCommandBadExitCode):
@@ -184,7 +215,10 @@ def run_command(  # noqa: C901
         raise RunCommandUnhandledError(error_string) from err
 
     if verbose:
-        log(f"running: done: {label}: {cmd_string}")
+        log(
+            f"running: done: [blue]{label}[/blue]: [yellow]{cmd_string}[/yellow]",
+            prefix=decorate_logs,
+        )
 
     if record_sub_job_time is not None:
         # Capture how long this run took