runem 0.4.0__py3-none-any.whl → 0.6.0__py3-none-any.whl

runem/VERSION

@@ -1 +1 @@
-0.4.0
+0.6.0

runem/blocking_print.py

@@ -13,7 +13,14 @@ def _reset_console() -> Console:
 
     RICH_CONSOLE = Console(
         log_path=False,  # Do NOT print the source path.
-        markup=False,  # Do NOT print markup e.g. `[blink]Don't Panic![/blink]`.
+        # We allow markup here, BUT stdout/stderr from other procs should have
+        # `escape()` called on them so they don't error here.
+        # This means 'rich' effects/colors can be judiciously applied:
+        # e.g. `[blink]Don't Panic![/blink]`.
+        markup=True,
+        # `highlight` is what colourises string and number in print() calls.
+        # We do not want this to be auto-magic.
+        highlight=False,
     )
     return RICH_CONSOLE
 
@@ -31,7 +38,8 @@ def _reset_console_for_tests() -> None:
     RICH_CONSOLE = Console(
         log_path=False,  # Do NOT print the source path.
         log_time=False,  # Do not prefix with log time e.g. `[time] log message`.
-        markup=False,  # Do NOT print markup e.g. `[blink]Don't Panic![/blink]`.
+        markup=True,  # Allow some markup e.g. `[blink]Don't Panic![/blink]`.
+        highlight=False,
         width=999,  # A very wide width.
     )
 

runem/cli/initialise_options.py

@@ -0,0 +1,26 @@
+import argparse
+
+from runem.config_metadata import ConfigMetadata
+from runem.informative_dict import InformativeDict
+from runem.types.options import OptionsWritable
+
+
+def initialise_options(
+    config_metadata: ConfigMetadata,
+    args: argparse.Namespace,
+) -> OptionsWritable:
+    """Initialises and returns the set of options to use for this run.
+
+    Returns the options dictionary
+    """
+
+    options: OptionsWritable = InformativeDict(
+        {option["name"]: option["default"] for option in config_metadata.options_config}
+    )
+    if config_metadata.options_config and args.overrides_on:  # pragma: no branch
+        for option_name in args.overrides_on:  # pragma: no branch
+            options[option_name] = True
+    if config_metadata.options_config and args.overrides_off:  # pragma: no branch
+        for option_name in args.overrides_off:
+            options[option_name] = False
+    return options

runem/command_line.py

@@ -53,6 +53,14 @@ def _get_argparse_help_formatter() -> typing.Any:
     return argparse.HelpFormatter
 
 
+def error_on_log_logic(verbose: bool, silent: bool) -> None:
+    """Simply errors if we get logical inconsistencies in the logging-logic."""
+    if verbose and silent:
+        log("cannot parse '--verbose' and '--silent'")
+        # error exit
+        sys.exit(1)
+
+
 def parse_args(
     config_metadata: ConfigMetadata, argv: typing.List[str]
 ) -> ConfigMetadata:
@@ -130,7 +138,9 @@ def parse_args(
         nargs="+",
         default=config_metadata.all_job_tags,
         help=(
-            "Only jobs with the given tags. "
+            # TODO: clarify the logic by which we add/remove jobs based on tags
+            #       e.g. exclusive-in, union, x-or etc.
+            "Only run jobs with the given tags. "
             f"Defaults to '{sorted(config_metadata.all_job_tags)}'."
         ),
         required=False,
@@ -238,6 +248,16 @@ def parse_args(
         required=False,
     )
 
+    parser.add_argument(
+        "--silent",
+        "-s",
+        dest="silent",
+        action=argparse.BooleanOptionalAction,
+        default=False,
+        help=("Whether to show warning messages or not. "),
+        required=False,
+    )
+
     parser.add_argument(
         "--spinner",
         dest="show_spinner",
@@ -271,6 +291,8 @@ def parse_args(
 
     args = parser.parse_args(argv[1:])
 
+    error_on_log_logic(args.verbose, args.silent)
+
     if args.show_root_path_and_exit:
         log(str(config_metadata.cfg_filepath.parent), decorate=False)
         # cleanly exit

runem/config_parse.py

@@ -149,6 +149,7 @@ def parse_job_config(
     in_out_job_names: JobNames,
     in_out_phases: JobPhases,
     phase_order: OrderedPhases,
+    silent: bool = False,
 ) -> None:
     """Parses and validates a job-entry read in from disk.
 
@@ -208,6 +209,7 @@ def parse_job_config(
                 in_out_job_names,
                 in_out_phases,
                 phase_order,
+                warn_missing_phase=(not silent),
             )
     except KeyError as err:
         raise ValueError(
@@ -215,8 +217,11 @@ def parse_job_config(
         ) from err
 
 
-def parse_config(
-    config: Config, cfg_filepath: pathlib.Path, hooks_only: bool = False
+def parse_config(  # noqa: C901
+    config: Config,
+    cfg_filepath: pathlib.Path,
+    silent: bool = False,
+    hooks_only: bool = False,
 ) -> typing.Tuple[
     Hooks,  # hooks:
     OrderedPhases,  # phase_order:
@@ -282,7 +287,10 @@ def parse_config(
 
     if not phase_order:
         if not hooks_only:
-            warn("phase ordering not configured! Runs will be non-deterministic!")
+            if silent:  # pragma: no cover
+                pass
+            else:
+                warn("phase ordering not configured! Runs will be non-deterministic!")
             phase_order = tuple(job_phases)
 
     # now parse out the job_configs
@@ -301,6 +309,7 @@ def parse_config(
             in_out_job_names=job_names,
             in_out_phases=job_phases,
             phase_order=phase_order,
+            silent=silent,
         )
     return (
         hooks,
@@ -341,7 +350,9 @@ def generate_config(
 
 
 def _load_user_hooks_from_config(
-    user_config: Config, cfg_filepath: pathlib.Path
+    user_config: Config,
+    cfg_filepath: pathlib.Path,
+    silent: bool,
 ) -> Hooks:
     hooks: Hooks
     (
@@ -353,7 +364,7 @@ def _load_user_hooks_from_config(
         _,
         _,
         _,
-    ) = parse_config(user_config, cfg_filepath, hooks_only=True)
+    ) = parse_config(user_config, cfg_filepath, silent, hooks_only=True)
     return hooks
 
 
@@ -361,6 +372,7 @@ def load_config_metadata(
     config: Config,
     cfg_filepath: pathlib.Path,
     user_configs: typing.List[typing.Tuple[Config, pathlib.Path]],
+    silent: bool = False,
     verbose: bool = False,
 ) -> ConfigMetadata:
     hooks: Hooks
@@ -380,12 +392,14 @@ def load_config_metadata(
         job_names,
         job_phases,
         tags,
-    ) = parse_config(config, cfg_filepath)
+    ) = parse_config(config, cfg_filepath, silent)
 
     user_config: Config
     user_config_path: pathlib.Path
     for user_config, user_config_path in user_configs:
-        user_hooks: Hooks = _load_user_hooks_from_config(user_config, user_config_path)
+        user_hooks: Hooks = _load_user_hooks_from_config(
+            user_config, user_config_path, silent
+        )
         if user_hooks:
             if verbose:
                 log(f"hooks: loading user hooks from '{str(user_config_path)}'")

runem/job_execute.py

@@ -11,7 +11,7 @@ from runem.config_metadata import ConfigMetadata
 from runem.informative_dict import ReadOnlyInformativeDict
 from runem.job import Job
 from runem.job_wrapper import get_job_wrapper
-from runem.log import error, log
+from runem.log import error, log, warn
 from runem.types.common import FilePathList, JobTags
 from runem.types.filters import FilePathListLookup
 from runem.types.runem_config import JobConfig
@@ -50,7 +50,8 @@ def job_execute_inner(
 
     if not file_list:
         # no files to work on
-        log(f"WARNING: skipping job '{label}', no files for job")
+        if not config_metadata.args.silent:
+            warn(f"skipping job '{label}', no files for job")
         return {
             "job": (f"{label}: no files!", timedelta(0)),
             "commands": [],

runem/log.py

@@ -3,13 +3,32 @@ import typing
 from runem.blocking_print import blocking_print
 
 
-def log(msg: str = "", decorate: bool = True, end: typing.Optional[str] = None) -> None:
+def log(
+    msg: str = "",
+    decorate: bool = True,
+    end: typing.Optional[str] = None,
+) -> None:
     """Thin wrapper around 'print', change the 'msg' & handles system-errors.
 
     One way we change it is to decorate the output with 'runem'
+
+    Parameters:
+        msg: str - the message to log out. Any `rich` markup will be escaped
+                   and not applied.
+        decorate: str - whether to add runem-specific prefix text. We do this
+                        to identify text that comes from the app vs text that
+                        comes from hooks or other third-parties.
+        end: Optional[str] - same as the end option used by `print()` and
+                             `rich`
+    Returns:
+        Nothing
     """
+    # Remove any markup as it will probably error, if unsanitised.
+    # msg = escape(msg)
+
     if decorate:
-        msg = f"runem: {msg}"
+        # Make it clear that the message comes from `runem` internals.
+        msg = f"[light_slate_grey]runem[/light_slate_grey]: {msg}"
 
     # print in a blocking manner, waiting for system resources to free up if a
     # runem job is contending on stdout or similar.
@@ -17,8 +36,8 @@ def log(msg: str = "", decorate: bool = True, end: typing.Optional[str] = None)
 
 
 def warn(msg: str) -> None:
-    log(f"WARNING: {msg}")
+    log(f"[yellow]WARNING[/yellow]: {msg}")
 
 
 def error(msg: str) -> None:
-    log(f"ERROR: {msg}")
+    log(f"[red]ERROR[/red]: {msg}")

runem/report.py

@@ -211,7 +211,10 @@ def _print_reports_by_phase(
         for job_report_url_info in report_urls:
             if not job_report_url_info:
                 continue
-            log(f"report: {str(job_report_url_info[0])}: {str(job_report_url_info[1])}")
+            log(
+                f"report: [blue]{str(job_report_url_info[0])}[/blue]: "
+                f"{str(job_report_url_info[1])}"
+            )
 
 
 def report_on_run(

runem/run_command.py

@@ -7,17 +7,33 @@ from subprocess import STDOUT as SUBPROCESS_STDOUT
 from subprocess import Popen
 from timeit import default_timer as timer
 
+from rich.markup import escape
+
 from runem.log import log
 
 TERMINAL_WIDTH = 86
 
 
-class RunCommandBadExitCode(RuntimeError):
-    pass
+class RunemJobError(RuntimeError):
+    """An exception type that stores the stdout/stderr.
+
+    Designed so that we do not print the full stdout via the exception stack, instead,
+    allows an opportunity to parse the markup in it.
+    """
+
+    def __init__(self, friendly_message: str, stdout: str):
+        self.stdout = stdout
+        super().__init__(friendly_message)
 
 
-class RunCommandUnhandledError(RuntimeError):
-    pass
+class RunCommandBadExitCode(RunemJobError):
+    def __init__(self, stdout: str):
+        super().__init__(friendly_message="Bad exit-code", stdout=stdout)
+
+
+class RunCommandUnhandledError(RunemJobError):
+    def __init__(self, stdout: str):
+        super().__init__(friendly_message="Unhandled job error", stdout=stdout)
 
 
 # A function type for recording timing information.
@@ -27,25 +43,25 @@ RecordSubJobTimeType = typing.Callable[[str, timedelta], None]
 def parse_stdout(stdout: str, prefix: str) -> str:
     """Prefixes each line of the output with a given label, except trailing new
     lines."""
+
     # Edge case: Return the prefix immediately for an empty string
     if not stdout:
         return prefix
 
-    # Split stdout into lines, noting if it ends with a newline
-    ends_with_newline = stdout.endswith("\n")
+    # Stop errors in `rich` by parsing out anything that might look like
+    # rich-markup.
+    stdout = escape(stdout)
+
+    # Split stdout into lines
     lines = stdout.split("\n")
 
     # Apply prefix to all lines except the last if it's empty (due to a trailing newline)
-    modified_lines = [f"{prefix}{line}" for line in lines[:-1]] + (
-        [lines[-1]]
-        if lines[-1] == "" and ends_with_newline
-        else [f"{prefix}{lines[-1]}"]
+    modified_lines = [f"{prefix}{escape(line)}" for line in lines[:-1]] + (
+        [f"{prefix}{escape(lines[-1])}"]
     )
 
     # Join the lines back together, appropriately handling the final newline
-    modified_stdout = "\n".join(modified_lines)
-    # if ends_with_newline:
-    #     modified_stdout += "\n"
+    modified_stdout: str = "\n".join(modified_lines)
 
     return modified_stdout
 
@@ -69,24 +85,34 @@ def _log_command_execution(
     label: str,
     env_overrides: typing.Optional[typing.Dict[str, str]],
     valid_exit_ids: typing.Optional[typing.Tuple[int, ...]],
+    decorate_logs: bool,
     verbose: bool,
     cwd: typing.Optional[pathlib.Path] = None,
 ) -> None:
     """Logs out useful debug information on '--verbose'."""
     if verbose:
-        log(f"running: start: {label}: {cmd_string}")
+        log(
+            f"running: start: [blue]{label}[/blue]: [yellow]{cmd_string}[yellow]",
+            decorate=decorate_logs,
+        )
         if valid_exit_ids is not None:
             valid_exit_strs = ",".join(str(exit_code) for exit_code in valid_exit_ids)
-            log(f"\tallowed return ids are: {valid_exit_strs}")
+            log(
+                f"\tallowed return ids are: [green]{valid_exit_strs}[/green]",
+                decorate=decorate_logs,
+            )
 
         if env_overrides:
             env_overrides_as_string = " ".join(
                 [f"{key}='{value}'" for key, value in env_overrides.items()]
             )
-            log(f"ENV OVERRIDES: {env_overrides_as_string} {cmd_string}")
+            log(
+                f"ENV OVERRIDES: [yellow]{env_overrides_as_string} {cmd_string}[/yellow]",
+                decorate=decorate_logs,
+            )
 
         if cwd:
-            log(f"cwd: {str(cwd)}")
+            log(f"cwd: {str(cwd)}", decorate=decorate_logs)
 
 
 def run_command(  # noqa: C901
@@ -98,6 +124,7 @@ def run_command(  # noqa: C901
     valid_exit_ids: typing.Optional[typing.Tuple[int, ...]] = None,
     cwd: typing.Optional[pathlib.Path] = None,
     record_sub_job_time: typing.Optional[RecordSubJobTimeType] = None,
+    decorate_logs: bool = True,
     **kwargs: typing.Any,
 ) -> str:
     """Runs the given command, returning stdout or throwing on any error."""
@@ -115,6 +142,7 @@ def run_command(  # noqa: C901
         label,
         env_overrides,
         valid_exit_ids,
+        decorate_logs,
         verbose,
         cwd,
     )
@@ -143,7 +171,12 @@ def run_command(  # noqa: C901
                 stdout += line
                 if verbose:
                     # print each line of output, assuming that each has a newline
-                    log(parse_stdout(line, prefix=f"{label}: "))
+                    log(
+                        parse_stdout(
+                            line, prefix=f"[green]| [/green][blue]{label}[/blue]: "
+                        ),
+                        decorate=False,
+                    )
 
             # Wait for the subprocess to finish and get the exit code
             process.wait()
@@ -154,15 +187,15 @@ def run_command(  # noqa: C901
                 )
                 raise RunCommandBadExitCode(
                     (
-                        f"non-zero exit {process.returncode} (allowed are "
-                        f"{valid_exit_strs}) from {cmd_string}"
+                        f"non-zero exit [red]{process.returncode}[/red] (allowed are "
+                        f"[green]{valid_exit_strs}[/green]) from {cmd_string}"
                     )
                 )
     except BaseException as err:
         if ignore_fails:
             return ""
         parsed_stdout: str = (
-            parse_stdout(stdout, prefix=f"{label}: ERROR: ") if process else ""
+            parse_stdout(stdout, prefix="[red]| [/red]") if process else ""
         )
         env_overrides_as_string = ""
         if env_overrides:
@@ -171,11 +204,11 @@ def run_command(  # noqa: C901
             )
             env_overrides_as_string = f"{env_overrides_as_string} "
         error_string = (
-            f"runem: test: FATAL: command failed: {label}"
-            f"\n\t{env_overrides_as_string}{cmd_string}"
-            f"\nERROR"
+            f"runem: [red bold]FATAL[/red bold]: command failed: [blue]{label}[/blue]"
+            f"\n\t[yellow]{env_overrides_as_string}{cmd_string}[/yellow]"
+            f"\n[red underline]| ERROR[/red underline]"
             f"\n{str(parsed_stdout)}"
-            f"\nERROR END"
+            f"\n[red underline]| ERROR END[/red underline]"
         )
 
         if isinstance(err, RunCommandBadExitCode):
@@ -184,7 +217,10 @@ def run_command(  # noqa: C901
         raise RunCommandUnhandledError(error_string) from err
 
     if verbose:
-        log(f"running: done: {label}: {cmd_string}")
+        log(
+            f"running: done: [blue]{label}[/blue]: [yellow]{cmd_string}[/yellow]",
+            decorate=decorate_logs,
+        )
 
     if record_sub_job_time is not None:
         # Capture how long this run took