runem 0.0.27__py3-none-any.whl → 0.0.29__py3-none-any.whl

runem/VERSION

@@ -1 +1 @@
-0.0.27
+0.0.29

runem/command_line.py

@@ -5,9 +5,10 @@ import sys
 import typing
 
 from runem.config_metadata import ConfigMetadata
+from runem.informative_dict import InformativeDict
 from runem.log import log
 from runem.runem_version import get_runem_version
-from runem.types import JobNames, OptionConfig, Options
+from runem.types import JobNames, OptionConfig, OptionsWritable
 from runem.utils import printable_set
 
 
@@ -175,7 +176,7 @@ def parse_args(
         # cleanly exit
         sys.exit(0)
 
-    options: Options = initialise_options(config_metadata, args)
+    options: OptionsWritable = initialise_options(config_metadata, args)
 
     if not _validate_filters(config_metadata, args):
         sys.exit(1)
@@ -245,15 +246,15 @@ def _validate_filters(
 def initialise_options(
     config_metadata: ConfigMetadata,
     args: argparse.Namespace,
-) -> Options:
+) -> OptionsWritable:
     """Initialises and returns the set of options to use for this run.
 
     Returns the options dictionary
     """
 
-    options: Options = {
-        option["name"]: option["default"] for option in config_metadata.options_config
-    }
+    options: OptionsWritable = InformativeDict(
+        {option["name"]: option["default"] for option in config_metadata.options_config}
+    )
     if config_metadata.options_config and args.overrides_on:  # pragma: no branch
         for option_name in args.overrides_on:  # pragma: no branch
             options[option_name] = True

runem/config_metadata.py

@@ -1,12 +1,13 @@
 import argparse
 import pathlib
 
+from runem.informative_dict import InformativeDict
 from runem.types import (
     JobNames,
     JobPhases,
     JobTags,
     OptionConfigs,
-    Options,
+    OptionsWritable,
     OrderedPhases,
     PhaseGroupedJobs,
     TagFileFilters,
@@ -24,7 +25,7 @@ class ConfigMetadata:
     all_job_phases: JobPhases  # the set of job-phases (should be subset of 'phases')
     all_job_tags: JobTags  # the set of job-tags (used for filtering)
 
-    options: Options  # the final configured options to pass to jobs
+    options: OptionsWritable  # the final configured options to pass to jobs
 
     args: argparse.Namespace  # the raw cli args, probably missing information
     jobs_to_run: JobNames  # superset of job-name candidates to run, from cli+config
@@ -52,7 +53,7 @@ class ConfigMetadata:
         self.all_job_phases = all_job_phases
         self.all_job_tags = all_job_tags
 
-        self.options = {}  # will be defined after cli argument parsing
+        self.options = InformativeDict()  # shows useful errors on bad-option lookups
 
         self.args = (
             argparse.Namespace()
@@ -69,7 +70,7 @@ class ConfigMetadata:
         phases_to_run: JobPhases,
         tags_to_run: JobTags,
         tags_to_avoid: JobTags,
-        options: Options,
+        options: OptionsWritable,
     ) -> None:
         self.options = options
         self.args = args

runem/informative_dict.py

@@ -0,0 +1,42 @@
+import typing
+
+# Define type variables for key and value to be used in the custom dictionary
+K = typing.TypeVar("K")
+V = typing.TypeVar("V")
+
+
+class InformativeDict(typing.Dict[K, V], typing.Generic[K, V]):
+    """A dictionary type that prints out the available keys."""
+
+    def __getitem__(self, key: K) -> V:
+        """Attempt to retrieve an item, raising a detailed exception if the key is not
+        found."""
+        try:
+            return super().__getitem__(key)
+        except KeyError:
+            available_keys: typing.Iterable[str] = (str(k) for k in self.keys())
+            raise KeyError(
+                f"Key '{key}' not found. Available keys: {', '.join(available_keys)}"
+            ) from None
+
+
+class ReadOnlyInformativeDict(InformativeDict[K, V], typing.Generic[K, V]):
+    """A read-only variant of the above."""
+
+    def __setitem__(self, key: K, value: V) -> None:
+        raise NotImplementedError("This dictionary is read-only")
+
+    def __delitem__(self, key: K) -> None:
+        raise NotImplementedError("This dictionary is read-only")
+
+    def pop(self, *args: typing.Any, **kwargs: typing.Any) -> V:
+        raise NotImplementedError("This dictionary is read-only")
+
+    def popitem(self) -> typing.Tuple[K, V]:
+        raise NotImplementedError("This dictionary is read-only")
+
+    def clear(self) -> None:
+        raise NotImplementedError("This dictionary is read-only")
+
+    def update(self, *args: typing.Any, **kwargs: typing.Any) -> None:
+        raise NotImplementedError("This dictionary is read-only")

runem/job_execute.py

@@ -7,24 +7,34 @@ from datetime import timedelta
 from timeit import default_timer as timer
 
 from runem.config_metadata import ConfigMetadata
+from runem.informative_dict import ReadOnlyInformativeDict
 from runem.job import Job
 from runem.job_wrapper import get_job_wrapper
 from runem.log import log
-from runem.types import FilePathListLookup, JobConfig, JobFunction, JobReturn, JobTags
+from runem.types import (
+    FilePathListLookup,
+    JobConfig,
+    JobFunction,
+    JobReturn,
+    JobTags,
+    JobTiming,
+    TimingEntries,
+    TimingEntry,
+)
 
 
 def job_execute_inner(
     job_config: JobConfig,
     config_metadata: ConfigMetadata,
     file_lists: FilePathListLookup,
-) -> typing.Tuple[typing.Tuple[str, timedelta], JobReturn]:
+) -> typing.Tuple[JobTiming, JobReturn]:
     """Wrapper for running a job inside a sub-process.
 
     Returns the time information and any reports the job generated
     """
     label = Job.get_job_name(job_config)
     if config_metadata.args.verbose:
-        log(f"START: {label}")
+        log(f"START: '{label}'")
     root_path: pathlib.Path = config_metadata.cfg_filepath.parent
     function: JobFunction
     job_tags: typing.Optional[JobTags] = Job.get_job_tags(job_config)
@@ -37,7 +47,19 @@ def job_execute_inner(
     if not file_list:
         # no files to work on
         log(f"WARNING: skipping job '{label}', no files for job")
-        return (f"{label}: no files!", timedelta(0)), None
+        return {
+            "job": (f"{label}: no files!", timedelta(0)),
+            "commands": [],
+        }, None
+
+    sub_command_timings: TimingEntries = []
+
+    def _record_sub_job_time(label: str, timing: timedelta) -> None:
+        """Record timing information for sub-commands/tasks, atomically.
+
+        For example inside of run_command() calls
+        """
+        sub_command_timings.append((label, timing))
 
     if (
         "ctx" in job_config
@@ -53,29 +75,38 @@ def job_execute_inner(
     start = timer()
     func_signature = inspect.signature(function)
     if config_metadata.args.verbose:
-        log(f"job: running {Job.get_job_name(job_config)}")
+        log(f"job: running: '{Job.get_job_name(job_config)}'")
     reports: JobReturn
-    if "args" in func_signature.parameters:
-        reports = function(  # type: ignore  # FIXME: which function do we have?
-            config_metadata.args, config_metadata.options, file_list
-        )
-    else:
-        reports = function(
-            options=config_metadata.options,  # type: ignore
-            file_list=file_list,
-            procs=config_metadata.args.procs,
-            root_path=root_path,
-            verbose=config_metadata.args.verbose,
-            # unpack useful data points from the job_config
-            label=Job.get_job_name(job_config),
-            job=job_config,
-        )
+    try:
+        if "args" in func_signature.parameters:
+            reports = function(  # type: ignore  # FIXME: which function do we have?
+                config_metadata.args, config_metadata.options, file_list
+            )
+        else:
+            reports = function(
+                options=ReadOnlyInformativeDict(config_metadata.options),  # type: ignore
+                file_list=file_list,
+                procs=config_metadata.args.procs,
+                root_path=root_path,
+                verbose=config_metadata.args.verbose,
+                # unpack useful data points from the job_config
+                label=Job.get_job_name(job_config),
+                job=job_config,
+                record_sub_job_time=_record_sub_job_time,
+            )
+    except BaseException:  # pylint: disable=broad-exception-caught
+        # log that we hit an error on this job and re-raise
+        log(decorate=False)
+        log(f"job: ERROR: job '{Job.get_job_name(job_config)}' failed to complete!")
+        # re-raise
+        raise
+
     end = timer()
     time_taken: timedelta = timedelta(seconds=end - start)
     if config_metadata.args.verbose:
-        log(f"DONE: {label}: {time_taken}")
-    timing_data = (label, time_taken)
-    return (timing_data, reports)
+        log(f"job: DONE: '{label}': {time_taken}")
+    this_job_timing_data: TimingEntry = (label, time_taken)
+    return ({"job": this_job_timing_data, "commands": sub_command_timings}, reports)
 
 
 def job_execute(
@@ -83,7 +114,7 @@ def job_execute(
     running_jobs: typing.Dict[str, str],
     config_metadata: ConfigMetadata,
     file_lists: FilePathListLookup,
-) -> typing.Tuple[typing.Tuple[str, timedelta], JobReturn]:
+) -> typing.Tuple[JobTiming, JobReturn]:
     """Thin-wrapper around job_execute_inner needed for mocking in tests.
 
     Needed for faster tests.

runem/job_filter.py

@@ -65,6 +65,10 @@ def _get_jobs_matching(
     filtered_jobs: PhaseGroupedJobs,
     verbose: bool,
 ) -> None:
+    """Via filtered_jobs, filters 'jobs' that match the given phase and and tags.
+
+    Warns if the job-name isn't found in list of valid job-names.
+    """
     phase_jobs: typing.List[JobConfig] = jobs[phase]
 
     job: JobConfig
@@ -74,7 +78,10 @@ def _get_jobs_matching(
 
         job_name: str = Job.get_job_name(job)
         if job_name not in job_names:
-            if verbose:
+            # test test_get_jobs_matching_when_job_not_in_valid_job_names should
+            # cover the follow in Ci but does not for some reason I don't have
+            # time to look in to. /FH
+            if verbose:  # pragma: FIXME: add code coverage
                 log(
                     (
                         f"not running job '{job_name}' because it isn't in the "
@@ -118,7 +125,10 @@ def filter_jobs(  # noqa: C901
     filtered_jobs: PhaseGroupedJobs = defaultdict(list)
     for phase in config_metadata.phases:
         if phase not in phases_to_run:
-            if verbose:
+            # test test_get_jobs_matching_when_job_not_in_valid_job_names should
+            # cover the follow in Ci but does not for some reason I don't have
+            # time to look in to. /FH
+            if verbose:  # pragma: FIXME: add code coverage
                 log(f"skipping phase '{phase}'")
             continue
         _get_jobs_matching(
@@ -131,7 +141,10 @@ def filter_jobs(  # noqa: C901
             verbose=verbose,
         )
         if len(filtered_jobs[phase]) == 0:
-            if verbose:
+            # test test_get_jobs_matching_when_job_not_in_valid_job_names should
+            # cover the follow in Ci but does not for some reason I don't have
+            # time to look in to. /FH
+            if verbose:  # pragma: FIXME: add code coverage
                 log(f"No jobs for phase '{phase}' tags {printable_set(tags_to_run)}")
             continue
 

runem/report.py

@@ -14,6 +14,7 @@ from runem.types import (
     PhaseName,
     ReportUrlInfo,
     ReportUrls,
+    TimingEntries,
 )
 
 try:
@@ -22,7 +23,7 @@ except ImportError:  # pragma: FIXME: add code coverage
     termplotlib = None
 
 
-def _align_bar_graphs_workaround(original_text: str) -> None:
+def _align_bar_graphs_workaround(original_text: str) -> str:
     """Module termplotlib doesn't align floats, this fixes that.
 
     This makes it so we can align the point in the floating point string, without it,
@@ -39,40 +40,93 @@ def _align_bar_graphs_workaround(original_text: str) -> None:
         r"\[.*?(\d+)\.", lambda m: f"[{m.group(1):>{max_width}}.", original_text
     )
 
-    print(formatted_text)
+    return formatted_text
+
+
+def _replace_bar_characters(text: str, end_str: str, replace_char: str) -> str:
+    """Replaces block characters in lines containing `end_str` with give char.
+
+    Args:
+        text_lines (List[str]): A list of strings, each representing a line of text.
+        replace_char (str): The character to replace all bocks with
+
+    Returns:
+        List[str]: The modified list of strings with block characters replaced
+                   on specified lines.
+    """
+    # Define the block character and its light shade replacement
+    block_chars = (
+        "▏▎▋▊█▌▐▄▀─"  # Extend this string with any additional block characters you use
+    )
+
+    text_lines: typing.List[str] = text.split("\n")
+
+    # Process each line, replacing block characters if `end_str` is present
+    modified_lines = [
+        line.translate(str.maketrans(block_chars, replace_char * len(block_chars)))
+        if end_str in line
+        else line
+        for line in text_lines
+    ]
+
+    return "\n".join(modified_lines)
+
+
+def _semi_shade_phase_totals(text: str) -> str:
+    light_shade_char = "░"
+    return _replace_bar_characters(text, "(user-time)", light_shade_char)
+
+
+def _dot_jobs(text: str) -> str:
+    dot_char = "·"
+    return _replace_bar_characters(text, "(+)", dot_char)
 
 
 def _plot_times(
-    overall_run_time: timedelta,
+    wall_clock_for_runem_main: timedelta,
     phase_run_oder: OrderedPhases,
     timing_data: JobRunTimesByPhase,
-) -> timedelta:
+) -> typing.Tuple[timedelta, timedelta]:
     """Prints a report to terminal on how well we performed.
 
     Also calculates the wall-clock time-saved for the user.
+
+    Returns the total system time spent and the time-saved.     (system-time-spent,
+    wall-clock-time-saved)
     """
     labels: typing.List[str] = []
     times: typing.List[float] = []
-    job_time_sum: timedelta = timedelta()  # init to 0
-    for phase in phase_run_oder:
+
+    # Track active processing time for jobs, distinct from wall-clock time (the
+    # time the user experiences).
+    system_time_spent: timedelta = timedelta()  # init to 0
+
+    for idx, phase in enumerate(phase_run_oder):
+        not_last_phase: bool = idx < len(phase_run_oder) - 1
+        utf8_phase = "├" if not_last_phase else "└"
+        utf8_phase_group = "│" if not_last_phase else " "
         # log(f"Phase '{phase}' jobs took:")
-        phase_total_time: float = 0.0
         phase_start_idx = len(labels)
-        for label, job_time in timing_data[phase]:
-            if job_time.total_seconds() == 0:
-                continue
-            labels.append(f"│├{phase}.{label}")
-            times.append(job_time.total_seconds())
-            job_time_sum += job_time
-            phase_total_time += job_time.total_seconds()
-        labels.insert(phase_start_idx, f"├{phase} (total)")
-        times.insert(phase_start_idx, phase_total_time)
-
-    for label, job_time in reversed(timing_data["_app"]):
-        labels.insert(0, f"├runem.{label}")
-        times.insert(0, job_time.total_seconds())
-    labels.insert(0, "runem")
-    times.insert(0, overall_run_time.total_seconds())
+
+        phase_job_times: timedelta = _gen_jobs_report(
+            phase,
+            labels,
+            times,
+            utf8_phase_group,
+            timing_data[phase],
+        )
+        labels.insert(phase_start_idx, f"{utf8_phase}{phase} (user-time)")
+        times.insert(phase_start_idx, phase_job_times.total_seconds())
+        system_time_spent += phase_job_times
+
+    runem_app_timing: typing.List[JobTiming] = timing_data["_app"]
+    job_metadata: JobTiming
+    for job_metadata in reversed(runem_app_timing):
+        job_label, job_time_total = job_metadata["job"]
+        labels.insert(0, f"├runem.{job_label}")
+        times.insert(0, job_time_total.total_seconds())
+    labels.insert(0, "runem (total wall-clock)")
+    times.insert(0, wall_clock_for_runem_main.total_seconds())
     if termplotlib:
         fig = termplotlib.figure()
         # cspell:disable-next-line
@@ -81,14 +135,67 @@ def _plot_times(
             labels,
             force_ascii=False,
         )
+        shaded_bar_graph: str = _semi_shade_phase_totals(fig.get_string())
+        dotted_bar_graph: str = _dot_jobs(shaded_bar_graph)
+
         # ensure the graphs get aligned nicely.
-        _align_bar_graphs_workaround(fig.get_string())
+        final_bar_graph: str = _align_bar_graphs_workaround(dotted_bar_graph)
+        print(final_bar_graph)
     else:  # pragma: FIXME: add code coverage
-        for label, time in zip(labels, times):
-            log(f"{label}: {time}s")
+        for job_label, time in zip(labels, times):
+            log(f"{job_label}: {time}s")
 
-    time_saved: timedelta = job_time_sum - overall_run_time
-    return time_saved
+    wall_clock_time_saved: timedelta = system_time_spent - wall_clock_for_runem_main
+    return system_time_spent, wall_clock_time_saved
+
+
+def _gen_jobs_report(
+    phase: PhaseName,
+    labels: typing.List[str],
+    times: typing.List[float],
+    utf8_phase_group: str,
+    job_timings: typing.List[JobTiming],
+) -> timedelta:
+    """Gathers the reports for sub-jobs.
+
+    Split out from _plot_times as the code was getting complex
+    """
+    job_timing: JobTiming
+
+    # Filter out JobTiming instances with non-zero total_seconds
+    non_zero_timing_data: typing.List[JobTiming] = [
+        job_timing
+        for job_timing in job_timings
+        if job_timing["job"][1].total_seconds() != 0
+    ]
+
+    job_time_sum: timedelta = timedelta()  # init to 0
+    for idx, job_timing in enumerate(non_zero_timing_data):
+        not_last: bool = idx < len(non_zero_timing_data) - 1
+        utf8_job = "├" if not_last else "└"
+        utf8_sub_jobs = "│" if not_last else " "
+        job_label, job_time_total = job_timing["job"]
+        job_bar_label: str = f"{phase}.{job_label}"
+        labels.append(f"{utf8_phase_group}{utf8_job}{job_bar_label}")
+        times.append(job_time_total.total_seconds())
+        job_time_sum += job_time_total
+        sub_command_times: TimingEntries = job_timing["commands"]
+
+        if len(sub_command_times) <= 1:
+            # we only have one or fewer sub-commands, just show the job-time
+            continue
+
+        # also print the sub-components of the job as we have more than one
+        for idx, (sub_job_label, sub_job_time) in enumerate(sub_command_times):
+            sub_utf8 = "├"
+            if idx == len(sub_command_times) - 1:
+                sub_utf8 = "└"
+            labels.append(
+                f"{utf8_phase_group}{utf8_sub_jobs}{sub_utf8}{job_bar_label}"
+                f".{sub_job_label} (+)"
+            )
+            times.append(sub_job_time.total_seconds())
+    return job_time_sum
 
 
 def _print_reports_by_phase(
@@ -107,8 +214,8 @@ def _print_reports_by_phase(
 def report_on_run(
     phase_run_oder: OrderedPhases,
     job_run_metadatas: JobRunMetadatasByPhase,
-    overall_runtime: timedelta,
-) -> timedelta:
+    wall_clock_for_runem_main: timedelta,
+) -> typing.Tuple[timedelta, timedelta]:
     """Generate high-level reports AND prints out any reports returned by jobs.
 
     IMPORTANT: returns the wall-clock time saved to the user.
@@ -130,9 +237,8 @@ def report_on_run(
                 report_data[phase].extend(reports["reportUrls"])
 
     # Now plot the times on the terminal to give a visual report of the timing.
-    # Also, calculate the time saved by runem, a key selling-point metric
-    time_saved: timedelta = _plot_times(
-        overall_run_time=overall_runtime,
+    time_metrics: typing.Tuple[timedelta, timedelta] = _plot_times(
+        wall_clock_for_runem_main=wall_clock_for_runem_main,
         phase_run_oder=phase_run_oder,
         timing_data=timing_data,
     )
@@ -140,6 +246,7 @@ def report_on_run(
     # Penultimate-ly print out the available reports grouped by run-phase.
     _print_reports_by_phase(phase_run_oder, report_data)
 
-    # Return the key metric for runem, the wall-clock time saved to the user
+    # Return the key metrics for runem, the system vs wall-clock time saved to
+    # the user
     # TODO: write this to disk
-    return time_saved
+    return time_metrics

runem/run_command.py

@@ -1,9 +1,11 @@
 import os
 import pathlib
 import typing
+from datetime import timedelta
 from subprocess import PIPE as SUBPROCESS_PIPE
 from subprocess import STDOUT as SUBPROCESS_STDOUT
 from subprocess import Popen
+from timeit import default_timer as timer
 
 from runem.log import log
 
@@ -18,10 +20,34 @@ class RunCommandUnhandledError(RuntimeError):
     pass
 
 
+# A function type for recording timing information.
+RecordSubJobTimeType = typing.Callable[[str, timedelta], None]
+
+
 def parse_stdout(stdout: str, prefix: str) -> str:
-    """Prefixes each line of the output with a label."""
-    stdout = prefix + stdout.replace("\n", f"\n{prefix}")
-    return stdout
+    """Prefixes each line of the output with a given label, except trailing new
+    lines."""
+    # Edge case: Return the prefix immediately for an empty string
+    if not stdout:
+        return prefix
+
+    # Split stdout into lines, noting if it ends with a newline
+    ends_with_newline = stdout.endswith("\n")
+    lines = stdout.split("\n")
+
+    # Apply prefix to all lines except the last if it's empty (due to a trailing newline)
+    modified_lines = [f"{prefix}{line}" for line in lines[:-1]] + (
+        [lines[-1]]
+        if lines[-1] == "" and ends_with_newline
+        else [f"{prefix}{lines[-1]}"]
+    )
+
+    # Join the lines back together, appropriately handling the final newline
+    modified_stdout = "\n".join(modified_lines)
+    # if ends_with_newline:
+    #     modified_stdout += "\n"
+
+    return modified_stdout
 
 
 def _prepare_environment(
@@ -71,11 +97,16 @@ def run_command(  # noqa: C901
     ignore_fails: bool = False,
     valid_exit_ids: typing.Optional[typing.Tuple[int, ...]] = None,
     cwd: typing.Optional[pathlib.Path] = None,
+    record_sub_job_time: typing.Optional[RecordSubJobTimeType] = None,
     **kwargs: typing.Any,
 ) -> str:
     """Runs the given command, returning stdout or throwing on any error."""
     cmd_string = " ".join(cmd)
 
+    if record_sub_job_time is not None:
+        # start the capture of how long this sub-task takes.
+        start = timer()
+
     run_env: typing.Dict[str, str] = _prepare_environment(
         env_overrides,
     )
@@ -111,7 +142,7 @@ def run_command(  # noqa: C901
             for line in process.stdout:
                 stdout += line
                 if verbose:
-                    # print each line of output
+                    # print each line of output, assuming that each has a newline
                     log(parse_stdout(line, prefix=f"{label}: "))
 
             # Wait for the subprocess to finish and get the exit code
@@ -154,4 +185,11 @@ def run_command(  # noqa: C901
 
     if verbose:
         log(f"running: done: {label}: {cmd_string}")
+
+    if record_sub_job_time is not None:
+        # Capture how long this run took
+        end = timer()
+        time_taken: timedelta = timedelta(seconds=end - start)
+        record_sub_job_time(label, time_taken)
+
     return stdout

runem/runem.py

@@ -297,9 +297,11 @@ def _main(
     end = timer()
 
     job_run_metadatas: JobRunMetadatasByPhase = defaultdict(list)
-    job_run_metadatas["_app"].append(
-        (("pre-build", (timedelta(seconds=end - start))), None)
-    )
+    pre_build_time: JobTiming = {
+        "job": ("pre-build", (timedelta(seconds=end - start))),
+        "commands": [],
+    }
+    job_run_metadatas["_app"].append((pre_build_time, None))
 
     start = timer()
 
@@ -313,7 +315,10 @@ def _main(
 
     end = timer()
 
-    phase_run_timing: JobTiming = ("run-phases", timedelta(seconds=end - start))
+    phase_run_timing: JobTiming = {
+        "job": ("run-phases", timedelta(seconds=end - start)),
+        "commands": [],
+    }
     phase_run_report: JobReturn = None
     phase_run_metadata: JobRunMetadata = (phase_run_timing, phase_run_report)
     job_run_metadatas["_app"].append(phase_run_metadata)
@@ -333,11 +338,19 @@ def timed_main(argv: typing.List[str]) -> None:
     phase_run_oder, job_run_metadatas, failure_exception = _main(argv)
     end = timer()
     time_taken: timedelta = timedelta(seconds=end - start)
-    time_saved = report_on_run(phase_run_oder, job_run_metadatas, time_taken)
+    wall_clock_time_saved: timedelta
+    system_time_spent: timedelta
+    system_time_spent, wall_clock_time_saved = report_on_run(
+        phase_run_oder, job_run_metadatas, time_taken
+    )
+    message: str = "DONE: runem took"
+    if failure_exception:
+        message = "FAILED: your jobs failed after"
     log(
         (
-            f"DONE: runem took: {time_taken.total_seconds()}s, "
-            f"saving you {time_saved.total_seconds()}s"
+            f"{message}: {time_taken.total_seconds()}s, "
+            f"saving you {wall_clock_time_saved.total_seconds()}s, "
+            f"without runem you would have waited {system_time_spent.total_seconds()}s"
         )
     )
 

runem/types.py

@@ -3,6 +3,8 @@ import pathlib
 import typing
 from datetime import timedelta
 
+from runem.informative_dict import InformativeDict, ReadOnlyInformativeDict
+
 
 class FunctionNotFound(ValueError):
     """Thrown when the test-function cannot be found."""
@@ -30,7 +32,21 @@ class JobReturnData(typing.TypedDict, total=False):
     reportUrls: ReportUrls  # urls containing reports for the user
 
 
-JobTiming = typing.Tuple[str, timedelta]
+TimingEntry = typing.Tuple[str, timedelta]
+TimingEntries = typing.List[TimingEntry]
+
+
+class JobTiming(typing.TypedDict, total=True):
+    """A hierarchy of timing info. Job->JobCommands.
+
+    The overall time for a job is in 'job', the child calls to run_command are in
+    'commands'
+    """
+
+    job: TimingEntry  # the overall time for a job
+    commands: TimingEntries  # timing for each call to `run_command`
+
+
 JobReturn = typing.Optional[JobReturnData]
 JobRunMetadata = typing.Tuple[JobTiming, JobReturn]
 JobRunTimesByPhase = typing.Dict[PhaseName, typing.List[JobTiming]]
@@ -53,7 +69,9 @@ OptionName = str
 OptionValue = bool
 
 OptionConfigs = typing.Tuple[OptionConfig, ...]
-Options = typing.Dict[OptionName, OptionValue]
+OptionsWritable = InformativeDict[OptionName, OptionValue]
+OptionsReadOnly = ReadOnlyInformativeDict[OptionName, OptionValue]
+Options = OptionsReadOnly
 
 # P1: bool for verbose, P2: list of file paths to work on
 
@@ -70,7 +88,7 @@ FilePathListLookup = typing.DefaultDict[JobTag, FilePathList]
 
 # FIXME: this type is no-longer the actual spec of the test-functions
 JobFunction = typing.Union[
-    typing.Callable[[argparse.Namespace, Options, FilePathList], None],
+    typing.Callable[[argparse.Namespace, OptionsWritable, FilePathList], None],
     typing.Callable[[typing.Any], None],
 ]
 

{runem-0.0.27.dist-info → runem-0.0.29.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: runem
-Version: 0.0.27
+Version: 0.0.29
 Summary: Awesome runem created by lursight
 Home-page: https://github.com/lursight/runem/
 Author: lursight
@@ -26,6 +26,7 @@ Requires-Dist: pytest-cov ==4.1.0 ; extra == 'test'
 Requires-Dist: pytest-profiling ==1.7.0 ; extra == 'test'
 Requires-Dist: pytest-xdist ==3.3.1 ; extra == 'test'
 Requires-Dist: pytest ==7.4.3 ; extra == 'test'
+Requires-Dist: setuptools ; extra == 'test'
 Requires-Dist: termplotlib ==0.3.9 ; extra == 'test'
 Requires-Dist: types-PyYAML ==6.0.12.12 ; extra == 'test'
 Requires-Dist: requests-mock ==1.10.0 ; extra == 'test'
@@ -400,12 +401,12 @@ runem: reports:
 runem: runem: 8.820488s
 runem: ├runem.pre-build: 0.019031s
 runem: ├runem.run-phases: 8.801317s
-runem: ├pre-run (total): 0.00498s
+runem: ├pre-run (user-time): 0.00498s
 runem: │├pre-run.install python requirements: 2.6e-05s
 runem: │├pre-run.ls -alh runem: 0.004954s
-runem: ├edit (total): 0.557559s
+runem: ├edit (user-time): 0.557559s
 runem: │├edit.reformat py: 0.557559s
-runem: ├analysis (total): 21.526145s
+runem: ├analysis (user-time): 21.526145s
 runem: │├analysis.pylint py: 7.457029s
 runem: │├analysis.flake8 py: 0.693754s
 runem: │├analysis.mypy py: 1.071956s
@@ -430,12 +431,12 @@ runem: reports:
 runem                                  [14.174612]  ███████████████▋
 ├runem.pre-build                       [ 0.025858]
 ├runem.run-phases                      [14.148587]  ███████████████▋
-├pre-run (total)                       [ 0.005825]
+├pre-run (user-time)                   [ 0.005825]
 │├pre-run.install python requirements  [ 0.000028]
 │├pre-run.ls -alh runem                [ 0.005797]
-├edit (total)                          [ 0.579153]  ▋
+├edit (user-time)                      [ 0.579153]  ▋
 │├edit.reformat py                     [ 0.579153]  ▋
-├analysis (total)                      [36.231034]  ████████████████████████████████████████
+├analysis (user-time)                  [36.231034]  ████████████████████████████████████████
 │├analysis.pylint py                   [12.738303]  ██████████████▏
 │├analysis.flake8 py                   [ 0.798575]  ▉
 │├analysis.mypy py                     [ 0.335984]  ▍

{runem-0.0.27.dist-info → runem-0.0.29.dist-info}/RECORD

@@ -1,31 +1,32 @@
-runem/VERSION,sha256=4RaaVpocPRZzAO4OueuhfnfUYNX8-rNoVGqAXr5Yij4,7
+runem/VERSION,sha256=jph1-M7GbnVztMqqYOGloa6ZJHFW9vEnWp0NTfilvfk,7
 runem/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 runem/__main__.py,sha256=dsOiVZegpfK9JOs5n7UmbX5iwwbj7iFkEbLoVeEgAn4,136
 runem/base.py,sha256=EZfR7FIlwEdU9Vfe47Wk2DOO8GQqpKxxLNKp6YHueZ4,316
 runem/blocking_print.py,sha256=S9dtgAeuTzc2-ht-vk9Wl6l-0PwS2tYbHDHDQQitrlA,841
 runem/cli.py,sha256=LTkxwALR67TM8GlkwrfISoYxcBZ848d9Mp75lQtDTAE,133
-runem/command_line.py,sha256=AOZ9hH2MGSoyOWHUM68rZ6E5GzwjNKFpDyTFVcTFvcI,10279
+runem/command_line.py,sha256=gRiT_fdRt4_xh_kFVoEY9GyTCcg-LmoKSs4y7HVtHqY,10379
 runem/config.py,sha256=dBQks5ERVTwdxw0PlOm0ioDrsxfgTCfGlBTuE2RFtn8,3866
-runem/config_metadata.py,sha256=EjCEqx9-2mtMrFf3lJJ8HFhfmScioZKeY_c9Rzajne8,2836
+runem/config_metadata.py,sha256=FaMLZOUNKpg77yowyAKSDH9rmKbZfIwAmkDVbZMX4xU,2925
 runem/config_parse.py,sha256=B_hQPYSMIxzy5XUVWFuQ--0WZnCuDqgP3bQUYHwKc1A,8389
 runem/files.py,sha256=vAI17m-Y1tRGot6_vDpTuBTkKm8xdByGoh7HCfNkMFU,1900
+runem/informative_dict.py,sha256=U7p9z78UwOT4TAfng1iDXCEyeYz6C-XZlx9Z1pWNVrI,1548
 runem/job.py,sha256=QVXvzz67fJk__-h0womFQsB80-w41E3XRcHpxmRnv3o,2912
-runem/job_execute.py,sha256=QRGaG0S996I_bkHGvIBZolfgX2bqzosQ6RHK3X0_Fow,3245
-runem/job_filter.py,sha256=MvGuEqROgdAn7f4yZivJUS9ZuByQgrEDY_q4_D6Y0uQ,4493
+runem/job_execute.py,sha256=FLZ6KCcsI2mywBW-qMqrdEUhHURADxvUBXuOxJVSkss,4217
+runem/job_filter.py,sha256=-qinE3cmFsk6Q9N2Hxp_vm0olIzvRgrWczmvegcd7yc,5352
 runem/job_runner_simple_command.py,sha256=H4LuAgzLAiy_T2juRO0qL7ULH9OpWhcXmeuAALi8JaI,836
 runem/job_wrapper.py,sha256=c2im3YTBmrNlULbQs9kKWlIy45B6PRuogDZcfitmX7A,752
 runem/job_wrapper_python.py,sha256=sjKRUTnjiAR0JKmdzoKBPM5YHIjv2q86IUUyJmV7f0s,4263
 runem/log.py,sha256=YddguLhhLrdbEuHxJlolNdIg_LMTEN09nqiqEWWjDzU,517
 runem/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-runem/report.py,sha256=GdTavjbkaOX-cwXE0KFzTIYMN58vCQRDD2esaTqB6H0,4808
-runem/run_command.py,sha256=xKNXS2YAw67i9unNDSEud2kfcE35dig59ce7opU_ZVk,5051
-runem/runem.py,sha256=d7BZWNqADcg1QCeAVtFic8MS-1OnyVS4AemDz_fBrts,11991
+runem/report.py,sha256=2JD6b515FF-t040V2C6eKOp1PIS0w8AEWmpA-5WwSao,8758
+runem/run_command.py,sha256=Egl_j4bJ9mwi2JEFCsl0W6WH2IRgIdpMN7qdj8voClQ,6386
+runem/runem.py,sha256=Et3Jfg_qO5BBet43FI3LmmP2opuIZBKsraQJnAktPps,12443
 runem/runem_version.py,sha256=MbETwZO2Tb1Y3hX_OYZjKepEMKA1cjNvr-7Cqhz6e3s,271
-runem/types.py,sha256=V5f-HQxgEXdrNy0AuqozzDfWOoPeyRI3hRbACbiiHN4,5804
+runem/types.py,sha256=recvwKLWY4KB043jyGGdFU812O45d9KkN5yF8sZX2ug,6368
 runem/utils.py,sha256=3N_kel9LsriiMq7kOjT14XhfxUOgz4hdDg97wlLKm3U,221
-runem-0.0.27.dist-info/LICENSE,sha256=awOCsWJ58m_2kBQwBUGWejVqZm6wuRtCL2hi9rfa0X4,1211
-runem-0.0.27.dist-info/METADATA,sha256=XbQPBGyg6CiyCDkqXRihNWNh-_DAC9OmsKpu92zZ3wk,30525
-runem-0.0.27.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-runem-0.0.27.dist-info/entry_points.txt,sha256=nu0g_vBeuPihYtimbtlNusxWovylMppvJ8UxdJlJfvM,46
-runem-0.0.27.dist-info/top_level.txt,sha256=gK6iqh9OfHDDpErioCC9ul_zx2Q5zWTALtcuGU7Vil4,6
-runem-0.0.27.dist-info/RECORD,,
+runem-0.0.29.dist-info/LICENSE,sha256=awOCsWJ58m_2kBQwBUGWejVqZm6wuRtCL2hi9rfa0X4,1211
+runem-0.0.29.dist-info/METADATA,sha256=ftCbQO7zAiHhPhkpZuB9UBYmWi68u_sw59gkZ_DVwcE,30581
+runem-0.0.29.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+runem-0.0.29.dist-info/entry_points.txt,sha256=nu0g_vBeuPihYtimbtlNusxWovylMppvJ8UxdJlJfvM,46
+runem-0.0.29.dist-info/top_level.txt,sha256=gK6iqh9OfHDDpErioCC9ul_zx2Q5zWTALtcuGU7Vil4,6
+runem-0.0.29.dist-info/RECORD,,

{runem-0.0.27.dist-info → runem-0.0.29.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.42.0)
+Generator: bdist_wheel (0.43.0)
 Root-Is-Purelib: true
 Tag: py3-none-any