runem 0.6.0__tar.gz → 0.7.0__tar.gz

{runem-0.6.0 → runem-0.7.0}/HISTORY.md

@@ -4,6 +4,55 @@ Changelog
 
 (unreleased)
 ------------
+- Merge pull request #83 from
+  lursight/feat/no_runem_traceback_on_job_fail. [Frank Harrison]
+
+  Feat/no runem traceback on job fail
+- Feat(error-only): only shows the error not the runem error tracback.
+  [Frank Harrison]
+
+  The runem error traceback is irelevant if the sub-task fails. So we just
+  show that task's output instead of the traceback for where we handle the
+  error.
+- Feat(error-ctx): shows the failed job label as we show the causing
+  error. [Frank Harrison]
+- Feat(remove-failed): always remove the failed job from the list of
+  running jobs. [Frank Harrison]
+- Merge pull request #82 from lursight/feat/schema_validation. [Frank
+  Harrison]
+
+  feat(validation): validates the .runem.yml file against the schema
+- Feat(validation): validates the .runem.yml file against the schema.
+  [Frank Harrison]
+- Merge pull request #81 from lursight/chore/ruff. [Frank Harrison]
+
+  Chore/ruff
+- Chore(ruff): some formatting change made whilst configuring ruff.
+  [Frank Harrison]
+- Chore(ruff): use ruff as its faster/better. [Frank Harrison]
+- Merge pull request #80 from lursight/chore/update_deps. [Frank
+  Harrison]
+
+  Chore/update deps
+- Chore(deps): updates pylint 3.1.0 -> 3.3.6. [Frank Harrison]
+- Chore(deps): updates pytest 8.3.3 -> 8.3.5 and pytest-cov to latest.
+  [Frank Harrison]
+- Merge pull request #79 from
+  lursight/feat/removes_dectorate_param_from_log. [Frank Harrison]
+
+  feat(log): changes the semantics of log's 'decorate' to allow overriding
+- Feat(log): changes the semantics of log's 'decorate' to allow
+  overriding. [Frank Harrison]
+
+  Also renames the log API's param decorate -> prefix.
+
+  This better represents the intent of the param as decorate was adding a
+  default prefix.
+
+
+0.6.0 (2025-02-03)
+------------------
+- Release: version 0.6.0 🚀 [Frank Harrison]
 - Merge pull request #78 from lursight/feat/better_error_display. [Frank
   Harrison]
 

{runem-0.6.0 → runem-0.7.0}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: runem
-Version: 0.6.0
+Version: 0.7.0
 Summary: Awesome runem created by lursight
 Author: lursight
 License: Specify your license here
@@ -16,31 +16,32 @@ Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: packaging>=22.0
 Requires-Dist: PyYAML>=5.0.0
+Requires-Dist: jsonschema>=4.22
 Requires-Dist: rich>10.0.0
 Requires-Dist: typing_extensions>3.0.0
 Provides-Extra: tests
-Requires-Dist: black==24.10.0; extra == "tests"
 Requires-Dist: coverage==7.5; extra == "tests"
-Requires-Dist: docformatter==1.7.5; extra == "tests"
 Requires-Dist: flake8-bugbear==24.2.6; extra == "tests"
 Requires-Dist: flake8==7.0.0; extra == "tests"
 Requires-Dist: gitchangelog==3.0.4; extra == "tests"
-Requires-Dist: isort==5.13.2; extra == "tests"
 Requires-Dist: mkdocs==1.5.3; extra == "tests"
 Requires-Dist: mypy==1.9.0; extra == "tests"
 Requires-Dist: pydocstyle==6.3.0; extra == "tests"
-Requires-Dist: pylint==3.1.0; extra == "tests"
+Requires-Dist: pylint==3.3.6; extra == "tests"
 Requires-Dist: pylama==8.4.1; extra == "tests"
-Requires-Dist: pytest-cov==6.0.0; extra == "tests"
+Requires-Dist: pytest-cov==6.1.1; extra == "tests"
 Requires-Dist: pytest-profiling==1.7.0; extra == "tests"
 Requires-Dist: pytest-xdist==3.6.1; extra == "tests"
-Requires-Dist: pytest==8.3.3; extra == "tests"
+Requires-Dist: pytest==8.3.5; extra == "tests"
+Requires-Dist: ruff==0.11.6; extra == "tests"
 Requires-Dist: setuptools; extra == "tests"
 Requires-Dist: termplotlib==0.3.9; extra == "tests"
 Requires-Dist: tox; extra == "tests"
 Requires-Dist: types-PyYAML==6.0.12.20240311; extra == "tests"
 Requires-Dist: requests-mock==1.11.0; extra == "tests"
+Requires-Dist: types-jsonschema; extra == "tests"
 Requires-Dist: types-setuptools; extra == "tests"
+Dynamic: license-file
 
 <!-- [![codecov](https://codecov.io/gh/lursight/runem/branch/main/graph/badge.svg?token=run-test_token_here)](https://codecov.io/gh/lursight/runem) -->
 [![CI](https://github.com/lursight/runem/actions/workflows/main.yml/badge.svg)](https://github.com/lursight/runem/actions/workflows/main.yml)

{runem-0.6.0 → runem-0.7.0}/pyproject.toml

@@ -23,7 +23,10 @@ classifiers = [
 dependencies = [
   # `packaging` is probably not needed after moving to pyproject.toml
   "packaging>=22.0",
+
+  # For yml passingg and validation
   "PyYAML>=5.0.0",
+  "jsonschema>=4.22",
 
   # For UI Elements
   "rich>10.0.0",
@@ -49,26 +52,25 @@ runem = ["VERSION"]
 [project.optional-dependencies]
 tests = [
   # This requirements are for development and testing only, not for production.
-  "black==24.10.0",
   "coverage==7.5",
-  "docformatter==1.7.5",
   "flake8-bugbear==24.2.6",
   "flake8==7.0.0",
   "gitchangelog==3.0.4",
-  "isort==5.13.2",
   "mkdocs==1.5.3",
   "mypy==1.9.0",
   "pydocstyle==6.3.0",
-  "pylint==3.1.0",
+  "pylint==3.3.6",
   "pylama==8.4.1",
-  "pytest-cov==6.0.0",
+  "pytest-cov==6.1.1",
   "pytest-profiling==1.7.0",
   "pytest-xdist==3.6.1",
-  "pytest==8.3.3",
+  "pytest==8.3.5",
+  "ruff==0.11.6",
   "setuptools",
   "termplotlib==0.3.9",
   "tox",
   "types-PyYAML==6.0.12.20240311",
   "requests-mock==1.11.0",
+  "types-jsonschema",
   "types-setuptools",
 ]

runem-0.7.0/runem/VERSION

@@ -0,0 +1 @@
+0.7.0

{runem-0.6.0 → runem-0.7.0}/runem/cli/initialise_options.py

@@ -13,7 +13,6 @@ def initialise_options(
 
     Returns the options dictionary
     """
-
     options: OptionsWritable = InformativeDict(
         {option["name"]: option["default"] for option in config_metadata.options_config}
     )

{runem-0.6.0 → runem-0.7.0}/runem/command_line.py

@@ -43,10 +43,8 @@ def _get_argparse_help_formatter() -> typing.Any:
 
     if use_fixed_width:
         # Use custom formatter with the width specified in the environment variable
-        return (
-            lambda prog: HelpFormatterFixedWidth(  # pylint: disable=unnecessary-lambda
-                prog
-            )
+        return lambda prog: HelpFormatterFixedWidth(  # pylint: disable=unnecessary-lambda
+            prog
         )
 
     # Use default formatter
@@ -294,12 +292,12 @@ def parse_args(
     error_on_log_logic(args.verbose, args.silent)
 
     if args.show_root_path_and_exit:
-        log(str(config_metadata.cfg_filepath.parent), decorate=False)
+        log(str(config_metadata.cfg_filepath.parent), prefix=False)
         # cleanly exit
         sys.exit(0)
 
     if args.show_version_and_exit:
-        log(str(get_runem_version()), decorate=False)
+        log(str(get_runem_version()), prefix=False)
         # cleanly exit
         sys.exit(0)
 
@@ -383,7 +381,6 @@ def initialise_options(
 
     Returns the options dictionary
     """
-
     options: OptionsWritable = InformativeDict(
         {option["name"]: option["default"] for option in config_metadata.options_config}
     )

{runem-0.6.0 → runem-0.7.0}/runem/config.py

@@ -2,9 +2,9 @@ import pathlib
 import sys
 import typing
 
-import yaml
 from packaging.version import Version
 
+from runem.config_validate import validate_runem_file
 from runem.log import error, log
 from runem.runem_version import get_runem_version
 from runem.types.runem_config import (
@@ -13,6 +13,7 @@ from runem.types.runem_config import (
     GlobalSerialisedConfig,
     UserConfigMetadata,
 )
+from runem.yaml_utils import load_yaml_object
 
 CFG_FILE_YAML = pathlib.Path(".runem.yml")
 
@@ -46,7 +47,7 @@ def _search_up_multiple_dirs_for_file(
 
 
 def _find_config_file(
-    config_filename: typing.Union[str, pathlib.Path]
+    config_filename: typing.Union[str, pathlib.Path],
 ) -> typing.Tuple[typing.Optional[pathlib.Path], typing.Tuple[pathlib.Path, ...]]:
     """Searches up from the cwd for the given config file-name."""
     start_dirs = (pathlib.Path(".").absolute(),)
@@ -117,8 +118,11 @@ def _conform_global_config_types(
 
 def load_and_parse_config(cfg_filepath: pathlib.Path) -> Config:
     """For the given config file pass, project or user, load it & parse/conform it."""
-    with cfg_filepath.open("r+", encoding="utf-8") as config_file_handle:
-        all_config = yaml.full_load(config_file_handle)
+    all_config = load_yaml_object(cfg_filepath)
+    validate_runem_file(
+        cfg_filepath,
+        all_config,
+    )
 
     conformed_config: Config
     global_config: typing.Optional[GlobalConfig]

{runem-0.6.0 → runem-0.7.0}/runem/config_parse.py

@@ -162,7 +162,8 @@ def parse_job_config(
             ("cwd" in job["ctx"]) and (job["ctx"]["cwd"] is not None)
         )
         if (not have_ctw_cwd) or isinstance(
-            job["ctx"]["cwd"], str  # type: ignore # handled above
+            job["ctx"]["cwd"],  # type: ignore # handled above
+            str,
         ):
             # if
             # - we don't have a cwd, ctx

runem-0.7.0/runem/config_validate.py

@@ -0,0 +1,47 @@
+import pathlib
+import typing
+
+from runem.log import error, log
+from runem.types.errors import SystemExitBad
+from runem.yaml_utils import load_yaml_object
+from runem.yaml_validation import ValidationErrors, validate_yaml
+
+
+def _load_runem_schema() -> typing.Any:
+    """Loads and returns the yaml schema for runem.
+
+    Returns:
+        Any: the Draft202012Validator conformant schema.
+    """
+    schema_path: pathlib.Path = pathlib.Path(__file__).with_name("schema.yml")
+    if not schema_path.exists():
+        error(
+            (
+                "runem schema file not found, cannot continue! "
+                f"Is the install corrupt? {schema_path}"
+            )
+        )
+        raise SystemExitBad(1)
+    schema: typing.Any = load_yaml_object(schema_path)
+    return schema
+
+
+def validate_runem_file(
+    cfg_filepath: pathlib.Path,
+    all_config: typing.Any,
+) -> None:
+    """Validates the config Loader object against the runem schema.
+
+    Exits if the files does not validate.
+    """
+    schema: typing.Any = _load_runem_schema()
+    errors: ValidationErrors = validate_yaml(all_config, schema)
+    if not errors:
+        # aok
+        return
+
+    error(f"failed to validate runem config [yellow]{cfg_filepath}[/yellow]")
+    for err in errors:
+        path = ".".join(map(str, err.path)) or "<root>"
+        log(f" [yellow]{path}[/yellow]: {err.message}")
+    raise SystemExit("Config validation failed.")

{runem-0.6.0 → runem-0.7.0}/runem/informative_dict.py

@@ -9,8 +9,7 @@ class InformativeDict(typing.Dict[K, V], typing.Generic[K, V]):
     """A dictionary type that prints out the available keys."""
 
     def __getitem__(self, key: K) -> V:
-        """Attempt to retrieve an item, raising a detailed exception if the key is not
-        found."""
+        """Attempt to get item, raising a detailed exception if the key is not found."""
         try:
             return super().__getitem__(key)
         except KeyError:
@@ -24,19 +23,25 @@ class ReadOnlyInformativeDict(InformativeDict[K, V], typing.Generic[K, V]):
     """A read-only variant of the above."""
 
     def __setitem__(self, key: K, value: V) -> None:
+        """Readonly object, setitem disallowed."""
         raise NotImplementedError("This dictionary is read-only")
 
     def __delitem__(self, key: K) -> None:
+        """Readonly object, delitem disallowed."""
         raise NotImplementedError("This dictionary is read-only")
 
     def pop(self, *args: typing.Any, **kwargs: typing.Any) -> V:
+        """Readonly object, pop disallowed."""
         raise NotImplementedError("This dictionary is read-only")
 
     def popitem(self) -> typing.Tuple[K, V]:
+        """Readonly object, popitem disallowed."""
         raise NotImplementedError("This dictionary is read-only")
 
     def clear(self) -> None:
+        """Readonly object, clear disallowed."""
         raise NotImplementedError("This dictionary is read-only")
 
     def update(self, *args: typing.Any, **kwargs: typing.Any) -> None:
+        """Readonly object, update disallowed."""
         raise NotImplementedError("This dictionary is read-only")

{runem-0.6.0 → runem-0.7.0}/runem/job.py

@@ -71,7 +71,6 @@ class Job:
 
         TODO: make a non-static member function
         """
-
         # default to all file-tags
         tags_for_files: typing.Iterable[str] = file_lists.keys()
         use_default_tags: bool = job_tags is None
@@ -91,7 +90,6 @@ class Job:
 
         TODO: make a non-static member function
         """
-
         # First try one of the following keys.
         valid_name_keys = ("label", "command")
         for candidate in valid_name_keys:
@@ -101,6 +99,6 @@ class Job:
 
         # The try the python-wrapper address
         try:
-            return f'{job["addr"]["file"]}.{job["addr"]["function"]}'
+            return f"{job['addr']['file']}.{job['addr']['function']}"
         except KeyError:
             raise NoJobName()  # pylint: disable=raise-missing-from

{runem-0.6.0 → runem-0.7.0}/runem/job_execute.py

@@ -105,7 +105,7 @@ def job_execute_inner(
         reports = function(**all_k_args)
     except BaseException:  # pylint: disable=broad-exception-caught
         # log that we hit an error on this job and re-raise
-        log(decorate=False)
+        log(prefix=False)
         error(f"job: job '{Job.get_job_name(job_config)}' failed to complete!")
         # re-raise
         raise
@@ -132,12 +132,15 @@ def job_execute(
     """
     this_id: str = str(uuid.uuid4())
     running_jobs[this_id] = Job.get_job_name(job_config)
-    results = job_execute_inner(
-        job_config,
-        config_metadata,
-        file_lists,
-        **kwargs,
-    )
-    completed_jobs[this_id] = running_jobs[this_id]
-    del running_jobs[this_id]
+    try:
+        results = job_execute_inner(
+            job_config,
+            config_metadata,
+            file_lists,
+            **kwargs,
+        )
+    finally:
+        # Always tidy-up job statuses
+        completed_jobs[this_id] = running_jobs[this_id]
+        del running_jobs[this_id]
     return results

{runem-0.6.0 → runem-0.7.0}/runem/job_filter.py

@@ -101,21 +101,21 @@ def filter_jobs(  # noqa: C901
         if tags_to_run:
             log(
                 f"filtering for tags {printable_set(tags_to_run)}",
-                decorate=True,
+                prefix=True,
                 end="",
             )
         if tags_to_avoid:
             if tags_to_run:
-                log(", ", decorate=False, end="")
+                log(", ", prefix=False, end="")
             else:
-                log(decorate=True, end="")
+                log(prefix=True, end="")
             log(
                 f"excluding jobs with tags {printable_set(tags_to_avoid)}",
-                decorate=False,
+                prefix=False,
                 end="",
             )
         if tags_to_run or tags_to_avoid:
-            log(decorate=False)
+            log(prefix=False)
     filtered_jobs: PhaseGroupedJobs = defaultdict(list)
     for phase in config_metadata.phases:
         if phase not in phases_to_run:

{runem-0.6.0 → runem-0.7.0}/runem/job_wrapper_python.py

@@ -15,7 +15,6 @@ def _load_python_function_from_module(
     function_to_load: str,
 ) -> JobFunction:
     """Given a job-description dynamically loads the test-function so we can call it."""
-
     # first locate the module relative to the config file
     abs_module_file_path: pathlib.Path = (
         cfg_filepath.parent / module_file_path
@@ -109,9 +108,9 @@ def get_job_wrapper_py_func(
         ) from err
 
     anchored_file_path = cfg_filepath.parent / module_file_path
-    assert (
-        anchored_file_path.exists()
-    ), f"{module_file_path} not found at {anchored_file_path}!"
+    assert anchored_file_path.exists(), (
+        f"{module_file_path} not found at {anchored_file_path}!"
+    )
 
     module_name = module_file_path.stem.replace(" ", "_").replace("-", "_")
 

{runem-0.6.0 → runem-0.7.0}/runem/log.py

@@ -5,7 +5,7 @@ from runem.blocking_print import blocking_print
 
 def log(
     msg: str = "",
-    decorate: bool = True,
+    prefix: typing.Optional[bool] = None,
     end: typing.Optional[str] = None,
 ) -> None:
     """Thin wrapper around 'print', change the 'msg' & handles system-errors.
@@ -26,7 +26,10 @@ def log(
     # Remove any markup as it will probably error, if unsanitised.
     # msg = escape(msg)
 
-    if decorate:
+    if prefix is None:
+        prefix = True
+
+    if prefix:
         # Make it clear that the message comes from `runem` internals.
         msg = f"[light_slate_grey]runem[/light_slate_grey]: {msg}"
 

{runem-0.6.0 → runem-0.7.0}/runem/report.py

@@ -46,12 +46,13 @@ def replace_bar_graph_characters(text: str, end_str: str, replace_char: str) ->
     """Replaces block characters in lines containing `end_str` with give char.
 
     Args:
-        text_lines (List[str]): A list of strings, each representing a line of text.
+        text (str): Text containing lines of bar-graphs (perhaps)
+        end_str (str): If contained by a line, the bar-graph shapes are replaced.
         replace_char (str): The character to replace all bocks with
 
     Returns:
-        List[str]: The modified list of strings with block characters replaced
-                   on specified lines.
+        str: The modified `text` with block characters replaced on specific
+             lines.
     """
     # Define the block character and its light shade replacement
     block_chars = (

{runem-0.6.0 → runem-0.7.0}/runem/run_command.py

@@ -41,9 +41,7 @@ RecordSubJobTimeType = typing.Callable[[str, timedelta], None]
 
 
 def parse_stdout(stdout: str, prefix: str) -> str:
-    """Prefixes each line of the output with a given label, except trailing new
-    lines."""
-
+    """Prefixes each line of output with a given label, except trailing new lines."""
     # Edge case: Return the prefix immediately for an empty string
     if not stdout:
         return prefix
@@ -93,13 +91,13 @@ def _log_command_execution(
     if verbose:
         log(
             f"running: start: [blue]{label}[/blue]: [yellow]{cmd_string}[yellow]",
-            decorate=decorate_logs,
+            prefix=decorate_logs,
         )
         if valid_exit_ids is not None:
             valid_exit_strs = ",".join(str(exit_code) for exit_code in valid_exit_ids)
             log(
                 f"\tallowed return ids are: [green]{valid_exit_strs}[/green]",
-                decorate=decorate_logs,
+                prefix=decorate_logs,
             )
 
         if env_overrides:
@@ -108,11 +106,11 @@ def _log_command_execution(
             )
             log(
                 f"ENV OVERRIDES: [yellow]{env_overrides_as_string} {cmd_string}[/yellow]",
-                decorate=decorate_logs,
+                prefix=decorate_logs,
             )
 
         if cwd:
-            log(f"cwd: {str(cwd)}", decorate=decorate_logs)
+            log(f"cwd: {str(cwd)}", prefix=decorate_logs)
 
 
 def run_command(  # noqa: C901
@@ -175,7 +173,7 @@ def run_command(  # noqa: C901
                         parse_stdout(
                             line, prefix=f"[green]| [/green][blue]{label}[/blue]: "
                         ),
-                        decorate=False,
+                        prefix=False,
                     )
 
             # Wait for the subprocess to finish and get the exit code
@@ -206,7 +204,7 @@ def run_command(  # noqa: C901
         error_string = (
             f"runem: [red bold]FATAL[/red bold]: command failed: [blue]{label}[/blue]"
             f"\n\t[yellow]{env_overrides_as_string}{cmd_string}[/yellow]"
-            f"\n[red underline]| ERROR[/red underline]"
+            f"\n[red underline]| ERROR[/red underline]: [blue]{label}[/blue]"
             f"\n{str(parsed_stdout)}"
             f"\n[red underline]| ERROR END[/red underline]"
         )
@@ -219,7 +217,7 @@ def run_command(  # noqa: C901
     if verbose:
         log(
             f"running: done: [blue]{label}[/blue]: [yellow]{cmd_string}[/yellow]",
-            decorate=decorate_logs,
+            prefix=decorate_logs,
         )
 
     if record_sub_job_time is not None:

{runem-0.6.0 → runem-0.7.0}/runem/runem.py

@@ -19,6 +19,7 @@ We do:
 - time tests and tell you what used the most time, and how much time run-tests saved
   you
 """
+
 import contextlib
 import multiprocessing
 import os
@@ -48,6 +49,7 @@ from runem.log import error, log, warn
 from runem.report import report_on_run
 from runem.run_command import RunemJobError
 from runem.types.common import OrderedPhases, PhaseName
+from runem.types.errors import SystemExitBad
 from runem.types.filters import FilePathListLookup
 from runem.types.hooks import HookName
 from runem.types.runem_config import Config, Jobs, PhaseGroupedJobs
@@ -68,7 +70,6 @@ def _determine_run_parameters(argv: typing.List[str]) -> ConfigMetadata:
 
     Return a ConfigMetadata object with all the required information.
     """
-
     # Because we want to be able to show logging whilst parsing .runem.yml config, we
     # need to check the state of the logging-verbosity switches here, manually, as well.
     verbose = "--verbose" in argv
@@ -105,13 +106,14 @@ def _update_progress(
     """Updates progress report periodically for running tasks.
 
     Args:
-        label (str): The identifier.
+        phase (str): The currently running phase.
         running_jobs (Dict[str, str]): The currently running jobs.
+        completed_jobs (Dict[str, str]): The jobs that have finished work.
         all_jobs (Jobs): All jobs, encompassing both completed and running jobs.
         is_running (ValueProxy[bool]): Flag indicating if jobs are still running.
         num_workers (int): Indicates the number of workers performing the jobs.
+        show_spinner (bool): Whether to show the animated spinner or not.
     """
-
     last_running_jobs_set: typing.Set[str] = set()
 
     # Using the `rich` module to show a loading spinner on console
@@ -132,7 +134,8 @@ def _update_progress(
                 "blue",
             )  # Reflect current running jobs accurately
             report: str = (
-                f"[green]{phase}[/green]: {progress}({num_workers}): {running_jobs_list}"
+                f"[green]{phase}[/green]: {progress}({num_workers}): "
+                f"{running_jobs_list}"
             )
             if show_spinner:
                 assert isinstance(spinner_ctx, Status)
@@ -293,8 +296,8 @@ def _main(
         log(f"found {len(file_lists)} batches, ", end="")
         for tag in sorted(file_lists.keys()):
             file_list = file_lists[tag]
-            log(f"{len(file_list)} '{tag}' files, ", decorate=False, end="")
-        log(decorate=False)  # new line
+            log(f"{len(file_list)} '{tag}' files, ", prefix=False, end="")
+        log(prefix=False)  # new line
 
     filtered_jobs_by_phase: PhaseGroupedJobs = filter_jobs(
         config_metadata=config_metadata,
@@ -371,7 +374,7 @@ def timed_main(argv: typing.List[str]) -> None:
         # we got a failure somewhere, now that we've reported the timings we
         # re-raise.
         error(failure_exception.stdout)
-        raise failure_exception
+        raise SystemExitBad(1) from failure_exception
 
 
 if __name__ == "__main__":