dynamic-systems-testing-framework 1.0.1__py3-none-any.whl

dstf/plugins.py

@@ -0,0 +1,104 @@
+"""Entry-point plug-in loading — the seam third-party packages register through.
+
+Every extension registry in the framework (``backends.register``,
+``discovery.recognizer.register``, ``comparison.registry.register_metric`` /
+``register_combinator``) is populated by *importing* the module that calls it.
+Builtins are imported by their package ``__init__``; a third-party plug-in
+lives in a package we never import, so it needs a hook. That hook is a single
+entry-point group:
+
+.. code-block:: toml
+
+    # in the plug-in package's pyproject.toml
+    [project.entry-points."dstf.plugins"]
+    acme_backend = "acme_dstf.backend"          # module: import side effects register
+    acme_metrics = "acme_dstf.metrics:register" # callable: invoked with no args
+
+One group, not one per layer: a plug-in package routinely ships a backend
+*and* the metrics that score it, and splitting the declaration by layer only
+makes the author guess which group a given registration belongs to.
+
+A plug-in that fails to import or call is a **hard error**
+(:class:`~dstf.config.ConfigError`). A metric that silently failed to register
+does not disappear quietly — it changes scoring semantics, either by failing
+spec parse with a confusing "unknown metric" or by falling back to a default.
+Fail at startup, naming the entry point.
+"""
+
+from __future__ import annotations
+
+import importlib.metadata
+from types import ModuleType
+
+#: The single entry-point group third-party packages advertise plug-ins under.
+PLUGIN_GROUP = "dstf.plugins"
+
+# None until the first successful load. Loading is idempotent: the CLI calls
+# it once per process, but a library user may call it again from anywhere.
+_loaded: list[str] | None = None
+_completed: set[str] = set()
+
+
+def load_plugins() -> list[str]:
+    """Import every ``dstf.plugins`` entry point, returning the names loaded.
+
+    Each entry point's ``.load()`` result is either a **module** (the import
+    itself performed the registrations) or a **callable** (invoked with no
+    arguments). Anything else is an error.
+
+    Idempotent — the second and later calls return the same names without
+    re-importing. Raises :class:`~dstf.config.ConfigError` naming the failing
+    entry point, chaining the original exception.
+    """
+    global _loaded
+    if _loaded is not None:
+        return list(_loaded)
+
+    # Per-entry-point completion tracking, not one all-or-nothing flag: a
+    # library user who catches the ConfigError from a broken plug-in and
+    # retries must not re-invoke the HEALTHY plug-ins that already ran —
+    # their re-registration would raise the registry's duplicate error and
+    # blame the wrong plug-in (B2 review finding 4).
+    names: list[str] = []
+    for ep in importlib.metadata.entry_points(group=PLUGIN_GROUP):
+        if ep.name in _completed:
+            names.append(ep.name)
+            continue
+        try:
+            obj = ep.load()
+            if isinstance(obj, ModuleType):
+                pass  # import side effects did the registration
+            elif isinstance(obj, type):
+                # A class is callable, but instantiating it with no args is
+                # never what a plug-in author meant (B2 review finding 8).
+                raise TypeError(
+                    f"resolved to class {obj.__name__}; point the entry "
+                    f"point at a module or a no-argument callable"
+                )
+            elif callable(obj):
+                obj()
+            else:
+                raise TypeError(
+                    f"resolved to {type(obj).__name__}, expected a module or a callable"
+                )
+        except Exception as exc:
+            # Lazy — config imports enough of the package that a module-level
+            # import here would risk a cycle, and this is the cold path.
+            from .config import ConfigError
+
+            raise ConfigError(
+                f"Failed to load DSTF plug-in '{ep.name}' "
+                f"({PLUGIN_GROUP} = {ep.value!r}): {exc}"
+            ) from exc
+        _completed.add(ep.name)
+        names.append(ep.name)
+
+    _loaded = names
+    return list(names)
+
+
+def _reset_for_tests() -> None:
+    """Forget that plug-ins were loaded — unit tests only."""
+    global _loaded
+    _completed.clear()
+    _loaded = None

dstf/reporting/console_report.py

@@ -0,0 +1,187 @@
+"""Console (terminal) report for test comparison results."""
+
+import sys
+
+from ..comparison.comparator import TestComparison
+from .status import (
+    FAIL,
+    NO_REF,
+    NOT_RUN,
+    SIM_FAIL,
+    STATUS_LABELS,
+    FailedCriterion,
+    advisory_failures,
+    advisory_text,
+    classify,
+    exit_code,
+    failed_criteria,
+    group_failures,
+)
+
+# ANSI color codes
+_GREEN = "\033[92m"
+_RED = "\033[91m"
+_YELLOW = "\033[93m"
+_GREY = "\033[90m"
+_RESET = "\033[0m"
+_BOLD = "\033[1m"
+
+
+def _color(text: str, code: str) -> str:
+    """Wrap text in ANSI color if stdout is a terminal."""
+    if sys.stdout.isatty():
+        return f"{code}{text}{_RESET}"
+    return text
+
+
+def print_report(comparisons: list[TestComparison]) -> int:
+    """Print a console report and return exit code (0 when nothing FAILED or
+    failed to simulate — NOT RUN and NO_REF do not fail it — and something
+    ran, else 1: ``exit_code``)."""
+    if not comparisons:
+        print("No comparisons to report.")
+        return 1
+
+    n_passed = 0
+    n_failed = 0
+    n_no_ref = 0
+    n_sim_fail = 0
+    n_not_run = 0
+    failures: list[TestComparison] = []
+    warned: list[TestComparison] = []
+    advised: list[tuple[TestComparison, list[FailedCriterion]]] = []
+
+    for comp in comparisons:
+        # Ordering (SIM_FAIL > FAIL > NO_REF > PASS) and its rationale live
+        # in reporting.status — this is one of its call sites, not a copy.
+        state = classify(comp)
+        if state == NOT_RUN:
+            # No result: nothing ran, so nothing failed (D145).
+            n_not_run += 1
+            status = _color(STATUS_LABELS[NOT_RUN], _GREY)
+        elif state == SIM_FAIL:
+            n_sim_fail += 1
+            status = _color("SIM_FAIL", _RED)
+        elif state == FAIL:
+            n_failed += 1
+            status = _color("FAIL", _RED)
+        elif state == NO_REF:
+            n_no_ref += 1
+            status = _color("NO_REF", _YELLOW)
+        else:
+            n_passed += 1
+            status = _color("PASS", _GREEN)
+
+        # Show warning indicator if structural changes detected
+        warn_tag = ""
+        if comp.warnings:
+            warn_tag = _color(" [WARN]", _YELLOW)
+            warned.append(comp)
+
+        if comp.test_id:
+            id_tag = f"[{comp.test_id}] "
+        elif not comp.has_reference:
+            id_tag = "[new] "
+        else:
+            id_tag = ""
+        # A test that passes with a failure under `warn` says so: it read a
+        # plain PASS, with no word of the criterion that failed.
+        advisory = advisory_failures(comp)
+        advisory_tag = ""
+        if advisory:
+            advised.append((comp, advisory))
+            advisory_tag = _color(f" ({advisory_text(len(advisory))})", _YELLOW)
+        print(f"  {status}  {id_tag}{comp.model_id}{advisory_tag}{warn_tag}")
+
+        if not comp.passed and comp.sim_success:
+            failures.append(comp)
+
+    # Summary
+    total = len(comparisons)
+    print()
+    print(_color(f"Results: {n_passed}/{total} passed", _BOLD))
+    if n_failed:
+        print(_color(f"  Failed: {n_failed}", _RED))
+    if n_sim_fail:
+        print(_color(f"  Simulation errors: {n_sim_fail}", _RED))
+    if n_no_ref:
+        print(_color(f"  No baseline: {n_no_ref}", _YELLOW))
+    if n_not_run:
+        print(_color(f"  {STATUS_LABELS[NOT_RUN]}: {n_not_run}", _GREY))
+    if advised:
+        print(_color(f"  With advisory failures: {len(advised)}", _YELLOW))
+    if warned:
+        print(_color(f"  Structural warnings: {len(warned)}", _YELLOW))
+
+    # Print failure details
+    if failures:
+        print()
+        print(_color("Failure Details:", _BOLD))
+        print("-" * 80)
+        for comp in failures:
+            id_tag = f"[{comp.test_id}] " if comp.test_id else ""
+            print(f"\n  {id_tag}{comp.model_id}")
+            if comp.error_message:
+                print(f"    Error: {comp.error_message}")
+            for block in failure_blocks(comp):
+                for line in block:
+                    print(line)
+            # A group that fails on its own: the test's reason where no
+            # criterion failed.
+            for line in group_failures(comp):
+                print(f"    {line}")
+
+    # The criteria that failed under a `warn` in the tests that passed
+    if advised:
+        print()
+        print(_color("Advisory Failures (under `warn`: they fail no test):", _BOLD))
+        print("-" * 80)
+        for comp, advisory in advised:
+            id_tag = f"[{comp.test_id}] " if comp.test_id else ""
+            print(f"\n  {id_tag}{comp.model_id}")
+            for block in _blocks(advisory):
+                for line in block:
+                    print(line)
+
+    # Print structural warnings
+    if warned:
+        print()
+        print(_color("Structural Warnings:", _BOLD))
+        print("-" * 80)
+        for comp in warned:
+            print(f"\n  {comp.model_id}")
+            for w in comp.warnings:
+                print(f"    {w.text}")
+
+    return exit_code(comparisons)
+
+
+def failure_blocks(comp: TestComparison) -> list[list[str]]:
+    """One block of lines per criterion of ``comp`` that did not pass
+    (:func:`~.status.failed_criteria`, which JUnit's lines are read from
+    too), in TREE order: the line that says which criterion it is, the
+    criterion in its metric's own words — or why it was not scored — and
+    that metric's detail pairs.
+
+    The blocks used to be the same five generic numbers under the
+    variable's name alone: a start-up window's nrmse and an advisory
+    comparison against a soft check read as one variable failing twice,
+    with nothing to tell them apart."""
+    return _blocks(failed_criteria(comp))
+
+
+def _blocks(criteria: list[FailedCriterion]) -> list[list[str]]:
+    blocks: list[list[str]] = []
+    for criterion in criteria:
+        lines = [f"    {criterion.head}"]
+        if criterion.reason:
+            # A sentinel (config that could not build, no data to compare)
+            # has no meaningful numbers — the reason is the whole finding.
+            lines.append(f"      {'Not scored:':<20}{criterion.reason}")
+        else:
+            lines.append(f"      {criterion.sentence}")
+            lines += [
+                f"      {label + ':':<20}{value}" for label, value in criterion.details
+            ]
+        blocks.append(lines)
+    return blocks