dynamic-systems-testing-framework 1.0.1__py3-none-any.whl

dstf/comparison/types.py

@@ -0,0 +1,194 @@
+"""Comparison-result dataclasses + shared constants.
+
+Lives separately from :mod:`.comparator` (orchestration) and
+:mod:`.algorithms` (per-mode math) so both can import the result types
+without an import cycle. Re-exported from :mod:`.comparator` for
+historical-import compatibility, but new code should prefer
+``from dstf.comparison.types import VariableComparison``.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING
+
+import numpy as np
+
+from .geometry import Primitive
+from .sides import Sides, parse_sides
+
+if TYPE_CHECKING:
+    # Annotation-only — runtime import is inside compare_test() in
+    # comparator.py to break the comparator <-> metric_tree cycle.
+    from .metric_tree import MetricResult
+
+
+DEFAULT_TOLERANCE = 1e-4
+
+# Machine-epsilon guard — signals with range below this are treated as
+# constant by the NRMSE path.
+_EPS = 100 * np.finfo(np.float64).eps
+
+
+@dataclass
+class VariableComparison:
+    """Comparison result for a single tracked variable.
+
+    Conforms to the ``MetricResult`` contract in docs/extensibility.md:
+    carries a ``passed`` flag, a numeric score (``nrmse`` for NRMSE mode,
+    ``tube_points_inside`` for tube mode), and a structured ``diagnostics``
+    bag for metric-specific extras (e.g. event-timing deltas, spectral
+    peaks) that future metrics may attach without widening the schema.
+    """
+
+    index: int
+    name: str
+    passed: bool
+    nrmse: float  # Normalized RMSE (pass/fail metric)
+    rmse: float  # Raw RMSE
+    signal_range: float  # max(ref) - min(ref), for context
+    max_abs_error: float  # Largest absolute deviation
+    max_abs_error_time: float  # Time at which it occurs
+    reference_final: float
+    actual_final: float
+    is_constant: bool = False  # True if reference signal has zero range
+    # The tolerance applied for this variable, per side (a number is both).
+    tolerance_used: Sides = Sides(0.0, 0.0)
+    #: The metric this criterion was scored by — or, a config-error
+    #: sentinel's, could not be. THE one source of a criterion's metric name:
+    #: the console and JUnit blocks, the tree view and `verdicts.json` read
+    #: it and none substitutes a default. ``""`` only for a flat override
+    #: naming a mode nothing is registered under.
+    mode: str = "nrmse"
+    # Tube-specific metrics (populated when mode="tube")
+    tube_points_inside: float | None = None  # Fraction of points inside tube (0-1)
+    tube_worst_violation: float | None = None  # Largest violation (absolute)
+    tube_worst_violation_time: float | None = None  # Time of worst violation
+    # Open-ended structured extras — future metrics attach here instead of
+    # growing this dataclass (event-timing, spectral, domain-specific scores).
+    diagnostics: dict = field(default_factory=dict)
+    #: The baseline this criterion READ — ``"primary"``, a soft check's name
+    #: — or ``None`` for one that reads none: a ``range``, a criterion with
+    #: declared events / peaks / values, one whose config could not be
+    #: built. Stamped by the comparator and the tree evaluator from the mode
+    #: they built (``is_baseline_free``); THE one source of the baseline a
+    #: surface names beside a criterion (the console's and JUnit's ``·
+    #: against primary``, the tree view's ``against``), on both paths.
+    against: str | None = None
+    # The scored variable's expression (see ``VariableResult.expression``): a
+    # label, never a key. Stamped by the comparator and the tree evaluator
+    # from the result variable the leaf scored; "" when there is none.
+    expression: str = ""
+    # What the mode scored against, as drawable data (see geometry.py).
+    # Ids are local until the evaluator prefixes them with the leaf path.
+    geometry: tuple[Primitive, ...] = ()
+
+    def __post_init__(self) -> None:
+        # A number handed in (a fixture, a third-party metric) is both sides.
+        self.tolerance_used = parse_sides(self.tolerance_used, "tolerance_used")
+
+
+@dataclass(frozen=True)
+class ScoreSummary:
+    """One leaf's verdict in words: the pill text and the criterion sentence.
+
+    Produced per mode by :mod:`.summaries` and dispatched through the
+    metric descriptor, so the report's tree view and the terminal review
+    loop read the same mode-aware text.
+    """
+
+    display: str
+    criterion: str
+    score: float | None = None
+    score_label: str = ""
+    #: The mode's own key/value pairs behind the verdict — the largest error
+    #: and where, the finals, the worst violation, the failed checkpoints.
+    #: Each mode names only quantities that mean something for it: the
+    #: shared numeric fields are overloaded on some modes (a range leaf's
+    #: ``nrmse`` holds its worst violation), so no consumer prints them
+    #: under a generic label.
+    details: tuple[tuple[str, str], ...] = ()
+
+
+@dataclass
+class StructuralWarning:
+    """A warning beside a test's verdict, in one of two forms.
+
+    A CHANGE between the baseline and this run: ``field`` with
+    ``reference_value`` and ``current_value`` — ``Event count: 3 -> 5``.
+
+    A STATEMENT about this run alone: ``field`` and ``statement`` —
+    ``Tracked but not scored by any criterion: unitTests.x[2]``. It has no
+    baseline side; laid out as a change (``1 scored by the tree ->
+    unitTests.x[2]``) it read as one between two runs, and backwards.
+
+    A surface prints :attr:`text`, or asks :attr:`is_change` before it
+    lays the two values out in columns.
+    """
+
+    field: str
+    reference_value: str = ""
+    current_value: str = ""
+    statement: str = ""
+
+    @property
+    def is_change(self) -> bool:
+        return not self.statement
+
+    @property
+    def text(self) -> str:
+        if self.statement:
+            return f"{self.field}: {self.statement}"
+        return f"{self.field}: {self.reference_value} -> {self.current_value}"
+
+
+@dataclass
+class TestComparison:
+    """Comparison result for a full test model."""
+
+    model_id: str
+    passed: bool
+    test_id: str | None = None  # ref file ID (e.g., "0001")
+    variables: list[VariableComparison] = field(default_factory=list)
+    warnings: list[StructuralWarning] = field(default_factory=list)
+    error_message: str | None = None
+    sim_success: bool = True
+    has_reference: bool = True
+    # PTA.5 — this test's pass criterion is "the simulation ran"; no
+    # baseline is consulted and ``variables`` is empty by construction.
+    # Carried on the comparison (not just on the TestModel) so every
+    # report surface can classify it from the comparison alone —
+    # reporting.status is the one classifier, and this is the input it
+    # used to be missing (D117).
+    simulate_only: bool = False
+    #: False when there is no result to compare (D145): the last run never
+    #: ran this test, or never finished it. ``reporting.status`` reads it
+    #: first — NOT RUN, which is not a failure. ``sim_success`` is False
+    #: beside it: nothing succeeded.
+    ran: bool = True
+    # Phase 3.1: MetricTree root for this test. Today populated as the
+    # implicit flat-AND over per-variable comparisons (matches `passed`
+    # exactly). Phase 3.2+ replaces with user-authored trees from
+    # test_spec.json. None when no comparison ran (sim failure, no baseline).
+    metric_tree: MetricResult | None = None
+    #: Where the tolerance a leaf without its own was scored with came from:
+    #: ``"test"`` (the spec's ``comparison.tolerance``), ``"reference"`` (the
+    #: baseline JSON's) or ``"project"`` (``Config.tolerance``: the CLI's,
+    #: else testing.json's, else 1e-4 — the comparator is handed one number
+    #: and cannot tell those apart). None where nothing was scored.
+    tolerance_source: str | None = None
+    #: That base tolerance: what a leaf without its own is scored with.
+    base_tolerance: float | None = None
+
+    @property
+    def evaluated(self) -> bool:
+        """True when a comparison actually ran for this test.
+
+        Distinguishes a baseline-free or simulate-only test (which has a
+        metric tree / variables and a real verdict despite
+        ``has_reference=False``) from the NO_REF short-circuit where
+        nothing was scored. Report surfaces must classify ``not passed``
+        as FAILED whenever ``evaluated`` is true — review 2026-07-06,
+        finding 5 (failing baseline-free tests exited 0 as "NO_REF").
+        """
+        return self.metric_tree is not None or bool(self.variables)

dstf/comparison/validator.py

@@ -0,0 +1,232 @@
+"""Metric-tree validator enforcing D66 baseline-role rules.
+
+The comparator's MetricTree distinguishes three baseline roles (D66):
+**primary** = regression anchor; **soft_check** = advisory cross-check;
+**companion** = plot-only overlay. This module enforces that trees
+respect those semantics at spec-load time:
+
+1. Every tree has at least one leaf targeting ``primary`` outside
+   ``warn`` — guarantees a real regression anchor.
+2. Leaves with ``against: "<soft_check>"`` must sit under a ``warn``
+   ancestor — soft_checks never hard-fail.
+3. Leaves must never target companion names — companions are not
+   scorable, only displayable.
+4. Leaves must not target unknown baseline names.
+
+Callers supply a ``BaselineRegistry`` (a mapping of known names to
+their role) so validation is context-aware per test.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable, Sequence
+from dataclasses import dataclass
+from enum import Enum
+
+from .tree_spec import CombinatorSpec, LeafSpec, SpecNode
+
+
+class BaselineRole(Enum):
+    PRIMARY = "primary"
+    SOFT_CHECK = "soft_check"
+    COMPANION = "companion"
+
+
+@dataclass
+class ValidationError:
+    path: str
+    message: str
+
+    def __str__(self) -> str:
+        return f"{self.path}: {self.message}"
+
+
+BaselineRoleLookup = Callable[[str], BaselineRole | None]
+
+
+def validate_tree(
+    tree: SpecNode,
+    role_of: BaselineRoleLookup,
+    _path: str = "metrics",
+) -> list[ValidationError]:
+    """Return a list of validation errors for ``tree`` against D66 role rules.
+
+    Args:
+        tree: Parsed MetricTree spec (root node).
+        role_of: callable returning :class:`BaselineRole` for a known
+            baseline name, or ``None`` for an unknown name. Typical
+            caller builds this from a :class:`ReferenceStore` query.
+    """
+    errors: list[ValidationError] = []
+    _walk(tree, role_of, _path, in_warn=False, errors=errors)
+
+    # Tree-level rule: at least one primary leaf outside warn.
+    if not _has_primary_outside_warn(tree, role_of):
+        errors.append(
+            ValidationError(
+                _path,
+                "Every tree needs at least one criterion scored against the "
+                "primary baseline outside a 'warn': a tree of warnings alone "
+                "can never fail.",
+            )
+        )
+    return errors
+
+
+def _walk(
+    node: SpecNode,
+    role_of: BaselineRoleLookup,
+    path: str,
+    in_warn: bool,
+    errors: list[ValidationError],
+) -> None:
+    if isinstance(node, LeafSpec):
+        _check_leaf(node, role_of, path, in_warn, errors)
+        return
+    if isinstance(node, CombinatorSpec):
+        if node.combinator == "weighted":
+            _check_weighted_directions(node, path, errors)
+        child_in_warn = in_warn or node.combinator == "warn"
+        for i, child in enumerate(node.children):
+            _walk(child, role_of, f"{path}.children[{i}]", child_in_warn, errors)
+
+
+def _check_weighted_directions(
+    node: CombinatorSpec,
+    path: str,
+    errors: list[ValidationError],
+) -> None:
+    """Reject weighted nodes whose leaf children score in opposite directions.
+
+    ``weighted`` compares one summed score against one threshold with one
+    comparison direction, so summing a lower-is-better score (NRMSE-like
+    error) with a higher-is-better one (tube's in-band fraction) is
+    meaningless — the sum can move toward the threshold for opposite
+    reasons. The metric registry's ``score_direction`` is the authority
+    (D101).
+    """
+    from .registry import metric as registry_metric
+
+    directions: dict[str, list[str]] = {}
+    for child in node.children:
+        if not isinstance(child, LeafSpec):
+            continue
+        desc = registry_metric(child.metric)
+        if desc is None:
+            continue  # unknown metric fails loudly at parse/eval, not here
+        directions.setdefault(desc.score_direction, []).append(child.metric)
+    if len(directions) > 1:
+        parts = "; ".join(
+            f"{d}: {', '.join(sorted(set(ms)))}" for d, ms in sorted(directions.items())
+        )
+        errors.append(
+            ValidationError(
+                path,
+                f"A 'weighted' node cannot mix lower-is-better and "
+                f"higher-is-better scores ({parts}); split them into separate "
+                f"weighted nodes.",
+            )
+        )
+
+
+def _check_leaf(
+    leaf: LeafSpec,
+    role_of: BaselineRoleLookup,
+    path: str,
+    in_warn: bool,
+    errors: list[ValidationError],
+) -> None:
+    role = role_of(leaf.against)
+    if role is None:
+        errors.append(
+            ValidationError(
+                path,
+                f"This criterion scores against {leaf.against!r}, which is not "
+                f"a baseline of this test (the primary or a soft check); check "
+                f"the name.",
+            )
+        )
+        return
+    if role is BaselineRole.COMPANION:
+        errors.append(
+            ValidationError(
+                path,
+                f"This criterion scores against the companion {leaf.against!r}, "
+                f"and companions are only drawn; import it as a soft check "
+                f"(dstf import-baseline) to score against it.",
+            )
+        )
+        return
+    if role is BaselineRole.SOFT_CHECK and not in_warn:
+        errors.append(
+            ValidationError(
+                path,
+                f"This criterion scores against the soft check {leaf.against!r}, "
+                f"which must sit under a 'warn' so it can only warn.",
+            )
+        )
+        return
+
+
+def _has_primary_outside_warn(
+    node: SpecNode,
+    role_of: BaselineRoleLookup,
+    in_warn: bool = False,
+) -> bool:
+    if isinstance(node, LeafSpec):
+        if in_warn:
+            return False
+        return role_of(node.against) is BaselineRole.PRIMARY
+    if isinstance(node, CombinatorSpec):
+        child_in_warn = in_warn or node.combinator == "warn"
+        return any(
+            _has_primary_outside_warn(c, role_of, child_in_warn) for c in node.children
+        )
+    return False
+
+
+# ---------------------------------------------------------------------------
+# Convenience role-lookup factories
+# ---------------------------------------------------------------------------
+
+
+def role_lookup_from_store(store, model_id: str) -> BaselineRoleLookup:
+    """Build a role-lookup for ``store``'s view of ``model_id``.
+
+    Primary is always ``"primary"``. Soft_checks come from
+    :meth:`ReferenceStore.get_soft_checks`. Companions from
+    :meth:`ReferenceStore.get_companions`.
+    """
+    soft_check_names = set(store.get_soft_checks(model_id).keys())
+    companion_names = set(store.get_companions(model_id).keys())
+
+    def lookup(name: str) -> BaselineRole | None:
+        if name == "primary":
+            return BaselineRole.PRIMARY
+        if name in soft_check_names:
+            return BaselineRole.SOFT_CHECK
+        if name in companion_names:
+            return BaselineRole.COMPANION
+        return None
+
+    return lookup
+
+
+def role_lookup_from_names(
+    soft_checks: Sequence[str] = (),
+    companions: Sequence[str] = (),
+) -> BaselineRoleLookup:
+    """Build a role-lookup from explicit name lists (testing-friendly)."""
+    sc = set(soft_checks)
+    co = set(companions)
+
+    def lookup(name: str) -> BaselineRole | None:
+        if name == "primary":
+            return BaselineRole.PRIMARY
+        if name in sc:
+            return BaselineRole.SOFT_CHECK
+        if name in co:
+            return BaselineRole.COMPANION
+        return None
+
+    return lookup