dynamic-systems-testing-framework 1.0.1__py3-none-any.whl

dstf/comparison/metric_tree.py

@@ -0,0 +1,332 @@
+"""MetricTree: composition of metric results with Boolean / weighted combinators.
+
+See docs/vision.md "Metric composition" and docs/extensibility.md §6.
+
+A spec with no ``metrics`` tree collapses to a degenerate flat-AND over
+per-variable ``VariableComparison`` results (``implicit_and_tree``), which is
+exactly the implicit semantics. An explicit ``metrics`` tree in
+``test_spec.json`` reaches the and / or / k-of-n / warn / weighted
+combinators below.
+
+Design contract:
+
+* Combinators are pure functions of child ``MetricResult``s. No I/O, no mutation.
+* A combinator's output ``diagnostics`` must preserve enough of its children's
+  diagnostics that a report can show which branch failed and why.
+* Leaves wrap a ``VariableComparison``.
+"""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass, field
+from typing import Any, cast
+
+from .comparator import VariableComparison
+from .sides import as_written
+
+# ---------------------------------------------------------------------------
+# Uniform metric result (combinator output shape)
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class MetricResult:
+    """Uniform shape produced by both leaves and combinators.
+
+    ``VariableComparison`` is a richer, reporting-friendly leaf result; this
+    is the narrow type combinators pass around. A leaf is adapted to a
+    ``MetricResult`` on the way in via :func:`leaf_from_variable`.
+
+    Fields:
+      passed      — overall pass/fail after combinator logic.
+      score       — numeric score; interpretation is combinator-dependent
+                    (min NRMSE for AND, max NRMSE for OR, etc.). Not normalized
+                    in general. A value of ``None`` means "no meaningful score"
+                    (e.g. after a ``warn`` wrapper).
+      label       — short human-readable tag for reports (e.g. a variable name
+                    or a combinator name + children count).
+      diagnostics — open-ended structured extras. Reports render these.
+      children    — empty for leaves; populated for combinator nodes so the
+                    tree shape is reconstructible.
+    """
+
+    passed: bool
+    score: float | None
+    label: str = ""
+    diagnostics: dict[str, Any] = field(default_factory=dict)
+    children: list[MetricResult] = field(default_factory=list)
+
+
+# ---------------------------------------------------------------------------
+# Combinator protocol
+# ---------------------------------------------------------------------------
+
+
+class Combinator(ABC):
+    """Strategy interface for combining child metric results into one."""
+
+    #: Short identifier used in config / diagnostics.
+    name: str = ""
+
+    @abstractmethod
+    def combine(self, children: list[MetricResult]) -> MetricResult: ...
+
+
+class AndCombinator(Combinator):
+    """All children must pass. Score is ``min`` over the children's scores.
+
+    Matches the implicit behavior of a spec with no ``metrics`` tree: a test
+    passes iff every variable's metric passes.
+
+    ``min`` is only "the worst child" for lower-is-better metrics. Children
+    can mix directions (the registry's ``score_direction`` is ``"lower"``
+    for NRMSE-like scores, ``"higher"`` for tube coverage), and this node
+    does not consult it — ``passed`` is a pure Boolean AND and is always
+    right; the reported score is a summary number, not a ranking.
+    """
+
+    name = "and"
+
+    def combine(self, children: list[MetricResult]) -> MetricResult:
+        if not children:
+            # Vacuously true. Score undefined; pick None rather than lie.
+            return MetricResult(passed=True, score=None, label="and[0]", children=[])
+        passed = all(c.passed for c in children)
+        scores = [c.score for c in children if c.score is not None]
+        score = min(scores) if scores else None
+        return MetricResult(
+            passed=passed,
+            score=score,
+            label=f"and[{len(children)}]",
+            diagnostics={"n_failed": sum(1 for c in children if not c.passed)},
+            children=list(children),
+        )
+
+
+class OrCombinator(Combinator):
+    """At least one child must pass. Score is ``max`` over the children's scores.
+
+    Same caveat as :class:`AndCombinator`: ``max`` is only "the best child"
+    for higher-is-better metrics, and this node does not consult the
+    registry's ``score_direction``. ``passed`` (a Boolean OR) is unaffected.
+    """
+
+    name = "or"
+
+    def combine(self, children: list[MetricResult]) -> MetricResult:
+        if not children:
+            # Vacuously false for OR (no children = no passing child).
+            return MetricResult(passed=False, score=None, label="or[0]", children=[])
+        passed = any(c.passed for c in children)
+        scores = [c.score for c in children if c.score is not None]
+        score = max(scores) if scores else None
+        return MetricResult(
+            passed=passed,
+            score=score,
+            label=f"or[{len(children)}]",
+            diagnostics={"n_passed": sum(1 for c in children if c.passed)},
+            children=list(children),
+        )
+
+
+class KOfNCombinator(Combinator):
+    """At least K of N children must pass."""
+
+    name = "k-of-n"
+
+    def __init__(self, k: int):
+        if k < 0:
+            raise ValueError("k must be non-negative")
+        self.k = k
+
+    def combine(self, children: list[MetricResult]) -> MetricResult:
+        n_passed = sum(1 for c in children if c.passed)
+        passed = n_passed >= self.k
+        scores = [c.score for c in children if c.score is not None]
+        # Heuristic: score = K-th best child score (or worst if K > len).
+        score: float | None
+        if not scores:
+            score = None
+        else:
+            scores_sorted = sorted(scores, reverse=True)
+            idx = min(self.k - 1, len(scores_sorted) - 1) if self.k > 0 else 0
+            score = scores_sorted[idx] if scores_sorted else None
+        return MetricResult(
+            passed=passed,
+            score=score,
+            label=f"k-of-n[{self.k}/{len(children)}]",
+            diagnostics={"k": self.k, "n": len(children), "n_passed": n_passed},
+            children=list(children),
+        )
+
+
+class WeightedCombinator(Combinator):
+    """Weighted sum of child scores against a threshold (4.E).
+
+    Pass condition is direction-aware:
+      - ``direction="less"`` (default — NRMSE-like; lower is better):
+        ``sum(w_i * score_i) < threshold``.
+      - ``direction="greater"`` (tube-like; higher is better):
+        ``sum(w_i * score_i) > threshold``.
+
+    All children must produce a numeric score; if any child has ``score=None``,
+    the weighted node fails (with a diagnostic explaining why) since the
+    aggregate is undefined.
+    """
+
+    name = "weighted"
+
+    def __init__(self, weights: list[float], threshold: float, direction: str = "less"):
+        if direction not in ("less", "greater"):
+            raise ValueError(
+                f"weighted: direction must be 'less' or 'greater', got {direction!r}"
+            )
+        if not weights:
+            raise ValueError("weighted: requires at least one weight")
+        self.weights = list(weights)
+        self.threshold = float(threshold)
+        self.direction = direction
+
+    def combine(self, children: list[MetricResult]) -> MetricResult:
+        if len(children) != len(self.weights):
+            raise ValueError(
+                f"weighted: weights ({len(self.weights)}) must match "
+                f"children ({len(children)})"
+            )
+        if any(c.score is None for c in children):
+            return MetricResult(
+                passed=False,
+                score=None,
+                label=f"weighted[{len(children)}]",
+                diagnostics={
+                    "reason": "child has no numeric score; weighted sum undefined",
+                    "weights": list(self.weights),
+                    "threshold": self.threshold,
+                    "direction": self.direction,
+                },
+                children=list(children),
+            )
+        # Every child has a numeric score — the guard above returned otherwise.
+        weighted_sum = sum(
+            w * cast(float, c.score) for w, c in zip(self.weights, children)
+        )
+        passed = (
+            weighted_sum < self.threshold
+            if self.direction == "less"
+            else weighted_sum > self.threshold
+        )
+        return MetricResult(
+            passed=passed,
+            score=weighted_sum,
+            label=f"weighted[{len(children)}]",
+            diagnostics={
+                "weighted_sum": weighted_sum,
+                "threshold": self.threshold,
+                "direction": self.direction,
+                "weights": list(self.weights),
+            },
+            children=list(children),
+        )
+
+
+class WarnCombinator(Combinator):
+    """Single-child wrapper that always passes the parent but surfaces the
+    child's diagnostics as warnings.
+
+    Used when an additional baseline comparison is informational (e.g. an
+    experimental-data overlay) and should not gate pass/fail. A ``warn`` node
+    always reports ``passed=True``; the report renders the child's failure
+    (if any) as a warning rather than a failure.
+    """
+
+    name = "warn"
+
+    def combine(self, children: list[MetricResult]) -> MetricResult:
+        if len(children) != 1:
+            raise ValueError(
+                f"warn combinator expects exactly 1 child, got {len(children)}"
+            )
+        child = children[0]
+        # Surface child's failure (if any) as a warning in diagnostics; parent
+        # passes regardless. Score is intentionally None — a warn branch does
+        # not contribute to a meaningful aggregate score.
+        warned = not child.passed
+        return MetricResult(
+            passed=True,
+            score=None,
+            label="warn",
+            diagnostics={"warned": warned, "child_label": child.label},
+            children=[child],
+        )
+
+
+# ---------------------------------------------------------------------------
+# Leaf adapter + implicit-tree construction
+# ---------------------------------------------------------------------------
+
+
+def leaf_from_variable(vc: VariableComparison) -> MetricResult:
+    """Adapt a per-variable ``VariableComparison`` to a leaf ``MetricResult``.
+
+    The numeric ``score`` is chosen per mode:
+      * NRMSE    — ``vc.nrmse``                   (lower is better; worst = min)
+      * tube     — ``vc.tube_points_inside``      (higher is better; uses OR if
+                                                   composed with another tube node)
+      * final    — ``vc.nrmse`` (same semantics)
+
+    Callers that need the raw ``VariableComparison`` can recover it from
+    ``result.diagnostics['variable']``.
+    """
+    if vc.mode == "tube" and vc.tube_points_inside is not None:
+        score = vc.tube_points_inside
+    else:
+        score = vc.nrmse
+    diagnostics = {
+        "mode": vc.mode,
+        "tolerance": as_written(vc.tolerance_used),
+        "max_abs_error": vc.max_abs_error,
+        "variable": vc,
+    }
+    if vc.against is not None:
+        # The baseline the criterion read; none where it reads none.
+        diagnostics["against"] = vc.against
+    return MetricResult(
+        passed=vc.passed,
+        score=score,
+        label=vc.name,
+        diagnostics=diagnostics,
+        children=[],
+    )
+
+
+def stamp_leaf_path(leaf: MetricResult, path: str) -> None:
+    """Address a leaf result by its RFC 6902 path — the same path a patch op uses.
+
+    Prefixes the stashed ``VariableComparison``'s geometry ids with the path
+    so a figure handle and a spec edit share one name. Idempotent: a mode
+    emits local ids, and a stamped id starts with ``/``.
+    """
+    leaf.diagnostics["leaf_path"] = path
+    vc = leaf.diagnostics.get("variable")
+    if vc is not None:
+        vc.geometry = tuple(
+            p if p.id.startswith("/") else p.prefixed(path) for p in vc.geometry
+        )
+
+
+def implicit_and_tree(variables: list[VariableComparison]) -> MetricResult:
+    """Build the implicit flat-AND tree from a list of per-variable comparisons.
+
+    This is the degenerate MetricTree that matches the current pass/fail
+    semantics exactly: a test passes iff every variable passes. Leaves are
+    stamped ``/metrics/children/<i>`` — the paths
+    :func:`~.tree_spec.synthesize_implicit_tree` produces for the same list,
+    so the report can look leaves up by path on both evaluation routes.
+    """
+    leaves = []
+    for i, vc in enumerate(variables):
+        leaf = leaf_from_variable(vc)
+        stamp_leaf_path(leaf, f"/metrics/children/{i}")
+        leaves.append(leaf)
+    return AndCombinator().combine(leaves)

dstf/comparison/mode_schema.py

@@ -0,0 +1,176 @@
+"""UI schema derived from a mode's Config dataclass — what ``export-schema``
+writes as JSON-Schema and the authoring tool's form emitter renders.
+
+Each :class:`ComparisonMode`'s Config is a frozen dataclass with strictly
+typed fields. :func:`derive_schema` reads those types via
+:func:`typing.get_type_hints` and returns the introspected field list —
+types, defaults, ``Literal`` choices and the display metadata — as a data
+artifact. :mod:`dstf.reporting.schema_export` turns it into the published
+JSON Schema; the authoring tool builds its forms from the same
+introspection. The report has no forms and does not use this module.
+
+The :class:`ComparisonMode` ABC stays pure compute — no UI coupling on the
+mode class. The bridge lives here instead, keyed on mode name.
+"""
+
+from __future__ import annotations
+
+import dataclasses
+import types
+import typing
+from collections.abc import Mapping
+from dataclasses import dataclass, field
+from typing import Any, Union
+
+from .sides import Sides, as_written
+
+# ---------------------------------------------------------------------------
+# Schema types
+# ---------------------------------------------------------------------------
+
+#: Recognized field types. ``sides`` is a tolerance with two sides
+#: (:class:`~.sides.Sides`): a number, or a ``{"lower", "upper"}`` pair.
+#: ``passthrough`` is the fallback for complex types (nested dataclasses,
+#: list[dict], etc.) that introspection cannot reduce to a scalar or an
+#: enum — a consumer renders those as raw JSON.
+FIELD_TYPES = {"float", "int", "bool", "str", "enum", "sides", "passthrough"}
+
+#: What the two sides of a ``sides`` field are called, lower first, unless
+#: its metadata says (``side_labels``): a value's are below and above its
+#: target. A time tolerance names its own — ``early`` / ``late``.
+SIDE_LABELS = ("−", "+")
+
+
+@dataclass
+class FieldSpec:
+    name: str
+    type: str  # one of FIELD_TYPES
+    default: Any = None
+    optional: bool = False
+    choices: list[str] | None = None
+    label: str | None = None
+    help: str | None = None
+    # UI hints for number inputs. ``ui_max`` is a SOFT cap: a consumer that
+    # renders an input raises it to fit a larger current value rather than
+    # clamping the value down.
+    ui_min: float | None = None
+    ui_max: float | None = None
+    # A ``sides`` field's two labels, lower first; None on every other kind.
+    side_labels: tuple[str, str] | None = None
+
+
+@dataclass
+class Schema:
+    mode: str | None = None
+    fields: list[FieldSpec] = field(default_factory=list)
+
+    def to_dict(self) -> dict:
+        return {
+            "mode": self.mode,
+            "fields": [dataclasses.asdict(f) for f in self.fields],
+        }
+
+
+# ---------------------------------------------------------------------------
+# Introspection
+# ---------------------------------------------------------------------------
+
+
+def derive_schema(config_cls: type, *, mode: str | None = None) -> Schema:
+    """Inspect a dataclass and return its UI schema.
+
+    Handles ``float``, ``int``, ``bool``, ``str``, ``Optional[X]``,
+    ``Literal[...]``, and ``Optional[Literal[...]]``. Unknown shapes
+    (``list[dict]``, nested dataclasses, etc.) degrade to
+    ``type="passthrough"`` so callers can render a raw-JSON fallback
+    without crashing.
+
+    Reads ``metadata`` from :func:`dataclasses.field` for display hints:
+      - ``label``: human-readable label (defaults to a titleized name)
+      - ``help``: tooltip text
+      - ``side_labels``: what a ``sides`` field's two sides are called,
+        lower first (defaults to :data:`SIDE_LABELS`)
+    """
+    if not dataclasses.is_dataclass(config_cls):
+        raise TypeError(f"{config_cls!r} is not a dataclass")
+
+    hints = typing.get_type_hints(config_cls)
+    fields_out: list[FieldSpec] = []
+    for f in dataclasses.fields(config_cls):
+        hint = hints.get(f.name, f.type)
+        kind, optional, choices = _classify_type(hint)
+        default = _field_default(f)
+        meta: Mapping[str, Any] = f.metadata or {}
+        fields_out.append(
+            FieldSpec(
+                name=f.name,
+                type=kind,
+                default=default,
+                optional=optional,
+                choices=choices,
+                label=meta.get("label") or _titleize(f.name),
+                help=meta.get("help"),
+                ui_min=meta.get("ui_min"),
+                ui_max=meta.get("ui_max"),
+                side_labels=(
+                    tuple(meta.get("side_labels") or SIDE_LABELS)
+                    if kind == "sides"
+                    else None
+                ),
+            )
+        )
+    return Schema(mode=mode, fields=fields_out)
+
+
+def _classify_type(hint: Any) -> tuple[str, bool, list[str] | None]:
+    """Return (field_kind, optional, choices-if-enum) for a type hint."""
+    origin = typing.get_origin(hint)
+    args = typing.get_args(hint)
+
+    # Optional[X] — unwrap, mark optional, recurse. Accept both spellings:
+    # typing.Optional/Union[X, None] (origin is typing.Union) and the PEP 604
+    # form X | None (origin is types.UnionType on 3.10+).
+    if origin in (Union, types.UnionType) and type(None) in args:
+        non_none = [a for a in args if a is not type(None)]
+        if len(non_none) == 1:
+            inner_kind, _, choices = _classify_type(non_none[0])
+            return inner_kind, True, choices
+        # Multi-type Optional union: give up, passthrough.
+        return "passthrough", True, None
+
+    # Literal[...] — enum.
+    if origin is typing.Literal:
+        return "enum", False, [str(a) for a in args]
+
+    # Plain scalars.
+    if hint is float:
+        return "float", False, None
+    if hint is Sides:
+        return "sides", False, None
+    if hint is bool:
+        return "bool", False, None
+    if hint is int:
+        return "int", False, None
+    if hint is str:
+        return "str", False, None
+
+    # Everything else (list[dict], nested dataclass, forward ref, ...).
+    return "passthrough", False, None
+
+
+def _field_default(f: dataclasses.Field) -> Any:
+    if f.default is not dataclasses.MISSING:
+        if isinstance(f.default, Sides):
+            # As a spec would hold it: the number while the sides are equal.
+            return as_written(f.default)
+        return f.default
+    if f.default_factory is not dataclasses.MISSING:
+        try:
+            return f.default_factory()
+        except Exception:
+            return None
+    return None
+
+
+def _titleize(name: str) -> str:
+    return name.replace("_", " ").strip().capitalize()