dynamic-systems-testing-framework 1.0.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (185) hide show
  1. dstf/__init__.py +41 -0
  2. dstf/__main__.py +10 -0
  3. dstf/authoring/__init__.py +13 -0
  4. dstf/authoring/builtin/__init__.py +32 -0
  5. dstf/authoring/builtin/accept.py +205 -0
  6. dstf/authoring/builtin/edit_criteria.py +479 -0
  7. dstf/authoring/builtin/edit_family.py +228 -0
  8. dstf/authoring/builtin/edit_on_plot.py +1354 -0
  9. dstf/authoring/builtin/edit_tree.py +1399 -0
  10. dstf/authoring/builtin/enqueue.py +144 -0
  11. dstf/authoring/capabilities.py +463 -0
  12. dstf/authoring/errors.py +49 -0
  13. dstf/authoring/family_rows.py +526 -0
  14. dstf/authoring/forms.py +631 -0
  15. dstf/authoring/layout.py +104 -0
  16. dstf/authoring/live_rows.py +337 -0
  17. dstf/authoring/live_runs.py +385 -0
  18. dstf/authoring/needle.py +127 -0
  19. dstf/authoring/picker_rows.py +92 -0
  20. dstf/authoring/queue_rows.py +354 -0
  21. dstf/authoring/session.py +3016 -0
  22. dstf/authoring/shell/__init__.py +9 -0
  23. dstf/authoring/shell/actions.py +702 -0
  24. dstf/authoring/shell/app.py +53 -0
  25. dstf/authoring/shell/drawer_controls.py +190 -0
  26. dstf/authoring/shell/family_fold.py +468 -0
  27. dstf/authoring/shell/figure_hook.py +249 -0
  28. dstf/authoring/shell/grips.js +1000 -0
  29. dstf/authoring/shell/header.py +783 -0
  30. dstf/authoring/shell/live_view.py +93 -0
  31. dstf/authoring/shell/picker.py +267 -0
  32. dstf/authoring/shell/placing.py +122 -0
  33. dstf/authoring/shell/plot_card.py +1171 -0
  34. dstf/authoring/shell/queue_panel.py +433 -0
  35. dstf/authoring/shell/rail.py +593 -0
  36. dstf/authoring/shell/shell.py +528 -0
  37. dstf/authoring/shell/stats_card.py +501 -0
  38. dstf/authoring/shell/strip.py +204 -0
  39. dstf/authoring/shell/style.py +721 -0
  40. dstf/authoring/shell/suite.py +277 -0
  41. dstf/authoring/shell/suite_grid.js +414 -0
  42. dstf/authoring/shell/suite_grid.py +373 -0
  43. dstf/authoring/shell/tab_theme.py +110 -0
  44. dstf/authoring/shell/tabstate.py +146 -0
  45. dstf/authoring/shell/test_list.py +179 -0
  46. dstf/authoring/shell/test_page.py +1404 -0
  47. dstf/authoring/shell/tree_card.py +1171 -0
  48. dstf/authoring/shell/variables_drawer.py +589 -0
  49. dstf/authoring/shell/widgets.py +188 -0
  50. dstf/authoring/situation.py +287 -0
  51. dstf/authoring/spec_document.py +215 -0
  52. dstf/authoring/spec_preview.py +83 -0
  53. dstf/authoring/suite_rows.py +99 -0
  54. dstf/authoring/tree_move.py +516 -0
  55. dstf/authoring/tree_pointer.py +33 -0
  56. dstf/authoring/tree_text.py +61 -0
  57. dstf/authoring/typed_number.js +64 -0
  58. dstf/authoring/typed_number.py +76 -0
  59. dstf/authoring/unscorable.py +141 -0
  60. dstf/authoring/variable_rows.py +216 -0
  61. dstf/backends/__init__.py +283 -0
  62. dstf/backends/base.py +1117 -0
  63. dstf/backends/common/__init__.py +1 -0
  64. dstf/backends/common/json_driver.py +484 -0
  65. dstf/backends/common/live_mat.py +369 -0
  66. dstf/backends/common/mat_reader.py +632 -0
  67. dstf/backends/common/proc_kill.py +238 -0
  68. dstf/backends/common/proc_output.py +20 -0
  69. dstf/backends/common/result_files.py +109 -0
  70. dstf/backends/common/result_variables.py +329 -0
  71. dstf/backends/common/statistics.py +39 -0
  72. dstf/backends/cross_backend.py +193 -0
  73. dstf/backends/dymola/__init__.py +5 -0
  74. dstf/backends/dymola/backend.py +776 -0
  75. dstf/backends/dymola/interface_loader.py +241 -0
  76. dstf/backends/dymola/log_parser.py +396 -0
  77. dstf/backends/dymola/persistent_backend.py +949 -0
  78. dstf/backends/fmpy/__init__.py +11 -0
  79. dstf/backends/fmpy/backend.py +443 -0
  80. dstf/backends/julia/__init__.py +27 -0
  81. dstf/backends/julia/backend.py +156 -0
  82. dstf/backends/julia/persistent_backend.py +541 -0
  83. dstf/backends/julia/run_persistent.jl +167 -0
  84. dstf/backends/julia/run_test.jl +146 -0
  85. dstf/backends/openmodelica/__init__.py +21 -0
  86. dstf/backends/openmodelica/backend.py +368 -0
  87. dstf/backends/openmodelica/init_xml.py +76 -0
  88. dstf/backends/openmodelica/log_parser.py +119 -0
  89. dstf/backends/openmodelica/mos_generator.py +228 -0
  90. dstf/backends/openmodelica/persistent_backend.py +1045 -0
  91. dstf/backends/openmodelica/session_loader.py +63 -0
  92. dstf/backends/persistent.py +1811 -0
  93. dstf/backends/pool.py +602 -0
  94. dstf/backends/progress.py +395 -0
  95. dstf/backends/python/__init__.py +11 -0
  96. dstf/backends/python/backend.py +141 -0
  97. dstf/backends/python/run_test.py +141 -0
  98. dstf/backends/run_metadata.py +103 -0
  99. dstf/backends/types.py +310 -0
  100. dstf/cli/__init__.py +12 -0
  101. dstf/cli/author_cmd.py +61 -0
  102. dstf/cli/baseline_cmds.py +341 -0
  103. dstf/cli/common.py +231 -0
  104. dstf/cli/diag_cmds.py +350 -0
  105. dstf/cli/main.py +561 -0
  106. dstf/cli/review.py +398 -0
  107. dstf/cli/run_cmds.py +589 -0
  108. dstf/cli/spec_cmds.py +123 -0
  109. dstf/comparison/__init__.py +13 -0
  110. dstf/comparison/algorithms.py +2127 -0
  111. dstf/comparison/builtins.py +291 -0
  112. dstf/comparison/comparator.py +1189 -0
  113. dstf/comparison/geometry.py +164 -0
  114. dstf/comparison/handles.py +1194 -0
  115. dstf/comparison/metric_tree.py +332 -0
  116. dstf/comparison/mode_schema.py +176 -0
  117. dstf/comparison/modes.py +788 -0
  118. dstf/comparison/registry.py +230 -0
  119. dstf/comparison/seeds.py +281 -0
  120. dstf/comparison/sides.py +158 -0
  121. dstf/comparison/snap.py +62 -0
  122. dstf/comparison/strict_keys.py +246 -0
  123. dstf/comparison/summaries.py +354 -0
  124. dstf/comparison/tree_eval.py +868 -0
  125. dstf/comparison/tree_spec.py +678 -0
  126. dstf/comparison/types.py +194 -0
  127. dstf/comparison/validator.py +232 -0
  128. dstf/config.py +815 -0
  129. dstf/discovery/__init__.py +0 -0
  130. dstf/discovery/json_recognizer.py +702 -0
  131. dstf/discovery/library_state.py +248 -0
  132. dstf/discovery/mo_parser.py +794 -0
  133. dstf/discovery/patch_apply.py +308 -0
  134. dstf/discovery/patch_envelope.py +63 -0
  135. dstf/discovery/plots.py +318 -0
  136. dstf/discovery/recognizer.py +103 -0
  137. dstf/discovery/scan_cache.py +154 -0
  138. dstf/discovery/spec_parser.py +538 -0
  139. dstf/discovery/spec_shape.py +122 -0
  140. dstf/discovery/spec_update.py +493 -0
  141. dstf/discovery/test_registry.py +488 -0
  142. dstf/discovery/unit_tests.py +95 -0
  143. dstf/figures/__init__.py +14 -0
  144. dstf/figures/assemble.py +48 -0
  145. dstf/figures/decimate.py +150 -0
  146. dstf/figures/families.py +289 -0
  147. dstf/figures/palette.py +170 -0
  148. dstf/figures/plotly.py +1413 -0
  149. dstf/figures/view.py +233 -0
  150. dstf/pipeline.py +435 -0
  151. dstf/plugins.py +104 -0
  152. dstf/reporting/__init__.py +0 -0
  153. dstf/reporting/console_report.py +187 -0
  154. dstf/reporting/context_builder.py +979 -0
  155. dstf/reporting/dashboard_render.py +534 -0
  156. dstf/reporting/junit_report.py +192 -0
  157. dstf/reporting/overlay_loader.py +708 -0
  158. dstf/reporting/report_suite.py +1059 -0
  159. dstf/reporting/rows.py +458 -0
  160. dstf/reporting/schema_export.py +413 -0
  161. dstf/reporting/stats_rows.py +321 -0
  162. dstf/reporting/status.py +388 -0
  163. dstf/reporting/templates/axes.js +61 -0
  164. dstf/reporting/templates/dashboard.html +284 -0
  165. dstf/reporting/templates/dashboard.js +604 -0
  166. dstf/reporting/templates/filters.js +100 -0
  167. dstf/reporting/templates/interactive.html +420 -0
  168. dstf/reporting/templates/report.js +116 -0
  169. dstf/reporting/templates/vendor/plotly.min.js +8 -0
  170. dstf/reporting/theme.py +324 -0
  171. dstf/reporting/tree_rows.py +73 -0
  172. dstf/reporting/verdicts.py +124 -0
  173. dstf/storage/__init__.py +0 -0
  174. dstf/storage/accept.py +216 -0
  175. dstf/storage/baseline.py +43 -0
  176. dstf/storage/batch_manifest.py +204 -0
  177. dstf/storage/ref_index.py +295 -0
  178. dstf/storage/reference_store.py +661 -0
  179. dstf/storage/roles.py +576 -0
  180. dstf/storage/run_stats.py +85 -0
  181. dynamic_systems_testing_framework-1.0.1.dist-info/METADATA +266 -0
  182. dynamic_systems_testing_framework-1.0.1.dist-info/RECORD +185 -0
  183. dynamic_systems_testing_framework-1.0.1.dist-info/WHEEL +4 -0
  184. dynamic_systems_testing_framework-1.0.1.dist-info/entry_points.txt +2 -0
  185. dynamic_systems_testing_framework-1.0.1.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,230 @@
1
+ """Registries for comparison metrics and MetricTree combinators.
2
+
3
+ The extension seam docs/extensibility.md promises for the Metric layer.
4
+ Before this module the metric and combinator name spaces were closed
5
+ if/elif chains duplicated across four files (``modes.resolve_mode``,
6
+ ``tree_spec.VALID_METRICS`` / ``_MODE_TO_METRIC``,
7
+ ``tree_eval._build_combinator`` / ``_METRIC_TO_MODE_KEY``,
8
+ ``reporting/schema_export._MODES``) — adding a metric meant editing all
9
+ of them. Now each of those derives from the descriptors registered here,
10
+ so a third-party metric only has to call :func:`register_metric`.
11
+
12
+ Builtins register in :mod:`dstf.comparison.builtins`, imported by the
13
+ package ``__init__`` — mirroring how ``backends`` self-registers its
14
+ backends.
15
+ """
16
+
17
+ from __future__ import annotations
18
+
19
+ from collections.abc import Callable
20
+ from dataclasses import dataclass
21
+ from typing import TYPE_CHECKING, Literal
22
+
23
+ from ..backends.types import DatasetType
24
+ from .sides import Sides
25
+ from .summaries import summarize_default
26
+ from .types import ScoreSummary, VariableComparison
27
+
28
+ if TYPE_CHECKING:
29
+ from .handles import HandleEdit, RowPlace
30
+ from .metric_tree import Combinator
31
+ from .modes import ComparisonMode
32
+ from .seeds import SeedData
33
+ from .tree_spec import CombinatorSpec
34
+
35
+ #: The metric a leaf/override falls back to when none is named. Also the
36
+ #: one metric whose name is *absent* from the flat override dict (there is
37
+ #: no ``{"mode": "nrmse"}`` in stored specs — no key means NRMSE).
38
+ DEFAULT_METRIC = "nrmse"
39
+
40
+ #: The metric a flat override with no ``mode`` gets under ``--default-points``
41
+ #: instead (``modes.resolve_mode``).
42
+ DEFAULT_POINTS_METRIC = "points"
43
+
44
+
45
+ def unnamed_metrics() -> tuple[str, str]:
46
+ """The metrics an override that names no ``mode`` may be scored by: the
47
+ default, or ``points`` under ``--default-points``. What such an override
48
+ may carry is what either reads."""
49
+ return (DEFAULT_METRIC, DEFAULT_POINTS_METRIC)
50
+
51
+
52
+ #: Default ``accepted_datasets`` — every backend produces time series today.
53
+ TIME_SERIES_ONLY = frozenset({DatasetType.TIME_SERIES})
54
+
55
+
56
+ @dataclass(frozen=True)
57
+ class MetricDescriptor:
58
+ """Everything the framework needs to know about one leaf metric.
59
+
60
+ ``build`` takes the flat override dict (the ``variable_overrides``
61
+ payload shape, also used for a :class:`~.tree_spec.LeafSpec`'s params)
62
+ plus the resolved tolerance, and returns a ready-to-score mode.
63
+
64
+ ``score_direction`` says which way a better score runs: ``"lower"``
65
+ for error-like scores (NRMSE, point violations) and ``"higher"`` for
66
+ coverage-like ones (tube's in-band fraction). ``accepted_datasets``
67
+ declares which :class:`~dstf.backends.types.DatasetType` kinds the
68
+ metric can score — a leaf handed anything else hard-fails in
69
+ :func:`~.tree_eval._evaluate_leaf`.
70
+
71
+ ``accepts_quantity`` says the metric's ``quantity`` param names the
72
+ data it scores. For such a metric — and ONLY for such a metric — a
73
+ string ``quantity`` makes the evaluator resolve the value from the
74
+ run's statistics / diagnostics instead of a tracked trajectory, and
75
+ lets :func:`~.tree_spec._parse_leaf` default ``variable`` to the path.
76
+ Off by default: a metric that scores trajectories must keep scoring
77
+ trajectories, so an unrecognized ``quantity`` stays an inert param
78
+ rather than silently swapping that metric's data source.
79
+
80
+ ``summarize`` turns the mode's ``VariableComparison`` into the pill
81
+ text and criterion sentence; see :mod:`.summaries`.
82
+
83
+ ``edit_handle`` is the inverse of the mode's ``kind="handle"``
84
+ primitives: the raw leaf, the handle's local id and its dragged position
85
+ in, the new raw leaf out, :class:`~.handles.HandleEditError` for a drag
86
+ the leaf cannot take; see :mod:`.handles`. ``None`` for a metric no drag
87
+ edits (the builtins ``nrmse`` and ``event-timing``).
88
+
89
+ ``place_row`` is what a Ctrl+click on the plot adds to the metric's
90
+ table: the leaf with one more row, settled on the data it scores
91
+ (:class:`~.handles.RowPlace`); ``None`` for a metric with no rows.
92
+ ``row_noun`` says what that row is called for a leaf — "a point" — or
93
+ ``None`` where a click adds nothing to THAT leaf (a tube of constant
94
+ width); the editor's hint reads it.
95
+
96
+ ``seed`` gives a new leaf its first values from the data it will score
97
+ (the reference / actual series in, the params to add out), so a leaf
98
+ added with only a metric and a variable is valid and sensible; ``None``
99
+ for a metric valid bare (``nrmse``, ``points``, ``event-timing``). See
100
+ :mod:`.seeds`.
101
+
102
+ ``drawn_by_default`` says whether the Test page draws a closed criterion
103
+ of this metric that is not failing before its swatch is touched
104
+ (``range`` and ``scalar`` do not); a failing one draws regardless.
105
+ """
106
+
107
+ name: str
108
+ config_cls: type
109
+ build: Callable[[dict, Sides], ComparisonMode]
110
+ accepted_datasets: frozenset[DatasetType] = TIME_SERIES_ONLY
111
+ score_direction: Literal["lower", "higher"] = "lower"
112
+ accepts_quantity: bool = False
113
+ #: One leaf's verdict in words. The default is NRMSE-shaped; every
114
+ #: builtin supplies its own in :mod:`.builtins`.
115
+ summarize: Callable[[VariableComparison], ScoreSummary] = summarize_default
116
+ #: A drag on one of the mode's handles, as the new raw leaf.
117
+ edit_handle: Callable[[HandleEdit], dict] | None = None
118
+ #: Whether a closed criterion of this metric that is not failing draws on
119
+ #: the Test page until its swatch says otherwise (legend b). Off for
120
+ #: metrics whose drawing is a bound across the whole plot — range, scalar
121
+ #: — which would otherwise cover every plot they sit on. The open
122
+ #: criterion always draws, and a failing one draws by default.
123
+ drawn_by_default: bool = True
124
+ #: A new leaf's first values from the data it will score.
125
+ seed: Callable[[SeedData], dict] | None = None
126
+ #: A Ctrl+click on the plot, as the leaf with one more row.
127
+ place_row: Callable[[RowPlace], dict] | None = None
128
+ #: What that row is called for a leaf, or None where a click adds none.
129
+ row_noun: Callable[[dict], str | None] | None = None
130
+
131
+
132
+ @dataclass(frozen=True)
133
+ class CombinatorDescriptor:
134
+ """One MetricTree combinator: a name plus a spec → instance builder.
135
+
136
+ ``build`` reads whatever combinator-specific fields it needs off the
137
+ :class:`~.tree_spec.CombinatorSpec` (``k``, ``weights``, ...). The
138
+ *shape* validation of those fields stays in ``tree_spec`` at parse
139
+ time — this is dispatch only.
140
+ """
141
+
142
+ name: str
143
+ build: Callable[[CombinatorSpec], Combinator]
144
+
145
+
146
+ # Insertion-ordered: registration order is the canonical presentation order
147
+ # for the schema export, the reporter's mode-schema map, and the "valid: ..."
148
+ # list in resolve_mode's error.
149
+ _METRICS: dict[str, MetricDescriptor] = {}
150
+ _COMBINATORS: dict[str, CombinatorDescriptor] = {}
151
+
152
+
153
+ def register_metric(desc: MetricDescriptor) -> MetricDescriptor:
154
+ """Register a leaf metric. Raises on a duplicate name.
155
+
156
+ Duplicates are loud rather than last-writer-wins: two metrics sharing
157
+ a name would make spec parsing depend on import order.
158
+ """
159
+ if desc.name in _METRICS:
160
+ raise ValueError(f"metric {desc.name!r} is already registered")
161
+ _METRICS[desc.name] = desc
162
+ return desc
163
+
164
+
165
+ def register_combinator(desc: CombinatorDescriptor) -> CombinatorDescriptor:
166
+ """Register a MetricTree combinator. Raises on a duplicate name."""
167
+ if desc.name in _COMBINATORS:
168
+ raise ValueError(f"combinator {desc.name!r} is already registered")
169
+ _COMBINATORS[desc.name] = desc
170
+ return desc
171
+
172
+
173
+ def metric(name: str) -> MetricDescriptor | None:
174
+ """Return the descriptor for ``name``, or None if not registered."""
175
+ return _METRICS.get(name)
176
+
177
+
178
+ def combinator(name: str) -> CombinatorDescriptor | None:
179
+ """Return the descriptor for ``name``, or None if not registered."""
180
+ return _COMBINATORS.get(name)
181
+
182
+
183
+ def metric_names() -> frozenset[str]:
184
+ return frozenset(_METRICS)
185
+
186
+
187
+ def combinator_names() -> frozenset[str]:
188
+ return frozenset(_COMBINATORS)
189
+
190
+
191
+ def metrics() -> tuple[MetricDescriptor, ...]:
192
+ """All metric descriptors in registration order."""
193
+ return tuple(_METRICS.values())
194
+
195
+
196
+ def summarize(vc: VariableComparison) -> ScoreSummary:
197
+ """The pill text and criterion for one leaf, by its mode's descriptor.
198
+
199
+ Any sentinel carrying a reason — a leaf config that could not build, an
200
+ empty-data failure — shows that reason instead of a numeric score there
201
+ is none of. Mode-agnostic on purpose: the flat variable_overrides path
202
+ emits the same shape. Uses ``vc.mode`` raw: the unknown-mode sentinel
203
+ has no mode, and "nrmse leaf" would name the wrong metric.
204
+ """
205
+ error = (vc.diagnostics or {}).get("error")
206
+ if error:
207
+ label = f"{vc.mode} leaf" if vc.mode else "leaf"
208
+ return ScoreSummary(
209
+ display="not scored", criterion=f"{label} not scored — {error}"
210
+ )
211
+ # A mode nothing is registered under — none at all, or a plug-in's that
212
+ # is not loaded — reads as the default summary does.
213
+ desc = metric(vc.mode)
214
+ fn = desc.summarize if desc is not None else summarize_default
215
+ return fn(vc)
216
+
217
+
218
+ def combinators() -> tuple[CombinatorDescriptor, ...]:
219
+ """All combinator descriptors in registration order."""
220
+ return tuple(_COMBINATORS.values())
221
+
222
+
223
+ def _reset_for_tests() -> None:
224
+ """Drop user registrations and restore the builtins — unit tests only."""
225
+ _METRICS.clear()
226
+ _COMBINATORS.clear()
227
+
228
+ from .builtins import register_builtins
229
+
230
+ register_builtins()
@@ -0,0 +1,281 @@
1
+ """A new criterion's first values, from the data it will score.
2
+
3
+ A leaf of `range`, `tube`, `scalar` or `dominant-frequency` with only a
4
+ `metric` and a `variable` is misconfigured or always fails: `range` wants a
5
+ bound, `tube` a width mode, `scalar` a quantity, and `dominant-frequency`
6
+ scores but FAILs "no peaks declared". So the tool seeds one when it is
7
+ added or its metric is switched: a pure function of the reference series
8
+ (else the actual one), cut to the criterion's window when it has one, per
9
+ metric, registered on its :attr:`~.registry.MetricDescriptor.seed`.
10
+
11
+ A seed lands in a spec file a person reads, so its values are short — but
12
+ rounded AWAY from the data, at the leading digit of the margin it leaves:
13
+ rounded to 6 significant digits, a range over 101325 +- 0.001 got
14
+ ``min_value == max_value`` and failed on the data it was seeded from. Where
15
+ the short form would land back on the data, the value is left unrounded.
16
+ """
17
+
18
+ from __future__ import annotations
19
+
20
+ import math
21
+ from dataclasses import dataclass
22
+ from typing import Literal
23
+
24
+ import numpy as np
25
+
26
+ from .algorithms import _compute_fft_spectrum, _find_top_n_peaks
27
+ from .registry import metric as metric_descriptor
28
+
29
+ #: The margin a seed leaves around the data, as a fraction of its span.
30
+ SEED_MARGIN = 0.05
31
+ #: ... and never less than this fraction of the data's magnitude: a reference
32
+ #: stored at 7 significant digits is rounded coarser than 5% of a small span.
33
+ SEED_MARGIN_FLOOR = 1e-6
34
+ #: The widest relative window a peak is seeded with. Two bins around a peak
35
+ #: at FFT bin 1 is +-200%, a window that matches anything; a peak whose two
36
+ #: bins are wider (one in the lowest three bins) is not seeded at all.
37
+ MAX_PEAK_TOLERANCE = 0.5
38
+
39
+
40
+ @dataclass(frozen=True)
41
+ class SeedData:
42
+ """The ``(time, values)`` a new criterion will score, either may be
43
+ missing."""
44
+
45
+ ref: tuple[np.ndarray, np.ndarray] | None
46
+ act: tuple[np.ndarray, np.ndarray] | None
47
+
48
+ def within(self, window: dict | None) -> SeedData:
49
+ """Both series cut to ``window``'s ``start`` / ``end`` (either may
50
+ be absent), ends included as the scorer's window includes them: a
51
+ windowed criterion is seeded from the samples it scores."""
52
+ if not window:
53
+ return self
54
+ # float(), as `parse_metric_tree` reads them: a hand-written
55
+ # ``"start": "2"`` parses, and compared raw raised UFuncTypeError.
56
+ start = float(window.get("start", -math.inf))
57
+ end = float(window.get("end", math.inf))
58
+
59
+ def cut(s: tuple[np.ndarray, np.ndarray] | None):
60
+ if s is None:
61
+ return None
62
+ keep = (s[0] >= start) & (s[0] <= end)
63
+ return s[0][keep], s[1][keep]
64
+
65
+ return SeedData(ref=cut(self.ref), act=cut(self.act))
66
+
67
+ def series(self) -> tuple[np.ndarray, np.ndarray] | None:
68
+ """The reference, else the actual; None when neither has a finite value."""
69
+ for s in (self.ref, self.act):
70
+ if s is not None and np.isfinite(s[1]).any():
71
+ return s
72
+ return None
73
+
74
+
75
+ def _short(x: float) -> float:
76
+ return float(f"{float(x):.6g}")
77
+
78
+
79
+ def _outward(x: float, margin: float, up: bool) -> float:
80
+ """``x`` rounded away from the data at the margin's leading digit, when
81
+ that short form still lies on ``x``'s far side; else ``x`` itself.
82
+ Readable where it can be, and never inside the data: 101325 +- 0.001
83
+ kept a 6-digit rounding's min == max, and a margin below 12 significant
84
+ digits of the value (1e5 +- 1e-11) rounded the 12-digit form back onto
85
+ the data."""
86
+ q = 10.0 ** math.floor(math.log10(margin))
87
+ n = math.ceil(x / q) if up else math.floor(x / q)
88
+ short = float(f"{n * q:.12g}")
89
+ return short if (short >= x if up else short <= x) else x
90
+
91
+
92
+ def _span(v: np.ndarray) -> tuple[float, float, float]:
93
+ """The finite data's min, max and span; a flat series' span is its
94
+ magnitude (else 1), so a margin is never zero."""
95
+ finite = v[np.isfinite(v)]
96
+ lo, hi = float(finite.min()), float(finite.max())
97
+ span = hi - lo
98
+ if span <= 0.0:
99
+ span = abs(hi) or 1.0
100
+ return lo, hi, span
101
+
102
+
103
+ def _margin(v: np.ndarray) -> float:
104
+ """5% of the finite data's span, and at least `SEED_MARGIN_FLOOR` of its
105
+ magnitude."""
106
+ lo, hi, span = _span(v)
107
+ # OilWater_PTHX's `STHX.tube.mediums[2].h`: seeded from a 7-digit
108
+ # reference as [130450.58, 130451.02], its float32 result reached 130451.03125.
109
+ return max(SEED_MARGIN * span, SEED_MARGIN_FLOOR * max(abs(lo), abs(hi)))
110
+
111
+
112
+ def seed_range(data: SeedData) -> dict:
113
+ """Both bounds, the data's span plus `_margin` either side."""
114
+ s = data.series()
115
+ if s is None:
116
+ return {}
117
+ lo, hi, _ = _span(s[1])
118
+ m = _margin(s[1])
119
+ return {
120
+ "min_value": _outward(lo - m, m, up=False),
121
+ "max_value": _outward(hi + m, m, up=True),
122
+ }
123
+
124
+
125
+ def seed_tube(data: SeedData) -> dict:
126
+ """A band `_margin` wide around the reference."""
127
+ s = data.series()
128
+ if s is None:
129
+ return {}
130
+ m = _margin(s[1])
131
+ return {"tube_width_mode": "band", "tube_abs": _outward(m, m, up=True)}
132
+
133
+
134
+ def seed_scalar(data: SeedData) -> dict:
135
+ """The final value: within 1% of the baseline's where the seed reads
136
+ the reference, else between bounds 5% of its magnitude either side."""
137
+ s = data.series()
138
+ if s is None:
139
+ return {}
140
+ quantity = {"reduce": "final"}
141
+ # Not `data.ref is not None`: a reference with no finite sample (all
142
+ # NaN, or none inside the window) seeds from the actual, and a
143
+ # tolerance against it failed on its own data.
144
+ if s is data.ref:
145
+ return {"quantity": quantity, "tolerance_mode": "rel", "tolerance": 0.01}
146
+ final = float(s[1][np.isfinite(s[1])][-1])
147
+ half = SEED_MARGIN * (abs(final) or _span(s[1])[2])
148
+ return {
149
+ "quantity": quantity,
150
+ "min_value": _outward(final - half, half, up=False),
151
+ "max_value": _outward(final + half, half, up=True),
152
+ }
153
+
154
+
155
+ def seed_dominant_frequency(data: SeedData) -> dict:
156
+ """The strongest peak of the data's spectrum, 5% wide or two bins,
157
+ whichever is wider. None when two bins are wider than
158
+ `MAX_PEAK_TOLERANCE` — a peak in the lowest three bins is a trend's
159
+ leakage at ~1/span, not a frequency: over TRANSFORM, 279 of 322 varying
160
+ variables have their strongest peak there. The leaf stays bare, and
161
+ scoring says to declare a peak."""
162
+ s = data.series()
163
+ if s is None:
164
+ return {}
165
+ freqs, spectrum = _compute_fft_spectrum(*s)
166
+ peaks = _find_top_n_peaks(freqs, spectrum, 1)
167
+ if not peaks:
168
+ return {}
169
+ f = peaks[0][0]
170
+ bin_width = float(freqs[1] - freqs[0])
171
+ # At least two bins wide: the bins follow the SPAN, so a peak seeded on
172
+ # a 10 s reference sits between the bins of a 9 s result, and matching
173
+ # needs a local maximum inside the window.
174
+ two_bins = 2.0 * bin_width / f
175
+ if two_bins > MAX_PEAK_TOLERANCE:
176
+ return {}
177
+ tol = max(SEED_MARGIN, two_bins)
178
+ return {
179
+ "peaks": [
180
+ {"freq": _short(f), "tolerance": _short(tol), "tolerance_mode": "rel"}
181
+ ]
182
+ }
183
+
184
+
185
+ def seeded_leaf(metric: str, variable: str, data: SeedData) -> dict:
186
+ """``{"metric", "variable"}`` plus the metric's seed, if it has one."""
187
+ leaf = {"metric": metric, "variable": variable}
188
+ desc = metric_descriptor(metric)
189
+ if desc is None or desc.seed is None:
190
+ return leaf
191
+ return {**leaf, **desc.seed(data)}
192
+
193
+
194
+ #: Diagnostics whose kind is known by name (K1): a count is scored exact, a
195
+ #: time within 10%. A tiny model's CPUtime can end on a whole 0.0 or 1.0:
196
+ #: read by its values alone it is a count, and a next run's 0.01 s fails.
197
+ COUNTS = frozenset({"EventCounter"})
198
+ TIMES = frozenset({"CPUtime"})
199
+
200
+
201
+ def quantity_kind(
202
+ path: str, values: np.ndarray | None = None
203
+ ) -> Literal["count", "time"]:
204
+ """What kind of number ``path`` names (S5): a ``count`` is scored exact,
205
+ a ``time`` within 10%. A statistic is a time by its path — the
206
+ ``timing`` section, CPUtime, and the ``*_time`` / ``*_wall`` /
207
+ ``cpu_time*`` keys; every other number a backend records is a count,
208
+ whatever its type (a Dymola ``result_points`` can be written as a
209
+ float).
210
+
211
+ A diagnostic's summary (``CPUtime.final``) takes K1's rule: by name
212
+ when known, else a count when every finite sample of ``values`` is a
213
+ whole number. A top-level statistic given the run's samples of the
214
+ diagnostic it names (``CPUtime``, folded in from the diagnostics) is
215
+ that diagnostic's final and takes the same rule — the Statistics row
216
+ and the add form's ``X.final`` leaf are one kind."""
217
+ name, _, field = path.rpartition(".")
218
+ if not name and values is not None:
219
+ name, field = path, "final"
220
+ if name and "." not in name and field in ("final", "min", "max"):
221
+ if name in COUNTS:
222
+ return "count"
223
+ if name in TIMES:
224
+ return "time"
225
+ finite = values[np.isfinite(values)] if values is not None else np.array([])
226
+ whole = finite.size > 0 and bool(np.all(finite == np.round(finite)))
227
+ return "count" if whole else "time"
228
+ head = path.split(".", 1)[0]
229
+ last = path.rsplit(".", 1)[-1]
230
+ if (
231
+ head == "timing"
232
+ or last in TIMES
233
+ or last.endswith(("_time", "_wall"))
234
+ or last.startswith("cpu_time")
235
+ ):
236
+ return "time"
237
+ return "count"
238
+
239
+
240
+ def seed_quantity(
241
+ path: str,
242
+ kind: Literal["count", "time"],
243
+ *,
244
+ into: str,
245
+ variable: str | None = None,
246
+ ) -> dict:
247
+ """A ``scalar`` criterion on the stored number ``path`` names:
248
+ ``tolerance 0 abs`` for a count and ``0.1 rel`` for a time. With
249
+ ``variable`` (a diagnostic's name) the leaf names it: parsed without
250
+ one, its variable becomes the quantity text, and its row reads
251
+ ``scalar · CPUtime.final``.
252
+
253
+ A time is seeded inside a ``warn`` when it lands in an ``and``
254
+ (``into``, the kind of the combinator it is added to): a run's CPU time
255
+ moves with the machine's load — under ``--parallel`` most of all — and
256
+ 10% would fail a test whose model did not change. Anywhere else it is
257
+ bare: a ``warn`` child passes whatever it scores, so under an ``or`` or
258
+ a ``k-of-n`` it would pass its parent with nothing else passing. A
259
+ count stays bare: it only moves when the model does."""
260
+ count = kind == "count"
261
+ leaf: dict = {"metric": "scalar"}
262
+ if variable is not None:
263
+ leaf["variable"] = variable
264
+ leaf.update(
265
+ {
266
+ "quantity": path,
267
+ "tolerance": 0 if count else 0.1,
268
+ "tolerance_mode": "abs" if count else "rel",
269
+ }
270
+ )
271
+ if not count and into == "and":
272
+ return {"combinator": "warn", "children": [leaf]}
273
+ return leaf
274
+
275
+
276
+ def seed_diagnostic(name: str, values: np.ndarray | None, *, into: str) -> dict:
277
+ """A criterion on the diagnostic ``name`` (the plan's K1): its stored
278
+ final through a quantity path, seeded by :func:`seed_quantity` at the
279
+ kind :func:`quantity_kind` reads from ``values``."""
280
+ path = f"{name}.final"
281
+ return seed_quantity(path, quantity_kind(path, values), into=into, variable=name)
@@ -0,0 +1,158 @@
1
+ """A tolerance's two sides — the ONE place a number-or-pair is read or written.
2
+
3
+ A spec holds a tolerance as a number (both sides equal) or as the pair
4
+ ``{"lower": a, "upper": b}``. Every scorer takes :class:`Sides`; a number
5
+ ``v`` is ``Sides(v, v)``, so a spec holding only numbers scores as it did
6
+ before a pair existed. What the sides MEAN is the scorer's: ``actual −
7
+ target`` must lie in ``[−lower, upper]``; in time, ``lower`` is how much
8
+ earlier and ``upper`` how much later.
9
+ """
10
+
11
+ from __future__ import annotations
12
+
13
+ import json
14
+ import math
15
+ import numbers
16
+ from dataclasses import dataclass
17
+ from typing import TypeGuard
18
+
19
+
20
+ @dataclass(frozen=True)
21
+ class Sides:
22
+ lower: float
23
+ upper: float
24
+
25
+ @property
26
+ def equal(self) -> bool:
27
+ return self.lower == self.upper
28
+
29
+
30
+ def _is_number(value: object) -> TypeGuard[int | float]:
31
+ """A finite real number that is not a bool: stricter than
32
+ ``float(value)``, which also takes ``True`` and the string ``"0.01"``,
33
+ and no stricter on a number — a numpy scalar is one. An int too large
34
+ for a float is not: it has no finite float."""
35
+ if isinstance(value, bool) or not isinstance(value, numbers.Real):
36
+ return False
37
+ try:
38
+ return math.isfinite(float(value))
39
+ except OverflowError:
40
+ return False
41
+
42
+
43
+ def is_base_tolerance(value: object) -> bool:
44
+ """Whether ``value`` can be a test's BASE tolerance (the spec entry's or
45
+ the reference JSON's ``comparison.tolerance``, the project's): ONE finite
46
+ number. Not a pair — nrmse, the default metric, has one ceiling — and
47
+ not a bool, a string or a list, which ``float()`` took or raised on;
48
+ not NaN or an infinity, which every leaf then refused in its own words."""
49
+ return _is_number(value)
50
+
51
+
52
+ def _as_json(value: object) -> str:
53
+ """``value`` in a refusal, cut to 60 characters: as the JSON file holds
54
+ it (``true``, ``NaN``, ``{"lower": …}``), not as Python spells it; its
55
+ ``repr`` only where JSON cannot write it."""
56
+ try:
57
+ text = json.dumps(value, allow_nan=True, ensure_ascii=False)
58
+ except (TypeError, ValueError):
59
+ text = repr(value)
60
+ if len(text) > 60:
61
+ text = text[:60] + "…"
62
+ return text
63
+
64
+
65
+ def base_tolerance_refused(where: str, value: object) -> str:
66
+ """What a base tolerance that is not :func:`is_base_tolerance` is refused
67
+ with — a test's comparison's ``error_message``, a project's config
68
+ error: the rule, then where the value is written (``where``, in words)
69
+ and what it is (:func:`_as_json`). Where a pair belongs is said to a
70
+ dict alone: a string or a list was no attempt at one."""
71
+ rule = "the base tolerance must be a number"
72
+ if isinstance(value, dict):
73
+ rule += "; a lower/upper pair belongs on a criterion"
74
+ return f"{rule} ({where} is {_as_json(value)})"
75
+
76
+
77
+ #: The keys of a pair, and of nothing else: what :func:`parse_sides` reads.
78
+ PAIR_KEYS = ("lower", "upper")
79
+
80
+
81
+ def parse_sides(value: object, what: str) -> Sides:
82
+ """``value`` as :class:`Sides`: a number is both sides, a dict whose keys
83
+ are exactly ``lower`` and ``upper`` is the pair, a :class:`Sides` is
84
+ itself. Anything else is a ``ValueError`` naming ``what`` and the value
85
+ (:func:`_as_json`).
86
+ No sign check: a negative side is not refused here, as a negative number
87
+ never was."""
88
+ if isinstance(value, Sides):
89
+ return value
90
+ if _is_number(value):
91
+ return Sides(float(value), float(value))
92
+ if isinstance(value, dict) and set(value) == set(PAIR_KEYS):
93
+ lower, upper = value["lower"], value["upper"]
94
+ if _is_number(lower) and _is_number(upper):
95
+ return Sides(float(lower), float(upper))
96
+ raise ValueError(
97
+ f'{what} must be a number or {{"lower": …, "upper": …}}, got {_as_json(value)}'
98
+ )
99
+
100
+
101
+ def parse_number(value: object, what: str) -> float:
102
+ """``value`` as the ONE number a field without sides takes — a tube's
103
+ minimum width, a floor under both of its sides. Anything else — a
104
+ lower/upper pair, a string, a bool, a list, NaN or an infinity — is a
105
+ ``ValueError`` naming ``what`` and the value (:func:`_as_json`), the
106
+ way :func:`parse_sides` refuses what is no number or pair. No sign
107
+ check here either."""
108
+ if _is_number(value):
109
+ return float(value)
110
+ raise ValueError(f"{what} must be one number, got {_as_json(value)}")
111
+
112
+
113
+ def as_written(sides: Sides) -> float | dict:
114
+ """``sides`` as data leaving Python (a tree view, a leaf's diagnostics):
115
+ the number when the sides are equal, else the pair. NOT how the tool
116
+ writes a spec — there the shape is the author's."""
117
+ if sides.equal:
118
+ return sides.upper
119
+ return {"lower": sides.lower, "upper": sides.upper}
120
+
121
+
122
+ def sides_text(sides: Sides, fmt: str = ".3e") -> str:
123
+ """``sides`` in a sentence: the one number when equal (a minus as
124
+ U+2212; a zero reads as zero whatever its sign bit), else
125
+ :func:`signed_sides`."""
126
+ if sides.equal:
127
+ text = format(abs(sides.upper) if sides.upper == 0 else sides.upper, fmt)
128
+ return f"−{text[1:]}" if text.startswith("-") else text
129
+ return signed_sides(sides, fmt)
130
+
131
+
132
+ def signed_sides(
133
+ sides: Sides, fmt: str = ".3e", *, scale: float = 1.0, unit: str = ""
134
+ ) -> str:
135
+ """``−lower / +upper`` (U+2212, as the report's other signs) — the
136
+ band's two edges as signed numbers, each ``scale``d and followed by
137
+ ``unit`` (``100`` and ``%`` for a relative pair), so a negative lower
138
+ side, whose edge lies above the target, reads ``+…``. A zero side reads
139
+ by its side, not its sign bit: ``-0.0`` is ``0.0``."""
140
+ lower = _signed(-sides.lower * scale, fmt, zero="−")
141
+ upper = _signed(sides.upper * scale, fmt, zero="+")
142
+ return f"{lower}{unit} / {upper}{unit}"
143
+
144
+
145
+ def signed(value: float, fmt: str = ".3e") -> str:
146
+ """A deviation in a sentence: ``value`` with its sign always — ``+``, or
147
+ U+2212 for the minus, as :func:`signed_sides` writes an edge; a zero
148
+ reads ``+``."""
149
+ return _signed(value, fmt, zero="+")
150
+
151
+
152
+ def _signed(value: float, fmt: str, *, zero: str) -> str:
153
+ """``value`` with its sign, the minus as U+2212; a zero takes ``zero``.
154
+ The sign only: an exponent's ``-`` stays as the format wrote it."""
155
+ text = format(abs(value) if value == 0 else value, f"+{fmt}")
156
+ if value == 0:
157
+ return zero + text[1:]
158
+ return f"−{text[1:]}" if text.startswith("-") else text