dynamic-systems-testing-framework 1.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dstf/__init__.py +41 -0
- dstf/__main__.py +10 -0
- dstf/authoring/__init__.py +13 -0
- dstf/authoring/builtin/__init__.py +32 -0
- dstf/authoring/builtin/accept.py +205 -0
- dstf/authoring/builtin/edit_criteria.py +479 -0
- dstf/authoring/builtin/edit_family.py +228 -0
- dstf/authoring/builtin/edit_on_plot.py +1354 -0
- dstf/authoring/builtin/edit_tree.py +1399 -0
- dstf/authoring/builtin/enqueue.py +144 -0
- dstf/authoring/capabilities.py +463 -0
- dstf/authoring/errors.py +49 -0
- dstf/authoring/family_rows.py +526 -0
- dstf/authoring/forms.py +631 -0
- dstf/authoring/layout.py +104 -0
- dstf/authoring/live_rows.py +337 -0
- dstf/authoring/live_runs.py +385 -0
- dstf/authoring/needle.py +127 -0
- dstf/authoring/picker_rows.py +92 -0
- dstf/authoring/queue_rows.py +354 -0
- dstf/authoring/session.py +3016 -0
- dstf/authoring/shell/__init__.py +9 -0
- dstf/authoring/shell/actions.py +702 -0
- dstf/authoring/shell/app.py +53 -0
- dstf/authoring/shell/drawer_controls.py +190 -0
- dstf/authoring/shell/family_fold.py +468 -0
- dstf/authoring/shell/figure_hook.py +249 -0
- dstf/authoring/shell/grips.js +1000 -0
- dstf/authoring/shell/header.py +783 -0
- dstf/authoring/shell/live_view.py +93 -0
- dstf/authoring/shell/picker.py +267 -0
- dstf/authoring/shell/placing.py +122 -0
- dstf/authoring/shell/plot_card.py +1171 -0
- dstf/authoring/shell/queue_panel.py +433 -0
- dstf/authoring/shell/rail.py +593 -0
- dstf/authoring/shell/shell.py +528 -0
- dstf/authoring/shell/stats_card.py +501 -0
- dstf/authoring/shell/strip.py +204 -0
- dstf/authoring/shell/style.py +721 -0
- dstf/authoring/shell/suite.py +277 -0
- dstf/authoring/shell/suite_grid.js +414 -0
- dstf/authoring/shell/suite_grid.py +373 -0
- dstf/authoring/shell/tab_theme.py +110 -0
- dstf/authoring/shell/tabstate.py +146 -0
- dstf/authoring/shell/test_list.py +179 -0
- dstf/authoring/shell/test_page.py +1404 -0
- dstf/authoring/shell/tree_card.py +1171 -0
- dstf/authoring/shell/variables_drawer.py +589 -0
- dstf/authoring/shell/widgets.py +188 -0
- dstf/authoring/situation.py +287 -0
- dstf/authoring/spec_document.py +215 -0
- dstf/authoring/spec_preview.py +83 -0
- dstf/authoring/suite_rows.py +99 -0
- dstf/authoring/tree_move.py +516 -0
- dstf/authoring/tree_pointer.py +33 -0
- dstf/authoring/tree_text.py +61 -0
- dstf/authoring/typed_number.js +64 -0
- dstf/authoring/typed_number.py +76 -0
- dstf/authoring/unscorable.py +141 -0
- dstf/authoring/variable_rows.py +216 -0
- dstf/backends/__init__.py +283 -0
- dstf/backends/base.py +1117 -0
- dstf/backends/common/__init__.py +1 -0
- dstf/backends/common/json_driver.py +484 -0
- dstf/backends/common/live_mat.py +369 -0
- dstf/backends/common/mat_reader.py +632 -0
- dstf/backends/common/proc_kill.py +238 -0
- dstf/backends/common/proc_output.py +20 -0
- dstf/backends/common/result_files.py +109 -0
- dstf/backends/common/result_variables.py +329 -0
- dstf/backends/common/statistics.py +39 -0
- dstf/backends/cross_backend.py +193 -0
- dstf/backends/dymola/__init__.py +5 -0
- dstf/backends/dymola/backend.py +776 -0
- dstf/backends/dymola/interface_loader.py +241 -0
- dstf/backends/dymola/log_parser.py +396 -0
- dstf/backends/dymola/persistent_backend.py +949 -0
- dstf/backends/fmpy/__init__.py +11 -0
- dstf/backends/fmpy/backend.py +443 -0
- dstf/backends/julia/__init__.py +27 -0
- dstf/backends/julia/backend.py +156 -0
- dstf/backends/julia/persistent_backend.py +541 -0
- dstf/backends/julia/run_persistent.jl +167 -0
- dstf/backends/julia/run_test.jl +146 -0
- dstf/backends/openmodelica/__init__.py +21 -0
- dstf/backends/openmodelica/backend.py +368 -0
- dstf/backends/openmodelica/init_xml.py +76 -0
- dstf/backends/openmodelica/log_parser.py +119 -0
- dstf/backends/openmodelica/mos_generator.py +228 -0
- dstf/backends/openmodelica/persistent_backend.py +1045 -0
- dstf/backends/openmodelica/session_loader.py +63 -0
- dstf/backends/persistent.py +1811 -0
- dstf/backends/pool.py +602 -0
- dstf/backends/progress.py +395 -0
- dstf/backends/python/__init__.py +11 -0
- dstf/backends/python/backend.py +141 -0
- dstf/backends/python/run_test.py +141 -0
- dstf/backends/run_metadata.py +103 -0
- dstf/backends/types.py +310 -0
- dstf/cli/__init__.py +12 -0
- dstf/cli/author_cmd.py +61 -0
- dstf/cli/baseline_cmds.py +341 -0
- dstf/cli/common.py +231 -0
- dstf/cli/diag_cmds.py +350 -0
- dstf/cli/main.py +561 -0
- dstf/cli/review.py +398 -0
- dstf/cli/run_cmds.py +589 -0
- dstf/cli/spec_cmds.py +123 -0
- dstf/comparison/__init__.py +13 -0
- dstf/comparison/algorithms.py +2127 -0
- dstf/comparison/builtins.py +291 -0
- dstf/comparison/comparator.py +1189 -0
- dstf/comparison/geometry.py +164 -0
- dstf/comparison/handles.py +1194 -0
- dstf/comparison/metric_tree.py +332 -0
- dstf/comparison/mode_schema.py +176 -0
- dstf/comparison/modes.py +788 -0
- dstf/comparison/registry.py +230 -0
- dstf/comparison/seeds.py +281 -0
- dstf/comparison/sides.py +158 -0
- dstf/comparison/snap.py +62 -0
- dstf/comparison/strict_keys.py +246 -0
- dstf/comparison/summaries.py +354 -0
- dstf/comparison/tree_eval.py +868 -0
- dstf/comparison/tree_spec.py +678 -0
- dstf/comparison/types.py +194 -0
- dstf/comparison/validator.py +232 -0
- dstf/config.py +815 -0
- dstf/discovery/__init__.py +0 -0
- dstf/discovery/json_recognizer.py +702 -0
- dstf/discovery/library_state.py +248 -0
- dstf/discovery/mo_parser.py +794 -0
- dstf/discovery/patch_apply.py +308 -0
- dstf/discovery/patch_envelope.py +63 -0
- dstf/discovery/plots.py +318 -0
- dstf/discovery/recognizer.py +103 -0
- dstf/discovery/scan_cache.py +154 -0
- dstf/discovery/spec_parser.py +538 -0
- dstf/discovery/spec_shape.py +122 -0
- dstf/discovery/spec_update.py +493 -0
- dstf/discovery/test_registry.py +488 -0
- dstf/discovery/unit_tests.py +95 -0
- dstf/figures/__init__.py +14 -0
- dstf/figures/assemble.py +48 -0
- dstf/figures/decimate.py +150 -0
- dstf/figures/families.py +289 -0
- dstf/figures/palette.py +170 -0
- dstf/figures/plotly.py +1413 -0
- dstf/figures/view.py +233 -0
- dstf/pipeline.py +435 -0
- dstf/plugins.py +104 -0
- dstf/reporting/__init__.py +0 -0
- dstf/reporting/console_report.py +187 -0
- dstf/reporting/context_builder.py +979 -0
- dstf/reporting/dashboard_render.py +534 -0
- dstf/reporting/junit_report.py +192 -0
- dstf/reporting/overlay_loader.py +708 -0
- dstf/reporting/report_suite.py +1059 -0
- dstf/reporting/rows.py +458 -0
- dstf/reporting/schema_export.py +413 -0
- dstf/reporting/stats_rows.py +321 -0
- dstf/reporting/status.py +388 -0
- dstf/reporting/templates/axes.js +61 -0
- dstf/reporting/templates/dashboard.html +284 -0
- dstf/reporting/templates/dashboard.js +604 -0
- dstf/reporting/templates/filters.js +100 -0
- dstf/reporting/templates/interactive.html +420 -0
- dstf/reporting/templates/report.js +116 -0
- dstf/reporting/templates/vendor/plotly.min.js +8 -0
- dstf/reporting/theme.py +324 -0
- dstf/reporting/tree_rows.py +73 -0
- dstf/reporting/verdicts.py +124 -0
- dstf/storage/__init__.py +0 -0
- dstf/storage/accept.py +216 -0
- dstf/storage/baseline.py +43 -0
- dstf/storage/batch_manifest.py +204 -0
- dstf/storage/ref_index.py +295 -0
- dstf/storage/reference_store.py +661 -0
- dstf/storage/roles.py +576 -0
- dstf/storage/run_stats.py +85 -0
- dynamic_systems_testing_framework-1.0.1.dist-info/METADATA +266 -0
- dynamic_systems_testing_framework-1.0.1.dist-info/RECORD +185 -0
- dynamic_systems_testing_framework-1.0.1.dist-info/WHEEL +4 -0
- dynamic_systems_testing_framework-1.0.1.dist-info/entry_points.txt +2 -0
- dynamic_systems_testing_framework-1.0.1.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,3016 @@
|
|
|
1
|
+
"""The Session — the authoring tool with no UI (two-tier design, §4.5).
|
|
2
|
+
|
|
3
|
+
Opens a project, scores its tests from the last run on disk, holds the
|
|
4
|
+
spec with the ops staged against it, and applies the outcomes capabilities
|
|
5
|
+
return. It writes only through ``save`` (the spec) and through the outcomes
|
|
6
|
+
it applies (the store, the queue's work dir). It owns the simulation queue
|
|
7
|
+
(D150): one per process, over a worker pool that lives until the tool
|
|
8
|
+
stops. Nothing here imports a UI.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import copy
|
|
14
|
+
import json
|
|
15
|
+
import logging
|
|
16
|
+
import os
|
|
17
|
+
import threading
|
|
18
|
+
import time
|
|
19
|
+
from collections.abc import Callable, Collection, Iterable, Mapping, Sequence
|
|
20
|
+
from dataclasses import dataclass, field, replace
|
|
21
|
+
from pathlib import Path
|
|
22
|
+
from types import MappingProxyType
|
|
23
|
+
from typing import TYPE_CHECKING, Any, Literal, cast
|
|
24
|
+
|
|
25
|
+
from ..backends import BackendUnavailableError, dataset_kind_for, select_backend
|
|
26
|
+
from ..backends.base import Backend
|
|
27
|
+
from ..backends.common.result_variables import tracked_results
|
|
28
|
+
from ..backends.pool import ItemState, QueueItem, Refused, StartingWorker
|
|
29
|
+
from ..backends.progress import recorded_backend
|
|
30
|
+
from ..backends.types import (
|
|
31
|
+
DatasetType,
|
|
32
|
+
TestResult,
|
|
33
|
+
VariableText,
|
|
34
|
+
)
|
|
35
|
+
from ..comparison.comparator import DEFAULT_TOLERANCE, compare_one
|
|
36
|
+
from ..comparison.types import TestComparison
|
|
37
|
+
from ..config import Config, ConfigError
|
|
38
|
+
from ..discovery.library_state import (
|
|
39
|
+
LibraryState,
|
|
40
|
+
Stamp,
|
|
41
|
+
following,
|
|
42
|
+
source_stamps,
|
|
43
|
+
stamp,
|
|
44
|
+
)
|
|
45
|
+
from ..discovery.mo_parser import LocatedClass, find_class
|
|
46
|
+
from ..discovery.patch_apply import PatchError
|
|
47
|
+
from ..discovery.plots import (
|
|
48
|
+
PlotExtra,
|
|
49
|
+
ResolvedPlot,
|
|
50
|
+
plot_leaves,
|
|
51
|
+
plots_of_test,
|
|
52
|
+
tracked_names,
|
|
53
|
+
)
|
|
54
|
+
from ..discovery.scan_cache import load_scan, store_scan
|
|
55
|
+
from ..discovery.spec_parser import spec_models_from_document
|
|
56
|
+
from ..discovery.spec_update import (
|
|
57
|
+
UnscorableCriterion,
|
|
58
|
+
apply_validated_patches,
|
|
59
|
+
has_metrics,
|
|
60
|
+
implicit_tree_op,
|
|
61
|
+
is_literal_variable,
|
|
62
|
+
needs_materialization,
|
|
63
|
+
unknown_override_modes,
|
|
64
|
+
validate_entry_keys,
|
|
65
|
+
validate_entry_metric_tree,
|
|
66
|
+
)
|
|
67
|
+
from ..discovery.test_registry import (
|
|
68
|
+
SpecModelLocator,
|
|
69
|
+
TestModel,
|
|
70
|
+
merge_tests,
|
|
71
|
+
scan_files,
|
|
72
|
+
scan_source_tests,
|
|
73
|
+
spec_tests_by_model,
|
|
74
|
+
)
|
|
75
|
+
from ..figures.assemble import leaves_by_variable, view_from_trajectory
|
|
76
|
+
from ..figures.view import VariableView, error_series
|
|
77
|
+
from ..storage.baseline import PRIMARY_BASELINE
|
|
78
|
+
from ..storage.reference_store import ReferenceStore
|
|
79
|
+
from .capabilities import (
|
|
80
|
+
Artifact,
|
|
81
|
+
Enqueue,
|
|
82
|
+
OpenProject,
|
|
83
|
+
Outcome,
|
|
84
|
+
Patch,
|
|
85
|
+
StoreWrite,
|
|
86
|
+
plural,
|
|
87
|
+
)
|
|
88
|
+
from .live_rows import Partial, unscored_view
|
|
89
|
+
from .live_runs import LiveRuns
|
|
90
|
+
from .situation import unrecorded_variables
|
|
91
|
+
from .spec_document import PatchOp, SpecDocument, edit_count, json_equal
|
|
92
|
+
|
|
93
|
+
if TYPE_CHECKING:
|
|
94
|
+
import numpy as np
|
|
95
|
+
|
|
96
|
+
from ..backends.pool import SimulationPool
|
|
97
|
+
from ..comparison.tree_spec import LeafSpec, SpecNode
|
|
98
|
+
from ..comparison.validator import BaselineRoleLookup
|
|
99
|
+
from ..figures.plotly import Visibility
|
|
100
|
+
from ..pipeline import ItemRun
|
|
101
|
+
from ..reporting.stats_rows import StatRow
|
|
102
|
+
|
|
103
|
+
Event = Literal["spec", "results", "queue", "notice", "loading", "known", "overlays"]
|
|
104
|
+
Listener = Callable[[dict[str, Any]], None]
|
|
105
|
+
|
|
106
|
+
logger = logging.getLogger(__name__)
|
|
107
|
+
|
|
108
|
+
#: The states of a queue item that is still to end.
|
|
109
|
+
ACTIVE = (ItemState.WAITING, ItemState.RUNNING)
|
|
110
|
+
|
|
111
|
+
# Why a test has no tree to edit — `Session.tree_or_reason`, from
|
|
112
|
+
# `_materialization`'s refusal. Each is one short line: a Statistics row's
|
|
113
|
+
# `+` says it as its tooltip, a criterion's editor in its row.
|
|
114
|
+
#: No comparison scored a variable (no baseline, a failed run, no result).
|
|
115
|
+
NOTHING_SCORED = "nothing is scored"
|
|
116
|
+
#: A simulate-only test: the comparator scores nothing on it. Also why an
|
|
117
|
+
#: add form's Add is grey there, tree or not.
|
|
118
|
+
SIMULATE_ONLY = "simulate-only: nothing is scored"
|
|
119
|
+
#: A scored name the loaded result does not carry: the comparison is from
|
|
120
|
+
#: a run the results no longer match, and a leaf on it would resolve to
|
|
121
|
+
#: nothing.
|
|
122
|
+
STALE = "its last run no longer holds what it scored — queue it again"
|
|
123
|
+
#: Every other refusal: a flat setting of the spec or the baseline that a
|
|
124
|
+
#: tree would score differently.
|
|
125
|
+
NO_TREE = (
|
|
126
|
+
"its flat criteria can't become a tree without changing its scores — "
|
|
127
|
+
"write a metrics tree in test_spec.json"
|
|
128
|
+
)
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
@dataclass
|
|
132
|
+
class Project:
|
|
133
|
+
config: Config
|
|
134
|
+
store: ReferenceStore
|
|
135
|
+
backend: Backend
|
|
136
|
+
scanned: dict[str, TestModel] # the source scan, merged against every spec edit
|
|
137
|
+
#: Where a spec-only test's model is, with its experiment annotation —
|
|
138
|
+
#: asked on every merge and read once per model; replaced with the scan.
|
|
139
|
+
locate: SpecModelLocator
|
|
140
|
+
tests: list[TestModel]
|
|
141
|
+
results: dict[str, TestResult]
|
|
142
|
+
dataset_kind: DatasetType
|
|
143
|
+
#: Set once this project's sibling-backend overlay index is built (or
|
|
144
|
+
#: its build failed): the project's, so a read asks the flag of the
|
|
145
|
+
#: project it reads — a reopen has no window where a read of the new
|
|
146
|
+
#: project sees the old one's set flag.
|
|
147
|
+
overlays_ready: threading.Event = field(default_factory=threading.Event)
|
|
148
|
+
|
|
149
|
+
def test(self, model_id: str) -> TestModel | None:
|
|
150
|
+
return next((t for t in self.tests if t.model_id == model_id), None)
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
@dataclass(frozen=True)
|
|
154
|
+
class Selection:
|
|
155
|
+
"""What a page has selected, handed to a capability through its
|
|
156
|
+
SessionView (D129): the checked tests, the variable and the criterion's
|
|
157
|
+
tree path. A page builds a new view when it changes; two browser tabs
|
|
158
|
+
keep two. Not the Session's — that was the old shell's one-tab shape.
|
|
159
|
+
``node_path``: the tree node whose controls are being drawn (NODE) — a
|
|
160
|
+
leaf or a combinator; ``leaf_path`` only ever names a criterion;
|
|
161
|
+
``leaf_paths`` every member of a folded criteria row (FAMILY_CRITERION,
|
|
162
|
+
D152), in tree order, ``leaf_path`` the first of them — empty wherever
|
|
163
|
+
one criterion is selected;
|
|
164
|
+
``quantity`` the statistic a Statistics row shows (STATISTIC), by the
|
|
165
|
+
path a leaf would read it through (``translation.nonlinear_count``).
|
|
166
|
+
|
|
167
|
+
``texts`` is the page's ONE read of the selected test's label scope
|
|
168
|
+
(:meth:`Session.variable_texts`), handed down with the selection so a
|
|
169
|
+
plot's ``+`` can say what the variable READS as without deriving the
|
|
170
|
+
scope for itself. A page with forty cards did exactly that once per card:
|
|
171
|
+
eighty scope builds per rebuild, each re-resolving the test's plots and
|
|
172
|
+
its tracked names — 0.37 s at forty cards against 0.03 s at five. A
|
|
173
|
+
selection carrying no texts reads the variable's NAME; nothing here
|
|
174
|
+
picks a test to read a label off, which is also why a selection of two
|
|
175
|
+
tests (refused at VARIABLE anyway) cannot show one of them's
|
|
176
|
+
expression.
|
|
177
|
+
"""
|
|
178
|
+
|
|
179
|
+
tests: frozenset[str] = frozenset()
|
|
180
|
+
variable: str | None = None
|
|
181
|
+
leaf_path: str | None = None
|
|
182
|
+
leaf_paths: tuple[str, ...] = ()
|
|
183
|
+
node_path: str | None = None
|
|
184
|
+
quantity: str | None = None
|
|
185
|
+
texts: Mapping[str, VariableText] = MappingProxyType({})
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
@dataclass
|
|
189
|
+
class _ModelReads:
|
|
190
|
+
"""What one click re-read from disk before this cache (brief §2.2): the
|
|
191
|
+
primary reference and the overlays; plus the decimated base view per
|
|
192
|
+
variable — trajectories, overlays and error series, no leaves — held
|
|
193
|
+
here and shared by `figure` and `plot_figure` through `_editable_view`,
|
|
194
|
+
so an edit re-decimates only the criterion's geometry and a plot's
|
|
195
|
+
several series draw over the bases held here, one at a time."""
|
|
196
|
+
|
|
197
|
+
ref: dict | None
|
|
198
|
+
overlays: list
|
|
199
|
+
bases: dict[str, VariableView] = field(default_factory=dict)
|
|
200
|
+
#: Per variable, the decimated view with no Actual — the baseline's
|
|
201
|
+
#: trace and the overlays — that a run in progress, or one that left no
|
|
202
|
+
#: result, is drawn over (D151); dropped with the bases.
|
|
203
|
+
references: dict[str, VariableView] = field(default_factory=dict)
|
|
204
|
+
#: The scored list the bases were built from (`_scored_key`). A rescore
|
|
205
|
+
#: that changes it — a staged edit that starts or stops scoring a
|
|
206
|
+
#: variable, a discard, a save, the background rescan — may leave any
|
|
207
|
+
#: base of the model stale; the key does not say which, so
|
|
208
|
+
#: `_editable_view` drops them all when it moves — for both figure reads
|
|
209
|
+
#: (one extra decimation per variable).
|
|
210
|
+
scored: tuple[tuple[str | None, int], ...] = ()
|
|
211
|
+
#: Every name in the result file (`Backend.result_names`), once
|
|
212
|
+
#: `names_read`: `None` when the backend cannot list the file, `()` when
|
|
213
|
+
#: the file lists nothing.
|
|
214
|
+
names: tuple[str, ...] | None = None
|
|
215
|
+
names_read: bool = False
|
|
216
|
+
#: Untracked series read from the result file, by name; `None` is a miss
|
|
217
|
+
#: (the file lacks the name, or holds nothing drawable under it), cached
|
|
218
|
+
#: so a redraw does not re-open the file for it.
|
|
219
|
+
series: dict[str, tuple[np.ndarray, np.ndarray] | None] = field(
|
|
220
|
+
default_factory=dict
|
|
221
|
+
)
|
|
222
|
+
#: Whether the overlays include the sibling-backend ones: False for a
|
|
223
|
+
#: read taken before the project's sibling index was ready; the first
|
|
224
|
+
#: read after it reloads the overlays (`_reads_for`).
|
|
225
|
+
siblings: bool = True
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
def _scored_key(comp: TestComparison | None) -> tuple[tuple[str | None, int], ...]:
|
|
229
|
+
"""What `view` hands `build_trajectories`, reduced to what the base
|
|
230
|
+
traces depend on: the (name, index) of every scored variable, in order;
|
|
231
|
+
empty when the test was not evaluated. The WHOLE list, not "is this
|
|
232
|
+
variable in it": with no baseline, an empty scored list draws a variable
|
|
233
|
+
through the no-baseline path (overlays attached) and a non-empty one (a
|
|
234
|
+
baseline-free leaf on ANOTHER variable) draws it from the result file,
|
|
235
|
+
with none — or as the empty state when the file cannot be read."""
|
|
236
|
+
# `evaluated` is `metric_tree is not None or bool(variables)`
|
|
237
|
+
# (comparison/types.py), so an unevaluated comparison has no variables
|
|
238
|
+
# and the tuple is empty by construction — no separate check.
|
|
239
|
+
if comp is None:
|
|
240
|
+
return ()
|
|
241
|
+
return tuple((vc.name, vc.index) for vc in comp.variables)
|
|
242
|
+
|
|
243
|
+
|
|
244
|
+
def no_tree_error(model_id: str, why: str) -> PatchError:
|
|
245
|
+
"""The refusal of an edit that needs ``model_id``'s tree where there is
|
|
246
|
+
none (`Session.tree_or_reason`'s ``why``): a patch the Session will not
|
|
247
|
+
take, whose ``reason`` is ``why`` — what the tool shows alone beside the
|
|
248
|
+
edit it came from — and whose message names the model too."""
|
|
249
|
+
return PatchError(f"{model_id}: {why}", reason=why)
|
|
250
|
+
|
|
251
|
+
|
|
252
|
+
class QueueError(RuntimeError):
|
|
253
|
+
"""The simulation queue did not do what was asked, or stands in the way
|
|
254
|
+
of what was; ``str()`` is the sentence why."""
|
|
255
|
+
|
|
256
|
+
|
|
257
|
+
@dataclass(frozen=True)
|
|
258
|
+
class _Ask:
|
|
259
|
+
"""One test a Queue click names, as the click's read needs it."""
|
|
260
|
+
|
|
261
|
+
model_id: str
|
|
262
|
+
#: The file the test was read from; None for a test that has none (a
|
|
263
|
+
#: spec-only test whose model the library does not hold, any test of a
|
|
264
|
+
#: source that is not Modelica).
|
|
265
|
+
path: Path | None
|
|
266
|
+
#: That file's stamp when the Session last read the test from it; None
|
|
267
|
+
#: for a test only the scan at open read, at a moment nobody stamped.
|
|
268
|
+
read_at: Stamp | None
|
|
269
|
+
#: Whether the scan found it; else only ``test_spec.json`` declares it.
|
|
270
|
+
scanned: bool
|
|
271
|
+
|
|
272
|
+
|
|
273
|
+
@dataclass
|
|
274
|
+
class _ClickRead:
|
|
275
|
+
"""What one Queue click read of the library, off the UI loop."""
|
|
276
|
+
|
|
277
|
+
#: The source files' stamps (`library_state.source_stamps`); None for a
|
|
278
|
+
#: source no worker keeps loaded.
|
|
279
|
+
stamps: Mapping[Path, Stamp] | None
|
|
280
|
+
#: The stamp each test's file was read at, for the tests read again.
|
|
281
|
+
read_at: dict[str, Stamp] = field(default_factory=dict)
|
|
282
|
+
#: The scanned tests read again from their files.
|
|
283
|
+
scanned: dict[str, TestModel] = field(default_factory=dict)
|
|
284
|
+
#: The spec-only tests' models, found again.
|
|
285
|
+
located: dict[str, LocatedClass] = field(default_factory=dict)
|
|
286
|
+
#: By model, the sentence for a test that cannot be read any more.
|
|
287
|
+
refused: dict[str, str] = field(default_factory=dict)
|
|
288
|
+
|
|
289
|
+
|
|
290
|
+
def _read_click(config: Config, asks: Sequence[_Ask]) -> _ClickRead:
|
|
291
|
+
"""Sweep the source files (stat only) and read again each asked test
|
|
292
|
+
whose own file's stamp is not the one it was last read at. Touches
|
|
293
|
+
nothing of a Session: safe on any thread.
|
|
294
|
+
|
|
295
|
+
A stamp is taken BEFORE its file is read: a file edited between the two
|
|
296
|
+
is read newer than its stamp says, and the next click reads it once
|
|
297
|
+
more — never the other way round."""
|
|
298
|
+
stamps = source_stamps(config)
|
|
299
|
+
read = _ClickRead(stamps)
|
|
300
|
+
if stamps is None or config.source_type != "modelica":
|
|
301
|
+
return read
|
|
302
|
+
stale: dict[Path, list[tuple[str, Stamp]]] = {}
|
|
303
|
+
for ask in asks:
|
|
304
|
+
if ask.path is None:
|
|
305
|
+
continue
|
|
306
|
+
# Not in the sweep: spelled another way than the sweep spells it (a
|
|
307
|
+
# scan cached under another path), or gone.
|
|
308
|
+
now = stamps.get(ask.path) or stamp(ask.path)
|
|
309
|
+
if now is not None and now == ask.read_at:
|
|
310
|
+
continue
|
|
311
|
+
if ask.scanned:
|
|
312
|
+
if now is None:
|
|
313
|
+
read.refused[ask.model_id] = _unreadable(ask, "is gone")
|
|
314
|
+
else:
|
|
315
|
+
stale.setdefault(ask.path, []).append((ask.model_id, now))
|
|
316
|
+
continue
|
|
317
|
+
# Declared in the spec only: looked for again — its class may be in
|
|
318
|
+
# another file of the library by now.
|
|
319
|
+
assert config.library_name is not None # resolved in __post_init__
|
|
320
|
+
located = find_class(config.library_dir, config.library_name, ask.model_id)
|
|
321
|
+
if located is None:
|
|
322
|
+
read.refused[ask.model_id] = _unreadable(
|
|
323
|
+
ask, "is gone" if now is None else _no_longer(ask.path)
|
|
324
|
+
)
|
|
325
|
+
continue
|
|
326
|
+
read.located[ask.model_id] = located
|
|
327
|
+
at = stamps.get(located.mo_file) or stamp(located.mo_file)
|
|
328
|
+
if at is not None:
|
|
329
|
+
read.read_at[ask.model_id] = at
|
|
330
|
+
if stale:
|
|
331
|
+
fresh = scan_files(config, list(stale))
|
|
332
|
+
for path, models in stale.items():
|
|
333
|
+
for model_id, now in models:
|
|
334
|
+
test = fresh.get(model_id)
|
|
335
|
+
if test is None:
|
|
336
|
+
read.refused[model_id] = (
|
|
337
|
+
f"{model_id}: its file {path} {_no_longer(path)}"
|
|
338
|
+
)
|
|
339
|
+
else:
|
|
340
|
+
read.scanned[model_id] = test
|
|
341
|
+
read.read_at[model_id] = now
|
|
342
|
+
return read
|
|
343
|
+
|
|
344
|
+
|
|
345
|
+
def _unreadable(ask: _Ask, why: str) -> str:
|
|
346
|
+
return f"{ask.model_id}: its file {ask.path} {why}"
|
|
347
|
+
|
|
348
|
+
|
|
349
|
+
def _no_longer(path: Path) -> str:
|
|
350
|
+
"""Why a file that is there gave no test: ``no longer holds it`` — or,
|
|
351
|
+
for one that cannot be opened, ``cannot be read: Permission denied``
|
|
352
|
+
in the system's own words. A file made unreadable read as one edited
|
|
353
|
+
until the test was gone from it."""
|
|
354
|
+
try:
|
|
355
|
+
with open(path, "rb"):
|
|
356
|
+
pass
|
|
357
|
+
except OSError as e:
|
|
358
|
+
return f"cannot be read: {e.strerror or type(e).__name__}"
|
|
359
|
+
return "no longer holds it"
|
|
360
|
+
|
|
361
|
+
|
|
362
|
+
def resolve_project(
|
|
363
|
+
config_path: Path, *, backend: str | None = None, work_dir: Path | None = None
|
|
364
|
+
) -> tuple[Config, str | None, Backend]:
|
|
365
|
+
"""What an open of ``config_path`` works with: its Config, the warning
|
|
366
|
+
about the backend the last run recorded (or None), and the backend that
|
|
367
|
+
reads its results. Without an explicit ``backend`` the one the last run
|
|
368
|
+
recorded wins, as ``compare`` does. ``ConfigError`` for a testing.json
|
|
369
|
+
that cannot be read as one and for a backend this machine cannot
|
|
370
|
+
build."""
|
|
371
|
+
config_path = Path(config_path)
|
|
372
|
+
config = Config(config_file=config_path, backend_name=backend, work_dir=work_dir)
|
|
373
|
+
warning: str | None = None
|
|
374
|
+
if backend is None:
|
|
375
|
+
assert config.work_dir is not None # resolved in __post_init__
|
|
376
|
+
recorded, warning = recorded_backend(config.work_dir)
|
|
377
|
+
if recorded and recorded != config.backend_name:
|
|
378
|
+
config = Config(
|
|
379
|
+
config_file=config_path, backend_name=recorded, work_dir=work_dir
|
|
380
|
+
)
|
|
381
|
+
try:
|
|
382
|
+
# The Session's backend reads results; it is not the queue's
|
|
383
|
+
# backend, so the batch class is what it wants. `select_backend`
|
|
384
|
+
# is the seam that turns every "this machine can't build it"
|
|
385
|
+
# failure — an unregistered name, a missing executable, an
|
|
386
|
+
# absent optional dependency — into one class, so `dstf author`
|
|
387
|
+
# prints a line where a RuntimeError tracebacked.
|
|
388
|
+
backend_obj = select_backend(config, persistent=False)
|
|
389
|
+
except BackendUnavailableError as e:
|
|
390
|
+
raise ConfigError(str(e)) from e
|
|
391
|
+
# A UI process has no console to narrate a run or a read to.
|
|
392
|
+
backend_obj.quiet = True
|
|
393
|
+
return config, warning, backend_obj
|
|
394
|
+
|
|
395
|
+
|
|
396
|
+
def check_project(
|
|
397
|
+
config_path: Path, *, backend: str | None = None, work_dir: Path | None = None
|
|
398
|
+
) -> None:
|
|
399
|
+
"""Everything of an open that a config or spec error stops, and nothing
|
|
400
|
+
slow: the Config, the backend, and the spec file parsed as the open
|
|
401
|
+
parses it — no scan of the library, no result read. ``ConfigError``
|
|
402
|
+
with the sentence every other command prints.
|
|
403
|
+
|
|
404
|
+
``dstf author`` runs it BEFORE it serves: the project used to be opened
|
|
405
|
+
only once the server was up, on a thread, so `"theme": "neon"`, a
|
|
406
|
+
`baseline_points` of 1 or an unknown spec key started the server,
|
|
407
|
+
logged a traceback and showed the error in the page — where `run` and
|
|
408
|
+
`compare` print one `Config error: …` line and exit 2."""
|
|
409
|
+
config, _warning, _backend = resolve_project(
|
|
410
|
+
config_path, backend=backend, work_dir=work_dir
|
|
411
|
+
)
|
|
412
|
+
spec = SpecDocument.load(config.spec_path)
|
|
413
|
+
spec_models_from_document(
|
|
414
|
+
spec.loaded, base_dir=config.spec_path.parent, label=str(config.spec_path)
|
|
415
|
+
)
|
|
416
|
+
|
|
417
|
+
|
|
418
|
+
class StaleSpecError(RuntimeError):
|
|
419
|
+
"""The spec file is not the one the Session loaded: nothing that reads
|
|
420
|
+
the loaded spec to WRITE — Save, its edits; Accept, a baseline with the
|
|
421
|
+
entry's `baseline_points` and comparison settings — goes on. ``doing``
|
|
422
|
+
is what was refused, for the sentence."""
|
|
423
|
+
|
|
424
|
+
#: What each refused act did NOT do, as the dialog says it.
|
|
425
|
+
NOT_DONE = {"saving": "Nothing was saved.", "accepting": "Nothing was accepted."}
|
|
426
|
+
|
|
427
|
+
def __init__(self, path: Path, doing: str = "saving"):
|
|
428
|
+
self.path = path
|
|
429
|
+
self.not_done = self.NOT_DONE[doing]
|
|
430
|
+
super().__init__(
|
|
431
|
+
f"{path} changed on disk since it was loaded; reload before {doing}"
|
|
432
|
+
)
|
|
433
|
+
|
|
434
|
+
|
|
435
|
+
class UnscorableSaveError(RuntimeError):
|
|
436
|
+
"""Save refused: a pending edit leaves a criterion that can never be
|
|
437
|
+
scored. One sentence naming it; nothing was written and every edit is
|
|
438
|
+
still pending."""
|
|
439
|
+
|
|
440
|
+
def __init__(self, reason: str):
|
|
441
|
+
self.reason = reason
|
|
442
|
+
super().__init__(f"{reason} — fill it or remove it before saving")
|
|
443
|
+
|
|
444
|
+
|
|
445
|
+
class PendingEditsError(RuntimeError):
|
|
446
|
+
def __init__(self, count: int):
|
|
447
|
+
self.count = count
|
|
448
|
+
super().__init__(
|
|
449
|
+
f"{count} staged edit(s) would be lost; save or discard them first"
|
|
450
|
+
)
|
|
451
|
+
|
|
452
|
+
|
|
453
|
+
class _Background:
|
|
454
|
+
"""One daemon thread that runs what the Session hands it off the UI
|
|
455
|
+
loop — the drained queue's ``status.json`` and dashboard — one at a
|
|
456
|
+
time and in the order handed. Started when there is work, ended when there is none. What it
|
|
457
|
+
runs is logged when it raises, whatever it raises (a hook's SystemExit
|
|
458
|
+
too), never re-raised: it writes files beside a reload that already
|
|
459
|
+
stands, and a thread that died would leave ``wait`` hanging on it."""
|
|
460
|
+
|
|
461
|
+
def __init__(self) -> None:
|
|
462
|
+
self._cond = threading.Condition()
|
|
463
|
+
self._queue: list[tuple[str, Callable[[], None]]] = []
|
|
464
|
+
self._busy = False
|
|
465
|
+
|
|
466
|
+
def submit(self, what: str, fn: Callable[[], None]) -> None:
|
|
467
|
+
with self._cond:
|
|
468
|
+
self._queue.append((what, fn))
|
|
469
|
+
if self._busy:
|
|
470
|
+
return
|
|
471
|
+
self._busy = True
|
|
472
|
+
threading.Thread(
|
|
473
|
+
target=self._drain, name="dstf-background", daemon=True
|
|
474
|
+
).start()
|
|
475
|
+
|
|
476
|
+
def _drain(self) -> None:
|
|
477
|
+
while True:
|
|
478
|
+
with self._cond:
|
|
479
|
+
if not self._queue:
|
|
480
|
+
self._busy = False
|
|
481
|
+
self._cond.notify_all()
|
|
482
|
+
return
|
|
483
|
+
what, fn = self._queue.pop(0)
|
|
484
|
+
try:
|
|
485
|
+
fn()
|
|
486
|
+
except BaseException:
|
|
487
|
+
logger.exception("%s failed", what)
|
|
488
|
+
|
|
489
|
+
def wait(self, timeout: float | None = None) -> bool:
|
|
490
|
+
with self._cond:
|
|
491
|
+
return self._cond.wait_for(lambda: not self._busy, timeout)
|
|
492
|
+
|
|
493
|
+
|
|
494
|
+
class Session:
|
|
495
|
+
def __init__(self, post: Callable[[Callable[[], None]], None] | None = None):
|
|
496
|
+
"""``post`` hands what another thread has for the Session — the
|
|
497
|
+
pool's callbacks, the rescan's merge — to the UI loop, where every
|
|
498
|
+
mutation happens; without one it runs on the thread that has it."""
|
|
499
|
+
self.project: Project | None = None
|
|
500
|
+
self.spec: SpecDocument = SpecDocument(
|
|
501
|
+
Path("test_spec.json"), {"tests": []}, None
|
|
502
|
+
)
|
|
503
|
+
#: The simulation queue (D150): every item queued and not cleared
|
|
504
|
+
#: since, in run order — first in, first out. Read it; `enqueue`,
|
|
505
|
+
#: `remove`, `cancel` and `clear_finished` change it.
|
|
506
|
+
self.queue: list[QueueItem] = []
|
|
507
|
+
#: The pool the queue runs on: opened at the first `enqueue`, alive
|
|
508
|
+
#: — its workers with it — until `stop_queue`.
|
|
509
|
+
self._pool: SimulationPool | None = None
|
|
510
|
+
#: The queue's items still to end, by model: what the pool's
|
|
511
|
+
#: callbacks are matched against, so one for an item the Session
|
|
512
|
+
#: has already dealt with (a close's, a replaced pool's) is nothing.
|
|
513
|
+
self._live: dict[str, QueueItem] = {}
|
|
514
|
+
#: The items that ran since ``status.json`` was last written.
|
|
515
|
+
self._unwritten: list[QueueItem] = []
|
|
516
|
+
#: The workers the user asked for; None is ``Config.parallel``.
|
|
517
|
+
self._workers: int | None = None
|
|
518
|
+
#: The pool's ``write_status`` (`pipeline.ItemRun`), kept past the
|
|
519
|
+
#: pool: the last items' snapshot is written after it closed.
|
|
520
|
+
self._write_status: Callable[[Iterable[QueueItem]], None] | None = None
|
|
521
|
+
#: Why this project's backend cannot be queued on, once an
|
|
522
|
+
#: `enqueue` found out; None while nothing says so.
|
|
523
|
+
self.queue_refusal: str | None = None
|
|
524
|
+
#: The message the pool stopped with (no worker could start), until
|
|
525
|
+
#: `clear_finished` or the next item that starts.
|
|
526
|
+
self.queue_stopped: str | None = None
|
|
527
|
+
#: The source files as the last Queue click swept them: the next
|
|
528
|
+
#: click's items carry this same state when nothing changed since,
|
|
529
|
+
#: and no worker then reads anything again.
|
|
530
|
+
self._library: LibraryState | None = None
|
|
531
|
+
#: By model, the stamp of the test's file when a Queue click last
|
|
532
|
+
#: read the test from it. A test with none here was read by the
|
|
533
|
+
#: scan at open, at a moment nobody stamped.
|
|
534
|
+
self._read_at: dict[str, Stamp] = {}
|
|
535
|
+
#: The models of the Queue clicks whose read of the library is
|
|
536
|
+
#: still running off the UI loop: a second click on one is refused.
|
|
537
|
+
self._reading: set[str] = set()
|
|
538
|
+
#: The number of the last `stage` call (`PatchOp.gesture`).
|
|
539
|
+
self._gesture = 0
|
|
540
|
+
#: By model, the sentence a Queue click's read refused it with,
|
|
541
|
+
#: while it stands among the notices (`_forget_refusals`).
|
|
542
|
+
self._refusals: dict[str, str] = {}
|
|
543
|
+
#: Counts `stop_queue`: a click read before one is not queued after
|
|
544
|
+
#: it (the tool stopping must leave no worker behind).
|
|
545
|
+
self._queue_epoch = 0
|
|
546
|
+
self.comparisons: dict[str, TestComparison] = {}
|
|
547
|
+
self.artifacts: list[Artifact] = []
|
|
548
|
+
self.notices: list[str] = []
|
|
549
|
+
self._post = post or (lambda fn: fn())
|
|
550
|
+
#: Whether there is a UI loop to keep free: a Queue click's read of
|
|
551
|
+
#: the library then runs on the background thread (`_enqueue_later`).
|
|
552
|
+
self._has_loop = post is not None
|
|
553
|
+
self._listeners: dict[str, list[Listener]] = {}
|
|
554
|
+
#: The number of the last `spec` event, which carries it as
|
|
555
|
+
#: ``revision`` (`_emit_spec`): a page tells the event an act of
|
|
556
|
+
#: its own produced from one queued before or after it.
|
|
557
|
+
self.spec_revision = 0
|
|
558
|
+
self.rescan_done = threading.Event()
|
|
559
|
+
self._reads: dict[str, _ModelReads] = {}
|
|
560
|
+
self._background = _Background()
|
|
561
|
+
#: The runs whose result the Session does not have (D151): the
|
|
562
|
+
#: tests the queue is running, and the partial results of runs
|
|
563
|
+
#: that did not finish.
|
|
564
|
+
self.live_runs = LiveRuns(
|
|
565
|
+
self, lambda state, models: self._emit("queue", state=state, models=models)
|
|
566
|
+
)
|
|
567
|
+
|
|
568
|
+
# -- project -----------------------------------------------------------
|
|
569
|
+
|
|
570
|
+
def open(
|
|
571
|
+
self,
|
|
572
|
+
config_path: Path,
|
|
573
|
+
*,
|
|
574
|
+
backend: str | None = None,
|
|
575
|
+
work_dir: Path | None = None,
|
|
576
|
+
) -> None:
|
|
577
|
+
"""Open the project ``config_path`` describes. Without an explicit
|
|
578
|
+
``backend`` the one the last run recorded wins, as ``compare`` does.
|
|
579
|
+
Refused while the queue holds a test or while edits are pending —
|
|
580
|
+
save or discard first; a swap must never drop them silently."""
|
|
581
|
+
self._refuse_if_busy()
|
|
582
|
+
config, warning, backend_obj = resolve_project(
|
|
583
|
+
config_path, backend=backend, work_dir=work_dir
|
|
584
|
+
)
|
|
585
|
+
store = ReferenceStore(config)
|
|
586
|
+
spec = SpecDocument.load(config.spec_path)
|
|
587
|
+
# The models this open can name before the scan — the spec's and the
|
|
588
|
+
# last run's — so a page draws their rows while a cold scan runs.
|
|
589
|
+
self._emit("known", models=_known_models(config, spec.effective()))
|
|
590
|
+
try:
|
|
591
|
+
started = time.monotonic()
|
|
592
|
+
# "scanning library…" before the first file — and the only scan
|
|
593
|
+
# event for a source type that scans nothing (Python, FMU, Julia).
|
|
594
|
+
self._emit("loading", phase="scan", done=0, total=0)
|
|
595
|
+
# The reference index, read here — off the UI loop, after the
|
|
596
|
+
# known rows, before the commit point: a page reads it as soon
|
|
597
|
+
# as `project` is set (its baseline's name), and over TRANSFORM
|
|
598
|
+
# the lazy scan (every ref file read whole) held the loop ~4 s
|
|
599
|
+
# from that first read — with the sibling-index wait
|
|
600
|
+
# (`_reads_for`), past the browser's ping budget: the socket
|
|
601
|
+
# dropped and the reconnect left the page stale.
|
|
602
|
+
store.index.ensure_loaded()
|
|
603
|
+
cached = load_scan(config)
|
|
604
|
+
if cached is not None:
|
|
605
|
+
scanned = cached
|
|
606
|
+
self._emit("loading", phase="scan", cached=True, total=len(cached))
|
|
607
|
+
else:
|
|
608
|
+
scanned = scan_source_tests(
|
|
609
|
+
config,
|
|
610
|
+
progress=lambda done, total: self._emit(
|
|
611
|
+
"loading", phase="scan", done=done, total=total
|
|
612
|
+
),
|
|
613
|
+
)
|
|
614
|
+
store_scan(config, scanned)
|
|
615
|
+
locate = SpecModelLocator(config)
|
|
616
|
+
tests = self._merge(config, scanned, spec.effective(), locate, edited=False)
|
|
617
|
+
# The queue is the open project's, and its workers the open
|
|
618
|
+
# backend's: both end here, at the commit point (nothing is
|
|
619
|
+
# queued or running — refused above).
|
|
620
|
+
self._reset_queue()
|
|
621
|
+
self.project = Project(
|
|
622
|
+
config=config,
|
|
623
|
+
store=store,
|
|
624
|
+
backend=backend_obj,
|
|
625
|
+
scanned=scanned,
|
|
626
|
+
locate=locate,
|
|
627
|
+
tests=tests,
|
|
628
|
+
results={},
|
|
629
|
+
dataset_kind=dataset_kind_for(backend_obj),
|
|
630
|
+
)
|
|
631
|
+
self.spec = spec
|
|
632
|
+
self.comparisons = {}
|
|
633
|
+
self._reads = {}
|
|
634
|
+
# A notice is project-scoped text ("stored 2 baselines for LibA.X"), and
|
|
635
|
+
# so is an artifact's path; a swap calls open, never close, and must not
|
|
636
|
+
# carry either into LibB's panel. Assigned here, at the commit point,
|
|
637
|
+
# and not on entry: an open that fails leaves the live project's panel
|
|
638
|
+
# as it was. The recorded-backend warning is this project's first
|
|
639
|
+
# notice, so the queue panel shows it.
|
|
640
|
+
self.notices = [warning] if warning else []
|
|
641
|
+
self.artifacts = []
|
|
642
|
+
# Started at the commit point, not after the reload: the warm reads the
|
|
643
|
+
# SIBLING backends' reference trees and the reload reads this one's, so
|
|
644
|
+
# neither waits on the other and the warm gets its head start inside
|
|
645
|
+
# the reload's window (15.6 s over TRANSFORM) instead of after it. How
|
|
646
|
+
# much of that window it converts is NOT measured and is less than all
|
|
647
|
+
# of it — both sides are json.loads, whose C scanner holds the GIL, so
|
|
648
|
+
# they overlap on file IO and contend on parsing. Not earlier than this:
|
|
649
|
+
# everything above can still raise, and an open that fails must leave
|
|
650
|
+
# the live session's state as it was.
|
|
651
|
+
threading.Thread(
|
|
652
|
+
target=self._warm_overlay_index,
|
|
653
|
+
args=(self.project,),
|
|
654
|
+
name="dstf-overlay-index",
|
|
655
|
+
daemon=True,
|
|
656
|
+
).start()
|
|
657
|
+
self._emit_spec({t.model_id for t in tests})
|
|
658
|
+
done_event = threading.Event()
|
|
659
|
+
self.rescan_done = done_event
|
|
660
|
+
try:
|
|
661
|
+
self._reload_results()
|
|
662
|
+
finally:
|
|
663
|
+
# Whether or not one model's scoring raised: the project is open
|
|
664
|
+
# and its results complete (`_reload_results`), so `done` fires
|
|
665
|
+
# and the rescan starts — a shell waiting on either must not wait
|
|
666
|
+
# forever. The failure still propagates to the caller, after both.
|
|
667
|
+
self._emit("loading", phase="done", seconds=time.monotonic() - started)
|
|
668
|
+
if cached is not None:
|
|
669
|
+
threading.Thread(
|
|
670
|
+
target=self._rescan,
|
|
671
|
+
args=(self.project, cached, done_event),
|
|
672
|
+
name="dstf-rescan",
|
|
673
|
+
daemon=True,
|
|
674
|
+
).start()
|
|
675
|
+
else:
|
|
676
|
+
done_event.set()
|
|
677
|
+
except BaseException:
|
|
678
|
+
# An open that fails takes back what it named: a page draws the
|
|
679
|
+
# open project's rows (or none) again.
|
|
680
|
+
self._emit("known", models=[])
|
|
681
|
+
raise
|
|
682
|
+
|
|
683
|
+
@property
|
|
684
|
+
def overlay_index_ready(self) -> threading.Event:
|
|
685
|
+
"""The open project's `Project.overlays_ready`; with none open, an
|
|
686
|
+
Event nothing sets."""
|
|
687
|
+
project = self.project
|
|
688
|
+
return project.overlays_ready if project is not None else threading.Event()
|
|
689
|
+
|
|
690
|
+
def _warm_overlay_index(self, project: Project) -> None:
|
|
691
|
+
from ..reporting.overlay_loader import warm_sibling_backend_index
|
|
692
|
+
|
|
693
|
+
ready = project.overlays_ready
|
|
694
|
+
try:
|
|
695
|
+
warm_sibling_backend_index(project.config)
|
|
696
|
+
except Exception: # a broken sibling file must not kill the thread silently
|
|
697
|
+
logger.exception("warming the sibling-backend overlay index failed")
|
|
698
|
+
ready.set()
|
|
699
|
+
return
|
|
700
|
+
ready.set()
|
|
701
|
+
try:
|
|
702
|
+
self._post(lambda: self._overlays_ready(project))
|
|
703
|
+
except Exception: # the UI loop is gone (a shutdown): no page to redraw
|
|
704
|
+
logger.debug("the sibling index is ready and the UI loop is gone")
|
|
705
|
+
|
|
706
|
+
def _overlays_ready(self, project: Project) -> None:
|
|
707
|
+
"""The sibling index is built: name the models read without it
|
|
708
|
+
(`_reads_for`) in ONE ``overlays`` event, so a page redraws them
|
|
709
|
+
and their next read gains the sibling-backend overlays. The reads
|
|
710
|
+
are kept, not dropped: they hold what was read from the result file,
|
|
711
|
+
which a running model cannot read again until its run ends.
|
|
712
|
+
Nothing for a project no longer open."""
|
|
713
|
+
if self.project is not project:
|
|
714
|
+
return
|
|
715
|
+
stale = {m for m, reads in self._reads.items() if not reads.siblings}
|
|
716
|
+
if stale:
|
|
717
|
+
self._emit("overlays", models=stale)
|
|
718
|
+
|
|
719
|
+
def _rescan(
|
|
720
|
+
self, project: Project, cached: dict[str, TestModel], done: threading.Event
|
|
721
|
+
) -> None:
|
|
722
|
+
"""A cached open trusts the cache; this finds out whether it should
|
|
723
|
+
have. Runs the full scan off the UI thread; everything that touches
|
|
724
|
+
the Session — the re-merge AND the events — goes through ``_post``
|
|
725
|
+
(the UI loop, where every other mutation happens)."""
|
|
726
|
+
try:
|
|
727
|
+
fresh = scan_source_tests(project.config)
|
|
728
|
+
except Exception:
|
|
729
|
+
logger.exception("background rescan failed")
|
|
730
|
+
|
|
731
|
+
def failed() -> None:
|
|
732
|
+
try:
|
|
733
|
+
self._emit("loading", phase="rescan", changed=False, error=True)
|
|
734
|
+
finally:
|
|
735
|
+
done.set()
|
|
736
|
+
|
|
737
|
+
self._post_or_run(failed)
|
|
738
|
+
return
|
|
739
|
+
changed_models = {
|
|
740
|
+
m for m in set(fresh) | set(cached) if fresh.get(m) != cached.get(m)
|
|
741
|
+
}
|
|
742
|
+
if changed_models:
|
|
743
|
+
store_scan(project.config, fresh)
|
|
744
|
+
|
|
745
|
+
def apply() -> None:
|
|
746
|
+
try:
|
|
747
|
+
# The project may have been swapped or closed meanwhile.
|
|
748
|
+
if changed_models and self.project is project:
|
|
749
|
+
self._take_rescan(project, fresh, changed_models)
|
|
750
|
+
self._emit("loading", phase="rescan", changed=bool(changed_models))
|
|
751
|
+
finally:
|
|
752
|
+
done.set()
|
|
753
|
+
|
|
754
|
+
self._post_or_run(apply)
|
|
755
|
+
|
|
756
|
+
def _take_rescan(
|
|
757
|
+
self, project: Project, fresh: dict[str, TestModel], changed: set[str]
|
|
758
|
+
) -> None:
|
|
759
|
+
"""On the UI loop: the background rescan's scan in place of the
|
|
760
|
+
project's. What a Queue click read since the open stays: the click
|
|
761
|
+
read its tests' files at a stamp it kept, the rescan read them at a
|
|
762
|
+
moment nobody stamped — before the click's, for all anyone knows —
|
|
763
|
+
and a waiting item must not be handed the older test."""
|
|
764
|
+
kept = {m: project.scanned[m] for m in self._read_at if m in project.scanned}
|
|
765
|
+
located = {
|
|
766
|
+
m: found
|
|
767
|
+
for m in self._read_at
|
|
768
|
+
if m not in project.scanned and (found := project.locate(m)) is not None
|
|
769
|
+
}
|
|
770
|
+
scan = {**fresh, **kept}
|
|
771
|
+
if kept:
|
|
772
|
+
self._store_scan_later(project.config, scan)
|
|
773
|
+
changed = changed - set(kept)
|
|
774
|
+
if changed:
|
|
775
|
+
# Only what changed is rescored: a one-file edit over
|
|
776
|
+
# TRANSFORM must not run 331 comparisons on the UI loop.
|
|
777
|
+
# A running model is not re-read: its directory is being
|
|
778
|
+
# rewritten, and its result arrives when it ends.
|
|
779
|
+
# The changed models' results are re-read with their new
|
|
780
|
+
# TestModels before they are rescored: a model the cache
|
|
781
|
+
# did not name was read as "not found in discovery"
|
|
782
|
+
# (SIM_FAIL where a cold open says NO_REF), and a model
|
|
783
|
+
# whose tracked variables changed was read with the old
|
|
784
|
+
# list.
|
|
785
|
+
self._rebuild_tests(
|
|
786
|
+
changed - self._running_models(),
|
|
787
|
+
scanned=scan,
|
|
788
|
+
# A new scan is the library read again: so are the
|
|
789
|
+
# spec-only models.
|
|
790
|
+
locate=SpecModelLocator(project.config).having(located),
|
|
791
|
+
)
|
|
792
|
+
|
|
793
|
+
def _post_or_run(self, fn: Callable[[], None]) -> None:
|
|
794
|
+
"""Hand ``fn`` to the UI loop; when the loop is gone (``post``
|
|
795
|
+
raises) run it here, so a waiter on the Event is never left
|
|
796
|
+
hanging."""
|
|
797
|
+
try:
|
|
798
|
+
self._post(fn)
|
|
799
|
+
except BaseException:
|
|
800
|
+
fn()
|
|
801
|
+
|
|
802
|
+
def close(self) -> None:
|
|
803
|
+
self._refuse_if_busy()
|
|
804
|
+
self._reset_queue()
|
|
805
|
+
self.project = None
|
|
806
|
+
self.artifacts = []
|
|
807
|
+
self.notices = []
|
|
808
|
+
self.spec = SpecDocument(Path("test_spec.json"), {"tests": []}, None)
|
|
809
|
+
self.comparisons = {}
|
|
810
|
+
self._reads = {}
|
|
811
|
+
self._emit_spec(set())
|
|
812
|
+
|
|
813
|
+
def reload(self) -> None:
|
|
814
|
+
"""Re-read the spec from disk, dropping pending ops (what a refused
|
|
815
|
+
save offers). A reload that cannot be done leaves the Session as it
|
|
816
|
+
was — its spec, its pending ops, its tests: a file that is no spec
|
|
817
|
+
(`ConfigError`, the decoder's and the merge's), and a file that was
|
|
818
|
+
there when it was loaded and is gone now. `SpecDocument.load` reads
|
|
819
|
+
a missing file as the empty document, which is right for a project
|
|
820
|
+
that never had one; over a deleted file it emptied the Suite."""
|
|
821
|
+
project = self._require_project()
|
|
822
|
+
held = self.spec
|
|
823
|
+
spec = SpecDocument.load(project.config.spec_path)
|
|
824
|
+
if spec.content_hash is None and held.content_hash is not None:
|
|
825
|
+
raise ConfigError(
|
|
826
|
+
f"{spec.path} is gone: nothing was reloaded — put the file "
|
|
827
|
+
"back, then Reload"
|
|
828
|
+
)
|
|
829
|
+
self.spec = spec
|
|
830
|
+
try:
|
|
831
|
+
self._rebuild_tests(set(self.comparisons))
|
|
832
|
+
except BaseException:
|
|
833
|
+
# The merge reads before it replaces anything (`_rebuild_tests`).
|
|
834
|
+
self.spec = held
|
|
835
|
+
raise
|
|
836
|
+
|
|
837
|
+
def wait_background(self, timeout: float | None = None) -> bool:
|
|
838
|
+
"""Wait until what the Session runs off the UI loop — a drained
|
|
839
|
+
queue's ``status.json`` and dashboard — has finished; False on a
|
|
840
|
+
timeout."""
|
|
841
|
+
return self._background.wait(timeout)
|
|
842
|
+
|
|
843
|
+
def on(self, event: Event, cb: Listener) -> None:
|
|
844
|
+
self._listeners.setdefault(event, []).append(cb)
|
|
845
|
+
|
|
846
|
+
# -- scoring and figures ----------------------------------------------
|
|
847
|
+
|
|
848
|
+
def rescore(self, model_id: str) -> TestComparison:
|
|
849
|
+
"""Score ``model_id`` against the spec with pending ops applied. A
|
|
850
|
+
flat-override test first gets the same ``add /metrics`` op
|
|
851
|
+
``spec-update`` prepends, so the document the tool edits and the one
|
|
852
|
+
it saves never differ in shape — but only when that tree scores
|
|
853
|
+
exactly what the flat path scored (Conventions #7); otherwise the
|
|
854
|
+
test is scored as loaded and nothing is staged. That op does not
|
|
855
|
+
stay while it is the tree the saved entry scores by, whatever else
|
|
856
|
+
is pending: an entry's own tree written out is no edit of it
|
|
857
|
+
(`stage`). It stays where a pending op edited the overrides it is
|
|
858
|
+
built from."""
|
|
859
|
+
op, _why = self._materialization(model_id)
|
|
860
|
+
if op is not None:
|
|
861
|
+
# Marked as `stage` marks its own: the drawer previews it as
|
|
862
|
+
# nothing changed, with the note that Save writes it. And no
|
|
863
|
+
# act of its own: where it stays, a pending op of this test
|
|
864
|
+
# edited what it is built from, and it is part of THAT edit —
|
|
865
|
+
# one override staged, then a rescore, read "2 pending edits".
|
|
866
|
+
mine = [o.gesture for o in self.spec.pending if o.model_id == model_id]
|
|
867
|
+
self.stage(
|
|
868
|
+
[
|
|
869
|
+
PatchOp(
|
|
870
|
+
model_id,
|
|
871
|
+
op["op"],
|
|
872
|
+
op["path"],
|
|
873
|
+
op["value"],
|
|
874
|
+
materializes=self._saved_scoring_tree(self.spec, model_id),
|
|
875
|
+
)
|
|
876
|
+
],
|
|
877
|
+
part_of=mine[-1] if mine else None,
|
|
878
|
+
)
|
|
879
|
+
return self.comparisons[model_id]
|
|
880
|
+
comp = self._score(model_id)
|
|
881
|
+
self._emit("results", models={model_id})
|
|
882
|
+
return comp
|
|
883
|
+
|
|
884
|
+
def view(self, model_id: str, variable: str) -> VariableView:
|
|
885
|
+
"""The variable's view: its trajectories, overlays and every leaf
|
|
886
|
+
scoring it. A test with no result, a failed simulation or no baseline
|
|
887
|
+
gets the empty state — trajectories present or not, no leaves."""
|
|
888
|
+
from ..reporting.context_builder import build_trajectories
|
|
889
|
+
from ..reporting.overlay_loader import attach_overlays_to_trajectories
|
|
890
|
+
|
|
891
|
+
project = self._require_project()
|
|
892
|
+
if project.test(model_id) is None:
|
|
893
|
+
raise KeyError(model_id)
|
|
894
|
+
shown = self.live_runs.series(model_id, variable)
|
|
895
|
+
if shown is not None:
|
|
896
|
+
return self._unfinished_view(model_id, variable, *shown)
|
|
897
|
+
comp = self.comparisons.get(model_id)
|
|
898
|
+
result = project.results.get(model_id)
|
|
899
|
+
reads = self._reads_for(model_id)
|
|
900
|
+
ref = reads.ref
|
|
901
|
+
scored = list(comp.variables) if comp is not None and comp.evaluated else []
|
|
902
|
+
texts = self.variable_texts(model_id)
|
|
903
|
+
trajectories, _diag, nobaseline = build_trajectories(scored, result, ref, texts)
|
|
904
|
+
overlays = reads.overlays
|
|
905
|
+
attach_overlays_to_trajectories(trajectories, overlays)
|
|
906
|
+
attach_overlays_to_trajectories(nobaseline, overlays)
|
|
907
|
+
traj = next((t for t in trajectories if t["name"] == variable), None)
|
|
908
|
+
if traj is None:
|
|
909
|
+
nb = next((t for t in nobaseline if t["name"] == variable), None)
|
|
910
|
+
if nb is not None:
|
|
911
|
+
traj = {
|
|
912
|
+
"text": nb.get("text") or variable,
|
|
913
|
+
"act_time": nb["time"],
|
|
914
|
+
"act_values": nb["values"],
|
|
915
|
+
"overlays": nb["overlays"],
|
|
916
|
+
}
|
|
917
|
+
else:
|
|
918
|
+
series = self._file_series(model_id, variable)
|
|
919
|
+
# A series read straight out of the result file carries no
|
|
920
|
+
# expression — the result in memory is where one lives —
|
|
921
|
+
# but it is still in the page's label scope, so a
|
|
922
|
+
# plot-only name that collides with a tracked variable's
|
|
923
|
+
# expression reads the same here as in its plot note.
|
|
924
|
+
traj = (
|
|
925
|
+
{
|
|
926
|
+
"text": texts[variable].text if variable in texts else variable,
|
|
927
|
+
"act_time": series[0],
|
|
928
|
+
"act_values": series[1],
|
|
929
|
+
}
|
|
930
|
+
if series is not None
|
|
931
|
+
else {}
|
|
932
|
+
)
|
|
933
|
+
leaves = leaves_by_variable(comp.metric_tree if comp is not None else None).get(
|
|
934
|
+
variable, []
|
|
935
|
+
)
|
|
936
|
+
return view_from_trajectory(variable, traj, leaves)
|
|
937
|
+
|
|
938
|
+
def _unfinished_view(
|
|
939
|
+
self,
|
|
940
|
+
model_id: str,
|
|
941
|
+
variable: str,
|
|
942
|
+
series: tuple[np.ndarray, np.ndarray] | None,
|
|
943
|
+
samples: int,
|
|
944
|
+
) -> VariableView:
|
|
945
|
+
"""The view of a variable whose test has no result because of a
|
|
946
|
+
run (D151): the baseline's trace and the overlays as always, and as
|
|
947
|
+
the Actual ``series`` — the samples read so far of a run in
|
|
948
|
+
progress, a partial file's after one that did not finish — or none.
|
|
949
|
+
No leaf: nothing of it is scored. ``samples`` is how many the series
|
|
950
|
+
was decimated from, 0 for one handed in whole."""
|
|
951
|
+
from ..figures.view import decimate_view
|
|
952
|
+
from ..reporting.overlay_loader import attach_overlays_to_trajectories
|
|
953
|
+
|
|
954
|
+
reads = self._reads_for(model_id)
|
|
955
|
+
reference = reads.references.get(variable)
|
|
956
|
+
if reference is None:
|
|
957
|
+
texts = self.variable_texts(model_id)
|
|
958
|
+
ref = reads.ref or {}
|
|
959
|
+
_comp, kept = self._layout_source(model_id)
|
|
960
|
+
ref_var = next(
|
|
961
|
+
(v for v in ref.get("variables", []) if v.get("name") == variable),
|
|
962
|
+
None,
|
|
963
|
+
)
|
|
964
|
+
traj = {
|
|
965
|
+
"name": variable,
|
|
966
|
+
# What an unmapped companion is matched by (D132).
|
|
967
|
+
"expression": next(
|
|
968
|
+
(
|
|
969
|
+
v.expression
|
|
970
|
+
for v in (kept.variables if kept is not None else ())
|
|
971
|
+
if v.name == variable
|
|
972
|
+
),
|
|
973
|
+
"",
|
|
974
|
+
),
|
|
975
|
+
"text": texts[variable].text if variable in texts else variable,
|
|
976
|
+
"ref_time": ref.get("time", []) if ref_var else [],
|
|
977
|
+
"ref_values": ref_var["values"] if ref_var else [],
|
|
978
|
+
}
|
|
979
|
+
attach_overlays_to_trajectories([traj], reads.overlays)
|
|
980
|
+
reference = reads.references[variable] = decimate_view(
|
|
981
|
+
view_from_trajectory(variable, traj, []),
|
|
982
|
+
self._require_project().config.max_embedded_samples,
|
|
983
|
+
)
|
|
984
|
+
if series is None:
|
|
985
|
+
return reference
|
|
986
|
+
time, values = series
|
|
987
|
+
return replace(
|
|
988
|
+
reference,
|
|
989
|
+
act_time=time,
|
|
990
|
+
act_values=values,
|
|
991
|
+
# Nothing to compare before the run's first sample.
|
|
992
|
+
errors=error_series(reference.ref_time, reference.ref_values, time, values)
|
|
993
|
+
if len(time)
|
|
994
|
+
else (),
|
|
995
|
+
full_samples={**reference.full_samples, "act": samples or len(time)},
|
|
996
|
+
)
|
|
997
|
+
|
|
998
|
+
def _layout_source(
|
|
999
|
+
self, model_id: str
|
|
1000
|
+
) -> tuple[TestComparison | None, TestResult | None]:
|
|
1001
|
+
"""The comparison and the result ``model_id``'s LAYOUT is read from
|
|
1002
|
+
— the scored names its plots derive from, its tracked names, its
|
|
1003
|
+
labels: the Session's own, or, while a run has taken them (in
|
|
1004
|
+
progress, or ended with no result), the ones that run replaced
|
|
1005
|
+
(D151). Never a verdict's or a trajectory's source."""
|
|
1006
|
+
kept = self.live_runs.kept(model_id)
|
|
1007
|
+
if kept is not None:
|
|
1008
|
+
return kept
|
|
1009
|
+
return (
|
|
1010
|
+
self.comparisons.get(model_id),
|
|
1011
|
+
self._require_project().results.get(model_id),
|
|
1012
|
+
)
|
|
1013
|
+
|
|
1014
|
+
def _reads_for(self, model_id: str) -> _ModelReads:
|
|
1015
|
+
from ..reporting.overlay_loader import load_overlays
|
|
1016
|
+
|
|
1017
|
+
project = self._require_project()
|
|
1018
|
+
reads = self._reads.get(model_id)
|
|
1019
|
+
if reads is not None and not reads.siblings and project.overlays_ready.is_set():
|
|
1020
|
+
# Read before the index was ready: the overlays again, now with
|
|
1021
|
+
# the sibling-backend ones; the decimated bases carry the
|
|
1022
|
+
# overlays and go with them. What came from the result file stays.
|
|
1023
|
+
reads.overlays = load_overlays(
|
|
1024
|
+
project.store, model_id, config=project.config
|
|
1025
|
+
)
|
|
1026
|
+
reads.siblings = True
|
|
1027
|
+
reads.bases.clear()
|
|
1028
|
+
reads.references.clear()
|
|
1029
|
+
if reads is None:
|
|
1030
|
+
# Never wait on the sibling-backend index: while the open's warm
|
|
1031
|
+
# thread builds it, it holds the index's lock, and a read on the
|
|
1032
|
+
# UI loop waited as long (over TRANSFORM, with the reference
|
|
1033
|
+
# index's scan, past the browser's ping budget). Until it is
|
|
1034
|
+
# ready the read has no sibling overlays; `_overlays_ready`
|
|
1035
|
+
# names the model then, and its `overlays` event redraws the page.
|
|
1036
|
+
siblings = project.overlays_ready.is_set()
|
|
1037
|
+
reads = _ModelReads(
|
|
1038
|
+
ref=project.store.get_reference(model_id),
|
|
1039
|
+
overlays=load_overlays(
|
|
1040
|
+
project.store,
|
|
1041
|
+
model_id,
|
|
1042
|
+
config=project.config if siblings else None,
|
|
1043
|
+
),
|
|
1044
|
+
siblings=siblings,
|
|
1045
|
+
)
|
|
1046
|
+
self._reads[model_id] = reads
|
|
1047
|
+
return reads
|
|
1048
|
+
|
|
1049
|
+
def _test_dir(self, model_id: str) -> Path | None:
|
|
1050
|
+
from ..backends.base import last_test_keys
|
|
1051
|
+
|
|
1052
|
+
project = self._require_project()
|
|
1053
|
+
work_dir = project.config.work_dir
|
|
1054
|
+
key = last_test_keys(work_dir).get(model_id) if work_dir is not None else None
|
|
1055
|
+
return work_dir / key if key is not None and work_dir is not None else None
|
|
1056
|
+
|
|
1057
|
+
def tracked(self, model_id: str) -> list[str]:
|
|
1058
|
+
"""The test's TRACKED names, in order: the scored variables first,
|
|
1059
|
+
then the rest of what the test tracks of the loaded result
|
|
1060
|
+
(`tracked_results`: its UnitTests variables and patterns — not a
|
|
1061
|
+
variable a discarded criterion had re-read, which the result keeps),
|
|
1062
|
+
through `tracked_names`, the one rule. With no tree the plots derive
|
|
1063
|
+
from them; the Variables drawer groups them "Tracked"; and a
|
|
1064
|
+
variable a criterion may be added for is one of them — the scored
|
|
1065
|
+
names alone would lose a variable whose last criterion was just
|
|
1066
|
+
removed, and with it the way back to one."""
|
|
1067
|
+
from ..comparison.tree_spec import collect_variables, tree_variables
|
|
1068
|
+
|
|
1069
|
+
project = self._require_project()
|
|
1070
|
+
comp, result = self._layout_source(model_id)
|
|
1071
|
+
# A statistics leaf scores a stored number: its parsed `variable`
|
|
1072
|
+
# (its quantity path, or a diagnostic's name) is no variable, and
|
|
1073
|
+
# listed here the add form's picker offered `translation.*` paths
|
|
1074
|
+
# as "Other variables". A name a trajectory leaf scores too stays.
|
|
1075
|
+
spec = self._entry_tree(model_id)
|
|
1076
|
+
numbers = (
|
|
1077
|
+
set(collect_variables(spec)) - set(tree_variables(spec)) if spec else set()
|
|
1078
|
+
)
|
|
1079
|
+
scored = (
|
|
1080
|
+
[vc.name for vc in comp.variables if vc.name not in numbers]
|
|
1081
|
+
if comp is not None
|
|
1082
|
+
else []
|
|
1083
|
+
)
|
|
1084
|
+
test = project.test(model_id)
|
|
1085
|
+
if result is not None and test is not None:
|
|
1086
|
+
result = replace(result, variables=tracked_results(test, result))
|
|
1087
|
+
return tracked_names(scored, result)
|
|
1088
|
+
|
|
1089
|
+
def variable_texts(self, model_id: str) -> dict[str, VariableText]:
|
|
1090
|
+
"""``{name: VariableText}`` for this test — both readings of every
|
|
1091
|
+
variable its page can show, from the ONE scope builder the report
|
|
1092
|
+
uses (:func:`~dstf.reporting.context_builder.test_texts`).
|
|
1093
|
+
|
|
1094
|
+
The scope is the whole TEST: the loaded result's variables, the
|
|
1095
|
+
scored comparisons' and every series of the test's resolved plots,
|
|
1096
|
+
plot-only names included. Nothing narrower: a card title, its
|
|
1097
|
+
criteria rows, its pills and its legend entries each disambiguated
|
|
1098
|
+
over their own little set, and the same variable read four
|
|
1099
|
+
different ways on one page (the b147918 review).
|
|
1100
|
+
|
|
1101
|
+
A name the scope does not carry is its own label, and the page
|
|
1102
|
+
reads it that way. Nothing is ever keyed by either reading —
|
|
1103
|
+
titles, rows and captions read them; markers and ops do not."""
|
|
1104
|
+
from ..reporting.context_builder import test_texts
|
|
1105
|
+
|
|
1106
|
+
project = self._require_project()
|
|
1107
|
+
if project.test(model_id) is None:
|
|
1108
|
+
return {}
|
|
1109
|
+
comp, result = self._layout_source(model_id)
|
|
1110
|
+
return test_texts(
|
|
1111
|
+
list(comp.variables) if comp is not None else [],
|
|
1112
|
+
result,
|
|
1113
|
+
[name for plot in self.plots(model_id) for name in plot.series],
|
|
1114
|
+
)
|
|
1115
|
+
|
|
1116
|
+
def labels(self, model_id: str) -> dict[str, str]:
|
|
1117
|
+
"""``{name: what every human-facing surface of this test's page
|
|
1118
|
+
reads for it}`` — the ``text`` of :meth:`variable_texts`, so the
|
|
1119
|
+
criteria rows, the pills, the focus bar, the "Add a criterion for
|
|
1120
|
+
x" captions and the add form's options all say the same thing.
|
|
1121
|
+
|
|
1122
|
+
NOT the plain label: a surface that shows the muted name beside
|
|
1123
|
+
its title (the card title) reads ``variable_texts(...)[name].label``
|
|
1124
|
+
instead — the name is already there, so the title does not repeat
|
|
1125
|
+
it in parentheses."""
|
|
1126
|
+
return {
|
|
1127
|
+
name: entry.text for name, entry in self.variable_texts(model_id).items()
|
|
1128
|
+
}
|
|
1129
|
+
|
|
1130
|
+
def variable_names(self, model_id: str) -> list[str]:
|
|
1131
|
+
"""Every variable the last result holds: the tracked ones first, in
|
|
1132
|
+
the page's order, then the rest of the result file in its order."""
|
|
1133
|
+
project = self._require_project()
|
|
1134
|
+
if project.test(model_id) is None:
|
|
1135
|
+
raise KeyError(model_id)
|
|
1136
|
+
tracked = self.tracked(model_id)
|
|
1137
|
+
have = set(tracked)
|
|
1138
|
+
return tracked + [n for n in self._file_names(model_id) if n not in have]
|
|
1139
|
+
|
|
1140
|
+
def _file_names(self, model_id: str) -> tuple[str, ...]:
|
|
1141
|
+
"""Every name in the last result FILE, listed once and kept with the
|
|
1142
|
+
model's reads; ``()`` when the backend keeps no file it can list, or
|
|
1143
|
+
the model is running and it was not listed before."""
|
|
1144
|
+
project = self._require_project()
|
|
1145
|
+
reads = self._reads_for(model_id)
|
|
1146
|
+
if reads.names_read:
|
|
1147
|
+
return reads.names or ()
|
|
1148
|
+
if self._is_running(model_id):
|
|
1149
|
+
# The file its run is writing, once that can be listed (D151):
|
|
1150
|
+
# kept with reads that go when the run ends, so the finished
|
|
1151
|
+
# file is listed again.
|
|
1152
|
+
live = self.live_runs.names(model_id)
|
|
1153
|
+
if live is not None:
|
|
1154
|
+
reads.names, reads.names_read = live, True
|
|
1155
|
+
return reads.names or ()
|
|
1156
|
+
if self.live_runs.partial(model_id) is not None:
|
|
1157
|
+
# What its last run left, through the one parse its plots are
|
|
1158
|
+
# read through (`LiveRuns.names`): the same file, not listed a
|
|
1159
|
+
# second time.
|
|
1160
|
+
reads.names = self.live_runs.names(model_id)
|
|
1161
|
+
reads.names_read = True
|
|
1162
|
+
return reads.names or ()
|
|
1163
|
+
test_dir = self._test_dir(model_id)
|
|
1164
|
+
listed = (
|
|
1165
|
+
type(project.backend).result_names(test_dir)
|
|
1166
|
+
if test_dir is not None
|
|
1167
|
+
else None
|
|
1168
|
+
)
|
|
1169
|
+
reads.names = None if listed is None else tuple(listed)
|
|
1170
|
+
reads.names_read = True
|
|
1171
|
+
return reads.names or ()
|
|
1172
|
+
|
|
1173
|
+
def _unread_in_file(
|
|
1174
|
+
self, models: Iterable[str], tests: Sequence[TestModel]
|
|
1175
|
+
) -> set[str]:
|
|
1176
|
+
"""The ``models`` whose criteria — as ``tests`` merge them — name a
|
|
1177
|
+
variable the loaded result lacks and its result FILE holds — a
|
|
1178
|
+
backend that records every variable (Dymola, the JSON driver) has
|
|
1179
|
+
it already, and a re-read with the merged TestModel extracts it
|
|
1180
|
+
(the owner's O2). A name the file lacks (OpenModelica, FMPy, a
|
|
1181
|
+
typo) only a rerun records; the strip asks for that
|
|
1182
|
+
(`situation.new_variables`)."""
|
|
1183
|
+
project = self._require_project()
|
|
1184
|
+
by_id = {t.model_id: t for t in tests}
|
|
1185
|
+
out: set[str] = set()
|
|
1186
|
+
for model_id in models:
|
|
1187
|
+
missing = unrecorded_variables(
|
|
1188
|
+
by_id.get(model_id),
|
|
1189
|
+
project.results.get(model_id),
|
|
1190
|
+
project.config.diagnostic_variables,
|
|
1191
|
+
)
|
|
1192
|
+
if missing and not set(missing).isdisjoint(self._file_names(model_id)):
|
|
1193
|
+
out.add(model_id)
|
|
1194
|
+
return out
|
|
1195
|
+
|
|
1196
|
+
def _is_running(self, model_id: str) -> bool:
|
|
1197
|
+
"""The queue is running this model: its work dir is being rewritten,
|
|
1198
|
+
so the file reads serve what they already hold and open nothing —
|
|
1199
|
+
as `view` and `figure` always have, from memory. Read off the item
|
|
1200
|
+
itself, which the pool marks RUNNING before it clears the
|
|
1201
|
+
directory: the Session hears of the start a tick later."""
|
|
1202
|
+
item = self._live.get(model_id)
|
|
1203
|
+
return item is not None and item.state is ItemState.RUNNING
|
|
1204
|
+
|
|
1205
|
+
def _running_models(self) -> frozenset[str]:
|
|
1206
|
+
return frozenset(m for m in self._live if self._is_running(m))
|
|
1207
|
+
|
|
1208
|
+
def _file_series(
|
|
1209
|
+
self, model_id: str, variable: str
|
|
1210
|
+
) -> tuple[np.ndarray, np.ndarray] | None:
|
|
1211
|
+
reads = self._reads_for(model_id)
|
|
1212
|
+
if variable not in reads.series:
|
|
1213
|
+
if self._is_running(model_id):
|
|
1214
|
+
return None # not cached: the run's end answers it
|
|
1215
|
+
test_dir = self._test_dir(model_id)
|
|
1216
|
+
got = (
|
|
1217
|
+
type(self._require_project().backend).result_series(
|
|
1218
|
+
test_dir, [variable]
|
|
1219
|
+
)
|
|
1220
|
+
if test_dir is not None
|
|
1221
|
+
else {}
|
|
1222
|
+
)
|
|
1223
|
+
reads.series[variable] = got.get(variable)
|
|
1224
|
+
return reads.series[variable]
|
|
1225
|
+
|
|
1226
|
+
def layout(self, model_id: str) -> tuple[PlotExtra, ...]:
|
|
1227
|
+
"""The extras a gesture edits (`authoring.layout`): the effective
|
|
1228
|
+
``plots`` key with pending ops applied, ``()`` without one. It keeps
|
|
1229
|
+
a name the result lacks — it is what the spec says."""
|
|
1230
|
+
test = self._require_project().test(model_id)
|
|
1231
|
+
if test is None:
|
|
1232
|
+
raise KeyError(model_id)
|
|
1233
|
+
return tuple(test.plots or ())
|
|
1234
|
+
|
|
1235
|
+
def _plot_leaves(self, model_id: str) -> list[LeafSpec] | None:
|
|
1236
|
+
""":func:`~dstf.discovery.plots.plot_leaves` for this test: the
|
|
1237
|
+
leaves its plots derive from, ``None`` for one plot per tracked
|
|
1238
|
+
name. From the tree SPEC and what was compared — no baseline read."""
|
|
1239
|
+
test = self._require_project().test(model_id)
|
|
1240
|
+
if test is None:
|
|
1241
|
+
raise KeyError(model_id)
|
|
1242
|
+
comp, _result = self._layout_source(model_id)
|
|
1243
|
+
compared = [vc.name for vc in comp.variables] if comp is not None else []
|
|
1244
|
+
return plot_leaves(test, compared)
|
|
1245
|
+
|
|
1246
|
+
def plots_from_tree(self, model_id: str) -> bool:
|
|
1247
|
+
"""Whether the test's plots derive from a tree — the authored one,
|
|
1248
|
+
or the implicit tree of what was compared — rather than one per
|
|
1249
|
+
tracked variable or array. NOT whether there is a tree VIEW: a test
|
|
1250
|
+
with no baseline compares nothing, and an authored tree still says
|
|
1251
|
+
what its plots are."""
|
|
1252
|
+
return self._plot_leaves(model_id) is not None
|
|
1253
|
+
|
|
1254
|
+
def plots(self, model_id: str) -> tuple[ResolvedPlot, ...]:
|
|
1255
|
+
"""The test's plots as the page draws them
|
|
1256
|
+
(:func:`~dstf.discovery.plots.plots_of_test`, the rule the report draws
|
|
1257
|
+
by): one per scored variable or array family, in the order the tree
|
|
1258
|
+
first names it, decorated by the effective ``plots`` extras. The
|
|
1259
|
+
leaves come from the tree SPEC with pending ops applied — never
|
|
1260
|
+
:meth:`tree`, which materializes the tree and reads the baseline;
|
|
1261
|
+
no tree is one plot per tracked name or array. A UnitTests element
|
|
1262
|
+
never folds (O3); a failed run has none. While a model runs whose
|
|
1263
|
+
names were never read, nothing is missing."""
|
|
1264
|
+
project = self._require_project()
|
|
1265
|
+
test = project.test(model_id)
|
|
1266
|
+
if test is None:
|
|
1267
|
+
raise KeyError(model_id)
|
|
1268
|
+
self.variable_names(model_id) # reads the file's names, once
|
|
1269
|
+
reads = self._reads_for(model_id)
|
|
1270
|
+
comp, result = self._layout_source(model_id)
|
|
1271
|
+
plots = plots_of_test(
|
|
1272
|
+
test,
|
|
1273
|
+
[vc.name for vc in comp.variables] if comp is not None else [],
|
|
1274
|
+
result,
|
|
1275
|
+
listed=reads.names,
|
|
1276
|
+
)
|
|
1277
|
+
if reads.names_read:
|
|
1278
|
+
return plots
|
|
1279
|
+
# The model is running and its names were never read: a listing
|
|
1280
|
+
# not taken is not a file without the context and watch series, so
|
|
1281
|
+
# nothing is missing until its run has ended and the file is listed.
|
|
1282
|
+
return tuple(replace(plot, missing=()) for plot in plots)
|
|
1283
|
+
|
|
1284
|
+
def plot_figure(
|
|
1285
|
+
self,
|
|
1286
|
+
model_id: str,
|
|
1287
|
+
index: int,
|
|
1288
|
+
*,
|
|
1289
|
+
editing: Collection[str] | None = None,
|
|
1290
|
+
visibility: Visibility | None = None,
|
|
1291
|
+
theme: str = "default",
|
|
1292
|
+
) -> dict:
|
|
1293
|
+
"""The editable figure of plot ``index`` (IndexError past the end).
|
|
1294
|
+
A subject of one variable with no context is :meth:`figure`; a
|
|
1295
|
+
family, or context, is drawn over the same cached bases. A context
|
|
1296
|
+
view draws as a trace only — a context variable scored on another
|
|
1297
|
+
plot signs no leaf onto this one. ``editing``, ``visibility`` and
|
|
1298
|
+
``theme`` as in :meth:`figure`."""
|
|
1299
|
+
from ..figures.plotly import plot_figure
|
|
1300
|
+
|
|
1301
|
+
if index < 0:
|
|
1302
|
+
raise IndexError(index)
|
|
1303
|
+
plot = self.plots(model_id)[index]
|
|
1304
|
+
if len(plot.subject) == 1 and not plot.context:
|
|
1305
|
+
return self.figure(
|
|
1306
|
+
model_id,
|
|
1307
|
+
plot.subject[0],
|
|
1308
|
+
editing=editing,
|
|
1309
|
+
visibility=visibility,
|
|
1310
|
+
theme=theme,
|
|
1311
|
+
)
|
|
1312
|
+
return self._dressed(
|
|
1313
|
+
model_id,
|
|
1314
|
+
plot_figure(
|
|
1315
|
+
[self._editable_view(model_id, name) for name in plot.subject],
|
|
1316
|
+
None,
|
|
1317
|
+
context=[
|
|
1318
|
+
self._editable_view(model_id, name)
|
|
1319
|
+
for name in plot.context
|
|
1320
|
+
if name not in plot.missing
|
|
1321
|
+
],
|
|
1322
|
+
editable=True,
|
|
1323
|
+
rings_for=editing,
|
|
1324
|
+
visibility=visibility,
|
|
1325
|
+
theme=theme,
|
|
1326
|
+
),
|
|
1327
|
+
theme,
|
|
1328
|
+
)
|
|
1329
|
+
|
|
1330
|
+
def _dressed(self, model_id: str, figure: dict, theme: str) -> dict:
|
|
1331
|
+
"""``figure`` with what a run without a result adds to it
|
|
1332
|
+
(`LiveRuns.dress`), in the theme's muted text; as it is for every
|
|
1333
|
+
other model."""
|
|
1334
|
+
from ..figures.palette import palette_of
|
|
1335
|
+
|
|
1336
|
+
return self.live_runs.dress(model_id, figure, palette_of(theme).context)
|
|
1337
|
+
|
|
1338
|
+
def figure(
|
|
1339
|
+
self,
|
|
1340
|
+
model_id: str,
|
|
1341
|
+
variable: str,
|
|
1342
|
+
*,
|
|
1343
|
+
editing: Collection[str] | None = None,
|
|
1344
|
+
visibility: Visibility | None = None,
|
|
1345
|
+
theme: str = "default",
|
|
1346
|
+
) -> dict:
|
|
1347
|
+
"""The tool's figure is the editable one — handle rings a user can
|
|
1348
|
+
drag (``variable_figure(..., editable=True)``); the report's is not.
|
|
1349
|
+
``editing``: the leaf paths whose handles get a ring, ``None`` meaning
|
|
1350
|
+
every leaf; the Plotly ``config`` never depends on it. ``visibility``:
|
|
1351
|
+
what the page chose to show (legend b); the cached base does not
|
|
1352
|
+
depend on it. ``theme``: the bundled theme the figure is drawn in —
|
|
1353
|
+
the TAB's (D148), which is why it is an argument and nothing the
|
|
1354
|
+
Session holds; the cached base does not depend on it either."""
|
|
1355
|
+
from ..figures.plotly import variable_figure
|
|
1356
|
+
|
|
1357
|
+
if self._require_project().test(model_id) is None:
|
|
1358
|
+
raise KeyError(model_id) # before the cache: never cache a read for it
|
|
1359
|
+
return self._dressed(
|
|
1360
|
+
model_id,
|
|
1361
|
+
variable_figure(
|
|
1362
|
+
self._editable_view(model_id, variable),
|
|
1363
|
+
None,
|
|
1364
|
+
editable=True,
|
|
1365
|
+
rings_for=editing,
|
|
1366
|
+
visibility=visibility,
|
|
1367
|
+
theme=theme,
|
|
1368
|
+
),
|
|
1369
|
+
theme,
|
|
1370
|
+
)
|
|
1371
|
+
|
|
1372
|
+
def _editable_view(self, model_id: str, variable: str) -> VariableView:
|
|
1373
|
+
"""The decimated base (cached) joined to the leaves decimated on
|
|
1374
|
+
their own — what both figure reads draw, WITHOUT a budget.
|
|
1375
|
+
The base traces are decimated once per model and variable and
|
|
1376
|
+
cached until the model's scored list moves (`_ModelReads.scored`);
|
|
1377
|
+
only the leaves' geometry is decimated per call. The exception is a
|
|
1378
|
+
running model: nothing is cached for it, so every call there
|
|
1379
|
+
re-decimates the base."""
|
|
1380
|
+
from ..figures.view import build_view, decimate_view
|
|
1381
|
+
|
|
1382
|
+
project = self._require_project()
|
|
1383
|
+
budget = project.config.max_embedded_samples
|
|
1384
|
+
reads = self._reads_for(model_id)
|
|
1385
|
+
scored = _scored_key(self.comparisons.get(model_id))
|
|
1386
|
+
if reads.scored != scored:
|
|
1387
|
+
reads.bases.clear()
|
|
1388
|
+
reads.references.clear()
|
|
1389
|
+
reads.scored = scored
|
|
1390
|
+
base = reads.bases.get(variable)
|
|
1391
|
+
if base is None:
|
|
1392
|
+
base = decimate_view(
|
|
1393
|
+
replace(self.view(model_id, variable), leaves=()), budget
|
|
1394
|
+
)
|
|
1395
|
+
# Not cached while the model runs, for `_file_series`' reason: it
|
|
1396
|
+
# answers None there and caches nothing, so an untracked series'
|
|
1397
|
+
# base is EMPTY.
|
|
1398
|
+
if not self._is_running(model_id):
|
|
1399
|
+
reads.bases[variable] = base
|
|
1400
|
+
comp = self.comparisons.get(model_id)
|
|
1401
|
+
leaves = leaves_by_variable(comp.metric_tree if comp is not None else None).get(
|
|
1402
|
+
variable, []
|
|
1403
|
+
)
|
|
1404
|
+
# The leaves are decimated on their own (no arrays but their
|
|
1405
|
+
# geometry) and joined to the cached base; the figure is then built
|
|
1406
|
+
# WITHOUT a budget, or `variable_figure` would decimate the base a
|
|
1407
|
+
# second time. The base's counts win the merge: a leaves-only view
|
|
1408
|
+
# stamps "act"/"ref" as 0.
|
|
1409
|
+
leaves_view = decimate_view(build_view(variable, leaves=leaves), budget)
|
|
1410
|
+
full = dict(leaves_view.full_samples)
|
|
1411
|
+
full.update(base.full_samples)
|
|
1412
|
+
return replace(base, leaves=leaves_view.leaves, full_samples=full)
|
|
1413
|
+
|
|
1414
|
+
def tree_view(self, model_id: str) -> dict | None:
|
|
1415
|
+
"""The read-only tree the report shows for this test, built by the
|
|
1416
|
+
report's own function; ``None`` when nothing was compared."""
|
|
1417
|
+
from ..reporting.context_builder import build_tree_view
|
|
1418
|
+
|
|
1419
|
+
project = self._require_project()
|
|
1420
|
+
test = project.test(model_id)
|
|
1421
|
+
if test is None:
|
|
1422
|
+
raise KeyError(model_id)
|
|
1423
|
+
comp = self.comparisons.get(model_id)
|
|
1424
|
+
if comp is None or not comp.evaluated:
|
|
1425
|
+
return self._unscored_view(test)
|
|
1426
|
+
base = (
|
|
1427
|
+
(comp.base_tolerance, comp.tolerance_source)
|
|
1428
|
+
if comp.base_tolerance is not None and comp.tolerance_source is not None
|
|
1429
|
+
else None
|
|
1430
|
+
)
|
|
1431
|
+
return build_tree_view(
|
|
1432
|
+
list(comp.variables),
|
|
1433
|
+
test,
|
|
1434
|
+
comp.metric_tree,
|
|
1435
|
+
self.variable_texts(model_id),
|
|
1436
|
+
# the editor's "in effect" hints (K3); the report shows none
|
|
1437
|
+
base,
|
|
1438
|
+
)
|
|
1439
|
+
|
|
1440
|
+
def _unscored_view(self, test: TestModel) -> dict | None:
|
|
1441
|
+
"""The criteria of a test with nothing scored BECAUSE OF A RUN —
|
|
1442
|
+
in progress, or ended without finishing (D151) — as a tree view
|
|
1443
|
+
with a word in place of every verdict (`live_rows.unscored_view`):
|
|
1444
|
+
the tree as authored, pending edits included, else the implicit
|
|
1445
|
+
tree of what its last result scored. None for any other test, and
|
|
1446
|
+
for one with neither."""
|
|
1447
|
+
from ..comparison.tree_spec import synthesize_implicit_tree
|
|
1448
|
+
|
|
1449
|
+
model_id = test.model_id
|
|
1450
|
+
word = self.live_runs.word(model_id)
|
|
1451
|
+
if word is None:
|
|
1452
|
+
return None
|
|
1453
|
+
spec = test.metric_tree_spec
|
|
1454
|
+
if spec is None:
|
|
1455
|
+
comp, _result = self._layout_source(model_id)
|
|
1456
|
+
compared = [vc.name for vc in comp.variables] if comp is not None else []
|
|
1457
|
+
if not compared:
|
|
1458
|
+
return None
|
|
1459
|
+
spec = synthesize_implicit_tree(
|
|
1460
|
+
compared, variable_overrides=test.variable_overrides
|
|
1461
|
+
)
|
|
1462
|
+
return unscored_view(spec, self.variable_texts(model_id), word)
|
|
1463
|
+
|
|
1464
|
+
def partial(self, model_id: str) -> Partial | None:
|
|
1465
|
+
"""What ``model_id``'s last run left when it did not finish — how
|
|
1466
|
+
it ended and the time its file reached — or None. Shown on its
|
|
1467
|
+
page; never scored, never a baseline."""
|
|
1468
|
+
return self.live_runs.partial(model_id)
|
|
1469
|
+
|
|
1470
|
+
def statistics(self, model_id: str) -> list[StatRow]:
|
|
1471
|
+
"""The test's Statistics rows (:func:`~dstf.reporting.stats_rows.
|
|
1472
|
+
statistics_rows`): the last run's statistics beside its baseline's,
|
|
1473
|
+
plus a row for every quantity a statistics leaf of the entry's tree
|
|
1474
|
+
reads that no recorded statistic holds. The baseline's rows alone
|
|
1475
|
+
with no result; with neither, only those leaves' "other scored"
|
|
1476
|
+
rows (``[]`` when the tree reads no statistic). Reads the CACHED
|
|
1477
|
+
reference, which `statistics_rows` never writes. An unknown model is a
|
|
1478
|
+
KeyError, as for :meth:`tree`."""
|
|
1479
|
+
from ..comparison.tree_spec import statistic_paths
|
|
1480
|
+
from ..reporting.stats_rows import statistics_rows
|
|
1481
|
+
|
|
1482
|
+
project = self._require_project()
|
|
1483
|
+
if project.test(model_id) is None:
|
|
1484
|
+
raise KeyError(model_id)
|
|
1485
|
+
result = project.results.get(model_id)
|
|
1486
|
+
ref = self._reads_for(model_id).ref or {}
|
|
1487
|
+
return statistics_rows(
|
|
1488
|
+
result.statistics if result is not None else None,
|
|
1489
|
+
ref.get("statistics"),
|
|
1490
|
+
run_diagnostics=(result.diagnostics or []) if result is not None else (),
|
|
1491
|
+
baseline_diagnostics=ref.get("diagnostics") or (),
|
|
1492
|
+
extra_paths=statistic_paths(self._entry_tree(model_id)),
|
|
1493
|
+
)
|
|
1494
|
+
|
|
1495
|
+
def statistic_leaves(self, model_id: str) -> list[tuple[LeafSpec, str]]:
|
|
1496
|
+
"""The statistics leaves of the entry's OWN tree, staged edits
|
|
1497
|
+
included, with their paths (:func:`~dstf.comparison.tree_spec.
|
|
1498
|
+
statistic_leaves`): which Statistics rows are scored. Read from the
|
|
1499
|
+
tree as authored, not from :meth:`tree_view` — a test with nothing
|
|
1500
|
+
compared (no baseline) has no view, and its criteria are criteria
|
|
1501
|
+
all the same. An unknown model is a KeyError."""
|
|
1502
|
+
from ..comparison.tree_spec import statistic_leaves
|
|
1503
|
+
|
|
1504
|
+
if self._require_project().test(model_id) is None:
|
|
1505
|
+
raise KeyError(model_id)
|
|
1506
|
+
return statistic_leaves(self._entry_tree(model_id))
|
|
1507
|
+
|
|
1508
|
+
def _entry_tree(self, model_id: str) -> SpecNode | None:
|
|
1509
|
+
"""The entry's OWN tree, parsed; None when it has none. Never the
|
|
1510
|
+
tree :meth:`tree` materializes for a flat entry: that reads the
|
|
1511
|
+
baseline, and an implicit tree holds no statistics leaf."""
|
|
1512
|
+
from ..comparison.tree_spec import parse_metric_tree
|
|
1513
|
+
|
|
1514
|
+
raw = (self.spec.entry(model_id) or {}).get("metrics")
|
|
1515
|
+
return parse_metric_tree(raw) if raw is not None else None
|
|
1516
|
+
|
|
1517
|
+
def tree(self, model_id: str) -> dict | None:
|
|
1518
|
+
"""The raw tree a Patch edits (:meth:`tree_or_reason`'s tree)."""
|
|
1519
|
+
return self.tree_or_reason(model_id)[0]
|
|
1520
|
+
|
|
1521
|
+
def tree_or_reason(self, model_id: str) -> tuple[dict | None, str | None]:
|
|
1522
|
+
"""``(tree, None)`` — the raw tree a Patch edits: the entry's
|
|
1523
|
+
``metrics`` block, else the tree ``rescore`` would materialize (the
|
|
1524
|
+
report's leaves) — or ``(None, why)``: why there is none, the ONE
|
|
1525
|
+
text every surface says it with (`stage`'s refusal, a criterion's
|
|
1526
|
+
editor, Edit tree, Edit on plot, a Statistics row's `+`), from
|
|
1527
|
+
`_materialization`'s own refusal, so the baseline is read once for
|
|
1528
|
+
both. The tree is a copy: a capability may build its next op from it
|
|
1529
|
+
without touching the document. An unknown model is a KeyError, as
|
|
1530
|
+
for ``tree_view``."""
|
|
1531
|
+
if self._require_project().test(model_id) is None:
|
|
1532
|
+
raise KeyError(model_id)
|
|
1533
|
+
entry = self.spec.entry(model_id)
|
|
1534
|
+
if has_metrics(entry):
|
|
1535
|
+
assert entry is not None # `has_metrics` said so
|
|
1536
|
+
return copy.deepcopy(entry["metrics"]), None
|
|
1537
|
+
op, why = self._materialization(model_id)
|
|
1538
|
+
return (op["value"], None) if op is not None else (None, why)
|
|
1539
|
+
|
|
1540
|
+
# -- the spec ----------------------------------------------------------
|
|
1541
|
+
|
|
1542
|
+
def stage(self, ops: Sequence[PatchOp], *, part_of: int | None = None) -> None:
|
|
1543
|
+
"""Stage ``ops`` and rescore their models. Materializes the implicit
|
|
1544
|
+
tree where a per-leaf op needs one — or an op that edits the tree
|
|
1545
|
+
whole (`PatchOp.edits_tree`); validates each touched entry
|
|
1546
|
+
(paths, the parsed tree, the D66 roles). On any error the Session
|
|
1547
|
+
is unchanged. A queued test takes every edit, and runs with the spec
|
|
1548
|
+
as it stands when it starts; a running one takes its criteria's —
|
|
1549
|
+
scored when its result arrives — and none of its simulation
|
|
1550
|
+
settings', which its run has already taken (D150).
|
|
1551
|
+
|
|
1552
|
+
One call is one edit (`PatchOp.gesture`), unless ``part_of`` names
|
|
1553
|
+
the edit these ops belong to: `rescore`'s, which no one did."""
|
|
1554
|
+
project = self._require_project()
|
|
1555
|
+
models = {op.model_id for op in ops}
|
|
1556
|
+
for op in ops:
|
|
1557
|
+
if self._is_running(op.model_id) and _in_simulation(op.path):
|
|
1558
|
+
raise QueueError(
|
|
1559
|
+
f"{op.model_id} is running: its simulation settings are "
|
|
1560
|
+
"locked until its run ends"
|
|
1561
|
+
)
|
|
1562
|
+
staged: list[PatchOp] = []
|
|
1563
|
+
# One call is one act of the user's, however many ops it takes
|
|
1564
|
+
# (`PatchOp.gesture`): what every count of pending edits counts.
|
|
1565
|
+
self._gesture += 1
|
|
1566
|
+
gesture = self._gesture if part_of is None else part_of
|
|
1567
|
+
for model_id in _in_order(op.model_id for op in ops):
|
|
1568
|
+
raw = [op.raw() for op in ops if op.model_id == model_id]
|
|
1569
|
+
entry = self.spec.entry(model_id) or {"model": model_id}
|
|
1570
|
+
# A gesture on a folded row gives the tree WHOLE (D152): on a
|
|
1571
|
+
# flat entry it is an edit of the implicit tree all the same,
|
|
1572
|
+
# and that tree is written out first, as for a per-leaf op — or
|
|
1573
|
+
# the preview showed every line of the tree as added, with no
|
|
1574
|
+
# note that Save writes it.
|
|
1575
|
+
edits_whole = not has_metrics(entry) and any(
|
|
1576
|
+
op.edits_tree for op in ops if op.model_id == model_id
|
|
1577
|
+
)
|
|
1578
|
+
if needs_materialization(entry, raw) or edits_whole:
|
|
1579
|
+
# The same tree rescore stages — from what the flat path
|
|
1580
|
+
# scored — never one from the entry's list: on both tiers
|
|
1581
|
+
# leaf i is /metrics/children/i of the report's tree.
|
|
1582
|
+
tree_op, why = self._materialization(model_id)
|
|
1583
|
+
if tree_op is None:
|
|
1584
|
+
assert why is not None # the entry has no tree
|
|
1585
|
+
raise no_tree_error(model_id, why)
|
|
1586
|
+
# Marked: the drawer diffs against the saved entry with this
|
|
1587
|
+
# tree in it (`SpecDocument.preview_base`) — but only when it
|
|
1588
|
+
# is the tree the SAVED entry scores. Built from overrides an
|
|
1589
|
+
# earlier pending op edited, it is not: unmarked, it shows as
|
|
1590
|
+
# added, and the note does not say "they score the same".
|
|
1591
|
+
staged.append(
|
|
1592
|
+
PatchOp(
|
|
1593
|
+
model_id,
|
|
1594
|
+
tree_op["op"],
|
|
1595
|
+
tree_op["path"],
|
|
1596
|
+
tree_op.get("value"),
|
|
1597
|
+
materializes=self._saved_scoring_tree(self.spec, model_id),
|
|
1598
|
+
gesture=gesture,
|
|
1599
|
+
)
|
|
1600
|
+
)
|
|
1601
|
+
# The ops as given: `rescore`'s materialization carries its mark.
|
|
1602
|
+
# Each value a copy: the caller's dict stays the caller's, and
|
|
1603
|
+
# editing it after staging changes neither export nor Save.
|
|
1604
|
+
staged += [
|
|
1605
|
+
replace(op, value=copy.deepcopy(op.value), gesture=gesture)
|
|
1606
|
+
for op in ops
|
|
1607
|
+
if op.model_id == model_id
|
|
1608
|
+
]
|
|
1609
|
+
# Convention 8: an edit that lands on the path the last pending op
|
|
1610
|
+
# wrote supersedes it, so the pending count and the envelope carry
|
|
1611
|
+
# one op, never a keystroke log (the drawer shows the entry's diff
|
|
1612
|
+
# either way): a replace after an add or a replace of the same path
|
|
1613
|
+
# is that op with the new value, a remove after a replace is the
|
|
1614
|
+
# remove, and an add after an add on an OBJECT member is the second
|
|
1615
|
+
# add — it replaces what the first put there (two moves in a row:
|
|
1616
|
+
# two whole trees). An add after an add at a list INDEX inserts
|
|
1617
|
+
# again, and both stay. A remove after an ADD is
|
|
1618
|
+
# kept as two: when the member existed, the add replaced it, and
|
|
1619
|
+
# dropping both would leave it. A batch is never collapsed: its ops
|
|
1620
|
+
# belong together. A replace at an append path (``/-``) names no
|
|
1621
|
+
# member and is left for `effective` to refuse, never folded in. A
|
|
1622
|
+
# fold onto the op that materialized the tree is no longer that
|
|
1623
|
+
# tree: the folded op is not marked (`PatchOp.materializes`). What
|
|
1624
|
+
# the replaced op said it did (`PatchOp.notes`) is kept with what
|
|
1625
|
+
# the new one says: under one key the new sentence, else both.
|
|
1626
|
+
pending = list(self.spec.pending)
|
|
1627
|
+
if len(staged) == 1 and pending:
|
|
1628
|
+
new, last = staged[0], pending[-1]
|
|
1629
|
+
same = (new.model_id, new.path) == (last.model_id, last.path)
|
|
1630
|
+
if same and not new.path.endswith("/-"):
|
|
1631
|
+
# The op that supersedes is the SAME edit as the one it
|
|
1632
|
+
# replaces (its `gesture`): a value typed over is one.
|
|
1633
|
+
if new.op == "replace" and last.op in ("add", "replace"):
|
|
1634
|
+
pending.pop()
|
|
1635
|
+
staged = [
|
|
1636
|
+
PatchOp(
|
|
1637
|
+
new.model_id,
|
|
1638
|
+
last.op,
|
|
1639
|
+
new.path,
|
|
1640
|
+
new.value,
|
|
1641
|
+
gesture=last.gesture,
|
|
1642
|
+
)
|
|
1643
|
+
]
|
|
1644
|
+
elif (new.op, last.op) == ("remove", "replace"):
|
|
1645
|
+
pending.pop()
|
|
1646
|
+
staged = [replace(new, gesture=last.gesture)]
|
|
1647
|
+
elif (new.op, last.op) == ("add", "add") and not new.path.rsplit(
|
|
1648
|
+
"/", 1
|
|
1649
|
+
)[-1].isdigit():
|
|
1650
|
+
pending.pop()
|
|
1651
|
+
said = {**dict(last.notes), **dict(new.notes)}
|
|
1652
|
+
staged = [
|
|
1653
|
+
replace(new, notes=tuple(said.items()), gesture=last.gesture)
|
|
1654
|
+
]
|
|
1655
|
+
candidate = SpecDocument(
|
|
1656
|
+
self.spec.path,
|
|
1657
|
+
self.spec.loaded,
|
|
1658
|
+
self.spec.content_hash,
|
|
1659
|
+
[*pending, *staged],
|
|
1660
|
+
)
|
|
1661
|
+
effective = candidate.effective() # PatchError on a bad path
|
|
1662
|
+
for model_id in models:
|
|
1663
|
+
# A key nothing reads is refused at once, as Save's own check
|
|
1664
|
+
# refuses it, with the sentence alone. A criterion that can
|
|
1665
|
+
# never be scored is NOT refused here: it is staged, its row
|
|
1666
|
+
# says why it is not scored, and Save refuses the write that
|
|
1667
|
+
# makes it (`spec_update.refuse_newly_unscorable`). Refused
|
|
1668
|
+
# here, a bare `range` could not be added for its editor to
|
|
1669
|
+
# fill, and an entry that already held one took no edit at all
|
|
1670
|
+
# — not the one that filled it.
|
|
1671
|
+
validate_entry_keys(effective, model_id) # PatchError
|
|
1672
|
+
validate_entry_metric_tree(
|
|
1673
|
+
effective, model_id, project.config, project.store
|
|
1674
|
+
) # PatchError
|
|
1675
|
+
# An edit that leaves an entry exactly as saved is no pending edit
|
|
1676
|
+
# of it — a value typed and cleared again, a refused one put back:
|
|
1677
|
+
# its ops counted ("1 pending edit", Save enabled), and Save rewrote
|
|
1678
|
+
# the file for nothing. Decided HERE, on the folded list, the one
|
|
1679
|
+
# thing the count, the strip, the Tests drawer's mark, Save and the
|
|
1680
|
+
# preview read: every op of such a model goes. After the checks
|
|
1681
|
+
# above, so a stage that is refused is refused as it was; the model
|
|
1682
|
+
# is still rescored below, back to its saved verdict. A flat entry's
|
|
1683
|
+
# own tree written out is no edit of it either
|
|
1684
|
+
# (`_its_tree_written_out`): the first per-leaf edit writes that
|
|
1685
|
+
# tree, and the edit put back left it — "2 unsaved", and a Save that
|
|
1686
|
+
# added a `metrics` block scoring as the entry already did. Those
|
|
1687
|
+
# ops go FIRST and the rest is then asked whether it is as saved:
|
|
1688
|
+
# beside a plots edit the tree goes and the plots edit stays, and
|
|
1689
|
+
# with nothing else changed nothing is left.
|
|
1690
|
+
gone = [op for m in models for op in self._its_tree_written_out(candidate, m)]
|
|
1691
|
+
if gone:
|
|
1692
|
+
candidate = SpecDocument(
|
|
1693
|
+
self.spec.path,
|
|
1694
|
+
self.spec.loaded,
|
|
1695
|
+
self.spec.content_hash,
|
|
1696
|
+
# By identity: an op equal to a dropped one (a second
|
|
1697
|
+
# `add /metrics` of the same tree) is dropped as itself.
|
|
1698
|
+
[op for op in candidate.pending if not any(op is g for g in gone)],
|
|
1699
|
+
)
|
|
1700
|
+
put_back = {m for m in models if candidate.as_saved(m)}
|
|
1701
|
+
if put_back:
|
|
1702
|
+
candidate = SpecDocument(
|
|
1703
|
+
self.spec.path,
|
|
1704
|
+
self.spec.loaded,
|
|
1705
|
+
self.spec.content_hash,
|
|
1706
|
+
[op for op in candidate.pending if op.model_id not in put_back],
|
|
1707
|
+
)
|
|
1708
|
+
if gone or put_back:
|
|
1709
|
+
effective = candidate.effective()
|
|
1710
|
+
tests = self._merge(
|
|
1711
|
+
project.config,
|
|
1712
|
+
project.scanned,
|
|
1713
|
+
effective,
|
|
1714
|
+
project.locate,
|
|
1715
|
+
edited=bool(candidate.pending),
|
|
1716
|
+
) # ConfigError on a bad tree
|
|
1717
|
+
# A criterion on a variable the result file holds but the last read
|
|
1718
|
+
# did not extract: read again with the merged TestModel, which now
|
|
1719
|
+
# tracks it, so it scores at once — no rerun (the owner's O2). Read
|
|
1720
|
+
# BEFORE anything is replaced: a read that raises (the disk went
|
|
1721
|
+
# away) leaves the Session as it was, as the docstring promises.
|
|
1722
|
+
# Read directly, not through `_rebuild_tests(scanned=...)`: the
|
|
1723
|
+
# effective spec is merged just above (on TRANSFORM a merge is most
|
|
1724
|
+
# of a stage's cost; that would do it twice), and it would re-read
|
|
1725
|
+
# AND rescore only the models it is handed — the other staged models
|
|
1726
|
+
# must be rescored whether or not theirs is re-read.
|
|
1727
|
+
reread = self._unread_in_file(models, tests)
|
|
1728
|
+
fresh = self._read_again(reread, tests) if reread else {}
|
|
1729
|
+
self.spec = candidate
|
|
1730
|
+
project.tests = tests
|
|
1731
|
+
self._update_waiting()
|
|
1732
|
+
if reread:
|
|
1733
|
+
self._keep_read(reread, fresh)
|
|
1734
|
+
self._emit_spec(models)
|
|
1735
|
+
self._score_all(models)
|
|
1736
|
+
|
|
1737
|
+
def discard(self, models: set[str] | None = None) -> None:
|
|
1738
|
+
"""Drop the pending ops of ``models`` — every pending op when
|
|
1739
|
+
``None`` — and restore their loaded verdicts; the other models' ops
|
|
1740
|
+
stay, in order. ``models`` none of which has a pending op change
|
|
1741
|
+
nothing, and emit nothing."""
|
|
1742
|
+
pending = self.spec.pending
|
|
1743
|
+
gone = {op.model_id for op in pending}
|
|
1744
|
+
if models is not None:
|
|
1745
|
+
gone &= models
|
|
1746
|
+
if not gone:
|
|
1747
|
+
return
|
|
1748
|
+
kept = [op for op in pending if op.model_id not in gone]
|
|
1749
|
+
self.spec = SpecDocument(
|
|
1750
|
+
self.spec.path, self.spec.loaded, self.spec.content_hash, kept
|
|
1751
|
+
)
|
|
1752
|
+
self._rebuild_tests(gone)
|
|
1753
|
+
|
|
1754
|
+
def saved_comparison(self, model_id: str) -> TestComparison | None:
|
|
1755
|
+
"""The test's comparison as SAVED — the spec as loaded, no pending
|
|
1756
|
+
op applied — beside ``comparisons``, which is the edited spec's.
|
|
1757
|
+
With nothing pending for the model it IS ``comparisons[model_id]``;
|
|
1758
|
+
otherwise it is scored afresh, as `_score` scores, writing nothing:
|
|
1759
|
+
not ``comparisons``, not ``project.tests``. ``None`` for a model the
|
|
1760
|
+
saved spec does not make a test (an entry a pending op creates).
|
|
1761
|
+
Uncached: the strip asks for its one test, per paint."""
|
|
1762
|
+
return self.saved_comparisons({model_id})[model_id]
|
|
1763
|
+
|
|
1764
|
+
def saved_comparisons(
|
|
1765
|
+
self, models: Collection[str]
|
|
1766
|
+
) -> dict[str, TestComparison | None]:
|
|
1767
|
+
"""`saved_comparison` of each of ``models``, the loaded spec merged
|
|
1768
|
+
ONCE for all of them (the pending drawer asks for every edited
|
|
1769
|
+
test's; a merge of TRANSFORM's spec is most of a call's cost)."""
|
|
1770
|
+
edited = {op.model_id for op in self.spec.pending}
|
|
1771
|
+
out: dict[str, TestComparison | None] = {
|
|
1772
|
+
m: self.comparisons.get(m) for m in models if m not in edited
|
|
1773
|
+
}
|
|
1774
|
+
rest = [m for m in models if m in edited]
|
|
1775
|
+
if rest:
|
|
1776
|
+
project = self._require_project()
|
|
1777
|
+
tests = {
|
|
1778
|
+
t.model_id: t
|
|
1779
|
+
for t in self._merge(
|
|
1780
|
+
project.config,
|
|
1781
|
+
project.scanned,
|
|
1782
|
+
self.spec.loaded,
|
|
1783
|
+
project.locate,
|
|
1784
|
+
edited=False,
|
|
1785
|
+
)
|
|
1786
|
+
}
|
|
1787
|
+
for m in rest:
|
|
1788
|
+
test = tests.get(m)
|
|
1789
|
+
out[m] = None if test is None else self._compare(test)
|
|
1790
|
+
return out
|
|
1791
|
+
|
|
1792
|
+
def save(self) -> None:
|
|
1793
|
+
"""Apply exactly the pending ops through ``spec-update``'s
|
|
1794
|
+
read-modify-write, then reload. Refuses when the file changed on
|
|
1795
|
+
disk since it was loaded, and when the edits leave a criterion that
|
|
1796
|
+
can never be scored (`UnscorableSaveError`: `stage` took it, for its
|
|
1797
|
+
editor to fill)."""
|
|
1798
|
+
project = self._require_project()
|
|
1799
|
+
if self.spec.on_disk_changed():
|
|
1800
|
+
raise StaleSpecError(self.spec.path)
|
|
1801
|
+
groups = self.spec.grouped()
|
|
1802
|
+
if groups:
|
|
1803
|
+
try:
|
|
1804
|
+
apply_validated_patches(
|
|
1805
|
+
self.spec.path, groups, project.config, project.store
|
|
1806
|
+
)
|
|
1807
|
+
except UnscorableCriterion as e:
|
|
1808
|
+
raise UnscorableSaveError(e.reason) from e
|
|
1809
|
+
models = {model_id for model_id, _ in groups}
|
|
1810
|
+
self.spec = SpecDocument.load(self.spec.path)
|
|
1811
|
+
self._rebuild_tests(models)
|
|
1812
|
+
|
|
1813
|
+
def export(self) -> dict:
|
|
1814
|
+
from ..discovery.patch_envelope import ModelPatch, build_envelope
|
|
1815
|
+
|
|
1816
|
+
return build_envelope([ModelPatch(m, ops) for m, ops in self.spec.grouped()])
|
|
1817
|
+
|
|
1818
|
+
# -- outcomes ----------------------------------------------------------
|
|
1819
|
+
|
|
1820
|
+
def apply(self, outcome: Outcome | Sequence[Outcome]) -> None:
|
|
1821
|
+
"""Apply what a capability returned. ``None`` is nothing; a sequence
|
|
1822
|
+
is applied in order."""
|
|
1823
|
+
if outcome is None:
|
|
1824
|
+
return
|
|
1825
|
+
if isinstance(outcome, (list, tuple)):
|
|
1826
|
+
for one in outcome:
|
|
1827
|
+
self.apply(one)
|
|
1828
|
+
return
|
|
1829
|
+
if isinstance(outcome, Patch):
|
|
1830
|
+
# A capability's op is never the Session's materialization: only
|
|
1831
|
+
# `stage` and `rescore` know the tree they built scores the same.
|
|
1832
|
+
self.stage([replace(op, materializes=False) for op in outcome.ops])
|
|
1833
|
+
elif isinstance(outcome, StoreWrite):
|
|
1834
|
+
# A queued or running test's result is about to be replaced:
|
|
1835
|
+
# a baseline written from the one in memory would be the last
|
|
1836
|
+
# run's, stored under the next one's settings.
|
|
1837
|
+
for model_id in sorted(outcome.models):
|
|
1838
|
+
item = self._live.get(model_id)
|
|
1839
|
+
if item is not None:
|
|
1840
|
+
raise QueueError(
|
|
1841
|
+
f"{model_id} is {_doing(item)}: accept it when its run "
|
|
1842
|
+
"has ended"
|
|
1843
|
+
)
|
|
1844
|
+
# The check Save makes. A baseline is written with the test as
|
|
1845
|
+
# the Session holds it — its `baseline_points`, the comparison
|
|
1846
|
+
# settings stored beside the data — which is the spec as
|
|
1847
|
+
# LOADED: after an edit of the file by hand an Accept stored
|
|
1848
|
+
# the old entry's baseline (50 points where the file said 77)
|
|
1849
|
+
# and said nothing.
|
|
1850
|
+
if self.spec.on_disk_changed():
|
|
1851
|
+
raise StaleSpecError(self.spec.path, "accepting")
|
|
1852
|
+
message = outcome.run(self._require_project().store)
|
|
1853
|
+
self._reload_results()
|
|
1854
|
+
# The reload first, so a listener that reads verdicts on
|
|
1855
|
+
# the notice sees what the write did to them.
|
|
1856
|
+
if message:
|
|
1857
|
+
self.notices.append(message)
|
|
1858
|
+
self._emit("notice", message=message)
|
|
1859
|
+
elif isinstance(outcome, Enqueue):
|
|
1860
|
+
if self._has_loop:
|
|
1861
|
+
self._enqueue_later(outcome.models)
|
|
1862
|
+
else:
|
|
1863
|
+
self.enqueue(outcome.models)
|
|
1864
|
+
elif isinstance(outcome, Artifact):
|
|
1865
|
+
self.artifacts.append(outcome)
|
|
1866
|
+
self._emit("notice", artifact=outcome)
|
|
1867
|
+
elif isinstance(outcome, OpenProject):
|
|
1868
|
+
self.open(outcome.config_path, work_dir=outcome.work_dir)
|
|
1869
|
+
else:
|
|
1870
|
+
raise TypeError(f"not an Outcome: {outcome!r}")
|
|
1871
|
+
|
|
1872
|
+
# -- the simulation queue ----------------------------------------------
|
|
1873
|
+
|
|
1874
|
+
@property
|
|
1875
|
+
def workers(self) -> int:
|
|
1876
|
+
"""How many tests the queue runs at once: what `set_workers` was
|
|
1877
|
+
given, else ``Config.parallel``."""
|
|
1878
|
+
if self._workers is not None:
|
|
1879
|
+
return self._workers
|
|
1880
|
+
project = self.project
|
|
1881
|
+
return max(1, int(project.config.parallel)) if project is not None else 1
|
|
1882
|
+
|
|
1883
|
+
def set_workers(self, count: int) -> None:
|
|
1884
|
+
"""Run ``count`` tests at once from now on, between 1 and the
|
|
1885
|
+
machine's cores. Workers already started stay: more start as items
|
|
1886
|
+
need them, and a surplus one leaves when its item ends."""
|
|
1887
|
+
self._workers = max(1, min(int(count), os.cpu_count() or 1))
|
|
1888
|
+
if self._pool is not None:
|
|
1889
|
+
self._pool.workers = self._workers
|
|
1890
|
+
self._emit("queue", state="workers", models=set())
|
|
1891
|
+
|
|
1892
|
+
@property
|
|
1893
|
+
def starting_workers(self) -> tuple[StartingWorker, ...]:
|
|
1894
|
+
"""The queue's workers that are not ready right now (`pool.starting`):
|
|
1895
|
+
being brought up, or reading the library again — a large library is
|
|
1896
|
+
minutes of load, and its tests wait meanwhile."""
|
|
1897
|
+
pool = self._pool
|
|
1898
|
+
return pool.starting if pool is not None else ()
|
|
1899
|
+
|
|
1900
|
+
@property
|
|
1901
|
+
def checking(self) -> tuple[str, ...]:
|
|
1902
|
+
"""The tests a Queue click named whose click is still being read
|
|
1903
|
+
(`_enqueue_later`) and that the queue does not hold yet, in project
|
|
1904
|
+
order — the order they will be queued in. Each is TAKEN, and every
|
|
1905
|
+
surface says so until the read is back: the Suite row reads QUEUED,
|
|
1906
|
+
the panel lists it (`queue_rows.CHECKING`) and the header counts
|
|
1907
|
+
it. For the tenth of a second to several the files are being looked
|
|
1908
|
+
at — a large library on a slow mount — nothing on the page moved,
|
|
1909
|
+
and a click that went nowhere is clicked again."""
|
|
1910
|
+
project = self.project
|
|
1911
|
+
if project is None or not self._reading:
|
|
1912
|
+
return ()
|
|
1913
|
+
return tuple(
|
|
1914
|
+
t.model_id
|
|
1915
|
+
for t in project.tests
|
|
1916
|
+
if t.model_id in self._reading and t.model_id not in self._live
|
|
1917
|
+
)
|
|
1918
|
+
|
|
1919
|
+
def queue_item(self, model_id: str) -> QueueItem | None:
|
|
1920
|
+
"""The model's item in the queue — the last one listed — or None."""
|
|
1921
|
+
return next((i for i in reversed(self.queue) if i.model_id == model_id), None)
|
|
1922
|
+
|
|
1923
|
+
def item_state(self, item: QueueItem) -> ItemState:
|
|
1924
|
+
"""``item``'s state as the Session has taken it in. The pool ends an
|
|
1925
|
+
item on its own thread; until the Session has read its result
|
|
1926
|
+
(`_item_ended`, a tick later) it is still in the queue here."""
|
|
1927
|
+
if self._live.get(item.model_id) is item and item.state not in ACTIVE:
|
|
1928
|
+
return (
|
|
1929
|
+
ItemState.RUNNING if item.started_at is not None else ItemState.WAITING
|
|
1930
|
+
)
|
|
1931
|
+
return item.state
|
|
1932
|
+
|
|
1933
|
+
def queue_state(self, model_id: str) -> ItemState | None:
|
|
1934
|
+
"""Where ``model_id`` stands in the queue (`item_state` of its
|
|
1935
|
+
item): WAITING or RUNNING while it is in it, how its last run there
|
|
1936
|
+
ended until `clear_finished`, else None."""
|
|
1937
|
+
item = self.queue_item(model_id)
|
|
1938
|
+
return self.item_state(item) if item is not None else None
|
|
1939
|
+
|
|
1940
|
+
@property
|
|
1941
|
+
def cannot_cancel(self) -> str | None:
|
|
1942
|
+
"""Why a running test cannot be stopped on the pool's backend, or
|
|
1943
|
+
None where it can (and before there is a pool)."""
|
|
1944
|
+
pool = self._pool
|
|
1945
|
+
if pool is None or pool.can_cancel:
|
|
1946
|
+
return None
|
|
1947
|
+
return (
|
|
1948
|
+
f"{pool.backend.label} cannot stop a running simulation: it runs "
|
|
1949
|
+
"to its end, or to its timeout"
|
|
1950
|
+
)
|
|
1951
|
+
|
|
1952
|
+
def enqueue(self, model_ids: Sequence[str]) -> list[QueueItem]:
|
|
1953
|
+
"""Append ``model_ids`` to the queue, in that order; returns their
|
|
1954
|
+
items. A test already queued or running is skipped (the pool's
|
|
1955
|
+
refusal: a test is never queued twice); when that leaves nothing to
|
|
1956
|
+
add, the refusals are raised as one :class:`QueueError`. The first
|
|
1957
|
+
call opens the pool — on a backend that cannot run one test at a
|
|
1958
|
+
time, or cannot be built here, a :class:`QueueError` with that
|
|
1959
|
+
backend's sentence, kept as `queue_refusal`.
|
|
1960
|
+
|
|
1961
|
+
Each call is one click, and what it queues runs the library AS IT
|
|
1962
|
+
IS ON DISK NOW. The source files are swept once (stat only); a
|
|
1963
|
+
queued test whose own file changed is read from it again — its
|
|
1964
|
+
annotation's values, its UnitTests expressions — into the
|
|
1965
|
+
project's scan; and the click's items carry the swept state, so a
|
|
1966
|
+
live worker that read the library before a change reads it again
|
|
1967
|
+
before it runs one of them. A test whose file is gone, or no longer
|
|
1968
|
+
holds it, is refused with one sentence — a notice when others were
|
|
1969
|
+
queued — and the rest are queued.
|
|
1970
|
+
|
|
1971
|
+
The read happens HERE, on the calling thread; a Session with a UI
|
|
1972
|
+
loop queues an `Enqueue` outcome through `_enqueue_later`."""
|
|
1973
|
+
project = self._require_project()
|
|
1974
|
+
self._known(project, model_ids)
|
|
1975
|
+
self._open_pool()
|
|
1976
|
+
read = _read_click(project.config, self._asks(project, model_ids))
|
|
1977
|
+
return self._queue_read(project, model_ids, read)
|
|
1978
|
+
|
|
1979
|
+
def _enqueue_later(self, model_ids: Sequence[str]) -> None:
|
|
1980
|
+
"""`enqueue` for a Session with a UI loop: this call returns at
|
|
1981
|
+
once, the click's read of the library — seconds over a large one on
|
|
1982
|
+
a slow mount — runs on the background thread, and the tests are
|
|
1983
|
+
queued on the loop when it is done. What `enqueue` raises after its
|
|
1984
|
+
read is a notice here; what it raises before (an unknown test, a
|
|
1985
|
+
backend that cannot be queued on) is raised here too. A test whose
|
|
1986
|
+
click is still being read is not read twice."""
|
|
1987
|
+
project = self._require_project()
|
|
1988
|
+
self._known(project, model_ids)
|
|
1989
|
+
models = [m for m in model_ids if m not in self._reading]
|
|
1990
|
+
if not model_ids:
|
|
1991
|
+
return
|
|
1992
|
+
if not models:
|
|
1993
|
+
raise QueueError(
|
|
1994
|
+
f"{model_ids[0]} is already being queued"
|
|
1995
|
+
if len(model_ids) == 1
|
|
1996
|
+
else f"all {len(model_ids)} tests are already being queued"
|
|
1997
|
+
)
|
|
1998
|
+
self._open_pool()
|
|
1999
|
+
asks = self._asks(project, models)
|
|
2000
|
+
epoch = self._queue_epoch
|
|
2001
|
+
self._reading.update(models)
|
|
2002
|
+
# Said at once: these tests were taken (`checking`).
|
|
2003
|
+
self._emit("queue", state="checking", models=set(models))
|
|
2004
|
+
|
|
2005
|
+
def read() -> None:
|
|
2006
|
+
got: _ClickRead | Exception
|
|
2007
|
+
try:
|
|
2008
|
+
got = _read_click(project.config, asks)
|
|
2009
|
+
except Exception as e: # said on the loop, never lost on a thread
|
|
2010
|
+
got = e
|
|
2011
|
+
self._post(lambda: self._click_read(project, models, epoch, got))
|
|
2012
|
+
|
|
2013
|
+
self._background.submit("a Queue click's read of the library", read)
|
|
2014
|
+
|
|
2015
|
+
def _click_read(
|
|
2016
|
+
self,
|
|
2017
|
+
project: Project,
|
|
2018
|
+
models: Sequence[str],
|
|
2019
|
+
epoch: int,
|
|
2020
|
+
got: _ClickRead | Exception,
|
|
2021
|
+
) -> None:
|
|
2022
|
+
"""On the UI loop: a click's read is back (`_enqueue_later`)."""
|
|
2023
|
+
if self.project is not project or epoch != self._queue_epoch:
|
|
2024
|
+
return # the project was swapped, or the queue stopped, meanwhile
|
|
2025
|
+
self._reading.difference_update(models)
|
|
2026
|
+
added: list[QueueItem] = []
|
|
2027
|
+
try:
|
|
2028
|
+
if isinstance(got, Exception):
|
|
2029
|
+
raise got
|
|
2030
|
+
added = self._queue_read(project, models, got)
|
|
2031
|
+
except QueueError as e:
|
|
2032
|
+
self._say(str(e))
|
|
2033
|
+
except Exception as e:
|
|
2034
|
+
logger.exception("queueing %s failed", ", ".join(models))
|
|
2035
|
+
self._say(f"Queue failed: {type(e).__name__}: {e}")
|
|
2036
|
+
finally:
|
|
2037
|
+
# A test the read refused was shown as taken (`checking`): its
|
|
2038
|
+
# row goes, and the sentence above says why.
|
|
2039
|
+
dropped = set(models) - {item.model_id for item in added}
|
|
2040
|
+
if dropped:
|
|
2041
|
+
self._emit("queue", state="refused", models=dropped)
|
|
2042
|
+
|
|
2043
|
+
def _say(self, message: str) -> None:
|
|
2044
|
+
self.notices.append(message)
|
|
2045
|
+
self._emit("notice", message=message)
|
|
2046
|
+
|
|
2047
|
+
def _known(self, project: Project, model_ids: Sequence[str]) -> None:
|
|
2048
|
+
for model_id in model_ids:
|
|
2049
|
+
if project.test(model_id) is None:
|
|
2050
|
+
raise KeyError(model_id)
|
|
2051
|
+
|
|
2052
|
+
def _asks(self, project: Project, model_ids: Sequence[str]) -> list[_Ask]:
|
|
2053
|
+
"""What a click's read needs of each test it names: the file the
|
|
2054
|
+
test came from and the stamp it was last read at."""
|
|
2055
|
+
asks = []
|
|
2056
|
+
for model_id in dict.fromkeys(model_ids):
|
|
2057
|
+
scanned = project.scanned.get(model_id)
|
|
2058
|
+
if scanned is not None:
|
|
2059
|
+
path: Path | None = scanned.source_file
|
|
2060
|
+
else:
|
|
2061
|
+
# Asked at every merge since the open: answered from memory.
|
|
2062
|
+
located = project.locate(model_id)
|
|
2063
|
+
path = located.mo_file if located is not None else None
|
|
2064
|
+
asks.append(
|
|
2065
|
+
_Ask(model_id, path, self._read_at.get(model_id), scanned is not None)
|
|
2066
|
+
)
|
|
2067
|
+
return asks
|
|
2068
|
+
|
|
2069
|
+
def _queue_read(
|
|
2070
|
+
self, project: Project, model_ids: Sequence[str], read: _ClickRead
|
|
2071
|
+
) -> list[QueueItem]:
|
|
2072
|
+
"""Queue ``model_ids`` against what their click ``read``."""
|
|
2073
|
+
library: LibraryState | None = None
|
|
2074
|
+
if read.stamps is not None:
|
|
2075
|
+
# The last click's own object when no file differs from it: a
|
|
2076
|
+
# worker then compares two serials and reads nothing.
|
|
2077
|
+
library = self._library = following(self._library, read.stamps)
|
|
2078
|
+
self._take_read(project, read)
|
|
2079
|
+
pool = self._open_pool()
|
|
2080
|
+
added: list[QueueItem] = []
|
|
2081
|
+
said = list(read.refused.values())
|
|
2082
|
+
self._refusals.update(read.refused)
|
|
2083
|
+
for model_id in model_ids:
|
|
2084
|
+
test = project.test(model_id)
|
|
2085
|
+
if model_id in read.refused or test is None:
|
|
2086
|
+
continue
|
|
2087
|
+
held = self._live.get(model_id)
|
|
2088
|
+
if held is not None:
|
|
2089
|
+
# Already queued: while it waits, it runs what THIS click
|
|
2090
|
+
# read, on a worker that has read the library since.
|
|
2091
|
+
pool.update(held, test, library)
|
|
2092
|
+
try:
|
|
2093
|
+
item = pool.submit(test, library)
|
|
2094
|
+
except Refused as e:
|
|
2095
|
+
said.append(str(e))
|
|
2096
|
+
continue
|
|
2097
|
+
self._live[model_id] = item
|
|
2098
|
+
self.queue.append(item)
|
|
2099
|
+
added.append(item)
|
|
2100
|
+
# A test that is queued again was read: what an earlier click said
|
|
2101
|
+
# of its file — gone, unreadable, no longer holding it — is over,
|
|
2102
|
+
# and stood in the panel after the file was back and the test had
|
|
2103
|
+
# passed.
|
|
2104
|
+
self._forget_refusals([item.model_id for item in added])
|
|
2105
|
+
if not added:
|
|
2106
|
+
if said:
|
|
2107
|
+
raise QueueError("; ".join(said))
|
|
2108
|
+
return []
|
|
2109
|
+
for sentence in read.refused.values():
|
|
2110
|
+
self._say(sentence)
|
|
2111
|
+
self._emit("queue", state="queued", models={i.model_id for i in added})
|
|
2112
|
+
return added
|
|
2113
|
+
|
|
2114
|
+
def _forget_refusals(self, model_ids: Sequence[str]) -> None:
|
|
2115
|
+
"""Take the notes the read's refusals of ``model_ids`` left out of
|
|
2116
|
+
the notices — each its own notice, or one part of a notice that
|
|
2117
|
+
says several (a click all of whose tests were refused)."""
|
|
2118
|
+
gone = {s for m in model_ids if (s := self._refusals.pop(m, None)) is not None}
|
|
2119
|
+
if not gone:
|
|
2120
|
+
return
|
|
2121
|
+
kept = []
|
|
2122
|
+
for notice in self.notices:
|
|
2123
|
+
parts = [part for part in notice.split("; ") if part not in gone]
|
|
2124
|
+
if parts:
|
|
2125
|
+
kept.append("; ".join(parts))
|
|
2126
|
+
if kept != self.notices:
|
|
2127
|
+
self.notices = kept
|
|
2128
|
+
self._emit("notice")
|
|
2129
|
+
|
|
2130
|
+
def _store_scan_later(self, config: Config, scan: dict[str, TestModel]) -> None:
|
|
2131
|
+
"""The scan cache follows the project's scan, off the UI loop."""
|
|
2132
|
+
|
|
2133
|
+
def store() -> None:
|
|
2134
|
+
store_scan(config, scan)
|
|
2135
|
+
|
|
2136
|
+
self._background.submit("the scan cache", store)
|
|
2137
|
+
|
|
2138
|
+
def _take_read(self, project: Project, read: _ClickRead) -> None:
|
|
2139
|
+
"""What a click read of its tests' files, into the project's scan
|
|
2140
|
+
(and the scan cache) — as the rescan's is, and NOT into the queue's
|
|
2141
|
+
items alone: every later re-merge (a spec edit, `_update_waiting`)
|
|
2142
|
+
then hands a waiting item the test as the click read it, where it
|
|
2143
|
+
would otherwise hand back the one read at open."""
|
|
2144
|
+
self._read_at.update(read.read_at)
|
|
2145
|
+
scanned = {
|
|
2146
|
+
m: test
|
|
2147
|
+
for m, test in read.scanned.items()
|
|
2148
|
+
if project.scanned.get(m) != test
|
|
2149
|
+
}
|
|
2150
|
+
located = {
|
|
2151
|
+
m: found for m, found in read.located.items() if project.locate(m) != found
|
|
2152
|
+
}
|
|
2153
|
+
if not scanned and not located:
|
|
2154
|
+
return
|
|
2155
|
+
scan = {**project.scanned, **scanned}
|
|
2156
|
+
if scanned:
|
|
2157
|
+
self._store_scan_later(project.config, scan)
|
|
2158
|
+
# The `spec` event of the re-merge is what tells a page a test's
|
|
2159
|
+
# definition changed: its Suite row and its Test page are redrawn.
|
|
2160
|
+
self._rebuild_tests(
|
|
2161
|
+
(set(scanned) | set(located)) - self._running_models(),
|
|
2162
|
+
scanned=scan,
|
|
2163
|
+
locate=project.locate.having(located),
|
|
2164
|
+
)
|
|
2165
|
+
|
|
2166
|
+
def remove(self, item: QueueItem) -> bool:
|
|
2167
|
+
"""Take a waiting ``item`` out of the queue, its row with it.
|
|
2168
|
+
``False`` once it has started: nothing is changed."""
|
|
2169
|
+
pool = self._pool
|
|
2170
|
+
if pool is None or not pool.remove(item):
|
|
2171
|
+
return False
|
|
2172
|
+
self._forget(item)
|
|
2173
|
+
self._emit("queue", state="removed", models={item.model_id})
|
|
2174
|
+
self._drained()
|
|
2175
|
+
return True
|
|
2176
|
+
|
|
2177
|
+
def cancel(self, item: QueueItem) -> bool:
|
|
2178
|
+
"""Stop a running ``item``'s simulation; it ends CANCELLED once its
|
|
2179
|
+
thread says so — at once on most backends, after its worker's grace
|
|
2180
|
+
where the simulation does not look at the request. Said meanwhile:
|
|
2181
|
+
a ``queue`` event, state ``cancelling``, so its row and its strip
|
|
2182
|
+
read that until the end. ``False`` when it is not running; the
|
|
2183
|
+
pool's ``Refused``, a sentence, on a backend that cannot stop a
|
|
2184
|
+
test."""
|
|
2185
|
+
pool = self._pool
|
|
2186
|
+
if pool is None or not pool.cancel(item):
|
|
2187
|
+
return False
|
|
2188
|
+
self._emit("queue", state="cancelling", models={item.model_id})
|
|
2189
|
+
return True
|
|
2190
|
+
|
|
2191
|
+
def clear_finished(self) -> None:
|
|
2192
|
+
"""Drop the rows of the items that ended — finished, failed or
|
|
2193
|
+
cancelled — and the stopped pool's message. Their results stay."""
|
|
2194
|
+
gone = {i.model_id for i in self.queue if self._live.get(i.model_id) is not i}
|
|
2195
|
+
self.queue = [i for i in self.queue if self._live.get(i.model_id) is i]
|
|
2196
|
+
self.queue_stopped = None
|
|
2197
|
+
self._emit("queue", state="cleared", models=gone)
|
|
2198
|
+
|
|
2199
|
+
def stop_workers(self) -> None:
|
|
2200
|
+
"""Close the pool: its workers end and their licence seats are
|
|
2201
|
+
free; the next `enqueue` starts new ones. Tests still waiting leave
|
|
2202
|
+
the queue. Refused while a test runs."""
|
|
2203
|
+
count = len(self._running_models())
|
|
2204
|
+
if count:
|
|
2205
|
+
raise QueueError(
|
|
2206
|
+
f"{plural(count, 'test')} {'is' if count == 1 else 'are'} running — "
|
|
2207
|
+
f"cancel {'it' if count == 1 else 'them'} or wait, then stop the "
|
|
2208
|
+
"workers"
|
|
2209
|
+
)
|
|
2210
|
+
self.stop_queue()
|
|
2211
|
+
|
|
2212
|
+
def stop_queue(self) -> None:
|
|
2213
|
+
"""End the queue's pool, whatever it holds — what the tool does
|
|
2214
|
+
when it stops: waiting tests are taken out, running ones stopped
|
|
2215
|
+
(CANCELLED, where the backend can; else it finishes them first),
|
|
2216
|
+
every worker closed and every pool thread joined. The rows of what
|
|
2217
|
+
ended stay until `clear_finished`."""
|
|
2218
|
+
# A click whose read is still running is not queued after this.
|
|
2219
|
+
self._queue_epoch += 1
|
|
2220
|
+
gone = set(self.checking)
|
|
2221
|
+
self._reading.clear()
|
|
2222
|
+
pool, self._pool = self._pool, None
|
|
2223
|
+
if pool is None:
|
|
2224
|
+
return
|
|
2225
|
+
pool.close()
|
|
2226
|
+
gone |= set(self._live)
|
|
2227
|
+
for item in list(self._live.values()):
|
|
2228
|
+
if item.state is ItemState.REMOVED:
|
|
2229
|
+
self._forget(item)
|
|
2230
|
+
else:
|
|
2231
|
+
# Stopped by the close, or ended while it closed: taken in
|
|
2232
|
+
# here. Its callback is still on its way, and finds nothing.
|
|
2233
|
+
self._item_ended(item)
|
|
2234
|
+
self._emit("queue", state="stopped", models=gone)
|
|
2235
|
+
self._drained()
|
|
2236
|
+
|
|
2237
|
+
def _reset_queue(self) -> None:
|
|
2238
|
+
"""The queue of a project that is closing: its pool ended, its rows
|
|
2239
|
+
and everything said about it gone."""
|
|
2240
|
+
self.stop_queue()
|
|
2241
|
+
self.live_runs.clear()
|
|
2242
|
+
self._library, self._read_at = None, {}
|
|
2243
|
+
self._refusals = {}
|
|
2244
|
+
self.queue, self._unwritten = [], []
|
|
2245
|
+
self._workers = self.queue_refusal = self.queue_stopped = None
|
|
2246
|
+
self._write_status = None
|
|
2247
|
+
|
|
2248
|
+
def _open_pool(self) -> SimulationPool:
|
|
2249
|
+
from ..pipeline import open_pool
|
|
2250
|
+
|
|
2251
|
+
project = self._require_project()
|
|
2252
|
+
if self.queue_refusal is not None:
|
|
2253
|
+
raise QueueError(self.queue_refusal)
|
|
2254
|
+
if self._pool is not None and self._pool.stopped is not None:
|
|
2255
|
+
# A stopped pool takes nothing more: a new one is the retry.
|
|
2256
|
+
self.stop_queue()
|
|
2257
|
+
if self._pool is None:
|
|
2258
|
+
# The tool reads a run's result WHILE it is in progress (D151): a
|
|
2259
|
+
# backend that would otherwise write it late writes it promptly
|
|
2260
|
+
# for this pool. Set here and nowhere else — no CLI run is.
|
|
2261
|
+
project.config.live_results = True
|
|
2262
|
+
try:
|
|
2263
|
+
# The pool's OWN backend, the persistent variant a CLI run
|
|
2264
|
+
# would choose — the instance `open` holds is the batch
|
|
2265
|
+
# class it reads results through — and silenced: the tool's
|
|
2266
|
+
# process never narrates a run to the console it was
|
|
2267
|
+
# started from.
|
|
2268
|
+
backend = select_backend(project.config, persistent=True)
|
|
2269
|
+
backend.quiet = True
|
|
2270
|
+
self._pool = open_pool(
|
|
2271
|
+
project.config,
|
|
2272
|
+
backend=backend,
|
|
2273
|
+
store=project.store,
|
|
2274
|
+
# On the POOL's threads: nothing of the Session is
|
|
2275
|
+
# touched there. Each is handed to the UI loop.
|
|
2276
|
+
on_start=lambda item: self._post(lambda: self._item_started(item)),
|
|
2277
|
+
on_end=lambda item: self._post(lambda: self._item_ended(item)),
|
|
2278
|
+
workers=self.workers,
|
|
2279
|
+
)
|
|
2280
|
+
except RuntimeError as e:
|
|
2281
|
+
# A backend that runs whole batches only, or cannot be
|
|
2282
|
+
# built here (`BackendUnavailableError` is a RuntimeError):
|
|
2283
|
+
# the Queue action carries the sentence from now on.
|
|
2284
|
+
self.queue_refusal = str(e)
|
|
2285
|
+
raise QueueError(self.queue_refusal) from e
|
|
2286
|
+
self._write_status = cast("ItemRun", self._pool.path).write_status
|
|
2287
|
+
return self._pool
|
|
2288
|
+
|
|
2289
|
+
def _forget(self, item: QueueItem) -> None:
|
|
2290
|
+
if self._live.get(item.model_id) is item:
|
|
2291
|
+
del self._live[item.model_id]
|
|
2292
|
+
self.queue.remove(item)
|
|
2293
|
+
|
|
2294
|
+
def _update_waiting(self) -> None:
|
|
2295
|
+
"""The waiting items follow the spec: a queued test runs with its
|
|
2296
|
+
test as it stands when it STARTS (D150), so every re-merge hands
|
|
2297
|
+
the pool the new one. A running item's is its run's."""
|
|
2298
|
+
pool, project = self._pool, self.project
|
|
2299
|
+
if pool is None or project is None:
|
|
2300
|
+
return
|
|
2301
|
+
for item in self._live.values():
|
|
2302
|
+
test = project.test(item.model_id)
|
|
2303
|
+
if test is not None and test != item.test:
|
|
2304
|
+
pool.update(item, test)
|
|
2305
|
+
|
|
2306
|
+
def _item_started(self, item: QueueItem) -> None:
|
|
2307
|
+
"""On the UI loop, once the pool's thread said ``item`` started. Its
|
|
2308
|
+
run cleared what the last one left on disk, so the result in memory
|
|
2309
|
+
is no file's any more: it goes, and the test reads NOT RUN until
|
|
2310
|
+
this run's arrives."""
|
|
2311
|
+
model_id, project = item.model_id, self.project
|
|
2312
|
+
if self._live.get(model_id) is not item or project is None:
|
|
2313
|
+
return
|
|
2314
|
+
# A worker took an item: whatever stopped the pool before is over.
|
|
2315
|
+
self.queue_stopped = None
|
|
2316
|
+
# Before the result goes: the page stays laid out by it (D151).
|
|
2317
|
+
self.live_runs.started(item)
|
|
2318
|
+
project.results.pop(model_id, None)
|
|
2319
|
+
self._reads.pop(model_id, None)
|
|
2320
|
+
self._score_one(model_id)
|
|
2321
|
+
self._emit("queue", state="started", models={model_id})
|
|
2322
|
+
|
|
2323
|
+
def _item_ended(self, item: QueueItem) -> None:
|
|
2324
|
+
"""On the UI loop, once the pool's thread said ``item`` ended: THAT
|
|
2325
|
+
test's result is taken from the item and it alone is rescored — with
|
|
2326
|
+
the criteria as they are now, pending edits included. Nothing else
|
|
2327
|
+
is read: a reload per item read TRANSFORM's 331 results for each."""
|
|
2328
|
+
model_id = item.model_id
|
|
2329
|
+
if self._live.get(model_id) is not item:
|
|
2330
|
+
return
|
|
2331
|
+
del self._live[model_id]
|
|
2332
|
+
if item.run_result is not None:
|
|
2333
|
+
self._unwritten.append(item)
|
|
2334
|
+
pool = self._pool
|
|
2335
|
+
if pool is not None and pool.stopped is not None:
|
|
2336
|
+
self.queue_stopped = pool.stopped
|
|
2337
|
+
# An item that never started is one the pool failed (no worker
|
|
2338
|
+
# could start): nothing ran, and the test's last result stands.
|
|
2339
|
+
# The queue says it ended, whatever its file reads as: the live
|
|
2340
|
+
# view goes, and what an unfinished run wrote is kept to be shown.
|
|
2341
|
+
self.live_runs.ended(item)
|
|
2342
|
+
if item.started_at is not None and self.project is not None:
|
|
2343
|
+
self._take_result(item)
|
|
2344
|
+
self._emit("queue", state="ended", models={model_id})
|
|
2345
|
+
self._drained()
|
|
2346
|
+
|
|
2347
|
+
def _take_result(self, item: QueueItem) -> None:
|
|
2348
|
+
project = self._require_project()
|
|
2349
|
+
model_id = item.model_id
|
|
2350
|
+
self._reads.pop(model_id, None)
|
|
2351
|
+
if item.result is None:
|
|
2352
|
+
# Cancelled, or its run raised: what it left on disk is no result.
|
|
2353
|
+
project.results.pop(model_id, None)
|
|
2354
|
+
else:
|
|
2355
|
+
project.results[model_id] = item.result
|
|
2356
|
+
# The result was read for the test as it started. A criterion
|
|
2357
|
+
# added while it ran may name a variable the file holds and
|
|
2358
|
+
# that read did not extract: read again, this test only. A
|
|
2359
|
+
# read that fails costs that variable, never the run: the
|
|
2360
|
+
# result that WAS read is scored, and the end is said.
|
|
2361
|
+
try:
|
|
2362
|
+
reread = self._unread_in_file({model_id}, project.tests)
|
|
2363
|
+
if reread:
|
|
2364
|
+
self._keep_read(reread, self._read_again(reread, project.tests))
|
|
2365
|
+
except Exception as e:
|
|
2366
|
+
logger.error(
|
|
2367
|
+
"reading %s again failed (%s: %s): its result stands as its run read it",
|
|
2368
|
+
model_id,
|
|
2369
|
+
type(e).__name__,
|
|
2370
|
+
e,
|
|
2371
|
+
)
|
|
2372
|
+
self._score_one(model_id)
|
|
2373
|
+
|
|
2374
|
+
def _score_one(self, model_id: str) -> None:
|
|
2375
|
+
"""Rescore ``model_id`` and say so; a scoring that raises is logged
|
|
2376
|
+
and leaves the model without a verdict, as a reload does."""
|
|
2377
|
+
project = self._require_project()
|
|
2378
|
+
try:
|
|
2379
|
+
if project.test(model_id) is not None:
|
|
2380
|
+
self._score(model_id)
|
|
2381
|
+
except Exception:
|
|
2382
|
+
logger.exception("scoring %s failed", model_id)
|
|
2383
|
+
self.comparisons.pop(model_id, None)
|
|
2384
|
+
self._emit("results", models={model_id})
|
|
2385
|
+
|
|
2386
|
+
def _drained(self) -> None:
|
|
2387
|
+
"""``status.json`` and the dashboard, once nothing is queued or
|
|
2388
|
+
running, for the items that ran since they were last written — not
|
|
2389
|
+
per item: both are one run's snapshot, and the queue's live view is
|
|
2390
|
+
the tool's own pages. Off the UI loop (the dashboard's render is
|
|
2391
|
+
1.6 s over TRANSFORM's work dir with --report sidecars), on a
|
|
2392
|
+
snapshot of the verdicts as they stand."""
|
|
2393
|
+
if self._live:
|
|
2394
|
+
return
|
|
2395
|
+
items, self._unwritten = self._unwritten, []
|
|
2396
|
+
write, project = self._write_status, self.project
|
|
2397
|
+
if not items or write is None or project is None:
|
|
2398
|
+
return
|
|
2399
|
+
work_dir = project.config.work_dir
|
|
2400
|
+
assert work_dir is not None # resolved in Config.__post_init__
|
|
2401
|
+
comparisons = list(self.comparisons.values())
|
|
2402
|
+
results = dict(project.results)
|
|
2403
|
+
theme = project.config.theme
|
|
2404
|
+
|
|
2405
|
+
def run() -> None:
|
|
2406
|
+
from ..backends.progress import stamp_verdicts
|
|
2407
|
+
from ..reporting.dashboard_render import render_final
|
|
2408
|
+
from ..reporting.verdicts import status_word
|
|
2409
|
+
|
|
2410
|
+
write(items)
|
|
2411
|
+
# Each row's verdict beside its simulation status, as a `run`
|
|
2412
|
+
# leaves it: "passed" there says only that the simulation ran.
|
|
2413
|
+
stamp_verdicts(work_dir, {c.model_id: status_word(c) for c in comparisons})
|
|
2414
|
+
# The page lists every test the Session scored, with its
|
|
2415
|
+
# verdict — not the queue's items alone.
|
|
2416
|
+
render_final(
|
|
2417
|
+
work_dir, comparisons=comparisons, results=results, theme=theme
|
|
2418
|
+
)
|
|
2419
|
+
|
|
2420
|
+
self._background.submit("the queue's status.json and dashboard", run)
|
|
2421
|
+
|
|
2422
|
+
# -- internals ---------------------------------------------------------
|
|
2423
|
+
|
|
2424
|
+
def _require_project(self) -> Project:
|
|
2425
|
+
if self.project is None:
|
|
2426
|
+
raise RuntimeError("no project is open")
|
|
2427
|
+
return self.project
|
|
2428
|
+
|
|
2429
|
+
def _merge(
|
|
2430
|
+
self,
|
|
2431
|
+
config: Config,
|
|
2432
|
+
scanned: dict[str, TestModel],
|
|
2433
|
+
document: dict,
|
|
2434
|
+
locate: SpecModelLocator,
|
|
2435
|
+
*,
|
|
2436
|
+
edited: bool,
|
|
2437
|
+
) -> list[TestModel]:
|
|
2438
|
+
"""The project's tests from the scan and the spec ``document``.
|
|
2439
|
+
``edited`` says whether that document holds pending edits: an error
|
|
2440
|
+
in one that does not is the FILE's, and names the file alone — it
|
|
2441
|
+
said "(in memory)" of a spec just read from disk."""
|
|
2442
|
+
label = str(config.spec_path)
|
|
2443
|
+
models = spec_models_from_document(
|
|
2444
|
+
document,
|
|
2445
|
+
base_dir=config.spec_path.parent,
|
|
2446
|
+
label=f"{label} (with unsaved edits)" if edited else label,
|
|
2447
|
+
)
|
|
2448
|
+
return merge_tests(scanned, spec_tests_by_model(models), locate=locate)
|
|
2449
|
+
|
|
2450
|
+
def _rebuild_tests(
|
|
2451
|
+
self,
|
|
2452
|
+
models: set[str],
|
|
2453
|
+
*,
|
|
2454
|
+
scanned: dict[str, TestModel] | None = None,
|
|
2455
|
+
locate: SpecModelLocator | None = None,
|
|
2456
|
+
) -> None:
|
|
2457
|
+
"""Re-merge the scan against the effective spec; models the spec no
|
|
2458
|
+
longer names drop out of ``comparisons``, models it newly names are
|
|
2459
|
+
scored (with whatever result the last reload holds for them).
|
|
2460
|
+
|
|
2461
|
+
``scanned`` and ``locate`` are the library READ AGAIN — the
|
|
2462
|
+
rescan's scan and a new locator, or a Queue click's read of the
|
|
2463
|
+
tests it queues: they are merged in place of the project's, and
|
|
2464
|
+
the ``models`` are re-read from the last run with their merged
|
|
2465
|
+
TestModels. The read comes BEFORE anything is replaced — the scan,
|
|
2466
|
+
the tests, the comparisons — so a read that raises (the disk went
|
|
2467
|
+
away) leaves the Session as it was."""
|
|
2468
|
+
project = self._require_project()
|
|
2469
|
+
again = scanned is not None or locate is not None
|
|
2470
|
+
scan = project.scanned if scanned is None else scanned
|
|
2471
|
+
locate = project.locate if locate is None else locate
|
|
2472
|
+
tests = self._merge(
|
|
2473
|
+
project.config,
|
|
2474
|
+
scan,
|
|
2475
|
+
self.spec.effective(),
|
|
2476
|
+
locate,
|
|
2477
|
+
edited=bool(self.spec.pending),
|
|
2478
|
+
)
|
|
2479
|
+
current = {t.model_id for t in tests}
|
|
2480
|
+
reread = models & current if again else set()
|
|
2481
|
+
fresh = self._read_again(reread, tests) if reread else {}
|
|
2482
|
+
project.scanned, project.locate, project.tests = scan, locate, tests
|
|
2483
|
+
self._update_waiting()
|
|
2484
|
+
gone = set(self.comparisons) - current
|
|
2485
|
+
new = current - set(self.comparisons)
|
|
2486
|
+
for model_id in gone:
|
|
2487
|
+
del self.comparisons[model_id]
|
|
2488
|
+
# Keyed off the reads, not the comparisons: a read cached for a model
|
|
2489
|
+
# that has a result but no comparison (a scoring failure leaves one)
|
|
2490
|
+
# must not outlive the model either.
|
|
2491
|
+
for model_id in set(self._reads) - current:
|
|
2492
|
+
del self._reads[model_id]
|
|
2493
|
+
if reread:
|
|
2494
|
+
self._keep_read(reread, fresh)
|
|
2495
|
+
self._emit_spec(models | gone | new)
|
|
2496
|
+
self._score_all((models | new) & current)
|
|
2497
|
+
|
|
2498
|
+
def _read_again(
|
|
2499
|
+
self, models: set[str], tests: Sequence[TestModel]
|
|
2500
|
+
) -> dict[str, TestResult]:
|
|
2501
|
+
"""``models`` read from the last run with their TestModels in
|
|
2502
|
+
``tests``; nothing is stored. Only they are kept: ``read_results``
|
|
2503
|
+
reports every manifest entry, and the others are not being re-read.
|
|
2504
|
+
A read that raises leaves every result and read as it was."""
|
|
2505
|
+
project = self._require_project()
|
|
2506
|
+
to_read = [t for t in tests if t.model_id in models]
|
|
2507
|
+
fresh: dict[str, TestResult] = {}
|
|
2508
|
+
|
|
2509
|
+
def on_result(model_id: str, result: TestResult) -> None:
|
|
2510
|
+
if model_id in models:
|
|
2511
|
+
fresh[model_id] = result
|
|
2512
|
+
|
|
2513
|
+
project.backend.read_last_results(to_read, on_result=on_result)
|
|
2514
|
+
return fresh
|
|
2515
|
+
|
|
2516
|
+
def _keep_read(self, models: set[str], fresh: dict[str, TestResult]) -> None:
|
|
2517
|
+
"""``fresh`` (`_read_again`) in place of what an earlier read stored
|
|
2518
|
+
for ``models``. A model it has no result for (no manifest entry, or
|
|
2519
|
+
one its last run never finished) ends with none and is scored NOT
|
|
2520
|
+
RUN by the caller (D145), as after a full reload. The result FILE did not change, so its listing
|
|
2521
|
+
(`_ModelReads.names`) is kept, and the next `variable_names` does
|
|
2522
|
+
not list it again; what came from the old result goes."""
|
|
2523
|
+
project = self._require_project()
|
|
2524
|
+
for model_id in models:
|
|
2525
|
+
if model_id in fresh:
|
|
2526
|
+
project.results[model_id] = fresh[model_id]
|
|
2527
|
+
else:
|
|
2528
|
+
project.results.pop(model_id, None)
|
|
2529
|
+
old = self._reads.pop(model_id, None)
|
|
2530
|
+
if old is not None and old.names_read:
|
|
2531
|
+
reads = self._reads_for(model_id)
|
|
2532
|
+
reads.names, reads.names_read = old.names, True
|
|
2533
|
+
|
|
2534
|
+
def _score(self, model_id: str) -> TestComparison:
|
|
2535
|
+
project = self._require_project()
|
|
2536
|
+
test = project.test(model_id)
|
|
2537
|
+
if test is None:
|
|
2538
|
+
raise KeyError(model_id)
|
|
2539
|
+
comp = self._compare(test)
|
|
2540
|
+
self.comparisons[model_id] = comp
|
|
2541
|
+
return comp
|
|
2542
|
+
|
|
2543
|
+
def _compare(self, test: TestModel) -> TestComparison:
|
|
2544
|
+
"""Score ``test`` against the last results; stores nothing."""
|
|
2545
|
+
project = self._require_project()
|
|
2546
|
+
tolerance = project.config.tolerance
|
|
2547
|
+
return compare_one(
|
|
2548
|
+
test,
|
|
2549
|
+
project.results.get(test.model_id),
|
|
2550
|
+
project.store,
|
|
2551
|
+
float(tolerance) if tolerance is not None else DEFAULT_TOLERANCE,
|
|
2552
|
+
project.config.default_points,
|
|
2553
|
+
dataset_kind=project.dataset_kind,
|
|
2554
|
+
)
|
|
2555
|
+
|
|
2556
|
+
def _score_all(self, models: Iterable[str]) -> None:
|
|
2557
|
+
project = self._require_project()
|
|
2558
|
+
wanted = set(models)
|
|
2559
|
+
for test in project.tests:
|
|
2560
|
+
if test.model_id in wanted:
|
|
2561
|
+
self._score(test.model_id)
|
|
2562
|
+
self._emit("results", models=wanted)
|
|
2563
|
+
|
|
2564
|
+
def _reload_results(self) -> None:
|
|
2565
|
+
"""Re-read the last run and rescore — every model except the ones
|
|
2566
|
+
the queue is running (their directories are being rewritten, so
|
|
2567
|
+
their in-memory result and verdict stand until their run ends).
|
|
2568
|
+
Each result is scored as it is read and
|
|
2569
|
+
announced as ``loading(phase="results")``; tests with no result are
|
|
2570
|
+
scored and announced after the read, so the last announcement is
|
|
2571
|
+
``done == total``; ONE ``results`` event follows for the batch. A
|
|
2572
|
+
model whose scoring raises does not abort the read: the results stay
|
|
2573
|
+
complete, its stale verdict is dropped, and the first failure is
|
|
2574
|
+
raised after the batch event; ``open`` propagates it after its
|
|
2575
|
+
``done`` event. Each announcement carries ``error``:
|
|
2576
|
+
True for a model whose scoring raised (its verdict is absent),
|
|
2577
|
+
False otherwise."""
|
|
2578
|
+
project = self._require_project()
|
|
2579
|
+
locked = self._running_models()
|
|
2580
|
+
to_read = [t for t in project.tests if t.model_id not in locked]
|
|
2581
|
+
project.results = {m: r for m, r in project.results.items() if m in locked}
|
|
2582
|
+
for model_id in list(self._reads):
|
|
2583
|
+
if model_id not in locked:
|
|
2584
|
+
del self._reads[model_id]
|
|
2585
|
+
announced: set[str] = set()
|
|
2586
|
+
failures: dict[str, Exception] = {}
|
|
2587
|
+
|
|
2588
|
+
def score_and_announce(model_id: str) -> None:
|
|
2589
|
+
failed = False
|
|
2590
|
+
try:
|
|
2591
|
+
self._score(model_id)
|
|
2592
|
+
except Exception as e:
|
|
2593
|
+
self.comparisons.pop(model_id, None)
|
|
2594
|
+
failures.setdefault(model_id, e)
|
|
2595
|
+
failed = True
|
|
2596
|
+
if model_id in announced:
|
|
2597
|
+
return # one model under two manifest keys is read twice
|
|
2598
|
+
announced.add(model_id)
|
|
2599
|
+
self._emit(
|
|
2600
|
+
"loading",
|
|
2601
|
+
phase="results",
|
|
2602
|
+
model=model_id,
|
|
2603
|
+
done=len(announced),
|
|
2604
|
+
total=len(to_read),
|
|
2605
|
+
error=failed,
|
|
2606
|
+
)
|
|
2607
|
+
|
|
2608
|
+
def on_result(model_id: str, result: TestResult) -> None:
|
|
2609
|
+
if model_id in locked:
|
|
2610
|
+
# `read_results` reports every manifest entry, and a running
|
|
2611
|
+
# model — excluded from `to_read` — comes back as "not found
|
|
2612
|
+
# in discovery": storing it would rescore the model to
|
|
2613
|
+
# SIM_FAIL in the middle of its own run.
|
|
2614
|
+
return
|
|
2615
|
+
project.results[model_id] = result
|
|
2616
|
+
if project.test(model_id) is not None:
|
|
2617
|
+
score_and_announce(model_id)
|
|
2618
|
+
|
|
2619
|
+
project.backend.read_last_results(to_read, on_result=on_result)
|
|
2620
|
+
# Tests with no result at all (never run, never finished, or
|
|
2621
|
+
# dropped from the manifest) are scored now — NOT RUN (D145).
|
|
2622
|
+
for test in to_read:
|
|
2623
|
+
if test.model_id not in project.results:
|
|
2624
|
+
score_and_announce(test.model_id)
|
|
2625
|
+
self._emit("results", models={t.model_id for t in to_read})
|
|
2626
|
+
if failures:
|
|
2627
|
+
model_id, error = next(iter(failures.items()))
|
|
2628
|
+
raise RuntimeError(
|
|
2629
|
+
f"scoring {model_id} failed ({len(failures)} of {len(to_read)} models)"
|
|
2630
|
+
) from error
|
|
2631
|
+
|
|
2632
|
+
def _its_tree_written_out(
|
|
2633
|
+
self, candidate: SpecDocument, model_id: str
|
|
2634
|
+
) -> list[PatchOp]:
|
|
2635
|
+
"""The model's ops in ``candidate`` that do nothing but write out,
|
|
2636
|
+
as ``metrics``, the tree its SAVED entry already scores by — the ops
|
|
2637
|
+
`stage` drops; ``[]`` when there are none to drop. They are every op
|
|
2638
|
+
of the model on ``/metrics``, when all of this holds:
|
|
2639
|
+
|
|
2640
|
+
- the saved entry has no tree of its own and the entry as edited
|
|
2641
|
+
has one, equal to the tree the saved entry scores by;
|
|
2642
|
+
- the entry would score by that same tree WITHOUT its ``metrics``:
|
|
2643
|
+
a flat entry's tree is built from its ``variable_overrides``
|
|
2644
|
+
(`_saved_scoring_tree`), so those are as saved. With an override
|
|
2645
|
+
edited, the entry the other ops leave scores by another tree, and
|
|
2646
|
+
taking this one away would change its verdict.
|
|
2647
|
+
|
|
2648
|
+
What is kept is then the entry as edited but for its ``metrics``,
|
|
2649
|
+
and nothing checks it: an op the applier admits is an add, a replace
|
|
2650
|
+
or a remove whose path is at or under exactly one of
|
|
2651
|
+
``/comparison``, ``/metrics`` and ``/plots``
|
|
2652
|
+
(``patch_apply.DEFAULT_WHITELIST``), so the ops under ``/metrics``
|
|
2653
|
+
neither read nor write another member, nor the others ``metrics``.
|
|
2654
|
+
|
|
2655
|
+
The other members need not be as saved: a watch plot added beside a
|
|
2656
|
+
tolerance typed and cleared is one pending edit, the plot's. It was
|
|
2657
|
+
the whole entry or nothing, and the tree stayed pending beside any
|
|
2658
|
+
other edit — counted, previewed as "Save also writes the tree", and
|
|
2659
|
+
saved as a ``metrics`` block where only ``plots`` had changed.
|
|
2660
|
+
|
|
2661
|
+
"The tree the saved entry scores by" is `_materialization`'s and no
|
|
2662
|
+
other's, so what is dropped here and what `stage` writes out cannot
|
|
2663
|
+
drift: the value of an op it built and marked (`PatchOp.materializes`
|
|
2664
|
+
— marked only while it IS the saved entry's tree,
|
|
2665
|
+
`_saved_scoring_tree`) where the pending list still holds one, and
|
|
2666
|
+
otherwise `_materialization` asked for the saved entry. The marked
|
|
2667
|
+
op is read first because it is free: every keystroke in a flat
|
|
2668
|
+
test's editor comes through here with one, and the other way scores
|
|
2669
|
+
the test as saved (`saved_comparison`: a merge and a compare). The
|
|
2670
|
+
other way is for a tree no op of the Session's wrote — a move or a
|
|
2671
|
+
wrap stages the whole tree (`edit_tree`), and moved back it is the
|
|
2672
|
+
implicit one again, unmarked."""
|
|
2673
|
+
entry = candidate.entry(model_id)
|
|
2674
|
+
if entry is None or not has_metrics(entry):
|
|
2675
|
+
return []
|
|
2676
|
+
if not self._saved_scoring_tree(candidate, model_id):
|
|
2677
|
+
return []
|
|
2678
|
+
mine = [op for op in candidate.pending if op.model_id == model_id]
|
|
2679
|
+
marked = [op.value for op in mine if op.materializes]
|
|
2680
|
+
if marked:
|
|
2681
|
+
tree = marked[0]
|
|
2682
|
+
else:
|
|
2683
|
+
op, _why = self._materialization(model_id, saved=True)
|
|
2684
|
+
if op is None:
|
|
2685
|
+
return []
|
|
2686
|
+
tree = op["value"]
|
|
2687
|
+
if not json_equal(entry["metrics"], tree):
|
|
2688
|
+
return []
|
|
2689
|
+
return [
|
|
2690
|
+
op
|
|
2691
|
+
for op in mine
|
|
2692
|
+
if op.path == "/metrics" or op.path.startswith("/metrics/")
|
|
2693
|
+
]
|
|
2694
|
+
|
|
2695
|
+
def _saved_scoring_tree(self, spec: SpecDocument, model_id: str) -> bool:
|
|
2696
|
+
"""Whether the tree `_materialization` builds for the model's entry
|
|
2697
|
+
in ``spec`` — the Session's document, or the candidate `stage` is
|
|
2698
|
+
deciding on — is the one the SAVED entry scores: what marks it
|
|
2699
|
+
(`PatchOp.materializes`), and what lets it go again
|
|
2700
|
+
(`_its_tree_written_out`). Never when the saved entry has its own
|
|
2701
|
+
``metrics``: it scores that tree, and one built after a pending op
|
|
2702
|
+
removed it is a change. Otherwise the tree is built from
|
|
2703
|
+
``comparison.variable_overrides`` (the tracked variables are not
|
|
2704
|
+
stageable), so it is when those are as saved."""
|
|
2705
|
+
|
|
2706
|
+
def overrides(entry: dict) -> dict:
|
|
2707
|
+
return (entry.get("comparison") or {}).get("variable_overrides") or {}
|
|
2708
|
+
|
|
2709
|
+
saved = spec.saved_entry(model_id) or {"model": model_id}
|
|
2710
|
+
if has_metrics(saved):
|
|
2711
|
+
return False
|
|
2712
|
+
return overrides(saved) == overrides(spec.entry(model_id) or {})
|
|
2713
|
+
|
|
2714
|
+
def _materialization(
|
|
2715
|
+
self, model_id: str, *, saved: bool = False
|
|
2716
|
+
) -> tuple[dict | None, str | None]:
|
|
2717
|
+
"""``(op, None)``: the ``add /metrics`` op for a test the flat path
|
|
2718
|
+
scored, built from the variables it scored — the leaves the report
|
|
2719
|
+
shows, in the report's order, so leaf ``i`` is
|
|
2720
|
+
``/metrics/children/i`` on both tiers. ``saved``: for the entry as
|
|
2721
|
+
the FILE holds it and the comparison of that (`saved_comparison`),
|
|
2722
|
+
whatever is pending — the tree `_its_tree_written_out` drops.
|
|
2723
|
+
For a test tracked by its
|
|
2724
|
+
``.mo`` annotation those names are in no spec entry (and the entry
|
|
2725
|
+
may not exist yet: the op creates it). ``(None, None)`` when the
|
|
2726
|
+
entry already has a tree (`has_metrics`: a null ``metrics`` is
|
|
2727
|
+
none). Otherwise ``(None, why)`` — and ``rescore`` stages nothing:
|
|
2728
|
+
:data:`SIMULATE_ONLY` or :data:`NOTHING_SCORED` when nothing was
|
|
2729
|
+
scored (NO_REF, SIM_FAIL, NOT RUN, simulate-only; and a model not in the
|
|
2730
|
+
project, which `stage` refuses with it as a patch — `tree_or_reason`
|
|
2731
|
+
raises a KeyError for one before asking); :data:`STALE` when a scored variable's name is not one the
|
|
2732
|
+
loaded result carries (a comparison from a run the results no
|
|
2733
|
+
longer match); :data:`NO_TREE` when a tree would not score what the
|
|
2734
|
+
flat path scored: an unnamed or non-literal scored name, a string
|
|
2735
|
+
``quantity`` in an override (the flat path refuses it, a tree would
|
|
2736
|
+
score it), an override naming an unregistered mode (the flat path
|
|
2737
|
+
fails it; a tree would score nrmse), or a reference-JSON override
|
|
2738
|
+
the spec does not shadow (the flat path merges it; a tree ignores
|
|
2739
|
+
it). Each case has its negative test in tests/test_session.py."""
|
|
2740
|
+
project = self._require_project()
|
|
2741
|
+
bare: dict = {"model": model_id}
|
|
2742
|
+
if saved:
|
|
2743
|
+
entry = self.spec.saved_entry(model_id) or bare
|
|
2744
|
+
else:
|
|
2745
|
+
entry = self.spec.entry(model_id) or bare
|
|
2746
|
+
if has_metrics(entry):
|
|
2747
|
+
return None, None
|
|
2748
|
+
comp = (
|
|
2749
|
+
self.saved_comparison(model_id) if saved else self.comparisons.get(model_id)
|
|
2750
|
+
)
|
|
2751
|
+
if comp is None or not comp.variables:
|
|
2752
|
+
test = project.test(model_id)
|
|
2753
|
+
simulate_only = test is not None and test.simulate_only
|
|
2754
|
+
return None, SIMULATE_ONLY if simulate_only else NOTHING_SCORED
|
|
2755
|
+
# A pattern-tracked spec materializes from the names the pattern
|
|
2756
|
+
# resolved today (owner ruling, D126): the pattern keeps tracking,
|
|
2757
|
+
# the tree scores what it names, and a variable that later matches
|
|
2758
|
+
# without a leaf raises the "tracked but not scored" warning.
|
|
2759
|
+
names = [vc.name for vc in comp.variables]
|
|
2760
|
+
if not all(n and is_literal_variable(n) for n in names):
|
|
2761
|
+
return None, NO_TREE
|
|
2762
|
+
result = project.results.get(model_id)
|
|
2763
|
+
resolved = {v.name for v in result.variables} if result is not None else set()
|
|
2764
|
+
if not all(n in resolved for n in names):
|
|
2765
|
+
# A leaf must name a variable the loaded result carries, or the
|
|
2766
|
+
# tree path resolves it to nothing.
|
|
2767
|
+
return None, STALE
|
|
2768
|
+
if unknown_override_modes(entry):
|
|
2769
|
+
return None, NO_TREE
|
|
2770
|
+
spec_overrides = (entry.get("comparison") or {}).get("variable_overrides") or {}
|
|
2771
|
+
if any(
|
|
2772
|
+
isinstance(ov, dict) and isinstance(ov.get("quantity"), str)
|
|
2773
|
+
for ov in spec_overrides.values()
|
|
2774
|
+
):
|
|
2775
|
+
# The flat path refuses a stats-path quantity ("needs a metrics
|
|
2776
|
+
# tree"); the tree path would score it — a FAIL that becomes a
|
|
2777
|
+
# PASS on a look. Authoring the tree is the user's act, not ours.
|
|
2778
|
+
return None, NO_TREE
|
|
2779
|
+
ref = project.store.get_reference(model_id) or {}
|
|
2780
|
+
ref_overrides = (ref.get("comparison") or {}).get("variable_overrides") or {}
|
|
2781
|
+
if any(n in ref_overrides and n not in spec_overrides for n in names):
|
|
2782
|
+
return None, NO_TREE
|
|
2783
|
+
return implicit_tree_op(entry, variables=names), None
|
|
2784
|
+
|
|
2785
|
+
def _refuse_if_busy(self) -> None:
|
|
2786
|
+
if self._live:
|
|
2787
|
+
raise QueueError(
|
|
2788
|
+
f"{_queued_or_running(len(self._live))}; remove or cancel them, or wait"
|
|
2789
|
+
)
|
|
2790
|
+
if self.spec.pending:
|
|
2791
|
+
raise PendingEditsError(edit_count(self.spec.pending))
|
|
2792
|
+
|
|
2793
|
+
def _emit_spec(self, models: set[str]) -> None:
|
|
2794
|
+
"""The `spec` event for ``models``, numbered one up from the last
|
|
2795
|
+
(`spec_revision`)."""
|
|
2796
|
+
self.spec_revision += 1
|
|
2797
|
+
self._emit("spec", models=models, revision=self.spec_revision)
|
|
2798
|
+
|
|
2799
|
+
def _emit(self, event: Event, **payload: Any) -> None:
|
|
2800
|
+
"""Every listener hears every event: one that raises (a deleted UI
|
|
2801
|
+
element, say) is logged and skipped, never allowed to stop the
|
|
2802
|
+
others or to unwind what emitted it."""
|
|
2803
|
+
for cb in list(self._listeners.get(event, [])):
|
|
2804
|
+
try:
|
|
2805
|
+
cb({"event": event, **payload})
|
|
2806
|
+
except Exception:
|
|
2807
|
+
logger.exception("listener for %r failed", event)
|
|
2808
|
+
|
|
2809
|
+
|
|
2810
|
+
@dataclass(frozen=True)
|
|
2811
|
+
class ProjectView:
|
|
2812
|
+
"""The project as a capability sees it: no store, no backend."""
|
|
2813
|
+
|
|
2814
|
+
config: Config
|
|
2815
|
+
tests: tuple[TestModel, ...]
|
|
2816
|
+
results: Mapping[str, TestResult]
|
|
2817
|
+
dataset_kind: DatasetType
|
|
2818
|
+
_store: ReferenceStore = field(repr=False, compare=False)
|
|
2819
|
+
|
|
2820
|
+
def reference(self, model_id: str) -> dict | None:
|
|
2821
|
+
"""The primary baseline's JSON, or None."""
|
|
2822
|
+
return self._store.get_reference(model_id)
|
|
2823
|
+
|
|
2824
|
+
def baseline_names(self, model_id: str) -> tuple[str, ...]:
|
|
2825
|
+
"""What a leaf's ``against`` may name: ``primary`` when a baseline
|
|
2826
|
+
is stored, then every soft_check, in store order."""
|
|
2827
|
+
names: list[str] = []
|
|
2828
|
+
if self._store.get_reference(model_id) is not None:
|
|
2829
|
+
names.append(PRIMARY_BASELINE)
|
|
2830
|
+
names.extend(self._store.get_soft_checks(model_id))
|
|
2831
|
+
return tuple(names)
|
|
2832
|
+
|
|
2833
|
+
def role_lookup(self, model_id: str) -> BaselineRoleLookup:
|
|
2834
|
+
"""Each baseline name's role for ``model_id`` — what the D66
|
|
2835
|
+
validator checks a tree against, as `Session.stage` does: a move
|
|
2836
|
+
checks its new tree with it before it is staged."""
|
|
2837
|
+
from ..comparison.validator import role_lookup_from_store
|
|
2838
|
+
|
|
2839
|
+
return role_lookup_from_store(self._store, model_id)
|
|
2840
|
+
|
|
2841
|
+
|
|
2842
|
+
class SessionView:
|
|
2843
|
+
"""What a capability receives: the Session's state and its readers, and
|
|
2844
|
+
the selection of the page that built the view — nothing that writes.
|
|
2845
|
+
The enforcement is structural — a capability never holds the Session."""
|
|
2846
|
+
|
|
2847
|
+
__slots__ = ("_session", "_selection")
|
|
2848
|
+
|
|
2849
|
+
def __init__(
|
|
2850
|
+
self,
|
|
2851
|
+
session: Session,
|
|
2852
|
+
# B008 is about a fresh mutable per call; Selection is frozen.
|
|
2853
|
+
selection: Selection = Selection(), # noqa: B008
|
|
2854
|
+
):
|
|
2855
|
+
self._session = session
|
|
2856
|
+
self._selection = selection
|
|
2857
|
+
|
|
2858
|
+
@property
|
|
2859
|
+
def project(self) -> ProjectView | None:
|
|
2860
|
+
p = self._session.project
|
|
2861
|
+
if p is None:
|
|
2862
|
+
return None
|
|
2863
|
+
return ProjectView(
|
|
2864
|
+
config=p.config,
|
|
2865
|
+
tests=tuple(p.tests),
|
|
2866
|
+
results=MappingProxyType(p.results),
|
|
2867
|
+
dataset_kind=p.dataset_kind,
|
|
2868
|
+
_store=p.store,
|
|
2869
|
+
)
|
|
2870
|
+
|
|
2871
|
+
@property
|
|
2872
|
+
def spec_path(self) -> Path:
|
|
2873
|
+
return self._session.spec.path
|
|
2874
|
+
|
|
2875
|
+
@property
|
|
2876
|
+
def pending(self) -> tuple[PatchOp, ...]:
|
|
2877
|
+
return tuple(self._session.spec.pending)
|
|
2878
|
+
|
|
2879
|
+
@property
|
|
2880
|
+
def selection(self) -> Selection:
|
|
2881
|
+
return self._selection
|
|
2882
|
+
|
|
2883
|
+
@property
|
|
2884
|
+
def comparisons(self) -> Mapping[str, TestComparison]:
|
|
2885
|
+
return MappingProxyType(self._session.comparisons)
|
|
2886
|
+
|
|
2887
|
+
@property
|
|
2888
|
+
def artifacts(self) -> tuple[Artifact, ...]:
|
|
2889
|
+
return tuple(self._session.artifacts)
|
|
2890
|
+
|
|
2891
|
+
@property
|
|
2892
|
+
def notices(self) -> tuple[str, ...]:
|
|
2893
|
+
"""The lines the queue panel shows, oldest first: what each store write
|
|
2894
|
+
applied so far reported, plus the open-time warning if there was one."""
|
|
2895
|
+
return tuple(self._session.notices)
|
|
2896
|
+
|
|
2897
|
+
def queue_state(self, model_id: str) -> ItemState | None:
|
|
2898
|
+
"""Where the test stands in the simulation queue
|
|
2899
|
+
(:meth:`Session.queue_state`): WAITING or RUNNING while it is in
|
|
2900
|
+
it, how its last run there ended until that row is cleared, else
|
|
2901
|
+
``None``."""
|
|
2902
|
+
return self._session.queue_state(model_id)
|
|
2903
|
+
|
|
2904
|
+
@property
|
|
2905
|
+
def queue_refusal(self) -> str | None:
|
|
2906
|
+
"""Why nothing can be queued on this project's backend, or ``None``."""
|
|
2907
|
+
return self._session.queue_refusal
|
|
2908
|
+
|
|
2909
|
+
def partial(self, model_id: str) -> Partial | None:
|
|
2910
|
+
"""What the test's last run left when it did not finish
|
|
2911
|
+
(:meth:`Session.partial`), or ``None``."""
|
|
2912
|
+
return self._session.partial(model_id)
|
|
2913
|
+
|
|
2914
|
+
def rescore(self, model_id: str) -> TestComparison:
|
|
2915
|
+
return self._session.rescore(model_id)
|
|
2916
|
+
|
|
2917
|
+
def view(self, model_id: str, variable: str) -> VariableView:
|
|
2918
|
+
return self._session.view(model_id, variable)
|
|
2919
|
+
|
|
2920
|
+
def figure(
|
|
2921
|
+
self,
|
|
2922
|
+
model_id: str,
|
|
2923
|
+
variable: str,
|
|
2924
|
+
*,
|
|
2925
|
+
editing: Collection[str] | None = None,
|
|
2926
|
+
visibility: Visibility | None = None,
|
|
2927
|
+
theme: str = "default",
|
|
2928
|
+
) -> dict:
|
|
2929
|
+
return self._session.figure(
|
|
2930
|
+
model_id, variable, editing=editing, visibility=visibility, theme=theme
|
|
2931
|
+
)
|
|
2932
|
+
|
|
2933
|
+
def variable_names(self, model_id: str) -> list[str]:
|
|
2934
|
+
return self._session.variable_names(model_id)
|
|
2935
|
+
|
|
2936
|
+
def variable_texts(self, model_id: str) -> dict[str, VariableText]:
|
|
2937
|
+
"""Both readings of every variable of the test's page
|
|
2938
|
+
(:meth:`Session.variable_texts`): a control that shows the muted
|
|
2939
|
+
name beside a variable reads the ``label``, as the card title does."""
|
|
2940
|
+
return self._session.variable_texts(model_id)
|
|
2941
|
+
|
|
2942
|
+
def labels(self, model_id: str) -> dict[str, str]:
|
|
2943
|
+
"""``{name: what a capability's own text reads for it}`` —
|
|
2944
|
+
:meth:`Session.labels`, so a panel's caption and the page's rows
|
|
2945
|
+
say the same thing. It is the TEXT, not the plain label: a
|
|
2946
|
+
capability's caption carries no muted name to disambiguate it."""
|
|
2947
|
+
return self._session.labels(model_id)
|
|
2948
|
+
|
|
2949
|
+
def plots(self, model_id: str) -> tuple[ResolvedPlot, ...]:
|
|
2950
|
+
return self._session.plots(model_id)
|
|
2951
|
+
|
|
2952
|
+
def plot_figure(
|
|
2953
|
+
self,
|
|
2954
|
+
model_id: str,
|
|
2955
|
+
index: int,
|
|
2956
|
+
*,
|
|
2957
|
+
editing: Collection[str] | None = None,
|
|
2958
|
+
visibility: Visibility | None = None,
|
|
2959
|
+
theme: str = "default",
|
|
2960
|
+
) -> dict:
|
|
2961
|
+
return self._session.plot_figure(
|
|
2962
|
+
model_id, index, editing=editing, visibility=visibility, theme=theme
|
|
2963
|
+
)
|
|
2964
|
+
|
|
2965
|
+
def tree_view(self, model_id: str) -> dict | None:
|
|
2966
|
+
return self._session.tree_view(model_id)
|
|
2967
|
+
|
|
2968
|
+
def tree(self, model_id: str) -> dict | None:
|
|
2969
|
+
return self._session.tree(model_id)
|
|
2970
|
+
|
|
2971
|
+
def tree_or_reason(self, model_id: str) -> tuple[dict | None, str | None]:
|
|
2972
|
+
return self._session.tree_or_reason(model_id)
|
|
2973
|
+
|
|
2974
|
+
|
|
2975
|
+
def _known_models(config: Config, document: object) -> list[str]:
|
|
2976
|
+
"""The model ids the spec document and the work dir's batch manifest
|
|
2977
|
+
name — spec first, each once — read without the scan and without the
|
|
2978
|
+
spec parser, which validates trees and can raise: a bad one is the
|
|
2979
|
+
merge's to raise. Both files are data; what is not an entry is
|
|
2980
|
+
skipped."""
|
|
2981
|
+
ids: list[str] = []
|
|
2982
|
+
tests = document.get("tests") if isinstance(document, dict) else None
|
|
2983
|
+
for entry in tests if isinstance(tests, list) else []:
|
|
2984
|
+
if isinstance(entry, dict) and isinstance(entry.get("model"), str):
|
|
2985
|
+
ids.append(entry["model"])
|
|
2986
|
+
work_dir = config.work_dir
|
|
2987
|
+
manifest = work_dir / "batch_manifest.json" if work_dir is not None else None
|
|
2988
|
+
if manifest is not None and manifest.is_file():
|
|
2989
|
+
try:
|
|
2990
|
+
entries = json.loads(manifest.read_text(encoding="utf-8"))
|
|
2991
|
+
except (OSError, ValueError):
|
|
2992
|
+
entries = {}
|
|
2993
|
+
for entry in entries.values() if isinstance(entries, dict) else ():
|
|
2994
|
+
if isinstance(entry, dict) and isinstance(entry.get("model_id"), str):
|
|
2995
|
+
ids.append(entry["model_id"])
|
|
2996
|
+
return list(dict.fromkeys(ids))
|
|
2997
|
+
|
|
2998
|
+
|
|
2999
|
+
def _in_simulation(path: str) -> bool:
|
|
3000
|
+
"""Whether a patch path names a test's simulation settings."""
|
|
3001
|
+
return path == "/simulation" or path.startswith("/simulation/")
|
|
3002
|
+
|
|
3003
|
+
|
|
3004
|
+
def _doing(item: QueueItem) -> str:
|
|
3005
|
+
return "running" if item.state is ItemState.RUNNING else "queued"
|
|
3006
|
+
|
|
3007
|
+
|
|
3008
|
+
def _queued_or_running(count: int) -> str:
|
|
3009
|
+
return f"{plural(count, 'test')} {'is' if count == 1 else 'are'} queued or running"
|
|
3010
|
+
|
|
3011
|
+
|
|
3012
|
+
def _in_order(ids: Iterable[str]) -> list[str]:
|
|
3013
|
+
seen: dict[str, None] = {}
|
|
3014
|
+
for i in ids:
|
|
3015
|
+
seen.setdefault(i, None)
|
|
3016
|
+
return list(seen)
|