dynamic-systems-testing-framework 1.0.1__py3-none-any.whl

dstf/storage/accept.py

@@ -0,0 +1,216 @@
+"""Accepting a run's results as baselines — the one function the CLI's two
+``--accept`` paths, its interactive review and the authoring tool's Accept
+call."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from ..backends.common.result_variables import criterion_variables, tracks
+from ..backends.types import TestResult
+from ..comparison.comparator import nothing_scored, refused_base_tolerance
+from ..discovery.test_registry import TestModel
+from .batch_manifest import unfinished_run_models
+from .ref_index import RefIndex
+from .reference_store import ReferenceStore
+
+#: A baseline file larger than this many bytes — 5 MB — is said by the accept
+#: that wrote it (D153). A constant, not a key: the remedy it names is the key.
+LARGE_BASELINE_BYTES = 5_000_000
+
+
+def large_baseline_warning(store: ReferenceStore, model_id: str) -> str | None:
+    """The sentence for ``model_id``'s reference file when it is larger than
+    `LARGE_BASELINE_BYTES`, else ``None``. A warning: the baseline IS stored.
+    The size is the file's, and its counts are read back from it — only for
+    a file that is over, so an accept of small baselines reads nothing."""
+    test_id = store.index.get_id(model_id)
+    if test_id is None:
+        return None
+    ref_file = store.ref_dir / RefIndex.ref_filename(test_id)
+    try:
+        size = ref_file.stat().st_size
+    except OSError:
+        return None
+    if size <= LARGE_BASELINE_BYTES:
+        return None
+    reference = store.get_reference(model_id) or {}
+    points = len(reference.get("time", []))
+    n_vars = len(reference.get("variables", []))
+    return (
+        f"{ref_file.name} ({model_id}) is {size / 1e6:.1f} MB ({points:,} points "
+        f"× {n_vars:,} variable{'' if n_vars == 1 else 's'}) — set "
+        '"baseline_points" for this test to store fewer.'
+    )
+
+
+#: How many names a sentence that lists them spells out
+#: (:func:`lost_variables_warnings`, :func:`simulate_only_note`).
+_NAMED = 6
+
+
+def _named(names: list[str]) -> str:
+    """Up to `_NAMED` of ``names``, then ``and N more``."""
+    named = ", ".join(names[:_NAMED])
+    if len(names) > _NAMED:
+        named += f" and {len(names) - _NAMED} more"
+    return named
+
+
+def simulate_only_note(model_ids: list[str]) -> str | None:
+    """``Simulate-only, nothing to store: MyLib.Tank`` for the tests an
+    accept passed over because they declare nothing to score
+    (``TestModel.is_simulate_only``), else ``None``. Neither a warning nor a
+    skip: such a test has no baseline to store, and its accept did not fail
+    — but `Accepted 0 test baselines` with no word of the test read as an
+    accept that did nothing, for no reason."""
+    if not model_ids:
+        return None
+    return f"Simulate-only, nothing to store: {_named(model_ids)}"
+
+
+def _variable_names(reference: dict | None) -> list[str]:
+    return [v.get("name") or "" for v in (reference or {}).get("variables", [])]
+
+
+def lost_variables_warnings(
+    store: ReferenceStore, test: TestModel, held_before: list[str]
+) -> list[str]:
+    """The sentences for a baseline just written that no longer holds a
+    variable the one it replaced held (``held_before``, its names); none
+    when it lost nothing. A warning: the baseline IS stored — accepting is
+    how a variable is dropped on purpose — but a run that LOST one rewrote
+    the baseline without it, and nothing said so.
+
+    Two sentences, since a variable leaves a baseline for two reasons, and
+    each is said of its own names. One the test still tracks — a UnitTests
+    variable, a ``variables`` pattern, a criterion's variable — the run did
+    not deliver: ``the accepted result does not hold it``. One it no longer
+    tracks was taken out on purpose, and the run may well still write it:
+    ``the test no longer tracks it`` — that half read ``the accepted result
+    does not hold it`` too, of a result file that held it."""
+    model_id = test.model_id
+    test_id = store.index.get_id(model_id)
+    if test_id is None:
+        return []
+    holds = set(_variable_names(store.get_reference(model_id)))
+    lost = [name for name in held_before if name and name not in holds]
+    tracked = tracks(test)
+    by_criterion = set(criterion_variables(test, store.config.diagnostic_variables))
+    asked = [name for name in lost if tracked(name) or name in by_criterion]
+    dropped = [name for name in lost if name not in asked]
+
+    def said(names: list[str], why: str) -> str:
+        return (
+            f"{RefIndex.ref_filename(test_id)} ({model_id}) no longer holds "
+            f"{_named(names)} — {why} {'it' if len(names) == 1 else 'them'}."
+        )
+
+    return [
+        *([said(asked, "the accepted result does not hold")] if asked else []),
+        *([said(dropped, "the test no longer tracks")] if dropped else []),
+    ]
+
+
+def accept_refusal(
+    store: ReferenceStore, test: TestModel, result: TestResult | None
+) -> str | None:
+    """Why the comparison REFUSES ``test`` — its base tolerance is no
+    number, or its result holds nothing it tracks — or ``None``: the
+    comparison's own decisions (`refused_base_tolerance`, `nothing_scored`),
+    asked of the test and its stored reference as ``compare_one`` asks
+    them. A test with no successful result is not refused: it has its own
+    reason."""
+    if result is None or not result.success:
+        return None
+    refused = refused_base_tolerance(
+        test, store.get_reference(test.model_id), store.config.tolerance
+    )
+    if refused is not None:
+        return refused.error_message
+    return nothing_scored(test, result)
+
+
+def accept_run_results(
+    store: ReferenceStore,
+    tests: list[TestModel],
+    results: dict[str, TestResult],
+    work_dir: Path,
+) -> tuple[int, list[tuple[str, str]], list[str], list[str]]:
+    """:meth:`ReferenceStore.accept_results` plus two refusals, two warnings
+    and a note.
+
+    A test the batch manifest in ``work_dir`` still records as pending was
+    in flight when a run was aborted: it has no recorded verdict, and its
+    truncated result file may still parse. Acceptance requires an
+    authoritative successful run, so it is skipped with the reason, never
+    stored. Deliberately NOT keyed on the comparison verdict — a test that
+    ran fine and now compares FAILED is exactly the drift case accepting
+    exists to re-baseline.
+
+    A test the comparison REFUSES (`accept_refusal`) is skipped with the
+    refusal's sentence: its result was scored against nothing, and the
+    baseline written would carry — or keep — a base no comparison reads.
+    Its reference file is not touched.
+
+    A baseline file just written that is larger than `LARGE_BASELINE_BYTES`
+    is said once (`large_baseline_warning`): it IS stored, and counts as
+    stored. Every accept path prints what this returns, so the CLI and the
+    tool say the same sentence. So is a baseline that no longer holds a
+    variable the one it replaced held (`lost_variables_warnings`).
+
+    A simulate-only test whose run completed is neither stored nor skipped:
+    it has no baseline to store. The tests passed over for that are named
+    in one line (`simulate_only_note`), which every accept path prints too.
+
+    Returns ``(stored, skipped, warnings, notes)``: how many baselines were
+    written; ``accept_results``'s skips with the refusals appended — the
+    refused tests in the order given, then the unfinished ones in model
+    order; the warnings in the order written: per test, what it lost, then
+    its size; and the lines that are neither — the simulate-only tests'.
+    """
+    unfinished = unfinished_run_models(work_dir) & {t.model_id for t in tests}
+    refused = {
+        t.model_id: reason
+        for t in tests
+        if t.model_id not in unfinished
+        and (reason := accept_refusal(store, t, results.get(t.model_id))) is not None
+    }
+    accepted = [
+        t for t in tests if t.model_id not in unfinished and t.model_id not in refused
+    ]
+    # What each baseline holds now: read before the accept replaces it.
+    held_before = {
+        t.model_id: _variable_names(store.get_reference(t.model_id)) for t in accepted
+    }
+    stored, skipped, simulate_only = store.accept_results(accepted, results)
+    by_model = {t.model_id: t for t in accepted}
+    note = simulate_only_note(simulate_only)
+    return (
+        len(stored),
+        [
+            *skipped,
+            *refused.items(),
+            *(
+                (
+                    model_id,
+                    f"last run did not complete for this test (aborted or "
+                    f"cancelled while it was in flight; {work_dir} records no "
+                    f"verdict for it) — re-run before accepting",
+                )
+                for model_id in sorted(unfinished)
+            ),
+        ],
+        [
+            warning
+            for model_id in stored
+            for warning in (
+                *lost_variables_warnings(
+                    store, by_model[model_id], held_before[model_id]
+                ),
+                large_baseline_warning(store, model_id),
+            )
+            if warning is not None
+        ],
+        [note] if note is not None else [],
+    )

dstf/storage/baseline.py

@@ -0,0 +1,43 @@
+"""The `Baseline` value type shared by the store and its role modules.
+
+A baseline is one named set of reference trajectories: a ref file's
+``"primary"`` (synthesized from the flat on-disk fields by
+:func:`~dstf.storage.reference_store._extract_baselines`) or one soft_check
+read from ``soft_checks/ref_NNNN/<name>.json`` (D66). It lives in its own
+module so both :mod:`dstf.storage.reference_store` and
+:mod:`dstf.storage.roles` can depend on it without depending on each other.
+"""
+
+from dataclasses import dataclass, field
+
+#: Name of a reference file's one built-in baseline.
+PRIMARY_BASELINE = "primary"
+
+
+@dataclass(frozen=True)
+class Baseline:
+    """A single named baseline: a ref file's primary, or one soft_check.
+
+    See docs/architecture.md "Multiple named baselines (D66)".
+    """
+
+    #: Baseline name (e.g. ``"primary"``, ``"experiment"``, ``"analytical"``).
+    name: str
+    #: Shared time vector (all variables in this baseline share one time grid).
+    time: list[float]
+    #: Variable entries: ``[{"index", "name", "expression", "values"}, ...]``.
+    #: ``name`` is the variable's name in the result file and the only key;
+    #: ``expression`` is the label beside it (see ``VariableResult``).
+    variables: list[dict]
+    #: Diagnostic variable entries (same shape as ``variables``).
+    diagnostics: list[dict] = field(default_factory=list)
+    #: Simulation parameters that produced this baseline.
+    simulation: dict = field(default_factory=dict)
+    #: Comparison configuration (tolerance, variable_overrides).
+    comparison: dict = field(default_factory=dict)
+    #: Statistics captured at baseline-acceptance time.
+    statistics: dict = field(default_factory=dict)
+    #: Provenance metadata: origin, captured_at, backend, os, notes,
+    #: citation, plus arbitrary user metadata. For a ref file's primary this
+    #: is synthesized from ``date_added`` / ``last_updated``.
+    provenance: dict = field(default_factory=dict)

dstf/storage/batch_manifest.py

@@ -0,0 +1,204 @@
+"""The per-run batch manifest: test_key ↔ model_id, and the run's results.
+
+Lives in :mod:`dstf.storage` with the other on-disk record keepers — it is a
+JSON file in the work dir, written through the same atomic primitive as the
+reference files. The backends package re-exports it (``from dstf.backends
+import BatchManifest``) because every backend's ``run_tests`` returns one.
+
+``TestRunResult`` is imported only for typing: keeping it out of the runtime
+import graph is what lets ``dstf.storage.batch_manifest`` be imported first,
+before ``dstf.backends`` has run its package ``__init__``.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import threading
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+from .ref_index import _atomic_write_text
+from .run_stats import write_run_stats
+
+if TYPE_CHECKING:
+    from ..backends.types import TestRunResult
+
+logger = logging.getLogger(__name__)
+
+#: A run that was stopped on request (the simulation queue's cancel). It is
+#: recorded, not left pending: a pending entry falls through to the
+#: status.json cross-check, where an earlier run's "failed" row for the same
+#: model would be applied to a test nobody failed.
+CANCELLED_RUN_STATUS = "cancelled"
+
+#: ``last_run_status`` values that mean "this run recorded what became of
+#: this test". Only these make a manifest entry authoritative for a reader.
+RECORDED_RUN_STATUSES = ("passed", "failed", "timed_out", CANCELLED_RUN_STATUS)
+
+#: Stamped on every in-scope entry when a run starts, and overwritten the
+#: instant that test's result lands. An entry still saying ``pending`` when a
+#: reader arrives means a run began and never reached a verdict for it — an
+#: abort, a crash, a Ctrl-C — so the reader must NOT treat it as authoritative
+#: and must fall back to the status.json cross-check. Without this stamp a
+#: PREVIOUS run's ``passed`` survived ``assign_test_keys``' carry-forward into
+#: an aborted run's manifest and reported green over a partial ``.mat``.
+PENDING_RUN_STATUS = "pending"
+
+#: The two statuses of a last run that left NO result — it never finished the
+#: test, or was told to stop. Whatever file such a run left is partial, and
+#: is neither read nor accepted.
+NO_RESULT_RUN_STATUSES = (PENDING_RUN_STATUS, CANCELLED_RUN_STATUS)
+
+
+@dataclass
+class BatchManifest:
+    """Maps test keys to model IDs (and optionally ref IDs) and tracks run results."""
+
+    work_dir: Path
+    manifest: dict[str, dict]  # test_NNNN -> {"model_id": ..., "ref_id": ...}
+    results: list[TestRunResult] = field(default_factory=list)
+    #: Serializes the read-modify-save in :meth:`record_result`, which every
+    #: dispatch thread calls as its test finishes.
+    _write_lock: threading.Lock = field(
+        default_factory=threading.Lock, repr=False, compare=False
+    )
+
+    def test_dir(self, test_key: str) -> Path:
+        """Per-test subdirectory for all simulation artifacts."""
+        return self.work_dir / test_key
+
+    def model_id(self, test_key: str) -> str:
+        """Get the model ID for a test key."""
+        return self.manifest[test_key]["model_id"]
+
+    def save(self) -> Path:
+        path = self.work_dir / "batch_manifest.json"
+        # review 2026-07-06 finding 66: atomic replace — a torn write made
+        # every subsequent run/compare crash at startup until the file was
+        # hand-deleted.
+        _atomic_write_text(path, json.dumps(self.manifest, indent=2))
+        return path
+
+    def record_result(self, result: TestRunResult) -> None:
+        """Stamp one test's verdict into its manifest entry and persist NOW.
+
+        The run's per-test outcome has to outlive the process that produced
+        it: a later ``compare`` holds no run results, so without this it can
+        only read the ``.mat`` on disk — and a partial/stale one from a failed
+        run reads back as a success (D93 follow-on). ``last_run_status`` is
+        that record, in status.json's vocabulary
+        (``passed`` / ``failed`` / ``timed_out``), plus
+        :data:`CANCELLED_RUN_STATUS` for a run that was stopped on request.
+
+        Written per result rather than in a run epilogue **because the
+        epilogue is exactly what an abort skips**: Ctrl-C or an exception in
+        the dispatch loop would leave every entry at
+        :data:`PENDING_RUN_STATUS`, discarding verdicts the run had already
+        reached. Each save is a small atomic replace, on the same order of
+        cost as the status.json write the progress reporter already does per
+        state change.
+
+        Entries this run produced no result for are untouched, which is what
+        makes ``--merge`` and ``--rerun`` reports honest about the tests they
+        did not re-run.
+        """
+        if result.cancelled:
+            status = CANCELLED_RUN_STATUS
+        elif result.timed_out:
+            status = "timed_out"
+        else:
+            status = "passed" if result.success else "failed"
+        with self._write_lock:
+            for entry in self.manifest.values():
+                if entry["model_id"] == result.model_id:
+                    entry["last_run_status"] = status
+            self.save()
+        # The run's walls and statistics, beside the result they describe
+        # (storage.run_stats). After the verdict is saved and outside its
+        # lock: the file is the test's own, and its write never raises.
+        write_run_stats(self.test_dir(result.test_key), result)
+
+    def mark_pending(self, model_ids: set[str]) -> None:
+        """Stamp :data:`PENDING_RUN_STATUS` on the entries a run is about to
+        (re-)run, so a stale verdict cannot survive into it.
+
+        ``assign_test_keys`` carries every prior entry forward, so without
+        this a previous run's ``passed`` would still be sitting on a test that
+        this run fails and then aborts before recording — and the reader would
+        trust it. Entries outside *model_ids* keep their own last verdict.
+
+        Does not save: the caller (``_prepare_run``) saves the manifest once
+        the keys are assigned.
+        """
+        for entry in self.manifest.values():
+            if entry["model_id"] in model_ids:
+                entry["last_run_status"] = PENDING_RUN_STATUS
+
+    def enrich_ref_ids(self, ref_index) -> None:
+        """Add ref_id to each entry using the reference index."""
+        for _test_key, entry in self.manifest.items():
+            ref_id = ref_index.get_id(entry["model_id"])
+            entry["ref_id"] = f"ref_{ref_id}" if ref_id else None
+        self.save()
+
+    @classmethod
+    def load(cls, path: Path) -> BatchManifest:
+        try:
+            data = json.loads(path.read_text(encoding="utf-8"))
+            if not isinstance(data, dict):
+                raise ValueError(f"expected a JSON object, got {type(data).__name__}")
+        except (ValueError, OSError) as exc:
+            # review 2026-07-06 finding 66: a corrupt manifest must not crash
+            # startup. Quarantine it (keeping the bytes for forensics) and
+            # start from an empty manifest — every caller already treats a
+            # missing manifest as empty, so test keys are simply re-assigned
+            # on the next run. (JSONDecodeError is a ValueError subclass.)
+            corrupt = path.with_name(f"{path.name}.corrupt")
+            where = ""
+            try:
+                path.replace(corrupt)
+                where = f"; moved to {corrupt.name}"
+            except OSError:
+                pass
+            logger.warning(
+                "Corrupt batch manifest %s (%s)%s — starting with an empty manifest",
+                path,
+                exc,
+                where,
+            )
+            return cls(work_dir=path.parent, manifest={})
+        # Handle legacy format: test_key -> model_id string
+        normalized = {}
+        for key, val in data.items():
+            if isinstance(val, str):
+                normalized[key] = {"model_id": val, "ref_id": None}
+            else:
+                normalized[key] = val
+        return cls(work_dir=path.parent, manifest=normalized)
+
+
+def unfinished_run_models(work_dir: Path) -> set[str]:
+    """Model IDs whose last run started them and never reached a verdict.
+
+    A manifest entry still reading :data:`PENDING_RUN_STATUS` means a run
+    stamped it at startup and was aborted (Ctrl-C, crash, killed worker pool)
+    before that test finished; one reading :data:`CANCELLED_RUN_STATUS` was
+    stopped on request. The accept path uses this: blessing such a
+    test's on-disk ``.mat`` would bake in whatever the simulator had flushed
+    at the moment it died, and a truncated result file can still parse.
+
+    Entries with no field at all are NOT included — a manifest written before
+    the field existed says nothing about completeness, so those keep the
+    pre-existing status.json-guard behaviour (old data, not old code).
+    Returns an empty set when the work dir has no manifest.
+    """
+    manifest_path = work_dir / "batch_manifest.json"
+    if not manifest_path.exists():
+        return set()
+    return {
+        entry["model_id"]
+        for entry in BatchManifest.load(manifest_path).manifest.values()
+        if entry.get("last_run_status") in NO_RESULT_RUN_STATUSES
+    }