dynamic-systems-testing-framework 1.0.1__py3-none-any.whl

dstf/backends/progress.py

@@ -0,0 +1,395 @@
+"""Backend-agnostic live progress reporter.
+
+Writes `status.json` and `dashboard.html` to the work directory on every
+state change so a user can watch progress live while tests run.
+
+Event contract:
+    register(test_key, model_id)  — before any work starts
+    on_start(test_key, worker_id) — test/batch begins
+    on_finish(test_key, success, elapsed, detail, timed_out)
+
+Thread-safe. Writes are atomic (temp-file + replace).
+"""
+
+import json
+import os
+import threading
+import time
+import uuid
+from collections.abc import Mapping
+from dataclasses import asdict, dataclass, field
+from pathlib import Path
+
+#: A test's ``phase`` once its run has ended. It used to keep the last
+#: phase it was seen in: a finished run's status.json said every test was
+#: still "finalizing".
+DONE_PHASE = "done"
+
+
+@dataclass
+class TestStatus:
+    test_key: str
+    model_id: str
+    #: What became of the test's SIMULATION: queued | running | passed |
+    #: failed | timed_out | cancelled. ``passed`` says the simulation
+    #: finished — NOT that the comparison passed: that is ``verdict``.
+    status: str = "queued"
+    #: The comparison's verdict, as the console prints it (``PASS``,
+    #: ``FAIL``, ``SIM_FAIL``, ``NO_REF``, ``NOT RUN``), once a ``run`` or
+    #: ``compare`` has scored the test (`stamp_verdicts`); None until then.
+    verdict: str | None = None
+    started_at: float | None = None  # time.monotonic — for accurate elapsed computation
+    started_wall: float | None = (
+        None  # time.time (epoch) — for JS to compute live "running for Ns"
+    )
+    elapsed: float | None = None
+    detail: str | None = None
+    worker_id: int | None = None
+    report_dir: str | None = (
+        None  # e.g., "ref_0042" or "test_0005" — matches generate_report_suite naming
+    )
+    #: Where a RUNNING test is: "translating" / "simulating" / ... — and
+    #: ``DONE_PHASE`` once it has ended, however it ended. None before it
+    #: starts.
+    phase: str | None = None
+    # Per-field provenance, plumbed from TestModel.field_sources via register().
+    # Surfaced in the dashboard's Resolution column.
+    field_sources: dict = field(default_factory=dict)
+
+
+class ProgressReporter:
+    """Thread-safe live progress tracker.
+
+    Writes status.json + dashboard.html to work_dir on every state change.
+    """
+
+    def __init__(
+        self,
+        work_dir: Path,
+        total: int,
+        rerun_prefix: str | None = None,
+        metadata: dict | None = None,
+        theme: str = "default",
+    ):
+        self.work_dir = work_dir
+        self.total = total
+        # The bundled theme the live page is rendered in (Config.theme).
+        self.theme = theme
+        # Stashed for inclusion in status.json so the dashboard's rerun-
+        # command builder produces an absolute-path command that works
+        # from any CWD. Computed by callers via dashboard_render.build_
+        # rerun_prefix(config).
+        self.rerun_prefix = rerun_prefix
+        # Run-level provenance (backend family/configured name/os/version). Serialized into
+        # status.json under "metadata" and rendered as a banner by the
+        # reporters. None → the "metadata" key stays null (backward compat with
+        # snapshots written before this field existed). See run_metadata.py.
+        self._metadata = dict(metadata) if metadata else None
+        self._lock = threading.Lock()
+        self._write_lock = threading.Lock()
+        self._tests: dict[str, TestStatus] = {}
+        self._start_time = time.monotonic()
+        # Wall-clock anchor (epoch seconds). Sent into status.json so the
+        # dashboard JS can compute live "elapsed = now - start_wall" between
+        # meta-refresh ticks (otherwise the header elapsed appears frozen at
+        # whatever it was when the last test state-change wrote the snapshot).
+        self._start_wall = time.time()
+        # A finished run's own wall time, where its rows were `load`ed.
+        self._loaded_elapsed: float | None = None
+        # Set by finalize: the run is over, and nothing writes again.
+        self._closed = False
+
+    def update_metadata(self, **fields: object) -> None:
+        """Patch run-level metadata after the fact (e.g. a backend that only
+        learns its ``tool_version`` once its worker is live) and re-emit the
+        snapshot. No-op when no base metadata was supplied — we never fabricate
+        a provenance block from a late partial update.
+        """
+        with self._lock:
+            if self._metadata is None:
+                return
+            self._metadata.update(fields)
+        self._write()
+
+    def register(
+        self,
+        test_key: str,
+        model_id: str,
+        report_dir: str | None = None,
+        field_sources: dict | None = None,
+    ) -> None:
+        with self._lock:
+            self._tests[test_key] = TestStatus(
+                test_key=test_key,
+                model_id=model_id,
+                report_dir=report_dir,
+                field_sources=field_sources or {},
+            )
+        self._write()
+
+    def on_start(self, test_key: str, worker_id: int | None = None) -> None:
+        with self._lock:
+            ts = self._tests.get(test_key)
+            if ts is not None:
+                ts.status = "running"
+                ts.started_at = time.monotonic()
+                ts.started_wall = time.time()
+                ts.worker_id = worker_id
+                ts.phase = None
+        self._write()
+
+    def on_phase(self, test_key: str, phase: str) -> None:
+        with self._lock:
+            ts = self._tests.get(test_key)
+            if ts is not None and ts.status == "running":
+                ts.phase = phase
+        self._write()
+
+    def on_finish(
+        self,
+        test_key: str,
+        success: bool,
+        elapsed: float,
+        detail: str | None = None,
+        timed_out: bool = False,
+    ) -> None:
+        with self._lock:
+            ts = self._tests.get(test_key)
+            if ts is not None:
+                if timed_out:
+                    ts.status = "timed_out"
+                elif success:
+                    ts.status = "passed"
+                else:
+                    ts.status = "failed"
+                ts.elapsed = elapsed
+                ts.detail = detail
+                ts.phase = DONE_PHASE
+        self._write()
+
+    def on_result(
+        self,
+        test_key: str,
+        success: bool,
+        elapsed: float,
+        detail: str | None = None,
+        timed_out: bool = False,
+    ) -> None:
+        """End the row of a test whose run has ended, for the caller that
+        holds the run's RESULT: as :meth:`on_finish` ends it — unless the
+        backend already did, whose words then stand.
+
+        A backend returns a failed result without starting anything for a
+        test it cannot run (no source file, no FMU), and says nothing here
+        on the way: such a row stayed ``"status": "queued"``, ``"phase":
+        null`` in a finished run's status.json, beside its verdict
+        ``SIM_FAIL``. The run loop that collects every result calls this,
+        so no path of a backend's can leave a row open."""
+        with self._lock:
+            ts = self._tests.get(test_key)
+            if ts is None or ts.phase == DONE_PHASE:
+                return
+        self.on_finish(test_key, success, elapsed, detail=detail, timed_out=timed_out)
+
+    def load(
+        self, tests: list[TestStatus], *, started_wall: float, ended_wall: float
+    ) -> None:
+        """Take rows that are already final, whole, with no write per row —
+        a queue that drained reports itself in one snapshot. ``finalize``
+        writes it. The run they report began at ``started_wall`` and ended
+        at ``ended_wall`` (epoch seconds): its snapshot's ``start_wall``
+        and ``elapsed`` are theirs, not this reporter's own — it was made
+        after the run, and counted 0.0 s for one of a minute."""
+        with self._lock:
+            self._tests = {t.test_key: t for t in tests}
+            self._start_wall = started_wall
+            self._loaded_elapsed = max(0.0, ended_wall - started_wall)
+
+    def _snapshot(self) -> dict:
+        with self._lock:
+            tests = [asdict(t) for t in self._tests.values()]
+            metadata = dict(self._metadata) if self._metadata else None
+        counts = {"queued": 0, "running": 0, "passed": 0, "failed": 0, "timed_out": 0}
+        for t in tests:
+            counts[t["status"]] = counts.get(t["status"], 0) + 1
+        elapsed = (
+            time.monotonic() - self._start_time
+            if self._loaded_elapsed is None
+            else self._loaded_elapsed
+        )
+        done = counts["passed"] + counts["failed"] + counts["timed_out"]
+        eta = None
+        if done > 0 and done < self.total:
+            rate = done / elapsed if elapsed > 0 else 0
+            remaining = self.total - done
+            eta = remaining / rate if rate > 0 else None
+        return {
+            "total": self.total,
+            "elapsed": elapsed,
+            "eta_seconds": eta,
+            "counts": counts,
+            "tests": tests,
+            "rerun_prefix": self.rerun_prefix,
+            "start_wall": self._start_wall,
+            "updated_at": time.time(),
+            "metadata": metadata,
+        }
+
+    def _write(self, *, closing: bool = False) -> None:
+        # Serialize all file writes — without this, two threads can race on
+        # the same tmp filename and `replace` fails on Windows when the file
+        # is still open by another thread.
+        # review 2026-07-06 (finding 25): snapshot INSIDE the write lock —
+        # a snapshot taken outside could be written after a newer one,
+        # regressing status.json / the live dashboard.
+        with self._write_lock:
+            # After finalize, the command that ended the run owns the page:
+            # an event from an exec thread the backend abandoned (a timed-out
+            # worker) would render it live and unscored again. Checked under
+            # the lock, so no write can land after finalize's.
+            if self._closed:
+                return
+            snapshot = self._snapshot()
+            self._write_json(snapshot)
+            self._render_dashboard()
+            self._closed = closing
+
+    def _atomic_write(self, path: Path, text: str) -> None:
+        # Unique tmp name per write so concurrent writers never share a tmp
+        # file even if a higher-level lock is bypassed.
+        tmp = path.with_name(f"{path.name}.{os.getpid()}.{uuid.uuid4().hex}.tmp")
+        tmp.write_text(text, encoding="utf-8")
+        # Retry replace on Windows — antivirus, search indexer, browser
+        # previews, and Explorer can briefly lock either the tmp or the
+        # target file and cause WinError 5/32. Short backoff usually clears it.
+        last_err: OSError | None = None
+        for delay in (0, 0.05, 0.1, 0.2, 0.5):
+            if delay:
+                time.sleep(delay)
+            try:
+                tmp.replace(path)
+                return
+            except OSError as e:
+                last_err = e
+        # Give up: clean up tmp and silently drop this update so progress
+        # writes never crash the run. Worst case the dashboard misses one tick.
+        try:
+            tmp.unlink()
+        except OSError:
+            pass
+        # Re-raise on the very first write so a misconfiguration is visible,
+        # but suppress later transient failures.
+        if not (path.parent / path.name).exists():
+            raise last_err
+
+    def _write_json(self, snapshot: dict) -> None:
+        path = self.work_dir / "status.json"
+        self._atomic_write(path, json.dumps(snapshot, indent=2, default=str))
+
+    def _render_dashboard(self) -> None:
+        """Defer HTML rendering to dashboard_render.
+
+        ProgressReporter's job is to keep status.json fresh + own atomic
+        writes; the page rendering lives in reporting/dashboard_render.py
+        so the live and final pages share one template.
+        """
+        from ..reporting.dashboard_render import render_live
+
+        render_live(self.work_dir, theme=self.theme)
+
+    def finalize(self) -> None:
+        """Write the last status.json and render dashboard.html — LIVE.
+
+        The simulations are done; the compare phase is not. A static page
+        here is where a watching browser stops, every row reading
+        SIMULATED while the verdicts are computed and never loaded. The
+        command that ends the run renders the final page: with the
+        comparisons it holds (``run``, ``run -i``, ``compare``), or
+        without them when it scores nothing (``run --accept``, an aborted
+        run, the tool's queue as it drains). Nothing this reporter hears after it is
+        written.
+        """
+        self._write(closing=True)
+
+
+def stamp_verdicts(work_dir: Path, verdicts: Mapping[str, str | None]) -> None:
+    """Set each test's ``verdict`` in ``status.json`` — ``{model_id: word}``,
+    None to clear one — beside its simulation ``status``, and leave every
+    other key as it is. For the command that scored the run, after the
+    reporter that wrote the snapshot has closed (and for a ``compare``,
+    which has no reporter at all). No snapshot, or no test of it named:
+    nothing is written."""
+    snapshot = read_status(work_dir)
+    if snapshot is None:
+        return
+    changed = False
+    for row in snapshot.get("tests") or []:
+        if isinstance(row, dict) and row.get("model_id") in verdicts:
+            verdict = verdicts[row["model_id"]]
+            if row.get("verdict") != verdict or "verdict" not in row:
+                row["verdict"] = verdict
+                changed = True
+    if not changed:
+        return
+    from ..storage.ref_index import _atomic_write_text
+
+    _atomic_write_text(
+        work_dir / "status.json", json.dumps(snapshot, indent=2, default=str)
+    )
+
+
+def read_status(work_dir: Path) -> dict | None:
+    """The last snapshot ``ProgressReporter`` wrote to *work_dir*, or ``None``
+    when there is none to read.
+
+    The one reader of ``status.json``: the dashboard renderer, the backends'
+    verdict fallback, the per-test banner and the authoring shell's job panel
+    all come through here. Never raises — the shell calls it from a UI timer,
+    and an unreadable snapshot is a page with no progress rows, not a
+    traceback. ``ValueError`` covers both a malformed document
+    (``JSONDecodeError``) and bytes that are not UTF-8
+    (``UnicodeDecodeError``). One behaviour changed when the three former
+    readers folded into this one: a document that is valid JSON but not an
+    object (``[]``, ``null``, a bare string) now reads as no snapshot, where
+    each of them raised ``AttributeError`` on ``.get``. Every caller
+    subscripts the snapshot by name, so a non-object is no more usable than an
+    unparseable one.
+    """
+    p = work_dir / "status.json"
+    if not p.exists():
+        return None
+    try:
+        snapshot = json.loads(p.read_text(encoding="utf-8"))
+    except (ValueError, OSError):
+        return None
+    return snapshot if isinstance(snapshot, dict) else None
+
+
+def recorded_backend(work_dir: Path) -> tuple[str | None, str | None]:
+    """``(label, warning)``: the backend the last run in *work_dir* used, per
+    ``status.json``'s metadata, and a warning for the caller to show when the
+    snapshot predates run provenance.
+
+    ``label`` is ``None`` when the work dir has no snapshot, an unreadable
+    one, or one written before provenance was stamped (``metadata: null``).
+    ``warning`` is set only when metadata exists but names no backend —
+    written before the Backend rename (D123), when the key was "simulator".
+    The guard this function exists for (D120) cannot fire then, so
+    ``compare`` would fall back to environment auto-detect and could score,
+    say, an OpenModelica work dir against the Dymola partition. Reading the
+    old key would be a compatibility path; saying so is not.
+    """
+    snap = read_status(work_dir)
+    if snap is None:
+        return None, None
+    meta = snap.get("metadata") or {}
+    label = meta.get("backend_name")
+    if isinstance(label, str) and label:
+        return label, None
+    if meta:
+        return None, (
+            f"Warning: {work_dir / 'status.json'} predates the Backend rename "
+            "and does not record which backend produced it. Pass --backend "
+            "explicitly, or re-run instead of comparing."
+        )
+    return None, None

dstf/backends/python/__init__.py

@@ -0,0 +1,11 @@
+"""Python subprocess backend (D80).
+
+Mirrors the Julia D77 pattern: framework-shipped driver script loads the
+user's ``.py`` file, calls ``simulate(stop_time, tolerance)``, and writes
+a JSON result. Primary motivation is validating that the backend
+abstraction is truly language/simulator-agnostic — see the ConstantCsv
+example under ``examples/python/PythonTestingLib/`` which uses zero ODE
+machinery.
+"""
+
+from . import backend  # noqa: F401  (import triggers @register side effect)

dstf/backends/python/backend.py

@@ -0,0 +1,141 @@
+"""Python / subprocess backend (D80).
+
+Invokes a shipped driver script (``run_test.py``) via the configured
+Python interpreter. The driver loads the user's ``.py`` file — which
+must define ``simulate(stop_time, tolerance) -> dict`` — executes it,
+and writes a JSON result. The run/read machinery is the shared JSON-driver
+skeleton (:class:`~..common.json_driver.JsonDriverBackend`); only the Python
+specifics live here.
+
+Batch-only for the MVP; each test spawns one Python subprocess. Startup
+cost is roughly 30-100 ms per test (importlib + any user-file imports);
+for trivial tests this dominates over the actual ``simulate()`` call.
+A persistent-worker path (long-lived Python subprocess with stdin-JSON
+dispatch) is a future enhancement if per-test overhead becomes an issue
+— mirrors the D77 → D78 Julia progression.
+"""
+
+from __future__ import annotations
+
+import logging
+import shutil
+import subprocess
+from dataclasses import dataclass
+from pathlib import Path
+
+from ...config import Config
+from .. import register
+from ..common.json_driver import JsonDriverBackend
+from ..types import Capability, DatasetType
+
+logger = logging.getLogger(__name__)
+
+
+# Shipped driver script.
+_DRIVER_PATH = Path(__file__).resolve().parent / "run_test.py"
+
+
+@dataclass(frozen=True)
+class PythonConfig:
+    """Python-specific settings extracted from the unified :class:`Config`.
+
+    ``python_binary`` is the absolute path to the Python interpreter that
+    will run user test scripts. It must have whatever packages the user's
+    scripts import (scipy, pandas, ...). The framework does not manage
+    the user's environment — pick a ``python`` that has their deps.
+    """
+
+    python_binary: Path
+
+    @classmethod
+    def from_config(cls, config: Config) -> PythonConfig:
+        resolved: Path | None = None
+        if config.backend_path:
+            p = Path(config.backend_path).expanduser()
+            if p.exists():
+                resolved = p
+        if resolved is None:
+            for name in ("python", "python3"):
+                on_path = shutil.which(name)
+                if on_path:
+                    resolved = Path(on_path)
+                    break
+        if resolved is None:
+            raise RuntimeError(
+                "Python binary not found. Ensure 'python' or 'python3' is on "
+                "PATH, or set an explicit path under testing.json's "
+                '\'backends\' map, e.g. {"Python": ["/path/to/venv/bin/python"]}.'
+            )
+        return cls(python_binary=resolved)
+
+
+@register("Python")
+class PythonBackend(JsonDriverBackend):
+    """Subprocess-per-test Python backend."""
+
+    capabilities = frozenset(
+        {
+            Capability.BATCH_FALLBACK,
+            # Each test is its own subprocess: killing it stops the test.
+            Capability.CANCEL,
+            # Deliberately absent:
+            #   PERSISTENT_WORKERS — stdin-driven long-lived Python process deferred.
+            #   FMU_EXPORT — not meaningful for arbitrary Python scripts.
+        }
+    )
+    produced_datasets = frozenset({DatasetType.TIME_SERIES})
+    artifact_files = (
+        ("result.json", "Simulation result (time + variables)"),
+        ("python_stdout.txt", "Python stdout"),
+        ("python_stderr.txt", "Python stderr"),
+    )
+
+    driver_path = _DRIVER_PATH
+    tool_label = "Python"
+    action_label = "execution"
+    stdio_prefix = "python"
+    source_suffix = ".py"
+
+    def __init__(self, config: Config):
+        super().__init__(config)
+        self.python_config = PythonConfig.from_config(config)
+
+    @property
+    def tool_binary(self) -> Path:
+        return self.python_config.python_binary
+
+    def build_command(
+        self,
+        binary: Path,
+        driver: Path,
+        user_file: Path,
+        stop_time: float,
+        tolerance: float,
+        result_path: Path,
+    ) -> list[str]:
+        return [
+            str(binary),
+            str(driver),
+            str(user_file),
+            str(stop_time),
+            str(tolerance),
+            str(result_path),
+        ]
+
+    def describe_tool_version(self) -> str | None:
+        """Version of the *configured* interpreter (``python_config.
+        python_binary``) that actually runs each test's subprocess — not this
+        host's Python, which may differ. Probed via ``--version`` with a short
+        timeout; ``None`` on any failure."""
+        try:
+            out = subprocess.run(
+                [str(self.python_config.python_binary), "--version"],
+                capture_output=True,
+                text=True,
+                timeout=10,
+            )
+            # Older CPython prints the banner to stderr, newer to stdout.
+            banner = (out.stdout or out.stderr or "").strip()
+            return banner or None
+        except Exception:
+            return None

dstf/backends/python/run_test.py

@@ -0,0 +1,141 @@
+"""Framework-shipped Python driver for a single test (D80).
+
+Invoked by PythonBackend as:
+    python run_test.py <user_file> <stop_time> <tolerance> <result_path>
+
+The user file must define:
+    simulate(stop_time: float, tolerance: float) -> dict
+
+returning a dict with:
+    "time":      list[float] (monotonically non-decreasing)
+    "variables": dict[str, list[float]]  (each same length as "time")
+
+Result file is JSON:
+    {"success": true,  "time": [...], "variables": {...}}
+or on failure:
+    {"success": false, "error": "...", "time": [], "variables": {}}
+
+Exceptions are caught so the framework always gets a structured result
+rather than a process-level crash.
+"""
+
+from __future__ import annotations
+
+import importlib.util
+import json
+import sys
+import traceback
+from pathlib import Path
+
+
+def _load_user_module(user_file: Path):
+    # Allow sibling imports from the user's file directory.
+    parent = str(user_file.parent.resolve())
+    if parent not in sys.path:
+        sys.path.insert(0, parent)
+    spec = importlib.util.spec_from_file_location("_user_test", user_file)
+    if spec is None or spec.loader is None:
+        raise ImportError(f"Cannot load {user_file}")
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+def _write_failure(result_path: Path, message: str) -> None:
+    result_path.write_text(
+        json.dumps(
+            {
+                "success": False,
+                "error": message,
+                "time": [],
+                "variables": {},
+            }
+        ),
+        encoding="utf-8",
+    )
+
+
+def main(argv: list[str]) -> int:
+    if len(argv) != 4:
+        print(
+            "Usage: run_test.py <user_file> <stop_time> <tolerance> <result_path>",
+            file=sys.stderr,
+        )
+        return 2
+
+    user_file = Path(argv[0])
+    try:
+        stop_time = float(argv[1])
+        tolerance = float(argv[2])
+    except ValueError as exc:
+        print(f"Invalid numeric arg: {exc}", file=sys.stderr)
+        return 2
+    result_path = Path(argv[3])
+
+    try:
+        module = _load_user_module(user_file)
+        if not hasattr(module, "simulate"):
+            raise AttributeError(
+                f"{user_file.name} must define simulate(stop_time, tolerance)"
+            )
+        payload = module.simulate(stop_time, tolerance)
+        if (
+            not isinstance(payload, dict)
+            or "time" not in payload
+            or "variables" not in payload
+        ):
+            raise ValueError(
+                "simulate() must return a dict with 'time' and 'variables' keys"
+            )
+        if not isinstance(payload["variables"], dict):
+            raise ValueError("'variables' must be a dict of name -> list[float]")
+
+        # review 2026-07-06 (finding 29): validate value TYPES, not just list
+        # lengths — a simulate() returning strings used to pass the run and
+        # then blow up np.asarray during read_results, aborting every test's
+        # read phase. Fail here, cleanly, naming the offending variable.
+        raw_time = list(payload["time"])
+        try:
+            time_list = [float(v) for v in raw_time]
+        except (TypeError, ValueError) as exc:
+            raise ValueError(f"'time' contains a non-numeric value: {exc}") from exc
+        variables = {}
+        for k, v in payload["variables"].items():
+            name = str(k)
+            raw_values = list(v)
+            if len(raw_values) != len(time_list):
+                raise ValueError(
+                    f"variable '{name}' has {len(raw_values)} samples; "
+                    f"expected {len(time_list)} to match 'time'"
+                )
+            try:
+                variables[name] = [float(x) for x in raw_values]
+            except (TypeError, ValueError) as exc:
+                raise ValueError(
+                    f"variable '{name}' contains a non-numeric value: {exc}"
+                ) from exc
+
+        result_path.write_text(
+            json.dumps(
+                {
+                    "success": True,
+                    "time": time_list,
+                    "variables": variables,
+                }
+            ),
+            encoding="utf-8",
+        )
+        print(f"OK: {len(variables)} variables, {len(time_list)} time points")
+        return 0
+    except BaseException:
+        tb = traceback.format_exc()
+        print(f"FAIL:\n{tb}", file=sys.stderr)
+        try:
+            _write_failure(result_path, tb)
+        except Exception:
+            return 2
+        return 1
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv[1:]))