physverify 0.1.0__py3-none-any.whl

physverify/cli.py

@@ -0,0 +1,105 @@
+"""Command line.
+
+    physverify run SPEC.py [SPEC.py ...] [--json FILE|-] [--html FILE] [--quiet]
+    physverify matrix [OUT.md]
+    physverify compare [OUT.md]
+    physverify hook        (Claude Code PostToolUse hook; see docs/agent.md)
+    physverify mcp         (MCP server on stdio; see docs/agent.md)
+
+`run` exits 0 when every check passes (PASS or EXACT), 1 when any does not,
+and 2 when a spec cannot be loaded. A spec is a Python file defining
+`checks()`; see examples/verify_damped_oscillator.py.
+"""
+
+from __future__ import annotations
+
+import argparse
+import sys
+from pathlib import Path
+
+from . import __version__
+
+
+def _cmd_run(args) -> int:
+    from .report import load_spec, run_checks, to_html, to_json, to_text
+
+    outcomes = []
+    for spec in args.specs:
+        try:
+            checks = load_spec(spec)
+        except Exception as exc:
+            print(f"physverify: cannot load {spec}: {type(exc).__name__}: {exc}", file=sys.stderr)
+            return 2
+        outcomes += run_checks(checks)
+    source = ", ".join(str(Path(s)) for s in args.specs)
+    if args.json == "-":
+        print(to_json(outcomes, source))
+    else:
+        if not args.quiet:
+            print(to_text(outcomes))
+        if args.json:
+            Path(args.json).write_text(to_json(outcomes, source), encoding="utf-8")
+    if args.html:
+        Path(args.html).write_text(to_html(outcomes, source=source), encoding="utf-8")
+    return 0 if all(o.result.ok for o in outcomes) else 1
+
+
+def _cmd_matrix(args) -> int:
+    from . import matrix
+
+    return matrix.main([args.out] if args.out else [])
+
+
+def _cmd_compare(args) -> int:
+    from . import compare
+
+    return compare.main([args.out] if args.out else [])
+
+
+def _cmd_hook(args) -> int:
+    from .agent import hook_main
+
+    return hook_main()
+
+
+def _cmd_mcp(args) -> int:
+    from .agent import mcp_main
+
+    return mcp_main()
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(prog="physverify", description="Verification checks for physics simulation code.")
+    parser.add_argument("--version", action="version", version=f"physverify {__version__}")
+    sub = parser.add_subparsers(dest="command", required=True)
+
+    run = sub.add_parser("run", help="run the checks defined in one or more spec files")
+    run.add_argument("specs", nargs="+", help="Python files that define checks()")
+    run.add_argument("--json", metavar="FILE", help="write results as JSON to FILE, or to stdout with -")
+    run.add_argument("--html", metavar="FILE", help="write a self-contained HTML report")
+    run.add_argument("--quiet", action="store_true", help="no text output")
+    run.set_defaults(func=_cmd_run)
+
+    m = sub.add_parser("matrix", help="run the seeded-bug benchmark")
+    m.add_argument("out", nargs="?", help="also write the tables next to this Markdown file")
+    m.set_defaults(func=_cmd_matrix)
+
+    c = sub.add_parser("compare", help="compare the families with plain tests on the seeded bugs")
+    c.add_argument("out", nargs="?", help="also write the table to this Markdown file")
+    c.set_defaults(func=_cmd_compare)
+
+    h = sub.add_parser("hook", help="Claude Code PostToolUse hook: reads the event on stdin, exits 2 on a failed check")
+    h.set_defaults(func=_cmd_hook)
+
+    s = sub.add_parser("mcp", help="run a minimal MCP server on stdio with a run_checks tool")
+    s.set_defaults(func=_cmd_mcp)
+    return parser
+
+
+def main(argv=None) -> int:
+    args = build_parser().parse_args(argv)
+    return args.func(args)
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

physverify/compare.py

@@ -0,0 +1,104 @@
+"""How the verification families compare with ordinary unit tests on the seeded bugs.
+
+    python -m physverify.compare results/comparison.md
+
+Strategies, each a set of checks run on every bug:
+
+plain tests        one run per case, compared with a reference or a known value
+                   within a fixed tolerance (each problem's `smoke_checks`)
+reference only     the `reference` family: an order study against an independent solution
+convergence only   the `mms` family
+full suite         every family
+
+A bug counts as caught by a strategy if at least one of its checks does not
+pass. Runtime is the total time of that strategy's checks over all bugs and
+baselines. The plain tests use tolerances of the size an ordinary test would
+(0.1% for the ODE problems, 1% for the heat solver); a tighter tolerance would
+catch more bugs and need a more accurate reference, which is what the order
+studies provide without having to pick a tolerance.
+"""
+
+from __future__ import annotations
+
+import sys
+import time
+from dataclasses import dataclass, field
+from pathlib import Path
+
+from .checks import run_check
+from .matrix import PROBLEMS
+
+STRATEGIES = {
+    "plain tests": ("smoke",),
+    "reference only": ("reference",),
+    "convergence only": ("mms",),
+    "full suite": ("mms", "reference", "invariant", "limit", "patch"),
+}
+
+
+@dataclass
+class Outcome:
+    problem: str
+    bug: str | None
+    failed: dict = field(default_factory=dict)  # family -> number of checks that did not pass
+    seconds: dict = field(default_factory=dict)  # family -> total runtime
+
+
+def run_case(module, bug) -> Outcome:
+    bugs = frozenset([bug]) if bug else frozenset()
+    out = Outcome(module.NAME, bug)
+    for check in module.checks(bugs) + module.smoke_checks(bugs):
+        start = time.perf_counter()
+        result = run_check(check)
+        out.seconds[check.family] = out.seconds.get(check.family, 0.0) + time.perf_counter() - start
+        if not result.ok:
+            out.failed[check.family] = out.failed.get(check.family, 0) + 1
+    return out
+
+
+def run_all() -> list:
+    outcomes = []
+    for module in PROBLEMS.values():
+        outcomes.append(run_case(module, None))
+        for bug in module.BUGS:
+            outcomes.append(run_case(module, bug))
+    return outcomes
+
+
+def caught(outcome: Outcome, strategy: str) -> bool:
+    return any(outcome.failed.get(f) for f in STRATEGIES[strategy])
+
+
+def to_markdown(outcomes) -> str:
+    bug_rows = [o for o in outcomes if o.bug]
+    base_rows = [o for o in outcomes if not o.bug]
+    lines = [
+        "| Strategy | Bugs caught | Missed | False alarms on baselines | Runtime (s) |",
+        "|---|---|---|---|---|",
+    ]
+    for name in STRATEGIES:
+        hits = sum(caught(o, name) for o in bug_rows)
+        alarms = sum(caught(o, name) for o in base_rows)
+        secs = sum(o.seconds.get(f, 0.0) for o in outcomes for f in STRATEGIES[name])
+        lines.append(f"| {name} | {hits} of {len(bug_rows)} | {len(bug_rows) - hits} | {alarms} of {len(base_rows)} | {secs:.1f} |")
+    lines += ["", "| Problem | Bug | " + " | ".join(STRATEGIES) + " |", "|---|---|" + "---|" * len(STRATEGIES)]
+    for o in bug_rows:
+        cells = " | ".join("caught" if caught(o, s) else "**missed**" for s in STRATEGIES)
+        lines.append(f"| {o.problem} | {o.bug} | {cells} |")
+    lines += ["", "Runtimes depend on the machine; the other columns do not (the checks are deterministic)."]
+    return "\n".join(lines) + "\n"
+
+
+def main(argv=None) -> int:
+    argv = sys.argv[1:] if argv is None else argv
+    text = to_markdown(run_all())
+    print(text)
+    if argv:
+        path = Path(argv[0])
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.write_text(text, encoding="utf-8")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

physverify/loaders.py

@@ -0,0 +1,49 @@
+"""Import a module from a path without touching its folder.
+
+Used by the adapters that test other people's code. Nothing is written next to
+the module (no __pycache__), and optional plotting imports can be stubbed so a
+simulation file can be imported on a machine without matplotlib.
+"""
+
+from __future__ import annotations
+
+import importlib
+import importlib.util
+import sys
+import types
+from pathlib import Path
+
+
+def _stub_module(name: str) -> types.ModuleType:
+    return types.ModuleType(name)
+
+
+def load_module_from_path(name: str, path, stubs=()):
+    path = Path(path)
+    if not path.is_file():
+        raise FileNotFoundError(path)
+
+    added = []
+    for stub in stubs:
+        try:
+            importlib.import_module(stub)
+        except ImportError:
+            sys.modules[stub] = _stub_module(stub)
+            added.append(stub)
+
+    old_flag = sys.dont_write_bytecode
+    sys.dont_write_bytecode = True
+    try:
+        spec = importlib.util.spec_from_file_location(name, path)
+        module = importlib.util.module_from_spec(spec)
+        sys.modules[name] = module
+        try:
+            spec.loader.exec_module(module)
+        except BaseException:
+            sys.modules.pop(name, None)  # do not leave a half-initialised module behind
+            raise
+    finally:
+        sys.dont_write_bytecode = old_flag
+        for stub in added:
+            sys.modules.pop(stub, None)
+    return module

physverify/matrix.py

@@ -0,0 +1,160 @@
+"""Run every check on every seeded bug and tabulate which checks notice which bug.
+
+    python -m physverify.matrix results/detection_matrix.md
+
+The bugs were written by the author of this package, so the catch rate below is
+a measure of these bugs, not of bugs in general.
+"""
+
+from __future__ import annotations
+
+import sys
+from dataclasses import dataclass, field
+from pathlib import Path
+
+from .checks import FAMILIES, Check, Result, run_check
+from .problems import heat1d, kepler, pendulum, projectile, sod
+
+PROBLEMS = {m.NAME: m for m in (projectile, pendulum, kepler, heat1d, sod)}
+
+
+@dataclass
+class Row:
+    problem: str
+    bug: str | None  # None is the bug-free baseline
+    results: list = field(default_factory=list)  # list of (Check, Result)
+
+    def flagged(self) -> dict:
+        """family -> labels of the checks that did not pass"""
+        out: dict = {}
+        for check, result in self.results:
+            if not result.ok:
+                out.setdefault(check.family, []).append(check.label)
+        return out
+
+    def families_present(self) -> set:
+        return {check.family for check, _ in self.results}
+
+
+def evaluate(module, bugs=frozenset()) -> list:
+    return [(check, run_check(check)) for check in module.checks(bugs)]
+
+
+def build_rows(problems=None) -> list:
+    rows = []
+    for name, module in PROBLEMS.items():
+        if problems and name not in problems:
+            continue
+        rows.append(Row(name, None, evaluate(module)))
+        for bug in module.BUGS:
+            rows.append(Row(name, bug, evaluate(module, frozenset([bug]))))
+    return rows
+
+
+def _cell(row: Row, family: str) -> str:
+    if family not in row.families_present():
+        return "n/a"
+    flagged = row.flagged().get(family)
+    if not flagged:
+        return "no" if row.bug else "ok"
+    return "FLAGGED" if row.bug else "FALSE ALARM"
+
+
+def to_markdown(rows) -> str:
+    head = "| Problem | Bug | " + " | ".join(FAMILIES) + " | Predicted by design |\n"
+    head += "|---|---|" + "---|" * (len(FAMILIES) + 1) + "\n"
+    lines = []
+    for row in rows:
+        predicted = "baseline (all should pass)" if row.bug is None else ", ".join(
+            sorted(PROBLEMS[row.problem].BUGS[row.bug].expected)
+        ) or "none (blind spot)"
+        cells = " | ".join(_cell(row, f) for f in FAMILIES)
+        lines.append(f"| {row.problem} | {row.bug or '(none)'} | {cells} | {predicted} |")
+    bug_rows = [r for r in rows if r.bug]
+    caught = [r for r in bug_rows if r.flagged()]
+    missed = [r for r in bug_rows if not r.flagged()]
+    false_alarms = [r for r in rows if r.bug is None and r.flagged()]
+    by_family = {f: sum(1 for r in bug_rows if f in r.flagged()) for f in FAMILIES}
+    only_one = {f: sum(1 for r in bug_rows if set(r.flagged()) == {f}) for f in FAMILIES}
+    summary = [
+        "",
+        f"Bugs seeded: {len(bug_rows)}. Flagged by at least one check: {len(caught)}. Missed by all: {len(missed)}"
+        + (" (" + ", ".join(f"{r.problem}/{r.bug}" for r in missed) + ")" if missed else "")
+        + ".",
+        f"Bug-free baselines with a false alarm: {len(false_alarms)} of {len(rows) - len(bug_rows)}.",
+        "Bugs flagged per family: " + ", ".join(f"{f} {n}" for f, n in by_family.items()) + ".",
+        "Bugs flagged by that family alone: " + ", ".join(f"{f} {n}" for f, n in only_one.items()) + ".",
+    ]
+    return head + "\n".join(lines) + "\n" + "\n".join(summary) + "\n"
+
+
+def details(rows) -> str:
+    """Per-check messages, for reading why a cell is what it is."""
+    out = []
+    for row in rows:
+        out.append(f"### {row.problem} / {row.bug or '(baseline)'}")
+        for check, result in row.results:
+            out.append(f"- {check.family} [{check.label}] {result.status}: {result.message}")
+        out.append("")
+    return "\n".join(out)
+
+
+def catalogue(rows) -> str:
+    """One entry per seeded bug: what is changed, the expected symptom, predicted and observed families."""
+    out = [
+        "# Seeded-bug catalogue",
+        "",
+        "Generated by `python -m physverify.matrix results/detection_matrix.md`. Each bug is switched on by",
+        "name in the problem's solver (search the source for the bug's name to see the exact change).",
+        "Predictions and symptoms were written from reasoning before the checks were run.",
+        "",
+    ]
+    for row in rows:
+        if row.bug is None:
+            continue
+        info = PROBLEMS[row.problem].BUGS[row.bug]
+        flagged = sorted(row.flagged())
+        out += [
+            f"## {row.problem} / {row.bug}",
+            "",
+            f"- Change: {info.description}",
+            f"- Expected symptom: {info.symptom or '(not recorded)'}",
+            f"- Predicted to be flagged by: {', '.join(sorted(info.expected)) or 'nothing (blind spot)'}",
+            f"- Flagged by: {', '.join(flagged) or 'nothing'}",
+            "",
+        ]
+    return "\n".join(out)
+
+
+def by_check_markdown(rows, problem: str) -> str:
+    """One column per individual check, for a problem whose checks differ in how they refine."""
+    rows = [r for r in rows if r.problem == problem]
+    labels = [(c.family, c.label) for c, _ in rows[0].results]
+    head = "| Bug | " + " | ".join(f"{fam}: {label}" for fam, label in labels) + " |\n"
+    head += "|---|" + "---|" * len(labels) + "\n"
+    lines = []
+    for row in rows:
+        cells = []
+        for _, result in row.results:
+            cells.append(("ok" if row.bug is None else "no") if result.ok else ("FALSE ALARM" if row.bug is None else "FLAGGED"))
+        lines.append(f"| {row.bug or '(none)'} | " + " | ".join(cells) + " |")
+    return head + "\n".join(lines) + "\n"
+
+
+def main(argv=None) -> int:
+    argv = sys.argv[1:] if argv is None else argv
+    rows = build_rows()
+    text = to_markdown(rows)
+    print(text)
+    if argv:
+        path = Path(argv[0])
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.write_text(text, encoding="utf-8")
+        path.with_name(path.stem + "_details.md").write_text(details(rows), encoding="utf-8")
+        path.with_name("heat1d_by_check.md").write_text(by_check_markdown(rows, "heat1d"), encoding="utf-8")
+        path.with_name("bug_catalogue.md").write_text(catalogue(rows), encoding="utf-8")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

physverify/mms.py

@@ -0,0 +1,118 @@
+"""Method of Manufactured Solutions for systems of ODEs.
+
+You pick a smooth function u(t). Substituting it into an *independent*
+reference operator f_ref leaves a residual s(t) = u'(t) - f_ref(t, u(t)).
+The code under test is then run on  y' = f_test(t, y) + s(t).  If f_test
+agrees with f_ref, u(t) is its exact solution and the error must shrink at the
+scheme's design order. If f_test differs, the error stops shrinking.
+
+Two points that are easy to get wrong:
+
+* The reference operator must be written independently of the code under
+  test. If s(t) is built from the code's own right-hand side, a wrong
+  right-hand side passes.
+* The error is measured at a fixed final time, so the code must be driven for
+  an exact number of steps; see the `integrate` contract below.
+
+The `integrate` contract used throughout:
+
+    integrate(source, y0, t_end, n_steps) -> state at t_end
+
+`source(t)` returns the additive forcing (a vector, or 0.0). The adapter for a
+given code decides how to inject it.
+"""
+
+from __future__ import annotations
+
+from typing import Callable
+
+import numpy as np
+
+from .checks import OrderResult
+from .order import check_order
+
+
+def zero_source(t):
+    return 0.0
+
+
+def complex_step_derivative(u: Callable, t: float, h: float = 1e-30) -> np.ndarray:
+    """du/dt to machine precision, for u written with NumPy functions of t.
+
+    u must accept a complex t and avoid abs(), max() and similar non-analytic
+    operations.
+    """
+    return np.imag(np.asarray(u(t + 1j * h), dtype=complex)) / h
+
+
+class ManufacturedODE:
+    def __init__(self, u: Callable, rhs_ref: Callable, du: Callable | None = None):
+        self._u = u
+        self.rhs_ref = rhs_ref
+        self._du = du
+
+    def value(self, t: float) -> np.ndarray:
+        return np.asarray(self._u(t), dtype=float)
+
+    def derivative(self, t: float) -> np.ndarray:
+        if self._du is not None:
+            return np.asarray(self._du(t), dtype=float)
+        return complex_step_derivative(self._u, t)
+
+    def forcing(self, t: float) -> np.ndarray:
+        return self.derivative(t) - np.asarray(self.rhs_ref(t, self.value(t)), dtype=float)
+
+
+def max_abs_error(a, b) -> float:
+    with np.errstate(all="ignore"):
+        return float(np.max(np.abs(np.asarray(a, dtype=float) - np.asarray(b, dtype=float))))
+
+
+def mms_order_study(
+    integrate: Callable,
+    mms: ManufacturedODE,
+    t_end: float,
+    ns,
+    expected_order: float,
+    **order_kwargs,
+) -> OrderResult:
+    """Run `integrate` with the manufactured forcing at several step counts."""
+    y0 = mms.value(0.0)
+    y_exact = mms.value(t_end)
+    order_kwargs.setdefault("scale", float(np.max(np.abs(y_exact))))
+    hs, errs = [], []
+    for n in ns:
+        y_end = integrate(mms.forcing, y0, t_end, n)
+        hs.append(t_end / n)
+        errs.append(max_abs_error(y_end, y_exact))
+    return check_order(hs, errs, expected_order, **order_kwargs)
+
+
+def mms_trajectory_study(
+    integrate_trajectory: Callable,
+    mms: ManufacturedODE,
+    t_end: float,
+    ns,
+    expected_order: float,
+    **order_kwargs,
+) -> OrderResult:
+    """Like `mms_order_study`, but the error is the largest one over the whole run.
+
+    `integrate_trajectory(source, y0, t_end, n_steps)` returns the n_steps + 1
+    states at t_k = k * t_end / n_steps, starting with y0. A code whose error
+    happens to cancel at the final time, or which goes wrong part way through
+    and recovers, is caught here and not by the final-state check.
+    """
+    y0 = mms.value(0.0)
+    hs, errs, scale = [], [], 0.0
+    for n in ns:
+        states = np.asarray(integrate_trajectory(mms.forcing, y0, t_end, n), dtype=float)
+        if states.shape[0] != n + 1:
+            raise ValueError(f"integrate_trajectory returned {states.shape[0]} states for {n} steps; expected {n + 1}")
+        times = np.linspace(0.0, t_end, n + 1)
+        exact = np.array([mms.value(t) for t in times])
+        scale = max(scale, float(np.max(np.abs(exact))))
+        hs.append(t_end / n)
+        errs.append(max_abs_error(states, exact))
+    order_kwargs.setdefault("scale", scale)
+    return check_order(hs, errs, expected_order, **order_kwargs)

physverify/order.py

@@ -0,0 +1,131 @@
+"""Observed order of accuracy from errors at several resolutions."""
+
+from __future__ import annotations
+
+import numpy as np
+
+from .checks import EXACT, FAIL, INCONCLUSIVE, PASS, OrderResult
+
+
+def pairwise_orders(hs, errors) -> np.ndarray:
+    """p_i = log(e_i / e_{i+1}) / log(h_i / h_{i+1}) for consecutive resolutions."""
+    hs = np.asarray(hs, dtype=float)
+    errs = np.asarray(errors, dtype=float)
+    return np.log(errs[:-1] / errs[1:]) / np.log(hs[:-1] / hs[1:])
+
+
+def fit_order(hs, errors) -> float:
+    """Least-squares slope of log(error) against log(h)."""
+    return float(np.polyfit(np.log(hs), np.log(errors), 1)[0])
+
+
+def effective_floor(floor: float, scale: float, rel: float = 1e-13) -> float:
+    """Round-off grows with the size of the solution, so the noise floor does too.
+
+    The floor is the larger of the absolute `floor` (right for solutions of size
+    up to about 10) and `rel` times the solution's size (about 500 machine
+    epsilons, room for round-off accumulated over a few hundred steps). A
+    non-finite scale (a diverged run) leaves the floor unchanged so that the
+    non-finite error is still reported.
+    """
+    scale = abs(float(scale))
+    if not np.isfinite(scale):
+        return floor
+    return max(floor, rel * scale)
+
+
+def check_exact(hs, errors, *, floor: float = 1e-12, scale: float = 1.0) -> OrderResult:
+    """Require the error to be at round-off level at every resolution.
+
+    For a manufactured solution that lies inside the scheme's discrete solution
+    space (a "patch test": for example, polynomials a finite-difference stencil
+    differentiates exactly), a correct code reproduces it to round-off whatever
+    the step sizes. Order of convergence cannot see an error that is as small
+    as the scheme's own truncation error and shrinks at the same rate; this
+    can, because the expected error is zero rather than small.
+
+    EXACT  every error is at most `floor`
+    FAIL   some error is larger, or non-finite
+
+    `scale` is the size of the solution; see `effective_floor`.
+    """
+    floor = effective_floor(floor, scale)
+    hs = np.asarray(hs, dtype=float)
+    errs = np.asarray(errors, dtype=float)
+    order = np.argsort(-hs)
+    hs, errs = hs[order], errs[order]
+    base = dict(expected=float("inf"), hs=tuple(hs), errors=tuple(errs))
+    if not np.all(np.isfinite(errs)):
+        return OrderResult(FAIL, "non-finite error: the solution diverged or produced NaN", **base)
+    worst = float(errs.max())
+    if worst <= floor:
+        return OrderResult(EXACT, f"reproduced to round-off (largest error {worst:.2g}), as designed", **base)
+    msg = f"expected an exact solution to round-off, but the error reaches {worst:.3g}"
+    measurable = errs > floor
+    fitted = float("nan")
+    if measurable.sum() >= 3:
+        fitted = fit_order(hs[measurable], errs[measurable])
+        msg += f" and shrinks at about order {fitted:.2f}"
+    return OrderResult(FAIL, msg, fitted=fitted, **base)
+
+
+def check_order(
+    hs,
+    errors,
+    expected: float,
+    *,
+    tol_low: float = 0.3,
+    tol_high: float = 1.0,
+    floor: float = 1e-12,
+    fit_points: int = 4,
+    scale: float = 1.0,
+) -> OrderResult:
+    """Compare the observed order of convergence with the order the code claims.
+
+    PASS      fitted order lies in [expected - tol_low, expected + tol_high]
+    EXACT     every error is below `floor`, so the order cannot be measured
+    FAIL      order too low or too high, error plateau, or non-finite error
+    INCONCLUSIVE  fewer than two resolutions are above the noise floor
+
+    `floor` is the error level below which round-off dominates, for a solution
+    of size about 1. Pass `scale` (the size of the solution, for example the
+    largest absolute value of the exact answer) and the floor grows with it.
+    """
+    floor = effective_floor(floor, scale)
+    hs = np.asarray(hs, dtype=float)
+    errs = np.asarray(errors, dtype=float)
+    order = np.argsort(-hs)  # coarse to fine
+    hs, errs = hs[order], errs[order]
+    base = dict(expected=float(expected), hs=tuple(hs), errors=tuple(errs))
+
+    if not np.all(np.isfinite(errs)):
+        return OrderResult(FAIL, "non-finite error: the solution diverged or produced NaN", **base)
+
+    measurable = errs > floor
+    if not measurable.any():
+        return OrderResult(EXACT, f"error below {floor:g} at every resolution; order not measurable", **base)
+
+    h_m, e_m = hs[measurable], errs[measurable]
+    if len(e_m) < 2:
+        return OrderResult(INCONCLUSIVE, "fewer than 2 resolutions above the noise floor", **base)
+
+    if e_m[-2] / e_m[-1] < 1.2:
+        return OrderResult(
+            FAIL,
+            f"error does not decrease with resolution (about {e_m[-1]:.3g} at the finest level)",
+            fitted=0.0,
+            **base,
+        )
+
+    h_f, e_f = h_m[-fit_points:], e_m[-fit_points:]
+    p = fit_order(h_f, e_f)
+    if p < expected - tol_low:
+        msg = f"observed order {p:.2f} is below the expected {expected:g}"
+        status = FAIL
+    elif p > expected + tol_high:
+        msg = f"observed order {p:.2f} is above the expected {expected:g}; check the expected order"
+        status = FAIL
+    else:
+        msg = f"observed order {p:.2f} (expected {expected:g})"
+        status = PASS
+    return OrderResult(status, msg, fitted=p, **base)

physverify/problems/__init__.py

@@ -0,0 +1,6 @@
+"""Seeded-bug benchmark problems.
+
+Each problem module defines a reference (independent) implementation of the
+physics, a solver with switchable bugs, and `checks(bugs)` returning the
+checks to run on that solver.
+"""