physverify 0.1.0__py3-none-any.whl

physverify/report.py

@@ -0,0 +1,189 @@
+"""Run a list of checks and report the results as text, JSON or a self-contained HTML page."""
+
+from __future__ import annotations
+
+import html
+import json
+import math
+import time
+from dataclasses import dataclass
+from pathlib import Path
+
+from .checks import Check, OrderResult, Result, run_check
+from .loaders import load_module_from_path
+
+
+@dataclass
+class Outcome:
+    check: Check
+    result: Result
+    seconds: float
+
+
+def run_checks(checks) -> list[Outcome]:
+    out = []
+    for check in checks:
+        start = time.perf_counter()
+        result = run_check(check)
+        out.append(Outcome(check, result, time.perf_counter() - start))
+    return out
+
+
+def load_spec(path):
+    """Import a spec file and return its checks.
+
+    A spec is a Python file that defines `checks()`, returning a list of
+    `physverify.Check`. Its folder is put on sys.path first, so it can import
+    the solver next to it.
+    """
+    import sys
+
+    path = Path(path).resolve()
+    folder = str(path.parent)
+    if folder not in sys.path:
+        sys.path.insert(0, folder)
+    name = "physverify_spec_" + "".join(c if c.isalnum() else "_" for c in str(path))
+    module = load_module_from_path(name, path)
+    if not callable(getattr(module, "checks", None)):
+        raise AttributeError(f"{path} does not define checks()")
+    checks = list(module.checks())
+    for check in checks:
+        if not isinstance(check, Check):
+            raise TypeError(f"{path}: checks() returned {type(check).__name__}, not physverify.Check")
+    return checks
+
+
+def _number(x):
+    x = float(x)
+    return x if math.isfinite(x) else None
+
+
+def outcome_to_dict(o: Outcome) -> dict:
+    d = {
+        "family": o.check.family,
+        "label": o.check.label,
+        "status": o.result.status,
+        "ok": o.result.ok,
+        "message": o.result.message,
+        "seconds": round(o.seconds, 4),
+    }
+    if isinstance(o.result, OrderResult):
+        d["expected_order"] = _number(o.result.expected)
+        d["observed_order"] = _number(o.result.fitted)
+        d["h"] = [_number(h) for h in o.result.hs]
+        d["error"] = [_number(e) for e in o.result.errors]
+    return d
+
+
+def summary(outcomes) -> dict:
+    return {
+        "total": len(outcomes),
+        "ok": sum(o.result.ok for o in outcomes),
+        "not_ok": sum(not o.result.ok for o in outcomes),
+        "by_status": {s: sum(o.result.status == s for o in outcomes) for s in sorted({o.result.status for o in outcomes})},
+    }
+
+
+def to_json(outcomes, source: str = "") -> str:
+    from . import __version__
+
+    payload = {
+        "physverify": __version__,
+        "source": source,
+        "summary": summary(outcomes),
+        "checks": [outcome_to_dict(o) for o in outcomes],
+    }
+    return json.dumps(payload, indent=2)
+
+
+def to_text(outcomes) -> str:
+    lines = [f"{o.check.family:10s} {o.result.status:12s} {o.check.label}: {o.result.message}" for o in outcomes]
+    s = summary(outcomes)
+    lines.append(f"{s['ok']} of {s['total']} checks passed")
+    return "\n".join(lines)
+
+
+def _svg_plot(result: OrderResult, width=220, height=150) -> str:
+    """Log-log plot of error against step size, with a line of the expected slope."""
+    pts = [(h, e) for h, e in zip(result.hs, result.errors) if h > 0 and e > 0 and math.isfinite(e)]
+    if len(pts) < 2:
+        return ""
+    lx = [math.log10(h) for h, _ in pts]
+    ly = [math.log10(e) for _, e in pts]
+    x0, x1 = min(lx), max(lx)
+    y0, y1 = min(ly), max(ly)
+    expected_line = []
+    if math.isfinite(result.expected):
+        # through the finest point, with the expected slope
+        yf = ly[-1] if lx[-1] == x0 else ly[lx.index(x0)]
+        expected_line = [(x0, yf), (x1, yf + result.expected * (x1 - x0))]
+        y0 = min(y0, *[p[1] for p in expected_line])
+        y1 = max(y1, *[p[1] for p in expected_line])
+    pad = 14
+    sx = (width - 2 * pad) / ((x1 - x0) or 1.0)
+    sy = (height - 2 * pad) / ((y1 - y0) or 1.0)
+
+    def xy(x, y):
+        return f"{pad + (x - x0) * sx:.1f},{height - pad - (y - y0) * sy:.1f}"
+
+    parts = [f'<svg viewBox="0 0 {width} {height}" width="{width}" height="{height}" role="img" aria-label="error against step size, log-log">']
+    parts.append(f'<rect x="0.5" y="0.5" width="{width - 1}" height="{height - 1}" class="frame"/>')
+    if expected_line:
+        parts.append(f'<polyline points="{xy(*expected_line[0])} {xy(*expected_line[1])}" class="expected"/>')
+    parts.append('<polyline points="' + " ".join(xy(x, y) for x, y in zip(lx, ly)) + '" class="observed"/>')
+    for x, y in zip(lx, ly):
+        cx, cy = xy(x, y).split(",")
+        parts.append(f'<circle cx="{cx}" cy="{cy}" r="2.5" class="point"/>')
+    parts.append("</svg>")
+    return "".join(parts)
+
+
+_CSS = """
+:root { --bg:#ffffff; --fg:#1d1d1f; --muted:#6b6b70; --line:#d9d9de; --ok:#1a7f37; --bad:#b42318; --warn:#9a6700; --accent:#2f5bd3; }
+@media (prefers-color-scheme: dark) { :root { --bg:#16171a; --fg:#ececf0; --muted:#a0a0a8; --line:#33343a; --ok:#4ac26b; --bad:#ff7b72; --warn:#d4a72c; --accent:#7aa2ff; } }
+body { background:var(--bg); color:var(--fg); font:15px/1.5 system-ui, sans-serif; margin:0 auto; max-width:1100px; padding:24px 16px; }
+h1 { font-size:22px; margin:0 0 4px; } p.meta { color:var(--muted); margin:0 0 20px; }
+table { border-collapse:collapse; width:100%; } th, td { border-bottom:1px solid var(--line); padding:8px 6px; text-align:left; vertical-align:top; }
+th { color:var(--muted); font-weight:600; font-size:13px; }
+.status { font-weight:700; white-space:nowrap; } .ok { color:var(--ok); } .bad { color:var(--bad); } .warn { color:var(--warn); }
+.wrap { overflow-x:auto; } svg .frame { fill:none; stroke:var(--line); } svg .observed { fill:none; stroke:var(--accent); stroke-width:2; }
+svg .expected { fill:none; stroke:var(--muted); stroke-dasharray:4 3; } svg .point { fill:var(--accent); }
+td.num { font-variant-numeric:tabular-nums; white-space:nowrap; }
+"""
+
+
+def to_html(outcomes, title: str = "physverify report", source: str = "") -> str:
+    from . import __version__
+
+    s = summary(outcomes)
+    rows = []
+    for o in outcomes:
+        r = o.result
+        cls = "ok" if r.ok else ("warn" if r.status == "INCONCLUSIVE" else "bad")
+        plot = _svg_plot(r) if isinstance(r, OrderResult) else ""
+        order = ""
+        if isinstance(r, OrderResult) and math.isfinite(r.fitted):
+            order = f"{r.fitted:.2f}" + (f" (expected {r.expected:g})" if math.isfinite(r.expected) else "")
+        rows.append(
+            "<tr>"
+            f'<td class="status {cls}">{html.escape(r.status)}</td>'
+            f"<td>{html.escape(o.check.family)}</td>"
+            f"<td>{html.escape(o.check.label)}</td>"
+            f"<td>{html.escape(r.message)}</td>"
+            f'<td class="num">{html.escape(order)}</td>'
+            f"<td>{plot}</td>"
+            "</tr>"
+        )
+    return (
+        "<!doctype html>\n<html lang=\"en\"><head><meta charset=\"utf-8\">"
+        '<meta name="viewport" content="width=device-width, initial-scale=1">'
+        f"<title>{html.escape(title)}</title><style>{_CSS}</style></head><body>"
+        f"<h1>{html.escape(title)}</h1>"
+        f'<p class="meta">{s["ok"]} of {s["total"]} checks passed. physverify {html.escape(__version__)}'
+        + (f" · {html.escape(source)}" if source else "")
+        + ". Plots: error against step size on log-log axes; dashed line has the expected slope.</p>"
+        '<div class="wrap"><table><thead><tr><th>Status</th><th>Family</th><th>Check</th><th>Message</th>'
+        "<th>Observed order</th><th>Convergence</th></tr></thead><tbody>"
+        + "".join(rows)
+        + "</tbody></table></div></body></html>\n"
+    )

physverify/steppers.py

@@ -0,0 +1,38 @@
+"""Reference time steppers and a fixed-step driver.
+
+These are used (a) to build independent reference solutions and (b) as the
+stepper when only a right-hand side, not a whole solver, is under test.
+"""
+
+from __future__ import annotations
+
+import numpy as np
+
+
+def euler(f, t, y, h):
+    return y + h * f(t, y)
+
+
+def midpoint(f, t, y, h):
+    k1 = f(t, y)
+    return y + h * f(t + 0.5 * h, y + 0.5 * h * k1)
+
+
+def rk4(f, t, y, h):
+    k1 = f(t, y)
+    k2 = f(t + 0.5 * h, y + 0.5 * h * k1)
+    k3 = f(t + 0.5 * h, y + 0.5 * h * k2)
+    k4 = f(t + h, y + h * k3)
+    return y + (h / 6.0) * (k1 + 2.0 * k2 + 2.0 * k3 + k4)
+
+
+STEPPERS = {"euler": (euler, 1), "midpoint": (midpoint, 2), "rk4": (rk4, 4)}
+
+
+def integrate_fixed(f, y0, t0, t1, n_steps, stepper=rk4):
+    """Advance y' = f(t, y) from t0 to t1 in n_steps equal steps."""
+    h = (t1 - t0) / n_steps
+    y = np.array(y0, dtype=float)
+    for i in range(n_steps):
+        y = stepper(f, t0 + i * h, y, h)
+    return y

physverify/stochastic/__init__.py

@@ -0,0 +1 @@
+"""Verification checks for stochastic (Monte Carlo) codes: the 2D Ising model. See docs/ising.md."""

physverify/stochastic/calibrate.py

@@ -0,0 +1,202 @@
+"""Calibration of the statistical checks, and detection power against run length.
+
+    python -m physverify.stochastic.calibrate results/ising_calibration.md
+
+Calibration (milestone M2): many independent replicate runs of the bug-free
+sampler, each analysed exactly as a check would. Two rates are counted:
+
+* per comparison, the fraction of single t-tests rejected at nominal level a.
+  This should be close to a; if it is much larger, the error bars are too small.
+* per check, the fraction of verdicts that are FAIL at the check's
+  false-alarm rate alpha (Bonferroni over its comparisons). It must not exceed
+  alpha, and is expected to be below it because Bonferroni is conservative.
+
+Power (milestone M3): for a few bugs, the probability that the exact-distribution
+check flags the bug, as a function of the number of sweeps per chain.
+
+The replicate runs are simulated together as one large set of independent chains.
+"""
+
+from __future__ import annotations
+
+import math
+import sys
+from pathlib import Path
+
+import numpy as np
+
+from ..checks import FAIL
+from . import exact as X
+from .checks import distribution_comparisons, moment_comparisons
+from .sampler import IsingSampler
+from .stats import t_critical, verdict
+
+
+def binomial_interval(k, n, z=1.96):
+    """Wilson score interval for a proportion."""
+    if n == 0:
+        return (0.0, 1.0)
+    p = k / n
+    centre = (p + z * z / (2 * n)) / (1 + z * z / n)
+    half = z * math.sqrt(p * (1 - p) / n + z * z / (4 * n * n)) / (1 + z * z / n)
+    return max(0.0, centre - half), min(1.0, centre + half)
+
+
+def _groups(e, m, R, C):
+    for g in range(R):
+        yield e[g * C:(g + 1) * C], m[g * C:(g + 1) * C]
+
+
+def calibrate(R=1000, C=32, n_sweeps=200, seed=2026, levels=(0.05, 0.01), alpha_check=0.05):
+    """Returns rows (test, T, per-comparison rejection rates at each level, FAIL rate of the check)."""
+    sampler = IsingSampler("metropolis")
+    rows = []
+    for name, T in (("exact_distribution", 2.5), ("moments", X.T_C), ("moments", 1.8)):
+        e, m = sampler.sample(4, T, 0.0, n_sweeps, seed=seed, n_chains=R * C)
+        seed += 1
+        per_level = {a: [0, 0] for a in levels}
+        fails = 0
+        for eg, mg in _groups(e, m, R, C):
+            if name == "exact_distribution":
+                comps = distribution_comparisons(eg, mg, 4, T)
+            else:
+                comps = moment_comparisons(eg, mg, X.exact_moments(4, T), "")
+            for c in comps:
+                for a in levels:
+                    crit = t_critical(a, float(c.dof))
+                    per_level[a][0] += abs(c.estimate - c.exact) > crit * c.se
+                    per_level[a][1] += 1
+            fails += verdict(comps, alpha_check).status == FAIL
+        rows.append((name, T, {a: tuple(v) for a, v in per_level.items()}, fails, R))
+    return rows
+
+
+POWER_BUGS = (
+    None,
+    "reject_zero_de",
+    "exp_de_times_t",
+    "acceptance_table_off_by_one",
+    "same_random_number",
+    "sweep_skips_last_row",
+    "heatbath_missing_2",
+    "synchronous_update",
+)
+POWER_SWEEPS = (10, 25, 50, 100, 200)
+
+
+def power(R=50, C=16, sweeps=POWER_SWEEPS, seed=3000, T=2.5):
+    """Detection probability of exact_distribution (alpha 0.001) for each bug and run length."""
+    from . import ising
+
+    table = {}
+    for bug in POWER_BUGS:
+        algorithm = ising.BUGS[bug].algorithm if bug else "metropolis"
+        sampler = IsingSampler(algorithm, frozenset([bug]) if bug else frozenset())
+        probs = []
+        for n in sweeps:
+            e, m = sampler.sample(4, T, 0.0, n, seed=seed, n_chains=R * C)
+            seed += 1
+            hits = sum(verdict(distribution_comparisons(eg, mg, 4, T)).status == FAIL for eg, mg in _groups(e, m, R, C))
+            probs.append(hits / R)
+        table[bug or "(no bug)"] = probs
+    return table
+
+
+def power_svg(table, sweeps=POWER_SWEEPS, width=560, height=300) -> str:
+    """A small line chart of detection probability against sweeps per chain (log x axis)."""
+    pad_l, pad_r, pad_t, pad_b = 46, 170, 14, 36
+    x0, x1 = math.log10(sweeps[0]), math.log10(sweeps[-1])
+
+    def px(n):
+        return pad_l + (math.log10(n) - x0) / (x1 - x0) * (width - pad_l - pad_r)
+
+    def py(p):
+        return pad_t + (1 - p) * (height - pad_t - pad_b)
+
+    colours = ["#6b6b70", "#9a6700", "#2f5bd3", "#1a7f37", "#b42318", "#8250df", "#0a7ea4", "#bf3989"]
+    parts = [f'<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 {width} {height}" font-family="sans-serif" font-size="11">']
+    parts.append(f'<rect width="{width}" height="{height}" fill="white"/>')
+    for p in (0, 0.5, 1):
+        parts.append(f'<line x1="{pad_l}" x2="{width - pad_r}" y1="{py(p):.1f}" y2="{py(p):.1f}" stroke="#ddd"/>')
+        parts.append(f'<text x="{pad_l - 6}" y="{py(p) + 4:.1f}" text-anchor="end">{p:g}</text>')
+    for n in sweeps:
+        parts.append(f'<text x="{px(n):.1f}" y="{height - pad_b + 16}" text-anchor="middle">{n}</text>')
+    parts.append(f'<text x="{(pad_l + width - pad_r) / 2:.1f}" y="{height - 4}" text-anchor="middle">sweeps per chain (16 chains per run)</text>')
+    parts.append(f'<text x="12" y="{(pad_t + height - pad_b) / 2:.1f}" transform="rotate(-90 12 {(pad_t + height - pad_b) / 2:.1f})" text-anchor="middle">P(flagged)</text>')
+    for k, (name, probs) in enumerate(table.items()):
+        col = colours[k % len(colours)]
+        pts = " ".join(f"{px(n):.1f},{py(p):.1f}" for n, p in zip(sweeps, probs))
+        dash = ' stroke-dasharray="4 3"' if name in ("(no bug)", "reject_zero_de") else ""
+        parts.append(f'<polyline points="{pts}" fill="none" stroke="{col}" stroke-width="2"{dash}/>')
+        parts.append(f'<text x="{width - pad_r + 8}" y="{pad_t + 12 + 15 * k}" fill="{col}">{name}</text>')
+    parts.append("</svg>")
+    return "\n".join(parts) + "\n"
+
+
+def to_markdown(cal, table, R_power=50) -> str:
+    lines = [
+        "# Ising checks: calibration and power",
+        "",
+        "Generated by `python -m physverify.stochastic.calibrate results/ising_calibration.md`.",
+        "",
+        "## Calibration on the bug-free Metropolis sampler",
+        "",
+        "Each replicate is 32 independent chains of 200 sweeps on L = 4 (after the sampler's burn-in of 100),",
+        "analysed exactly as the check analyses its chains. 95% Wilson intervals in brackets.",
+        "",
+        "| Test | T | Comparisons | Rejected at 0.05 | Rejected at 0.01 | Check FAIL rate at alpha = 0.05 |",
+        "|---|---|---|---|---|---|",
+    ]
+    for name, T, per, fails, R in cal:
+        k5, n5 = per[0.05]
+        k1, n1 = per[0.01]
+        lo5, hi5 = binomial_interval(k5, n5)
+        lo1, hi1 = binomial_interval(k1, n1)
+        lo, hi = binomial_interval(fails, R)
+        lines.append(
+            f"| {name} | {T:.4g} | {n5} | {k5 / n5:.4f} [{lo5:.4f}, {hi5:.4f}] | {k1 / n1:.4f} [{lo1:.4f}, {hi1:.4f}] "
+            f"| {fails} of {R} = {fails / R:.3f} [{lo:.3f}, {hi:.3f}] |"
+        )
+    lines += [
+        "",
+        "Reading: per-comparison rejection rates near the nominal level mean the error bars are right; the check-level",
+        "FAIL rate must stay at or below its alpha (Bonferroni makes it conservative). The checks themselves use",
+        "alpha = 0.001, which cannot be measured directly with 1000 replicates; the 0.05 runs test the same machinery.",
+        "",
+        "## Power: probability that `exact_distribution` flags a bug",
+        "",
+        f"{R_power} replicate runs per cell, each 16 chains on L = 4 at T = 2.5, alpha = 0.001.",
+        "",
+        "| Bug | " + " | ".join(f"{n} sweeps" for n in POWER_SWEEPS) + " |",
+        "|---|" + "---|" * len(POWER_SWEEPS),
+    ]
+    for name, probs in table.items():
+        lines.append(f"| {name} | " + " | ".join(f"{p:.2f}" for p in probs) + " |")
+    lines += [
+        "",
+        "`(no bug)` should stay near 0 (false alarms). `reject_zero_de` is the equivalent mutant: it keeps the",
+        "Boltzmann distribution, so a distribution check should not flag it at any run length.",
+        "",
+        "![power curves](ising_power.svg)",
+        "",
+    ]
+    return "\n".join(lines)
+
+
+def main(argv=None) -> int:
+    argv = sys.argv[1:] if argv is None else argv
+    cal = calibrate()
+    print("calibration done", file=sys.stderr)
+    table = power()
+    text = to_markdown(cal, table)
+    print(text)
+    if argv:
+        path = Path(argv[0])
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.write_text(text, encoding="utf-8")
+        path.with_name("ising_power.svg").write_text(power_svg(table), encoding="utf-8")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())