edgegate 0.2.0__py3-none-any.whl

edgegate/matrix.py

@@ -0,0 +1,367 @@
+"""Run a matrix of variants x repeats and tabulate crash rate and resources per cell.
+
+Matrix file (YAML):
+    version: 1
+    base: edgegate.yaml              # the config every cell starts from (path relative to this file)
+    repeats: 6
+    duration_s: 150                  # optional: overrides benchmark.duration_s for every cell
+    stale_processes: [myengine]      # optional: kill processes whose /proc comm starts with these before each run
+    cells:
+      - { name: baseline }
+      - { name: b2,   args: { set: { "-b": "2" } } }       # replace the value after a flag (or append flag+value)
+      - { name: noS,  args: { remove: ["-S"] } }           # drop a bare flag (use set to change a valued one)
+      - { name: trt,  args: { add: ["-S"] }, duration_s: 240 }
+      - { name: fp32, env: { PRECISION: fp32 } }
+      - { name: pm25, power_mode: 3 }                      # nvpmodel id; must switch without a reboot
+      - { name: prev, path: ../builds/engine-v4.0.4 }      # a different executable or model for this cell
+Cells are interleaved (rep 1 of every cell, then rep 2, ...) so slow drift cannot favour one cell, and a run
+whose results file already exists is skipped, so an interrupted matrix resumes where it stopped.
+"""
+import copy
+import datetime as dt
+import json
+import os
+import pathlib
+import re
+import signal
+import subprocess
+import time
+
+import yaml
+
+from .compare import evaluate, status
+from .config import ConfigError, load as load_config
+from .measure import run as measure_run
+
+
+class MatrixError(Exception):
+    pass
+
+
+def load_matrix(path):
+    path = pathlib.Path(path)
+    try:
+        m = yaml.safe_load(path.read_text()) or {}
+    except FileNotFoundError:
+        raise ConfigError(f"matrix file not found: {path}") from None
+    except yaml.YAMLError as e:
+        raise ConfigError(f"matrix file is not valid YAML: {e}") from None
+    if m.get("version") != 1:
+        raise ConfigError("matrix: 'version: 1' is required")
+    if not m.get("base"):
+        raise ConfigError("matrix: base (a config file) is required")
+    cells = m.get("cells")
+    if not isinstance(cells, list) or not cells:
+        raise ConfigError("matrix: cells must be a non-empty list")
+    names = set()
+    for i, c in enumerate(cells):
+        if not isinstance(c, dict) or not re.fullmatch(r"[A-Za-z0-9_.-]+", str(c.get("name", ""))):
+            raise ConfigError(f"matrix: cells[{i}].name is required (letters, digits, _ . -)")
+        if c["name"] in names:
+            raise ConfigError(f"matrix: duplicate cell name {c['name']}")
+        names.add(c["name"])
+        a = c.get("args") or {}
+        if not isinstance(a, dict) or not isinstance(a.get("set", {}), dict) or not isinstance(a.get("add", []), list) \
+                or not isinstance(a.get("remove", []), list):
+            raise ConfigError(f"matrix: cells[{i}].args must have 'set' (mapping), 'add' (list) and/or 'remove' (list)")
+        if "power_mode" in c and not isinstance(c["power_mode"], int):
+            raise ConfigError(f"matrix: cells[{i}].power_mode must be an nvpmodel id (integer)")
+        if "path" in c:
+            if not isinstance(c["path"], str) or not c["path"]:
+                raise ConfigError(f"matrix: cells[{i}].path must be a path string")
+            c["path"] = os.path.normpath(os.path.join(pathlib.Path(path).resolve().parent, c["path"]))
+            if not pathlib.Path(c["path"]).exists():
+                raise ConfigError(f"matrix: cells[{i}].path does not exist: {c['path']}")
+    m["repeats"] = int(m.get("repeats", 1))
+    if m["repeats"] < 1:
+        raise ConfigError("matrix: repeats must be >= 1")
+    m["_root"] = path.resolve().parent
+    m["base"] = str((m["_root"] / m["base"]).resolve())
+    m.setdefault("stale_processes", [])
+    m.setdefault("max_crash_rate", 0.0)        # a cell fails its JUnit case when crashed/n exceeds this
+    m.setdefault("require_gate_pass", True)    # ...or when any run failed its own gate (e.g. ran clean but under spec)
+    return m
+
+
+def apply_cell(base_cfg, cell, matrix):
+    """A fresh config for one cell: base config with the cell's argument, env, duration and note edits applied."""
+    cfg = copy.deepcopy(base_cfg)
+    model = cfg["model"]
+    a = cell.get("args") or {}
+    if a and model["backend"] != "command":
+        raise ConfigError(f"matrix: cell {cell['name']} edits args but the base backend is {model['backend']}")
+    args = list(model.get("args") or [])
+    for flag in (a.get("remove") or []):          # drop a bare flag such as -S; change valued flags with set
+        args = [x for x in args if x != flag]
+    for flag, val in (a.get("set") or {}).items():
+        val = str(val)
+        if flag in args and args.index(flag) + 1 < len(args):
+            args[args.index(flag) + 1] = val
+        else:
+            args += [flag, val]
+    args += [str(x) for x in (a.get("add") or [])]
+    model["args"] = args
+    if cell.get("env"):
+        model["env"] = {**(model.get("env") or {}), **{k: str(v) for k, v in cell["env"].items()}}
+    dur = cell.get("duration_s", matrix.get("duration_s"))
+    if dur is not None:
+        cfg["benchmark"]["duration_s"] = dur
+    if cell.get("path"):
+        model["path"] = cell["path"]          # a different build of the engine, or a different model file
+    return cfg
+
+
+# ---- nvpmodel helpers (Jetson); all tolerate a host without nvpmodel
+
+def power_modes(conf_text):
+    """{id: name} from /etc/nvpmodel.conf text."""
+    return {int(m.group(1)): m.group(2) for m in re.finditer(r"<\s*POWER_MODEL\s+ID=(\d+)\s+NAME=([A-Za-z0-9_]+)", conf_text)}
+
+
+def current_power_mode():
+    try:
+        out = subprocess.run(["sudo", "-n", "nvpmodel", "-q"], capture_output=True, text=True, timeout=15).stdout
+    except (OSError, subprocess.SubprocessError):
+        return None
+    for line in out.splitlines():
+        if "NV Power Mode" in line:
+            return line.split(":", 1)[-1].strip()
+    return None
+
+
+def set_power_mode(mode_id, modes, wait_s=20):
+    """Switch nvpmodel and wait until it reports the expected name. Refuses modes that need a reboot: the
+    matrix process would not survive one. Returns the mode name."""
+    name = modes.get(mode_id)
+    if name is None:
+        raise MatrixError(f"nvpmodel id {mode_id} is not in /etc/nvpmodel.conf (known: {modes})")
+    if current_power_mode() == name:
+        return name
+    p = subprocess.run(["sudo", "-n", "nvpmodel", "-m", str(mode_id)], input="no\n", capture_output=True, text=True, timeout=60)
+    if "Reboot required" in (p.stdout + p.stderr):
+        raise MatrixError(f"nvpmodel mode {mode_id} ({name}) requires a reboot on this board; run it as a separate matrix after rebooting into it")
+    t0 = time.monotonic()
+    while time.monotonic() - t0 < wait_s:
+        if current_power_mode() == name:
+            time.sleep(3)
+            return name
+        time.sleep(1)
+    raise MatrixError(f"nvpmodel did not report {name} within {wait_s} s (now: {current_power_mode()})")
+
+
+def kill_stale(prefixes):
+    """SIGKILL every process whose /proc/<pid>/comm starts with one of the prefixes. Returns how many."""
+    n = 0
+    if not prefixes:
+        return 0
+    for pid in os.listdir("/proc"):
+        if not pid.isdigit() or int(pid) == os.getpid():
+            continue
+        try:
+            comm = pathlib.Path(f"/proc/{pid}/comm").read_text().strip()
+        except OSError:
+            continue
+        if any(comm.startswith(pfx) for pfx in prefixes):
+            try:
+                os.kill(int(pid), signal.SIGKILL); n += 1
+            except (ProcessLookupError, PermissionError):
+                pass
+    return n
+
+
+# ---- the run
+
+def _now():
+    return dt.datetime.now(dt.timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z")
+
+
+def run_matrix(matrix, out_dir, log=print, dry_run=False):
+    base_cfg = load_config(matrix["base"])
+    out_dir = pathlib.Path(out_dir).resolve()
+    out_dir.mkdir(parents=True, exist_ok=True)
+    logf = open(out_dir / "LOG", "a")
+    def say(msg):
+        logf.write(f"{_now()} {msg}\n"); logf.flush(); log(msg)
+    needs_pm = any("power_mode" in c for c in matrix["cells"])
+    modes, start_mode = {}, None
+    if needs_pm:
+        try:
+            modes = power_modes(pathlib.Path("/etc/nvpmodel.conf").read_text())
+        except OSError:
+            raise MatrixError("a cell sets power_mode but /etc/nvpmodel.conf is not readable (not a Jetson?)") from None
+        start_mode = current_power_mode()
+        start_id = next((i for i, n in modes.items() if n == start_mode), None)
+    say(f"matrix start: {len(matrix['cells'])} cells x {matrix['repeats']} repeats -> {out_dir}")
+    done = skipped = 0
+    try:
+        for rep in range(1, matrix["repeats"] + 1):
+            for cell in matrix["cells"]:
+                cdir = out_dir / cell["name"]; cdir.mkdir(exist_ok=True)
+                rpath = cdir / f"r{rep}.json"
+                if rpath.exists():
+                    skipped += 1; continue
+                cfg = apply_cell(base_cfg, cell, matrix)
+                if dry_run:
+                    say(f"[dry] {cell['name']} r{rep}: {cfg['model']['path']} {' '.join(cfg['model'].get('args') or [])} duration={cfg['benchmark'].get('duration_s')}")
+                    continue
+                try:
+                    if "power_mode" in cell:
+                        set_power_mode(cell["power_mode"], modes)
+                    elif needs_pm and start_id is not None:
+                        set_power_mode(start_id, modes)
+                except MatrixError as e:
+                    say(f"== {cell['name']} r{rep}: SKIP {e}")
+                    rpath.write_text(json.dumps({"skipped": str(e), "cell": cell["name"], "rep": rep, "created_at": _now()}, indent=1))
+                    continue
+                k = kill_stale(matrix["stale_processes"])
+                if k:
+                    time.sleep(3)
+                say(f"== {cell['name']} r{rep} mode={current_power_mode() or '-'} killed_stale={k} {_now()}")
+                results = measure_run(cfg, notes=f"matrix cell {cell['name']} rep {rep}" + (f": {cell['note']}" if cell.get("note") else ""),
+                                      out_dir=cdir / f"r{rep}.d")
+                checks = evaluate(results, cfg["thresholds"])
+                results["gate"] = {"status": status(checks), "failed": [c.name for c in checks if c.status == "fail"],
+                                   "checked": [c.name for c in checks if c.status != "skip"], "baseline": None}
+                results["matrix"] = {"cell": cell["name"], "rep": rep}
+                rpath.write_text(json.dumps(results, indent=1) + "\n")
+                done += 1
+                say("   " + summarize_run(results))
+    finally:
+        if needs_pm and start_id is not None and not dry_run:
+            try:
+                set_power_mode(start_id, modes)
+            except MatrixError as e:
+                say(f"could not restore power mode {start_mode}: {e}")
+        kill_stale(matrix["stale_processes"])
+        say(f"matrix done: {done} new runs, {skipped} already present")
+        logf.close()
+    return tabulate(matrix, out_dir)
+
+
+def summarize_run(r):
+    if r.get("command"):
+        c = r["command"]
+        line = f"exit {c['exit_code']}{' (full window)' if c.get('timed_out') else ''} after {c['wall_s']} s"
+        for name, m in (r.get("metrics") or {}).items():
+            if m.get("count") is not None:
+                line += f" · {name} {m['count']}" + (f" ({m['per_s']}/s)" if m.get("per_s") is not None else "")
+    else:
+        lat = r["latency_ms"]
+        line = f"latency p50 {lat.get('p50')} ms, p99 {lat.get('p99')} ms"
+    if r.get("power"):
+        line += f" · power {r['power']['total_w']} W"
+    line += f" · gate {r.get('gate', {}).get('status', '-')}"
+    return line
+
+
+def _crashed(r):
+    """A run is a crash when the exit-code check failed; without that rule, when a command run exited early
+    with a non-zero code; for latency runs, when the gate failed. None for skipped cells."""
+    if "skipped" in r:
+        return None
+    gate = r.get("gate") or {}
+    if r.get("command"):
+        if gate.get("failed") is not None and any(c == "exit_code" for c in gate["failed"]):
+            return True
+        if "exit_code" in gate.get("checked", []):
+            return False
+        return not r["command"].get("timed_out") and r["command"].get("exit_code") != 0
+    return gate.get("status") == "fail"
+
+
+def tabulate(matrix, out_dir):
+    """Per-cell summary from whatever result files exist. Returns the summary dict and writes summary.json."""
+    out_dir = pathlib.Path(out_dir)
+    cells = []
+    for cell in matrix["cells"]:
+        rows = []
+        for f in sorted((out_dir / cell["name"]).glob("r*.json"), key=lambda p: int(p.stem[1:])):
+            r = json.loads(f.read_text())
+            if "skipped" in r:
+                rows.append({"rep": r["rep"], "skipped": r["skipped"]}); continue
+            cr = _crashed(r)
+            row = {"rep": r.get("matrix", {}).get("rep", int(f.stem[1:])), "crashed": cr,
+                   "gate": r.get("gate", {}).get("status"), "failed": r.get("gate", {}).get("failed", []),
+                   "power_w": (r.get("power") or {}).get("total_w"), "board_ram_peak_mb": r["memory"].get("board_ram_peak_mb"),
+                   "peak_rss_mb": r["memory"].get("peak_rss_mb")}
+            if r.get("command"):
+                row.update({"exit_code": r["command"]["exit_code"], "wall_s": r["command"]["wall_s"],
+                            "forensics": r["command"].get("forensics"),
+                            "metrics": {k: v.get("count") for k, v in (r.get("metrics") or {}).items()},
+                            "rates": {k: v.get("per_s") for k, v in (r.get("metrics") or {}).items() if v.get("per_s") is not None}})
+            else:
+                row.update({"latency_p99_ms": r["latency_ms"].get("p99"), "throughput_ips": r.get("throughput_ips")})
+            rows.append(row)
+        measured = [x for x in rows if "skipped" not in x]
+        crashed = [x for x in measured if x["crashed"]]
+        clean = [x for x in measured if not x["crashed"]]
+        gate_failed = [x for x in measured if x.get("gate") == "fail"]
+        # "alive but wrong": ran the full window yet failed a non-exit check (zero output, over power, ...)
+        under_spec = [x for x in clean if x.get("gate") == "fail"]
+        def mean(key, src):
+            vals = [x[key] for x in src if x.get(key) is not None]
+            return round(sum(vals) / len(vals), 2) if vals else None
+        rates = {}
+        for k in {k for x in clean for k in (x.get("rates") or {})}:
+            vals = [x["rates"][k] for x in clean if x.get("rates", {}).get(k) is not None]
+            rates[k] = round(sum(vals) / len(vals), 3) if vals else None
+        rate = len(crashed) / len(measured) if measured else None
+        failed_checks = sorted({c for x in under_spec for c in x.get("failed", []) if c != "exit_code"})
+        if not measured:
+            verdict = "skip"
+        elif rate > matrix.get("max_crash_rate", 0.0) or (matrix.get("require_gate_pass", True) and gate_failed):
+            verdict = "fail"
+        else:
+            verdict = "pass"
+        cells.append({"name": cell["name"], "note": cell.get("note"), "n": len(measured), "skipped": len(rows) - len(measured),
+                      "crashed": len(crashed), "crash_rate": round(rate, 3) if rate is not None else None,
+                      "gate_failed": len(gate_failed), "under_spec": len(under_spec), "under_spec_checks": failed_checks,
+                      "time_to_crash_s": [x["wall_s"] for x in crashed if "wall_s" in x],
+                      "clean_power_w": mean("power_w", clean), "clean_rates": rates,
+                      "clean_latency_p99_ms": mean("latency_p99_ms", clean),
+                      "board_ram_peak_mb": max((x["board_ram_peak_mb"] or 0 for x in measured), default=None),
+                      "forensic_patterns": sorted({(x.get("forensics") or {}).get("ram_pattern") for x in crashed if x.get("forensics")} - {None}),
+                      "status": verdict,
+                      "runs": rows})
+    summary = {"edgegate_matrix": 1, "created_at": _now(), "base": matrix["base"], "repeats": matrix["repeats"],
+               "max_crash_rate": matrix.get("max_crash_rate", 0.0), "require_gate_pass": matrix.get("require_gate_pass", True), "cells": cells}
+    (out_dir / "summary.json").write_text(json.dumps(summary, indent=1) + "\n")
+    return summary
+
+
+def markdown(summary):
+    L = [f"# edgegate matrix: {pathlib.Path(summary['base']).name}, {summary['repeats']} repeats per cell", "",
+         "| cell | crashed | alive but under spec | time to crash | clean rate | clean power W | board RAM peak MB | RAM before crash | verdict |",
+         "| --- | --- | --- | --- | --- | --- | --- | --- | --- |"]
+    for c in summary["cells"]:
+        tc = c["time_to_crash_s"]
+        tcs = "—" if not tc else (f"{min(tc):.0f} s" if round(min(tc)) == round(max(tc)) else f"{min(tc):.0f}–{max(tc):.0f} s")
+        rate = ", ".join(f"{k} {v}/s" for k, v in (c["clean_rates"] or {}).items()) or (f"p99 {c['clean_latency_p99_ms']} ms" if c.get("clean_latency_p99_ms") else "—")
+        us = f"{c['under_spec']} / {c['n']}" + (f" ({', '.join(c['under_spec_checks'])})" if c['under_spec_checks'] else "")
+        L.append(f"| {c['name']}{' (' + c['note'] + ')' if c.get('note') else ''} | {c['crashed']} / {c['n']} | {us} | {tcs} | {rate} | {c['clean_power_w'] or '—'} | {c['board_ram_peak_mb'] or '—'} | {', '.join(c['forensic_patterns']) or '—'} | {c['status'].upper()} |")
+    L += ["", f"A cell FAILS when its crash rate exceeds {summary['max_crash_rate']:.0%}"
+          + (" or when any run failed its own gate ('alive but under spec': it ran the full window but missed a check such as a metric minimum)." if summary.get("require_gate_pass", True) else ".")
+          + " 'RAM before crash' is the pattern in the last 30 s of each crashed run: plateau, step (one excursion) or cycling (repeated large swings, typically an allocation that keeps failing)."]
+    return "\n".join(L) + "\n"
+
+
+def junit(summary):
+    import xml.etree.ElementTree as ET
+    cells = summary["cells"]
+    ts = ET.Element("testsuite", name="edgegate-matrix", tests=str(len(cells)),
+                    failures=str(sum(c["status"] == "fail" for c in cells)), skipped=str(sum(c["status"] == "skip" for c in cells)),
+                    errors="0", timestamp=summary["created_at"].rstrip("Z"))
+    for c in cells:
+        tc = ET.SubElement(ts, "testcase", classname="edgegate.matrix", name=c["name"])
+        msg = f"{c['crashed']} of {c['n']} runs crashed (limit {summary['max_crash_rate']:.0%})"
+        if c.get("under_spec"):
+            msg += f"; {c['under_spec']} ran the full window but failed {', '.join(c['under_spec_checks']) or 'a check'}"
+        if c["status"] == "fail":
+            ET.SubElement(tc, "failure", message=msg, type="crash_rate").text = msg
+        elif c["status"] == "skip":
+            ET.SubElement(tc, "skipped", message="no measured runs")
+        props = ET.SubElement(tc, "properties")
+        for k in ("crashed", "n", "crash_rate", "under_spec", "clean_power_w", "board_ram_peak_mb"):
+            if c.get(k) is not None:
+                ET.SubElement(props, "property", name=k, value=str(c[k]))
+    return ET.tostring(ts, encoding="unicode", xml_declaration=True)

edgegate/measure.py

@@ -0,0 +1,267 @@
+"""Run the benchmark and the accuracy pass; produce the results document."""
+import datetime as dt
+import os
+import pathlib
+import platform
+import re
+import signal
+import subprocess
+import threading
+import time
+
+import numpy as np
+
+from . import __version__, dataset
+from .backends import BackendError, make_backend, model_sha256
+from .sampler import make_sampler, peak_rss_mb, write_timeline
+
+
+def _now():
+    return dt.datetime.now(dt.timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z")
+
+
+def latency_stats(samples):
+    a = np.asarray(samples, dtype=np.float64)
+    return {"p50": round(float(np.percentile(a, 50)), 3), "p90": round(float(np.percentile(a, 90)), 3),
+            "p99": round(float(np.percentile(a, 99)), 3), "mean": round(float(a.mean()), 3),
+            "min": round(float(a.min()), 3), "max": round(float(a.max()), 3), "n": int(a.size)}
+
+
+def _device_info(cfg):
+    info = {"name": cfg["device"].get("name") or platform.node(), "hostname": platform.node(),
+            "machine": platform.machine(), "python": platform.python_version()}
+    for p, key in (("/etc/nv_tegra_release", "l4t"), ("/proc/device-tree/model", "model")):
+        try:
+            info[key] = pathlib.Path(p).read_text(errors="replace").strip().split("\n")[0].rstrip("\x00")
+        except OSError:
+            pass
+    try:
+        out = subprocess.run(["nvpmodel", "-q"], capture_output=True, text=True, timeout=10).stdout
+        for line in out.splitlines():
+            if "NV Power Mode" in line:
+                info["power_mode"] = line.split(":", 1)[-1].strip()
+    except (OSError, subprocess.SubprocessError):
+        pass
+    return info
+
+
+def accuracy_top1(backend, ds):
+    correct = total = 0
+    for x, label in dataset.load(ds):
+        out = backend.infer(x)
+        pred = int(np.argmax(np.asarray(out).reshape(out.shape[0], -1), axis=1)[0])
+        correct += int(pred == label)
+        total += 1
+    if total == 0:
+        raise BackendError("dataset yielded no samples")
+    return {"metric": "top1", "value": round(correct / total, 4), "correct": correct, "n": total}
+
+
+class ProcessTreeRss:
+    """Polls /proc for every process in the session started for the command and records the peak summed RSS."""
+
+    def __init__(self, sid, interval_s=0.5):
+        self.sid, self.interval_s, self.peak_mb = sid, interval_s, 0.0
+        self.series = []                      # (t_s, rss_mb) for the timeline
+        self._t0 = time.monotonic()
+        self._stop = threading.Event()
+        self._thread = threading.Thread(target=self._loop, daemon=True)
+
+    def _sample(self):
+        total_kb = 0
+        for pid in os.listdir("/proc"):
+            if not pid.isdigit():
+                continue
+            try:
+                stat = pathlib.Path(f"/proc/{pid}/stat").read_text()
+                if int(stat.rsplit(")", 1)[1].split()[3]) != self.sid:     # field 6 = session id
+                    continue
+                for line in pathlib.Path(f"/proc/{pid}/status").read_text().splitlines():
+                    if line.startswith("VmRSS:"):
+                        total_kb += int(line.split()[1]); break
+            except (OSError, ValueError, IndexError):
+                continue
+        return total_kb / 1024.0
+
+    def _loop(self):
+        while not self._stop.is_set():
+            mb = self._sample()
+            self.series.append((round(time.monotonic() - self._t0, 3), round(mb, 1)))
+            self.peak_mb = max(self.peak_mb, mb)
+            self._stop.wait(self.interval_s)
+
+    def __enter__(self):
+        self._thread.start(); return self
+
+    def __exit__(self, *exc):
+        self._stop.set(); self._thread.join(timeout=3)
+        self.peak_mb = max(self.peak_mb, self._sample())
+
+
+def _count_metrics(metrics, out_dir, wall_s):
+    found = {}
+    for m in metrics:
+        path = pathlib.Path(m["file"].replace("{out}", str(out_dir)))
+        try:
+            text = path.read_text(errors="replace")
+        except OSError:
+            found[m["name"]] = {"count": None, "per_s": None, "file": str(path), "note": "file not found"}
+            continue
+        n = len(re.findall(m["pattern"], text, re.MULTILINE))   # ^ and $ anchor per line, as in grep
+        found[m["name"]] = {"count": n, "per_s": round(n / wall_s, 3) if wall_s else None, "file": str(path)}
+    return found
+
+
+def forensics(samples, rss_series, end_t_s, window_s=30.0):
+    """What the board looked like in the last `window_s` seconds before the process died (or before the window
+    closed). `samples` are tegrastats samples with t_s; `rss_series` is [(t_s, rss_mb)] for the process tree.
+    death_t_s = last moment the process tree still held real memory (above 5% of its peak); None if it never fell."""
+    death = None
+    if rss_series:
+        peak = max(v for _, v in rss_series)
+        # Alive means the tree still holds real memory. A warm-up spike that settles to a lower plateau is not
+        # a death (an engine that peaks at 12 GB loading models and then runs at 4.6 GB is healthy), so the
+        # floor is small and absolute: below 5% of peak or 64 MB, whichever is larger.
+        floor = max(peak * 0.05, 64.0)
+        alive = [t for t, v in rss_series if v > floor]
+        if alive and alive[-1] < rss_series[-1][0] - 1.0:      # it fell more than a second before sampling stopped
+            death = alive[-1]
+    t_end = death if death is not None else end_t_s
+    win = [x for x in samples if x.get("t_s") is not None and t_end - window_s <= x["t_s"] <= t_end]
+    out = {"window_s": window_s, "death_t_s": round(death, 1) if death is not None else None, "samples": len(win)}
+    rams = [x["ram_mb"] for x in win if x.get("ram_mb") is not None]
+    if rams:
+        out["ram_mb"] = {"min": min(rams), "max": max(rams), "mean": round(sum(rams) / len(rams))}
+        # swings: how many times RAM moved by more than a quarter of its range inside the window.
+        # 0 or 1 = a plateau or a single step; many = allocation cycling.
+        span = max(rams) - min(rams)
+        swings, last = 0, rams[0]
+        for v in rams[1:]:
+            if span and abs(v - last) > span * 0.25:
+                swings += 1; last = v
+        out["ram_swings"] = swings
+        # plateau: steady (a healthy engine after warm-up). step: one excursion, up or up-and-back (a load
+        # spike, or a crash right after one). cycling: repeated large swings (an allocation that keeps failing).
+        out["ram_pattern"] = "plateau" if span < 512 or swings <= 1 else ("step" if swings == 2 else "cycling")
+    rails = sorted({n for x in win for n in x["rails"]})
+    if rails:
+        out["rails_w"] = {r: round(sum(x["rails"].get(r, 0) for x in win) / len(win) / 1000.0, 2) for r in rails}
+    rss_win = [v for t, v in rss_series if t_end - window_s <= t <= t_end]
+    if rss_win:
+        out["process_rss_mb"] = {"min": round(min(rss_win)), "max": round(max(rss_win))}
+    return out
+
+
+def run_command(cfg, backend, out_dir, notes=None):
+    """The command backend: run the process for benchmark.duration_s (or until it exits), sampling meanwhile."""
+    bench = cfg["benchmark"]
+    duration = bench.get("duration_s") or 0
+    out_dir = pathlib.Path(out_dir).resolve()      # absolute: the command runs with its own cwd (the config dir)
+    out_dir.mkdir(parents=True, exist_ok=True)
+    sub = lambda x: x.replace("{out}", str(out_dir))
+    cmd = [backend.path, *[sub(a) for a in backend.args]]
+    env = {**os.environ, **{k: sub(str(v)) for k, v in backend.env.items()}}
+    stdout_path = out_dir / "stdout.log"
+    timed_out = False
+    started_at = _now()
+    t0 = time.monotonic()
+    with open(stdout_path, "wb") as log, make_sampler(cfg) as sampler:
+        proc = subprocess.Popen(cmd, stdout=log, stderr=subprocess.STDOUT, cwd=sub(backend.cwd) if backend.cwd else None,
+                                env=env, start_new_session=True)
+        with ProcessTreeRss(proc.pid) as rss:
+            try:
+                proc.wait(timeout=duration if duration > 0 else None)
+            except subprocess.TimeoutExpired:
+                timed_out = True
+                try:
+                    os.killpg(proc.pid, signal.SIGTERM)
+                    proc.wait(timeout=10)
+                except (ProcessLookupError, subprocess.TimeoutExpired):
+                    try:
+                        os.killpg(proc.pid, signal.SIGKILL)
+                    except ProcessLookupError:
+                        pass
+                    proc.wait(timeout=10)
+    wall_s = round(time.monotonic() - t0, 2)
+    exit_code = 124 if timed_out else proc.returncode      # 124 mirrors coreutils timeout: ran the full window
+    power = sampler.summary()
+    memory = {"peak_rss_mb": round(rss.peak_mb, 1)}
+    if power and power.get("ram_peak_mb") is not None:
+        memory["board_ram_peak_mb"] = power.pop("ram_peak_mb")
+    elif power:
+        power.pop("ram_peak_mb", None)
+    metrics = _count_metrics(cfg.get("metrics", []), out_dir, wall_s)
+    timeline = None
+    if write_timeline(out_dir / "timeline.csv", sampler.timeline(), {"process_rss_mb": rss.series}):
+        timeline = str(out_dir / "timeline.csv")
+    last = forensics(sampler.timeline(), rss.series, wall_s)
+    none = {"p50": None, "p90": None, "p99": None, "mean": None, "min": None, "max": None, "n": 0,
+            "source": "n/a: command backend has no per-inference latency; see metrics[*].per_s"}
+    return {
+        "edgegate_version": __version__,
+        "created_at": _now(),
+        "device": _device_info(cfg),
+        "model": {"path": os.path.relpath(cfg["model"]["path"], cfg["_root"]), "sha256": model_sha256(cfg["model"]["path"]),
+                  "input_shape": None, **backend.describe()},
+        "benchmark": {"warmup": 0, "iterations": 0, "duration_s": duration},
+        "command": {"exit_code": exit_code, "timed_out": timed_out, "wall_s": wall_s, "stdout": str(stdout_path),
+                    "argv": cmd, "timeline": timeline, "started_at": started_at, "forensics": last},
+        "metrics": metrics,
+        "latency_ms": none,
+        "throughput_ips": None,
+        "memory": memory,
+        "power": power,
+        "accuracy": None,
+        "notes": [n for n in (notes,) if n],
+    }
+
+
+def run(cfg, notes=None, out_dir=None):
+    backend = make_backend(cfg)
+    if backend.name == "command":
+        return run_command(cfg, backend, out_dir or (pathlib.Path(cfg["_root"]) / "edgegate-out"), notes)
+    bench = cfg["benchmark"]
+    with make_sampler(cfg) as sampler:
+        raw = backend.bench(bench["warmup"], bench["iterations"], bench.get("duration_s", 0))
+    power = sampler.summary()
+    if power and out_dir:
+        pathlib.Path(out_dir).mkdir(parents=True, exist_ok=True)
+        if write_timeline(pathlib.Path(out_dir) / "timeline.csv", sampler.timeline()):
+            power["timeline"] = str(pathlib.Path(out_dir) / "timeline.csv")
+
+    if isinstance(raw, dict):          # trtexec summary
+        latency = {"p50": raw["median"], "p90": None, "p99": raw.get("p99"), "mean": raw["mean"],
+                   "min": raw["min"], "max": raw["max"], "n": bench["iterations"], "source": "trtexec summary"}
+        throughput = raw.get("throughput_qps")
+    else:
+        latency = latency_stats(raw)
+        throughput = round(1000.0 / latency["mean"], 2) if latency["mean"] else None
+
+    acc = None
+    acc_note = None
+    if cfg.get("dataset"):
+        if backend.can_infer:
+            acc = accuracy_top1(backend, cfg["dataset"])
+        else:
+            acc_note = f"accuracy skipped: backend '{backend.name}' cannot run arbitrary inputs; install onnxruntime on the device to measure it"
+
+    memory = {"peak_rss_mb": peak_rss_mb()}
+    if power and power.get("ram_peak_mb") is not None:
+        memory["board_ram_peak_mb"] = power.pop("ram_peak_mb")
+    elif power:
+        power.pop("ram_peak_mb", None)
+
+    return {
+        "edgegate_version": __version__,
+        "created_at": _now(),
+        "device": _device_info(cfg),
+        "model": {"path": os.path.relpath(cfg["model"]["path"], cfg["_root"]), "sha256": model_sha256(cfg["model"]["path"]),
+                  "input_shape": cfg["model"]["input"]["shape"], **backend.describe()},
+        "benchmark": {"warmup": bench["warmup"], "iterations": bench["iterations"], "duration_s": bench.get("duration_s", 0)},
+        "latency_ms": latency,
+        "throughput_ips": throughput,
+        "memory": memory,
+        "power": power,
+        "accuracy": acc,
+        "notes": [n for n in (notes, acc_note) if n],
+    }