edgegate 0.2.0__py3-none-any.whl

edgegate/__init__.py

@@ -0,0 +1,3 @@
+"""CI gate for model-plus-firmware builds on edge devices."""
+
+__version__ = "0.2.0"

edgegate/backends.py

@@ -0,0 +1,183 @@
+"""Inference backends. Each exposes bench() -> latency samples and, if it can run arbitrary inputs, infer()."""
+import hashlib
+import pathlib
+import re
+import shutil
+import subprocess
+import time
+
+import numpy as np
+
+
+class BackendError(Exception):
+    pass
+
+
+def model_sha256(path):
+    h = hashlib.sha256()
+    with open(path, "rb") as f:
+        for chunk in iter(lambda: f.read(1 << 20), b""):
+            h.update(chunk)
+    return "sha256:" + h.hexdigest()
+
+
+def _random_input(shape, dtype):
+    return np.random.default_rng(0).standard_normal(shape).astype(dtype)
+
+
+# ------------------------------------------------------------------ ONNX Runtime (host, or Jetson with the NVIDIA wheel)
+
+class OnnxRuntimeBackend:
+    name = "onnxruntime"
+    can_infer = True
+
+    def __init__(self, model_path, input_spec, precision="fp32"):
+        try:
+            import onnxruntime as ort
+        except ImportError:
+            raise BackendError("onnxruntime is not installed (pip install edgegate[host])") from None
+        self.ort = ort
+        providers = ort.get_available_providers()
+        preferred = [p for p in ("TensorrtExecutionProvider", "CUDAExecutionProvider", "CPUExecutionProvider") if p in providers]
+        self.session = ort.InferenceSession(str(model_path), providers=preferred)
+        self.provider = self.session.get_providers()[0]
+        meta = self.session.get_inputs()[0]
+        self.input_name = input_spec["name"] or meta.name
+        self.shape = tuple(input_spec["shape"])
+        self.dtype = np.dtype(input_spec["dtype"])
+        self.output_name = self.session.get_outputs()[0].name
+        self.precision = "fp32"   # ORT runs the graph as exported
+
+    def describe(self):
+        return {"backend": self.name, "provider": self.provider, "version": self.ort.__version__, "precision": self.precision}
+
+    def infer(self, x):
+        return self.session.run([self.output_name], {self.input_name: x})[0]
+
+    def bench(self, warmup, iterations, duration_s=0):
+        x = _random_input(self.shape, self.dtype)
+        for _ in range(warmup):
+            self.infer(x)
+        samples = []
+        for _ in range(iterations):
+            t0 = time.perf_counter_ns()
+            self.infer(x)
+            samples.append((time.perf_counter_ns() - t0) / 1e6)
+        return samples
+
+
+# ------------------------------------------------------------------ TensorRT via trtexec (Jetson)
+
+TRTEXEC_LATENCY_RE = re.compile(
+    r"Latency:\s*min\s*=\s*([\d.]+)\s*ms,\s*max\s*=\s*([\d.]+)\s*ms,\s*mean\s*=\s*([\d.]+)\s*ms,"
+    r"\s*median\s*=\s*([\d.]+)\s*ms,\s*percentile\((\d+)%\)\s*=\s*([\d.]+)\s*ms")
+TRTEXEC_THROUGHPUT_RE = re.compile(r"Throughput:\s*([\d.]+)\s*qps")
+
+
+def parse_trtexec_output(text):
+    """Pull the latency summary out of trtexec's stdout. Returns dict or raises BackendError."""
+    m = TRTEXEC_LATENCY_RE.search(text)
+    if not m:
+        raise BackendError("could not find a 'Latency:' summary line in trtexec output")
+    pct = int(m.group(5))
+    out = {"min": float(m.group(1)), "max": float(m.group(2)), "mean": float(m.group(3)),
+           "median": float(m.group(4)), f"p{pct}": float(m.group(6))}
+    t = TRTEXEC_THROUGHPUT_RE.search(text)
+    if t:
+        out["throughput_qps"] = float(t.group(1))
+    return out
+
+
+class TrtexecBackend:
+    """Builds (and caches) a TensorRT engine with trtexec, then benchmarks it. Accuracy needs onnxruntime."""
+    name = "trtexec"
+    can_infer = False
+
+    def __init__(self, model_path, input_spec, precision="fp16", cache_dir=None):
+        self.trtexec = shutil.which("trtexec") or ("/usr/src/tensorrt/bin/trtexec" if pathlib.Path("/usr/src/tensorrt/bin/trtexec").exists() else None)
+        if not self.trtexec:
+            raise BackendError("trtexec not found on PATH or at /usr/src/tensorrt/bin/trtexec")
+        self.model_path = pathlib.Path(model_path)
+        self.shape = tuple(input_spec["shape"])
+        self.input_name = input_spec["name"]
+        self.precision = precision
+        self.cache_dir = pathlib.Path(cache_dir or self.model_path.parent / ".edgegate-engines")
+        self.cache_dir.mkdir(parents=True, exist_ok=True)
+        tag = model_sha256(self.model_path)[7:19]
+        self.engine = self.cache_dir / f"{self.model_path.stem}-{tag}-{precision}-{'x'.join(map(str, self.shape))}.engine"
+        self.version = self._version()
+
+    def _version(self):
+        try:
+            out = subprocess.run([self.trtexec, "--help"], capture_output=True, text=True, timeout=30).stdout
+        except (OSError, subprocess.SubprocessError):
+            return None
+        m = re.search(r"TensorRT\s+v?([\d.]+)", out)
+        return m.group(1) if m else None
+
+    def describe(self):
+        return {"backend": self.name, "trtexec": self.trtexec, "version": self.version,
+                "precision": self.precision, "engine": str(self.engine)}
+
+    def _run(self, args, timeout):
+        try:
+            p = subprocess.run([self.trtexec, *args], capture_output=True, text=True, timeout=timeout)
+        except subprocess.TimeoutExpired:
+            raise BackendError(f"trtexec timed out after {timeout}s") from None
+        if p.returncode != 0:
+            tail = "\n".join((p.stdout + p.stderr).splitlines()[-15:])
+            raise BackendError(f"trtexec failed (exit {p.returncode}):\n{tail}")
+        return p.stdout
+
+    def build(self):
+        if self.engine.exists():
+            return False
+        args = [f"--onnx={self.model_path}", f"--saveEngine={self.engine}"]
+        if self.precision == "fp16":
+            args.append("--fp16")
+        elif self.precision == "int8":
+            args.append("--int8")
+        if self.input_name:
+            dims = "x".join(map(str, self.shape))
+            args += [f"--shapes={self.input_name}:{dims}"]
+        self._run(args, timeout=1800)
+        return True
+
+    def bench(self, warmup, iterations, duration_s=0):
+        """trtexec runs for at least `iterations` AND at least `duration_s` seconds (its own default is 3 s).
+        Power sampling needs a window long enough for DVFS to settle, so the Jetson example sets 10 s."""
+        self.build()
+        out = self._run([f"--loadEngine={self.engine}", f"--warmUp={max(warmup * 10, 200)}",
+                         f"--iterations={iterations}", f"--duration={duration_s}", "--percentile=99"],
+                        timeout=max(600, duration_s * 3 + 120))
+        self.last = parse_trtexec_output(out)
+        # trtexec reports a summary, not per-iteration samples; synthesize the fields the gate needs.
+        return self.last
+
+
+class CommandBackend:
+    """Any process as the device under test: an inference engine replaying a recording, a service under a
+    fixed load, a whole pipeline. edgegate runs it for benchmark.duration_s, samples power and RAM meanwhile,
+    tracks the peak RSS of its process tree, then counts the metrics you name in its output files.
+    There is no per-inference latency; throughput comes from the counted events over the wall time."""
+    name = "command"
+    can_infer = False
+
+    def __init__(self, path, args, env, cwd=None):
+        self.path, self.args, self.env, self.cwd = path, list(args or []), dict(env or {}), cwd
+        self.last = None
+
+    def describe(self):
+        return {"backend": self.name, "precision": None, "version": None, "args": self.args}
+
+    def bench(self, warmup, iterations, duration_s=0):
+        raise BackendError("the command backend is driven by measure.run_command, not bench()")
+
+
+def make_backend(cfg):
+    m = cfg["model"]
+    if m["backend"] == "onnxruntime":
+        return OnnxRuntimeBackend(m["path"], m["input"], m.get("precision", "fp32"))
+    if m["backend"] == "command":
+        return CommandBackend(m["path"], m.get("args"), m.get("env"), m.get("cwd"))
+    return TrtexecBackend(m["path"], m["input"], m.get("precision", "fp16"))

edgegate/cli.py

@@ -0,0 +1,142 @@
+"""Command-line interface."""
+import argparse
+import json
+import pathlib
+import sys
+
+from . import __version__, config
+from .backends import BackendError
+from .compare import evaluate, status
+from .dataset import DatasetError
+from .measure import run as measure_run
+from .report import junit, markdown
+from . import matrix as matrix_mod
+
+EXIT_OK, EXIT_ERROR, EXIT_FAIL = 0, 1, 2
+
+
+def _err(msg):
+    print(msg, file=sys.stderr)
+
+
+def _write(path, text):
+    p = pathlib.Path(path)
+    p.parent.mkdir(parents=True, exist_ok=True)
+    p.write_text(text)
+
+
+def _load_json(path, what):
+    try:
+        return json.loads(pathlib.Path(path).read_text())
+    except (OSError, json.JSONDecodeError) as e:
+        raise config.ConfigError(f"{what} {path}: {e}") from None
+
+
+def _emit(a, results, checks, baseline):
+    for c in checks:
+        print(f"{c.status.upper():5} {c.name}: {c.message}")
+    print(f"{status(checks).upper()}: {sum(c.status == 'fail' for c in checks)} failing check(s), "
+          f"{sum(c.status == 'skip' for c in checks)} skipped")
+    if a.junit:
+        _write(a.junit, junit(results, checks)); print(f"wrote {a.junit}")
+    if a.markdown:
+        _write(a.markdown, markdown(results, checks, baseline)); print(f"wrote {a.markdown}")
+    return EXIT_FAIL if status(checks) == "fail" else EXIT_OK
+
+
+def cmd_run(a):
+    cfg = config.load(a.config)
+    out_dir = pathlib.Path(a.out).with_suffix(".d")
+    results = measure_run(cfg, notes=a.note, out_dir=out_dir)
+    _write(a.out, json.dumps(results, indent=1) + "\n")
+    print(f"wrote {a.out}")
+    lat = results["latency_ms"]
+    if results.get("command"):
+        c = results["command"]
+        line = f"exit {c['exit_code']}{' (full window)' if c['timed_out'] else ''} after {c['wall_s']} s"
+        for name, m in results.get("metrics", {}).items():
+            line += f" · {name} {m['count']}" + (f" ({m['per_s']}/s)" if m.get("per_s") is not None else "")
+    else:
+        line = f"latency p50 {lat['p50']} ms, p99 {lat['p99']} ms, mean {lat['mean']} ms"
+    if results.get("power"):
+        line += f" · power {results['power']['total_w']} W"
+    if results.get("accuracy"):
+        line += f" · {results['accuracy']['metric']} {results['accuracy']['value']}"
+    print(line)
+    for n in results.get("notes", []):
+        print(f"note: {n}")
+    if a.check:
+        baseline = _load_json(a.baseline, "baseline") if a.baseline else None
+        checks = evaluate(results, cfg["thresholds"], baseline)
+        # record the verdict in the results file so a later run can tell whether this one is fit to be a baseline
+        results["gate"] = {"status": status(checks), "failed": [c.name for c in checks if c.status == "fail"],
+                           "checked": [c.name for c in checks if c.status != "skip"], "baseline": a.baseline}
+        _write(a.out, json.dumps(results, indent=1) + "\n")
+        return _emit(a, results, checks, baseline)
+    return EXIT_OK
+
+
+def cmd_check(a):
+    cfg = config.load(a.config)
+    results = _load_json(a.results, "results")
+    baseline = _load_json(a.baseline, "baseline") if a.baseline else None
+    return _emit(a, results, evaluate(results, cfg["thresholds"], baseline), baseline)
+
+
+def cmd_matrix(a):
+    m = matrix_mod.load_matrix(a.config)
+    summary = matrix_mod.run_matrix(m, a.out, dry_run=a.dry_run)
+    if a.dry_run:
+        return EXIT_OK
+    md = matrix_mod.markdown(summary)
+    _write(pathlib.Path(a.out) / "summary.md", md)
+    print(md)
+    if a.junit:
+        _write(a.junit, matrix_mod.junit(summary)); print(f"wrote {a.junit}")
+    if a.markdown:
+        _write(a.markdown, md); print(f"wrote {a.markdown}")
+    print(f"wrote {pathlib.Path(a.out) / 'summary.json'}")
+    return EXIT_FAIL if any(c["status"] == "fail" for c in summary["cells"]) else EXIT_OK
+
+
+def main(argv=None):
+    ap = argparse.ArgumentParser(prog="edgegate", description="CI gate for model-plus-firmware builds on edge devices.")
+    ap.add_argument("--version", action="version", version=f"edgegate {__version__}")
+    sub = ap.add_subparsers(dest="cmd", required=True)
+
+    p = sub.add_parser("run", help="measure latency, memory, power and accuracy on this device")
+    p.add_argument("-c", "--config", default="edgegate.yaml")
+    p.add_argument("-o", "--out", default="results/latest.json")
+    p.add_argument("--note", help="free-text note stored with the results (e.g. firmware build id)")
+    p.add_argument("--check", action="store_true", help="also evaluate thresholds (and --baseline if given); exit 2 on failure")
+    p.add_argument("--baseline", help="results JSON to compare against")
+    p.add_argument("--junit", help="write JUnit XML for the checks")
+    p.add_argument("--markdown", help="write a Markdown summary")
+    p.set_defaults(func=cmd_run)
+
+    p = sub.add_parser("check", help="evaluate an existing results file against thresholds and a baseline")
+    p.add_argument("-c", "--config", default="edgegate.yaml")
+    p.add_argument("--results", required=True)
+    p.add_argument("--baseline")
+    p.add_argument("--junit")
+    p.add_argument("--markdown")
+    p.set_defaults(func=cmd_check)
+
+    p = sub.add_parser("matrix", help="run variants x repeats (interleaved, resumable) and tabulate crash rate per cell")
+    p.add_argument("-c", "--config", default="matrix.yaml", help="matrix file (see edgegate.matrix docstring)")
+    p.add_argument("-o", "--out", default="results/matrix", help="results directory; existing r<N>.json files are skipped")
+    p.add_argument("--junit", help="one JUnit testcase per cell")
+    p.add_argument("--markdown", help="copy of the summary table")
+    p.add_argument("--dry-run", action="store_true", help="print the command line of every pending run and exit")
+    p.set_defaults(func=cmd_matrix)
+
+    a = ap.parse_args(argv)
+    try:
+        return a.func(a)
+    except (config.ConfigError, BackendError, DatasetError, RuntimeError, matrix_mod.MatrixError) as e:
+        _err(f"error: {e}")
+        return EXIT_ERROR
+
+
+if __name__ == "__main__":
+    sys.exit(main())

edgegate/compare.py

@@ -0,0 +1,140 @@
+"""Turn a results document, optional baseline, and thresholds into pass/fail checks."""
+from dataclasses import dataclass, asdict
+
+
+@dataclass
+class Check:
+    name: str
+    status: str          # pass | fail | skip
+    value: object = None
+    limit: object = None
+    baseline: object = None
+    message: str = ""
+
+    def as_dict(self):
+        return asdict(self)
+
+
+# A percentage regression on a tiny absolute value is noise (a 15 µs model "regresses" 150% from
+# scheduler jitter). Regressions must also exceed an absolute floor to count. Overridable per metric.
+FLOORS = {"latency_p99_regression_pct": ("latency_regression_floor_ms", 0.5),
+          "memory_regression_pct": ("memory_regression_floor_mb", 16.0),
+          "power_regression_pct": ("power_regression_floor_w", 0.25)}
+
+
+def _pct_change(cur, base):
+    if base in (None, 0) or cur is None:
+        return None
+    return (cur - base) / base * 100.0
+
+
+def baseline_problem(baseline, thresholds):
+    """Why this baseline must not anchor regression checks, or None. A run that failed its own gate, or a
+    command run that exited outside ok_exit_codes, is not a reference: comparing against it produces verdicts
+    like 'power regressed 66%' when the baseline simply died early."""
+    if not baseline or not (thresholds or {}).get("baseline_must_pass", True):
+        return None
+    gate = baseline.get("gate")
+    if gate and gate.get("status") == "fail":
+        return "baseline failed its own gate: " + ", ".join(gate.get("failed", [])[:4])
+    cmd = baseline.get("command")
+    ok = (thresholds or {}).get("ok_exit_codes")
+    if cmd is not None and ok is not None and cmd.get("exit_code") not in ok:
+        return f"baseline process exited {cmd.get('exit_code')}, not in accepted {ok}"
+    return None
+
+
+def evaluate(results, thresholds, baseline=None):
+    checks = []
+    t = thresholds or {}
+    bad_baseline = baseline_problem(baseline, t)
+    if bad_baseline:
+        checks.append(Check("baseline_valid", "fail", None, None, baseline.get("created_at"), message=bad_baseline + "; regression checks skipped"))
+        baseline = None
+    lat = results["latency_ms"].get("p99")
+    mem = results["memory"].get("peak_rss_mb")
+    board_mem = results["memory"].get("board_ram_peak_mb")
+    pw = results["power"]["total_w"] if results.get("power") else None
+    acc = results["accuracy"]["value"] if results.get("accuracy") else None
+    b = baseline or {}
+    b_lat = b.get("latency_ms", {}).get("p99") if b else None
+    b_mem = b.get("memory", {}).get("peak_rss_mb") if b else None
+    b_pw = b["power"]["total_w"] if b and b.get("power") else None
+    b_acc = b["accuracy"]["value"] if b and b.get("accuracy") else None
+
+    def absolute(name, value, limit, higher_is_bad=True, unit=""):
+        if limit is None:
+            return
+        if value is None:
+            checks.append(Check(name, "skip", None, limit, message="not measured")); return
+        ok = value <= limit if higher_is_bad else value >= limit
+        word = "within" if ok else ("exceeds" if higher_is_bad else "below")
+        checks.append(Check(name, "pass" if ok else "fail", value, limit,
+                            message=f"{value}{unit} {word} limit {limit}{unit}"))
+
+    def regression(name, value, base, pct_limit, unit=""):
+        if pct_limit is None:
+            return
+        if value is None or base is None:
+            checks.append(Check(name, "skip", value, pct_limit, base,
+                                message="no baseline" if base is None else "not measured")); return
+        change = _pct_change(value, base)
+        floor_key, floor_default = FLOORS[name]
+        floor = t.get(floor_key, floor_default)
+        over_pct = change > pct_limit
+        over_floor = (value - base) > floor
+        ok = not (over_pct and over_floor)
+        note = f" (under the {floor}{unit} floor)" if over_pct and not over_floor else ""
+        checks.append(Check(name, "pass" if ok else "fail", value, pct_limit, base,
+                            message=f"{value}{unit} vs baseline {base}{unit} ({change:+.1f}%, limit +{pct_limit}%){note}"))
+
+    absolute("latency_p99_ms_max", lat, t.get("latency_p99_ms_max"), unit=" ms")
+    absolute("memory_peak_mb_max", board_mem if board_mem is not None else mem, t.get("memory_peak_mb_max"), unit=" MB")
+    absolute("power_w_max", pw, t.get("power_w_max"), unit=" W")
+    absolute("accuracy_min", acc, t.get("accuracy_min"), higher_is_bad=False)
+
+    regression("latency_p99_regression_pct", lat, b_lat, t.get("latency_p99_regression_pct"), unit=" ms")
+    regression("memory_regression_pct", mem, b_mem, t.get("memory_regression_pct"), unit=" MB")
+    regression("power_regression_pct", pw, b_pw, t.get("power_regression_pct"), unit=" W")
+
+    drop = t.get("accuracy_drop_max")
+    if drop is not None:
+        if acc is None or b_acc is None:
+            checks.append(Check("accuracy_drop_max", "skip", acc, drop, b_acc,
+                                message="no baseline" if b_acc is None else "not measured"))
+        else:
+            d = b_acc - acc
+            checks.append(Check("accuracy_drop_max", "pass" if d <= drop else "fail", acc, drop, b_acc,
+                                message=f"{acc} vs baseline {b_acc} (drop {d:+.4f}, limit {drop})"))
+
+    cmd = results.get("command")
+    ok_codes = t.get("ok_exit_codes")
+    if cmd is not None and ok_codes is not None:
+        rc = cmd["exit_code"]
+        checks.append(Check("exit_code", "pass" if rc in ok_codes else "fail", rc, ok_codes,
+                            message=f"exit {rc}{' after the full window' if cmd.get('timed_out') else ''}, accepted {ok_codes}"))
+    metrics = results.get("metrics") or {}
+    b_metrics = (b.get("metrics") or {}) if b else {}
+    for name, limit in (t.get("metric_min") or {}).items():
+        absolute(f"metric_min:{name}", (metrics.get(name) or {}).get("count"), limit, higher_is_bad=False)
+    for name, pct_limit in (t.get("metric_regression_pct") or {}).items():
+        cur = (metrics.get(name) or {}).get("count")
+        base = (b_metrics.get(name) or {}).get("count")
+        if cur is None or base is None:
+            checks.append(Check(f"metric_regression_pct:{name}", "skip", cur, pct_limit, base,
+                                message="no baseline" if base is None else "not measured")); continue
+        drop_pct = (base - cur) / base * 100.0 if base else 0.0           # positive when the count fell
+        floor = (t.get("metric_regression_floor") or {}).get(name, 0)
+        ok = not (drop_pct > pct_limit and (base - cur) > floor)
+        checks.append(Check(f"metric_regression_pct:{name}", "pass" if ok else "fail", cur, pct_limit, base,
+                            message=f"{cur} vs baseline {base} ({-drop_pct:+.1f}%, limit -{pct_limit}%)"))
+
+    if baseline and baseline.get("model", {}).get("sha256") == results["model"]["sha256"] \
+            and baseline.get("device", {}).get("name") != results["device"]["name"]:
+        checks.append(Check("baseline_device", "skip", results["device"]["name"], None, baseline["device"].get("name"),
+                            message="baseline was recorded on a different device; regression checks compare across devices"))
+    return checks
+
+
+def status(checks):
+    return "fail" if any(c.status == "fail" for c in checks) else "pass"

edgegate/config.py

@@ -0,0 +1,95 @@
+"""Project configuration: a small YAML file next to the model."""
+import pathlib
+import re
+
+import yaml
+
+DEFAULTS = {
+    "benchmark": {"warmup": 20, "iterations": 200, "batch": 1, "duration_s": 0},
+    "power": {"sampler": "auto", "interval_ms": 200},
+    "thresholds": {},
+    "device": {"name": None},
+}
+
+
+class ConfigError(Exception):
+    pass
+
+
+def load(path):
+    path = pathlib.Path(path)
+    try:
+        cfg = yaml.safe_load(path.read_text()) or {}
+    except FileNotFoundError:
+        raise ConfigError(f"config not found: {path}") from None
+    except yaml.YAMLError as e:
+        raise ConfigError(f"config is not valid YAML: {path}: {e}") from None
+
+    if cfg.get("version") != 1:
+        raise ConfigError("config: 'version: 1' is required")
+    model = cfg.get("model")
+    if not isinstance(model, dict) or "path" not in model:
+        raise ConfigError("config: model.path is required")
+    if model.get("backend") not in ("onnxruntime", "trtexec", "command"):
+        raise ConfigError("config: model.backend must be 'onnxruntime', 'trtexec' or 'command'")
+    if model["backend"] == "command":
+        if not isinstance(model.get("args", []), list) or not all(isinstance(x, str) for x in model.get("args", [])):
+            raise ConfigError("config: model.args must be a list of strings")
+        if not isinstance(model.get("env", {}), dict):
+            raise ConfigError("config: model.env must be a mapping")
+        model["input"] = {"name": None, "shape": None, "dtype": None}
+        model.setdefault("precision", None)
+        if cfg.get("dataset") is not None:
+            raise ConfigError("config: dataset is not supported with the command backend (count metrics instead)")
+    else:
+        inp = model.get("input") or {}
+        if "shape" not in inp or not isinstance(inp["shape"], list) or not all(isinstance(d, int) and d > 0 for d in inp["shape"]):
+            raise ConfigError("config: model.input.shape must be a list of positive integers (batch first)")
+        model["input"] = {"name": inp.get("name"), "shape": inp["shape"], "dtype": inp.get("dtype", "float32")}
+        model.setdefault("precision", "fp16")
+
+    metrics = cfg.get("metrics") or []
+    if not isinstance(metrics, list):
+        raise ConfigError("config: metrics must be a list")
+    for i, m in enumerate(metrics):
+        if not isinstance(m, dict) or not all(isinstance(m.get(k), str) and m.get(k) for k in ("name", "file", "pattern")):
+            raise ConfigError(f"config: metrics[{i}] needs string fields name, file and pattern")
+        try:
+            re.compile(m["pattern"])
+        except re.error as e:
+            raise ConfigError(f"config: metrics[{i}].pattern is not a valid regex: {e}") from None
+    cfg["metrics"] = metrics
+
+    ds = cfg.get("dataset")
+    if ds is not None:
+        if ds.get("type") not in ("npz", "image_folder"):
+            raise ConfigError("config: dataset.type must be 'npz' or 'image_folder'")
+        if "path" not in ds:
+            raise ConfigError("config: dataset.path is required")
+        ds.setdefault("limit", None)
+        ds.setdefault("preprocess", {})
+
+    for section, defaults in DEFAULTS.items():
+        merged = dict(defaults)
+        merged.update(cfg.get(section) or {})
+        cfg[section] = merged
+
+    t = cfg["thresholds"]
+    for key in ("latency_p99_ms_max", "memory_peak_mb_max", "power_w_max", "accuracy_min",
+                "latency_p99_regression_pct", "memory_regression_pct", "power_regression_pct", "accuracy_drop_max",
+                "latency_regression_floor_ms", "memory_regression_floor_mb", "power_regression_floor_w"):
+        if key in t and not isinstance(t[key], (int, float)):
+            raise ConfigError(f"config: thresholds.{key} must be a number")
+    if "ok_exit_codes" in t and (not isinstance(t["ok_exit_codes"], list) or not all(isinstance(x, int) for x in t["ok_exit_codes"])):
+        raise ConfigError("config: thresholds.ok_exit_codes must be a list of integers")
+    for key in ("metric_min", "metric_regression_pct", "metric_regression_floor"):
+        if key in t and (not isinstance(t[key], dict) or not all(isinstance(v, (int, float)) for v in t[key].values())):
+            raise ConfigError(f"config: thresholds.{key} must map metric names to numbers")
+
+    cfg["_root"] = path.resolve().parent
+    cfg["model"]["path"] = str((cfg["_root"] / model["path"]).resolve())
+    if model["backend"] == "command":        # relative args and files resolve against the config's directory
+        model["cwd"] = str((cfg["_root"] / model["cwd"]).resolve()) if model.get("cwd") else str(cfg["_root"])
+    if ds is not None:
+        ds["path"] = str((cfg["_root"] / ds["path"]).resolve())
+    return cfg

edgegate/dataset.py

@@ -0,0 +1,60 @@
+"""Labeled samples for accuracy measurement. Two formats: an .npz with x and y, or an image folder with labels.csv."""
+import csv
+import pathlib
+
+import numpy as np
+
+
+class DatasetError(Exception):
+    pass
+
+
+def load(ds):
+    """Yield (input_array, label) pairs. input_array has the model's input shape with batch dimension 1."""
+    kind, path, limit = ds["type"], pathlib.Path(ds["path"]), ds.get("limit")
+    if kind == "npz":
+        yield from _npz(path, limit)
+    else:
+        yield from _image_folder(path, limit, ds.get("preprocess") or {})
+
+
+def _npz(path, limit):
+    try:
+        data = np.load(path)
+    except FileNotFoundError:
+        raise DatasetError(f"dataset not found: {path}") from None
+    if "x" not in data or "y" not in data:
+        raise DatasetError(f"{path}: expected arrays 'x' and 'y'")
+    x, y = data["x"], data["y"]
+    if len(x) != len(y):
+        raise DatasetError(f"{path}: x has {len(x)} samples but y has {len(y)}")
+    n = len(x) if limit is None else min(limit, len(x))
+    for i in range(n):
+        yield x[i][None, ...].astype(np.float32), int(y[i])
+
+
+def _image_folder(path, limit, pre):
+    try:
+        from PIL import Image
+    except ImportError:
+        raise DatasetError("image_folder datasets need Pillow (pip install edgegate[host])") from None
+    labels = path / "labels.csv"
+    if not labels.is_file():
+        raise DatasetError(f"{labels}: missing (one 'path,label' row per image)")
+    size = tuple(pre.get("size", [224, 224]))
+    mean = np.array(pre.get("mean", [0.485, 0.456, 0.406]), dtype=np.float32)
+    std = np.array(pre.get("std", [0.229, 0.224, 0.225]), dtype=np.float32)
+    scale = float(pre.get("scale", 1.0 / 255.0))
+    layout = pre.get("layout", "NCHW")
+    with labels.open(newline="") as f:
+        rows = [r for r in csv.DictReader(f)]
+    if rows and not {"path", "label"} <= set(rows[0]):
+        raise DatasetError(f"{labels}: header must be 'path,label'")
+    if limit is not None:
+        rows = rows[:limit]
+    for r in rows:
+        img = Image.open(path / r["path"]).convert("RGB").resize(size)
+        arr = (np.asarray(img, dtype=np.float32) * scale - mean) / std      # HWC
+        if layout == "NCHW":
+            arr = arr.transpose(2, 0, 1)
+        yield arr[None, ...].astype(np.float32), int(r["label"])