ceteris 0.2.0__py3-none-any.whl

ceteris/__init__.py

@@ -0,0 +1,47 @@
+"""ceteris -- capture benchmark run identity, then gate comparisons between runs.
+
+Named for *ceteris paribus*: all other things being equal. That is precisely
+the claim a benchmark comparison makes, and precisely the claim this tool
+checks.
+
+Library use mirrors the CLI:
+
+    from ceteris import capture, compare, Config
+
+    a = capture(repo="~/codes/hpx")
+    b = capture(repo="~/codes/hpx")
+    report = compare([a, b], vary=["runtime.env.LCI_ATTR_PACKET_SIZE"])
+    assert report.exit_code == 0
+
+`capture()` returning a plain Fingerprint (rather than writing a file) is what
+would let a future `ceteris run -- mpirun ...` wrapper call it before and after
+a job to detect mid-run drift. Nothing in v1 uses that, but the shape is cheap
+to preserve now and expensive to retrofit.
+"""
+
+from .compare import Report, compare
+from .config import Config
+from .model import Field, Fingerprint, State
+
+__version__ = "0.2.0"
+
+def __getattr__(name: str):
+    # capture is resolved lazily so that `from ceteris import compare` does not
+    # pull in the subprocess-running collectors.
+    if name == "capture":
+        from .capture import capture
+
+        return capture
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
+
+
+__all__ = [
+    "capture",
+    "compare",
+    "Config",
+    "Field",
+    "Fingerprint",
+    "Report",
+    "State",
+    "__version__",
+]

ceteris/__main__.py

@@ -0,0 +1,3 @@
+from .cli import main
+
+raise SystemExit(main())

ceteris/adapters/__init__.py

@@ -0,0 +1,315 @@
+"""Harness adapters: read the numbers the benchmark already produces.
+
+`ceteris run -- hyperfine ...` should need no --metric. Each adapter knows
+one harness: how to recognise it on the command line, how to make it write
+machine-readable output if it was not going to, and how to turn that output
+into metrics. The metric values are the harness's own statistics; ceteris
+adds nothing to them.
+
+The harness stays in charge of measurement. ceteris only asks where the
+result went.
+
+Three fixture formats (hyperfine, Google Benchmark, pytest-benchmark) are
+recorded from real runs. The others were reconstructed from documentation
+and are labelled as such in their tests; a real file from any of them is a
+welcome contribution.
+"""
+
+from __future__ import annotations
+
+import glob
+import json
+import os
+import re
+import tempfile
+import time
+from dataclasses import dataclass
+from typing import Any, Callable
+
+from ..model import Field, unknown, value
+
+
+def _slug(text: str) -> str:
+    return re.sub(r"[^A-Za-z0-9._/-]+", "_", text).strip("_")[:60]
+
+
+def _arg_value(argv: list[str], *names: str) -> str | None:
+    for i, a in enumerate(argv):
+        for n in names:
+            if a == n and i + 1 < len(argv):
+                return argv[i + 1]
+            if a.startswith(n + "="):
+                return a[len(n) + 1 :]
+    return None
+
+
+def _num(x: Any) -> Any:
+    try:
+        return float(x)
+    except (TypeError, ValueError):
+        return x
+
+
+@dataclass
+class Plan:
+    """What the adapter decided before the run."""
+
+    adapter: str
+    argv: list[str]                 # possibly augmented command to actually run
+    output: str | None = None       # file to read afterwards, if any
+    added_output: bool = False      # we injected the export flag ourselves
+
+
+class Adapter:
+    name = "base"
+
+    def detect(self, argv: list[str]) -> bool:  # pragma: no cover - interface
+        return False
+
+    def plan(self, argv: list[str], cwd: str) -> Plan:
+        return Plan(self.name, list(argv))
+
+    def collect(self, plan: Plan, stdout: str, cwd: str, started: float) -> dict[str, Field]:  # pragma: no cover
+        return {}
+
+    # -- helpers -------------------------------------------------------------
+    def _read_json(self, path: str | None) -> tuple[Any, str | None]:
+        if not path:
+            return None, "no output file"
+        try:
+            with open(path, encoding="utf-8") as h:
+                return json.load(h), None
+        except OSError as exc:
+            return None, f"cannot read {path}: {exc.strerror or exc}"
+        except ValueError as exc:
+            return None, f"{path} is not valid JSON: {exc}"
+
+    def _failed(self, why: str) -> dict[str, Field]:
+        return {f"{self.name}._adapter": unknown(why, provenance=self.name)}
+
+
+def _basename(argv: list[str]) -> str:
+    return os.path.basename(argv[0]) if argv else ""
+
+
+class Hyperfine(Adapter):
+    name = "hyperfine"
+
+    def detect(self, argv):
+        return _basename(argv) == "hyperfine"
+
+    def plan(self, argv, cwd):
+        out = _arg_value(argv, "--export-json")
+        if out:
+            return Plan(self.name, list(argv), os.path.join(cwd, out))
+        path = tempfile.mktemp(prefix="ceteris-hyperfine-", suffix=".json", dir=cwd)
+        return Plan(self.name, list(argv) + ["--export-json", path], path, added_output=True)
+
+    def collect(self, plan, stdout, cwd, started):
+        data, err = self._read_json(plan.output)
+        if err:
+            return self._failed(err)
+        out: dict[str, Field] = {}
+        prov = f"hyperfine --export-json ({'injected' if plan.added_output else 'given'})"
+        results = data.get("results", [])
+        # Metric names must be stable across configurations, or the noise
+        # floor cannot compare them. The command is the thing that varies
+        # between configurations, so it must not be in the name; a single
+        # command is 'hyperfine.median_s', several are numbered in order.
+        for i, r in enumerate(results, 1):
+            key = "hyperfine" if len(results) == 1 else f"hyperfine.{i}"
+            for stat in ("median", "min"):
+                if stat in r:
+                    out[f"{key}.{stat}_s"] = value(_num(r[stat]), provenance=f"{prov}; command: {r.get('command', '?')}")
+        return out or self._failed("no results in export")
+
+
+class GoogleBenchmark(Adapter):
+    name = "gbench"
+
+    def detect(self, argv):
+        return any(a.startswith("--benchmark_") for a in argv[1:])
+
+    def plan(self, argv, cwd):
+        out = _arg_value(argv, "--benchmark_out")
+        if out:
+            return Plan(self.name, list(argv), os.path.join(cwd, out))
+        path = tempfile.mktemp(prefix="ceteris-gbench-", suffix=".json", dir=cwd)
+        return Plan(self.name, list(argv) + [f"--benchmark_out={path}", "--benchmark_out_format=json"], path, True)
+
+    def collect(self, plan, stdout, cwd, started):
+        data, err = self._read_json(plan.output)
+        if err:
+            return self._failed(err)
+        out: dict[str, Field] = {}
+        for b in data.get("benchmarks", []):
+            if b.get("run_type") == "aggregate":
+                continue
+            unit = b.get("time_unit", "ns")
+            out[f"gbench.{b.get('name', '?')}.real_time_{unit}"] = value(_num(b.get("real_time")), provenance="--benchmark_out json")
+        return out or self._failed("no benchmarks in output")
+
+
+class PytestBenchmark(Adapter):
+    name = "pytest"
+
+    def detect(self, argv):
+        return any(a.startswith("--benchmark") for a in argv[1:]) and (
+            _basename(argv) in ("pytest", "py.test") or "pytest" in argv[:3]
+        )
+
+    def plan(self, argv, cwd):
+        out = _arg_value(argv, "--benchmark-json")
+        if out:
+            return Plan(self.name, list(argv), os.path.join(cwd, out))
+        path = tempfile.mktemp(prefix="ceteris-pytest-", suffix=".json", dir=cwd)
+        return Plan(self.name, list(argv) + [f"--benchmark-json={path}"], path, True)
+
+    def collect(self, plan, stdout, cwd, started):
+        data, err = self._read_json(plan.output)
+        if err:
+            return self._failed(err)
+        out: dict[str, Field] = {}
+        for b in data.get("benchmarks", []):
+            st = b.get("stats", {})
+            out[f"pytest.{b.get('name', '?')}.median_s"] = value(_num(st.get("median")), provenance="--benchmark-json")
+        return out or self._failed("no benchmarks in output")
+
+
+class JMH(Adapter):
+    name = "jmh"
+
+    def detect(self, argv):
+        return "-rf" in argv and _arg_value(argv, "-rf") == "json"
+
+    def plan(self, argv, cwd):
+        out = _arg_value(argv, "-rff") or "jmh-result.json"
+        return Plan(self.name, list(argv), os.path.join(cwd, out))
+
+    def collect(self, plan, stdout, cwd, started):
+        data, err = self._read_json(plan.output)
+        if err:
+            return self._failed(err)
+        out: dict[str, Field] = {}
+        for b in data if isinstance(data, list) else []:
+            name = b.get("benchmark", "?").split(".")[-2:]
+            pm = b.get("primaryMetric", {})
+            unit = _slug(pm.get("scoreUnit", "")).replace("/", "_")
+            out[f"jmh.{'.'.join(name)}.{b.get('mode', 'score')}_{unit}"] = value(_num(pm.get("score")), provenance="jmh -rf json")
+        return out or self._failed("no benchmarks in output")
+
+
+class Criterion(Adapter):
+    name = "criterion"
+
+    def detect(self, argv):
+        return _basename(argv) == "cargo" and "bench" in argv[1:3]
+
+    def collect(self, plan, stdout, cwd, started):
+        out: dict[str, Field] = {}
+        for path in glob.glob(os.path.join(cwd, "target", "criterion", "**", "new", "estimates.json"), recursive=True):
+            if os.path.getmtime(path) < started - 1:
+                continue
+            data, err = self._read_json(path)
+            if err:
+                continue
+            bench = os.path.relpath(os.path.dirname(os.path.dirname(path)), os.path.join(cwd, "target", "criterion"))
+            out[f"criterion.{_slug(bench)}.median_ns"] = value(_num(data.get("median", {}).get("point_estimate")), provenance=path)
+        return out or self._failed("no fresh target/criterion/*/new/estimates.json")
+
+
+class OSU(Adapter):
+    name = "osu"
+
+    def detect(self, argv):
+        return any(os.path.basename(a).startswith("osu_") for a in argv)
+
+    def collect(self, plan, stdout, cwd, started):
+        out: dict[str, Field] = {}
+        unit = "value"
+        for line in stdout.splitlines():
+            m = re.match(r"^#\s*Size\s+(.+?)\s*$", line)
+            if m:
+                unit = re.sub(r"[^A-Za-z0-9]+", "_", m.group(1)).strip("_")
+                continue
+            m = re.match(r"^\s*(\d+)\s+([0-9.]+)\s*$", line)
+            if m:
+                out[f"osu.{unit}.{m.group(1)}B"] = value(float(m.group(2)), provenance="osu stdout table")
+        return out or self._failed("no size/value rows in stdout")
+
+
+class NCCLTests(Adapter):
+    name = "nccl"
+
+    def detect(self, argv):
+        return any(os.path.basename(a).endswith("_perf") for a in argv)
+
+    def collect(self, plan, stdout, cwd, started):
+        out: dict[str, Field] = {}
+        for line in stdout.splitlines():
+            m = re.match(r"^\s*(\d+)\s+\d+\s+\S+\s+\S+\s+\S+\s+([0-9.]+)\s+([0-9.]+)\s+([0-9.]+)", line)
+            if m:
+                out[f"nccl.busbw_GBps.{m.group(1)}B"] = value(float(m.group(4)), provenance="nccl-tests stdout (out-of-place busbw)")
+        m = re.search(r"Avg bus bandwidth\s*:\s*([0-9.]+)", stdout)
+        if m:
+            out["nccl.avg_busbw_GBps"] = value(float(m.group(1)), provenance="nccl-tests stdout")
+        return out or self._failed("no result rows in stdout")
+
+
+class MLPerf(Adapter):
+    name = "mlperf"
+
+    def detect(self, argv):
+        return any("mlperf" in a.lower() or "loadgen" in a.lower() for a in argv)
+
+    def collect(self, plan, stdout, cwd, started):
+        candidates = [p for p in glob.glob(os.path.join(cwd, "**", "mlperf_log_summary.txt"), recursive=True)
+                      if os.path.getmtime(p) >= started - 1]
+        if not candidates:
+            return self._failed("no fresh mlperf_log_summary.txt under the working directory")
+        path = max(candidates, key=os.path.getmtime)
+        try:
+            text = open(path, encoding="utf-8").read()
+        except OSError as exc:
+            return self._failed(str(exc))
+        out: dict[str, Field] = {}
+        for label, key in (("Samples per second", "samples_per_second"), ("QPS w/ loadgen overhead", "qps"),
+                           ("90th percentile latency \\(ns\\)", "p90_latency_ns"), ("Completed samples per second", "completed_samples_per_second")):
+            m = re.search(rf"{label}\s*:\s*([0-9.]+)", text)
+            if m:
+                out[f"mlperf.{key}"] = value(float(m.group(1)), provenance=path)
+        m = re.search(r"Result is\s*:\s*(\w+)", text)
+        if m:
+            out["mlperf.result"] = value(m.group(1), provenance=path)
+        return out or self._failed(f"no recognised fields in {path}")
+
+
+ADAPTERS: list[Adapter] = [Hyperfine(), GoogleBenchmark(), PytestBenchmark(), JMH(), Criterion(), OSU(), NCCLTests(), MLPerf()]
+BY_NAME = {a.name: a for a in ADAPTERS}
+
+
+def detect(argv: list[str]) -> Adapter | None:
+    for adapter in ADAPTERS:
+        if adapter.detect(argv):
+            return adapter
+    return None
+
+
+def ingest(path: str, fmt: str | None = None) -> dict[str, Field]:
+    """Explicit --ingest FILE[:format]: parse an output file without running."""
+    if fmt is None:
+        base = os.path.basename(path).lower()
+        fmt = "mlperf" if "mlperf" in base else "jmh" if "jmh" in base else None
+        if fmt is None:
+            data, err = Adapter()._read_json(path)
+            if isinstance(data, dict) and "results" in data:
+                fmt = "hyperfine"
+            elif isinstance(data, dict) and "benchmarks" in data:
+                fmt = "gbench" if data["benchmarks"] and "real_time" in data["benchmarks"][0] else "pytest"
+            elif isinstance(data, list):
+                fmt = "jmh"
+    adapter = BY_NAME.get(fmt or "")
+    if adapter is None:
+        return {"ingest._adapter": unknown(f"cannot determine format of {path}; pass FILE:format", provenance="--ingest")}
+    plan = Plan(adapter.name, [], path)
+    return adapter.collect(plan, "", os.path.dirname(os.path.abspath(path)), 0)

ceteris/capture.py

@@ -0,0 +1,62 @@
+"""Capture orchestration."""
+
+from __future__ import annotations
+
+import datetime as _dt
+import platform
+
+from . import nodes
+from .collectors import Context, run_all
+from .config import Config
+from .model import Fingerprint
+
+
+def capture(
+    repo: str | None = None,
+    cmake_cache: str | None = None,
+    compiler: str | None = None,
+    cxx_flags: str | None = None,
+    build_type: str | None = None,
+    label: str | None = None,
+    cfg: Config | None = None,
+) -> Fingerprint:
+    """Collect a fingerprint of the current environment.
+
+    Returns a Fingerprint rather than writing a file so the same call is usable
+    from a script, a test, or a future wrapper that captures either side of a
+    job.
+    """
+    from . import __version__
+
+    cfg = cfg or Config.load()
+    ctx = Context(
+        cfg=cfg,
+        repo=repo,
+        cmake_cache=cmake_cache,
+        compiler=compiler,
+        cxx_flags=cxx_flags,
+        build_type=build_type,
+    )
+    fields = run_all(ctx)
+    capture_args = []
+    for flag, val in (
+        ("--repo", repo), ("--cmake-cache", cmake_cache), ("--compiler", compiler),
+        ("--cxx-flags", cxx_flags), ("--build-type", build_type),
+    ):
+        if val is not None:
+            capture_args += [f"{flag}={val}"]
+    meta = {
+        "label": label or platform.uname().node,
+        # captured_at lives in meta and never in the comparable body, so two
+        # captures of an identical environment produce an identical hash.
+        "captured_at": _dt.datetime.now(_dt.timezone.utc).isoformat(
+            timespec="seconds"
+        ),
+        "tool": "ceteris",
+        "tool_version": __version__,
+    }
+    fingerprint = Fingerprint(fields=fields, meta=meta)
+    # Under a multi-node allocation this fans out one capture per node and
+    # merges the node-local fields; on a single host it only normalises the
+    # hostname fields so single- and multi-node records share a schema.
+    return nodes.apply(fingerprint, capture_args)

ceteris/certificate.py

@@ -0,0 +1,98 @@
+"""The certificate: one line that says a comparison was valid, and enough to
+recompute that claim from the records.
+
+    ceteris-certified v1 configs=2 n=5,5 vary=build.cxx_flags waive= verdict=ok noise=15% sha256:<h>
+
+The hash covers the records' content hashes, the declarations, and the
+verdict. `ceteris verify LINE FILES...` re-runs the comparison with the
+declarations parsed out of the line and checks the hash. A record edited
+after the fact, a declaration quietly widened, or a different set of files
+all fail verification.
+
+This is the thing that gets pasted into a README, a PR, or a paper's
+artifact appendix, which is how a check turns into a norm.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import re
+from dataclasses import dataclass
+from typing import Sequence
+
+from .compare import Report
+
+VERSION = 1
+
+
+def _hash(report: Report) -> str:
+    payload = {
+        "records": sorted(g.content_hash for g in report.configs for _ in g.members),
+        "vary": sorted(report.declared),
+        "waive": sorted(report.waived.items()),
+        "strict": report.strict,
+        "exit_code": report.exit_code,
+        "noise": [(v.metric, v.assessed, v.within_noise) for v in report.noise],
+    }
+    return hashlib.sha256(json.dumps(payload, sort_keys=True).encode()).hexdigest()
+
+
+def _noise_summary(report: Report) -> str:
+    assessed = [v for v in report.noise if v.assessed]
+    if not assessed:
+        return "unassessed"
+    return f"{max(v.noise for v in assessed):.0%}"
+
+
+def issue(report: Report) -> str:
+    verdict = {0: "ok", 1: "confounded", 2: "indeterminate", 4: "within-noise"}.get(report.exit_code, "invalid")
+    n = ",".join(str(g.n) for g in report.configs)
+    vary = ",".join(report.declared)
+    waive = ";".join(f"{k}:{v}" for k, v in report.waived.items()).replace(" ", "_")
+    return (
+        f"ceteris-certified v{VERSION} configs={len(report.configs)} n={n} "
+        f"vary={vary} waive={waive} strict={'1' if report.strict else '0'} "
+        f"verdict={verdict} noise={_noise_summary(report)} sha256:{_hash(report)}"
+    )
+
+
+@dataclass
+class Parsed:
+    vary: list[str]
+    waive: dict[str, str]
+    strict: bool
+    verdict: str
+    digest: str
+
+
+def parse(line: str) -> Parsed:
+    m = re.match(
+        r"ceteris-certified v(\d+) configs=\d+ n=[\d,]* vary=(\S*) waive=(\S*) strict=([01]) "
+        r"verdict=(\S+) noise=\S+ sha256:([0-9a-f]{64})\s*$",
+        line.strip(),
+    )
+    if not m:
+        raise ValueError("not a ceteris certificate line")
+    if int(m.group(1)) != VERSION:
+        raise ValueError(f"certificate version {m.group(1)} not supported")
+    waive = {}
+    if m.group(3):
+        for item in m.group(3).split(";"):
+            k, _, v = item.partition(":")
+            waive[k] = v.replace("_", " ")
+    return Parsed(
+        vary=[v for v in m.group(2).split(",") if v],
+        waive=waive,
+        strict=m.group(4) == "1",
+        verdict=m.group(5),
+        digest=m.group(6),
+    )
+
+
+def verify(line: str, report: Report) -> tuple[bool, str]:
+    parsed = parse(line)
+    actual = _hash(report)
+    if actual != parsed.digest:
+        return False, "hash mismatch: the records, declarations or verdict differ from what was certified"
+    return True, f"verified: {parsed.verdict}"