mlx-train-perf 0.1.0__py3-none-any.whl

mlx_train_perf/__init__.py

@@ -0,0 +1,12 @@
+from mlx_train_perf.core.loss import (
+    DenseHead,
+    HeadRef,
+    QuantizedHead,
+    Resolution,
+    linear_cross_entropy,
+    resolve_impl,
+    tied_head,
+)
+
+__all__ = ["DenseHead", "HeadRef", "QuantizedHead", "Resolution", "linear_cross_entropy",
+           "resolve_impl", "tied_head"]

mlx_train_perf/_compat.py

@@ -0,0 +1,21 @@
+from mlx_train_perf.errors import UnverifiedMlxError
+
+VERIFIED_MLX_VERSIONS: frozenset[str] = frozenset({"0.31.2"})
+
+
+def _installed_mlx_version() -> str:
+    import mlx.core as mx  # noqa: PLC0415
+
+    return str(mx.__version__)  # type: ignore[attr-defined]  # mlx stubs omit __version__
+
+
+def check_mlx_verified(*, allow_unverified: bool) -> None:
+    """Kernel impl gate: JIT codegen is version-sensitive (register cliffs are sharp)."""
+    ver = _installed_mlx_version()
+    if ver in VERIFIED_MLX_VERSIONS or allow_unverified:
+        return
+    raise UnverifiedMlxError(
+        f"mlx {ver} is not kernel-verified (verified: {sorted(VERIFIED_MLX_VERSIONS)}). "
+        f"Pass allow_unverified_mlx=True to proceed at your own risk (run the parity suite "
+        f"first: pytest --run-metal), or use impl='chunked'."
+    )

mlx_train_perf/adapters/mlx_lm.py

@@ -0,0 +1,214 @@
+"""mlx-lm trainer adapter: `split_model` + `make_loss_fn`.
+
+Verified against the installed mlx-lm==0.31.3 (mlx==0.31.2) source, 2026-07-04:
+
+- **Trainer loss contract** (site-packages/mlx_lm/tuner/trainer.py:86-99, `default_loss`)::
+
+      inputs = batch[:, :-1]
+      targets = batch[:, 1:]
+      logits = model(inputs)
+      steps = mx.arange(1, targets.shape[1] + 1)
+      mask = mx.logical_and(steps >= lengths[:, 0:1], steps <= lengths[:, 1:])
+      ce = nn.losses.cross_entropy(logits, targets) * mask
+      ntoks = mask.sum()
+      ce = ce.astype(mx.float32).sum() / ntoks
+      return ce, ntoks
+
+  `lengths` is a TWO-COLUMN `(offset, length)` int array, one row per batch example --
+  built by `iterate_batches` (trainer.py:170) as `mx.array(list(zip(offsets, lengths)))`.
+  This module reproduces that mask and denominator exactly (mirrors the contract; does
+  not import `trainer.py` itself, to keep `mlx_lm` a lazy import here).
+
+- **Injection point**: `mlx_lm.tuner.trainer.train(..., loss: callable = default_loss)`,
+  invoked as `nn.value_and_grad(model, loss)(model, *batch)` (trainer.py:218-240,
+  240). Per the installed `mlx.nn.value_and_grad` source, its wrapped function calls
+  `model.update(params)` **before** calling `loss(model, ...)` -- so by the time our
+  loss runs, `model` already carries that step's (gradient-updated, on steps after the
+  first) parameters. A loss closure that snapshots a trainable head's weight array once
+  at construction would keep computing against that first snapshot forever -- the
+  weight *attribute* on `model` gets reassigned by `model.update`, but a frozen
+  dataclass field holding the old array object does not follow the reassignment (this
+  was confirmed empirically against the installed mlx before relying on it). `loss_fn`
+  therefore calls `split_model(model)` fresh on every invocation rather than once at
+  `make_loss_fn` construction time.
+
+- **Model structure** (mlx_lm/models/llama.py, mlx_lm/models/qwen3.py -- identical
+  shape in both families): `Model.__call__` computes hidden states via `self.model(x)`
+  (the inner `LlamaModel` / `Qwen3Model`) and projects with `self.lm_head`
+  (`nn.Linear` or `nn.QuantizedLinear`) unless `args.tie_word_embeddings`, in which case
+  it projects via `self.model.embed_tokens.as_linear(hidden)`
+  (`nn.Embedding` / `nn.QuantizedEmbedding`). Both quantized module types expose
+  `weight, scales, biases, group_size, bits, mode` directly; `biases` is `None` and
+  `mode` is not `"affine"` for the non-affine quantization modes (`mxfp4`/`mxfp8`/
+  `nvfp4`) that this project's kernel and chunked paths do not implement.
+
+Only the Llama and Qwen3 model families are supported (matched by
+`type(model).__module__`); anything else raises `AdapterError` naming the support list.
+"""
+from collections.abc import Callable
+from typing import Any, Literal
+
+import mlx.core as mx
+from mlx import nn
+
+from mlx_train_perf.core.loss import (
+    DenseHead,
+    HeadRef,
+    QuantizedHead,
+    linear_cross_entropy,
+    resolve_impl,
+    tied_head,
+)
+from mlx_train_perf.errors import AdapterError, MissingDependencyError
+
+# Keyed by the exact `type(model).__module__` mlx-lm uses for each family (verified
+# against the installed mlx_lm.models.llama / mlx_lm.models.qwen3 above).
+_SUPPORTED_FAMILIES: dict[str, str] = {
+    "llama": "mlx_lm.models.llama",
+    "qwen3": "mlx_lm.models.qwen3",
+}
+
+
+def _require_mlx_lm() -> None:
+    try:
+        import mlx_lm  # noqa: F401, PLC0415
+    except ImportError as exc:
+        raise MissingDependencyError(
+            "mlx-lm is required for mlx_train_perf.adapters.mlx_lm; install the optional "
+            "'mlx-lm' extra (pip install 'mlx-train-perf[mlx-lm]')"
+        ) from exc
+
+
+def _quantized_head(module: nn.Module) -> QuantizedHead:
+    # `nn` resolves to `Any` here (see the pyproject.toml mypy override for `mlx.nn`),
+    # so these attribute reads are unchecked -- correctness is pinned by
+    # test_split_quantized_head_reads_affine_fields / test_split_rejects_non_affine_*.
+    mode = module.mode
+    if mode != "affine":
+        raise AdapterError(
+            f"quantized head/embedding mode {mode!r} is not supported; only "
+            "mode='affine' quantized heads are supported (the only mode this "
+            "project's kernel and chunked paths implement)"
+        )
+    biases = module.biases
+    if biases is None:
+        raise AdapterError(
+            "quantized head/embedding has no biases (mode without biases); "
+            "mlx_train_perf's QuantizedHead requires biases"
+        )
+    return QuantizedHead(
+        w_q=module.weight,
+        scales=module.scales,
+        biases=biases,
+        group_size=module.group_size,
+        bits=module.bits,
+    )
+
+
+def _head_from_module(module: nn.Module) -> HeadRef:
+    """`model.lm_head` case: a dedicated (untied) output projection."""
+    if isinstance(module, nn.QuantizedLinear):
+        return _quantized_head(module)
+    if isinstance(module, nn.Linear):
+        # nn.Module.trainable_parameters is itself untyped in mlx's source.
+        trainable = "weight" in module.trainable_parameters()  # type: ignore[no-untyped-call]
+        return DenseHead(weight=module.weight, trainable=trainable)
+    raise AdapterError(
+        f"unsupported head module type {type(module).__name__!r}; expected "
+        "nn.Linear or nn.QuantizedLinear"
+    )
+
+
+def _tied_head_from_embedding(embedding: nn.Module) -> HeadRef:
+    """`model.model.embed_tokens` case, used as the head when weights are tied."""
+    if isinstance(embedding, nn.QuantizedEmbedding):
+        return _quantized_head(embedding)
+    if isinstance(embedding, nn.Embedding):
+        # nn.Module.trainable_parameters is itself untyped in mlx's source.
+        trainable = "weight" in embedding.trainable_parameters()  # type: ignore[no-untyped-call]
+        return tied_head(embedding.weight, trainable=trainable)
+    raise AdapterError(
+        f"unsupported embedding module type {type(embedding).__name__!r}; expected "
+        "nn.Embedding or nn.QuantizedEmbedding"
+    )
+
+
+def split_model(model: Any) -> tuple[Callable[[mx.array], mx.array], HeadRef]:
+    """Split an mlx-lm `Model` into a hidden-state trunk and a `HeadRef`.
+
+    `model` is typed `Any` deliberately: importing mlx-lm's model classes here (just
+    for a type annotation) would defeat the point of `mlx_lm` being an optional, lazily
+    imported dependency. Support is instead verified structurally, at call time.
+    """
+    _require_mlx_lm()
+    module_name = type(model).__module__
+    if module_name not in _SUPPORTED_FAMILIES.values():
+        supported = ", ".join(sorted(_SUPPORTED_FAMILIES))
+        raise AdapterError(
+            f"unsupported model architecture (module {module_name!r}); "
+            f"mlx_train_perf's mlx-lm adapter supports: {supported}"
+        )
+    inner = model.model  # the inner LlamaModel / Qwen3Model -- yields hidden states
+
+    def trunk(x: mx.array) -> mx.array:
+        return inner(x)  # type: ignore[no-any-return]
+
+    if model.args.tie_word_embeddings:
+        head = _tied_head_from_embedding(inner.embed_tokens)
+    else:
+        head = _head_from_module(model.lm_head)
+    return trunk, head
+
+
+def make_loss_fn(
+    model: Any,
+    *,
+    impl: Literal["auto", "kernel", "chunked", "naive"] = "auto",
+    allow_unverified_mlx: bool = False,
+) -> Callable[[Any, mx.array, mx.array], tuple[mx.array, mx.array]]:
+    """Build a loss callable matching mlx-lm's trainer contract:
+    `loss(model, batch, lengths) -> (loss, ntoks)` (see the module docstring for the
+    exact, version-cited contract this reproduces).
+
+    Fails fast: an unsupported architecture (`AdapterError`) or a missing `mlx-lm`
+    install (`MissingDependencyError`) is raised immediately, before any training step
+    runs, rather than on the first call.
+    """
+    # Fail fast only -- the (trunk, head) pair itself is discarded. `loss_fn` below
+    # re-derives both from the live `model` argument on every call (see the module
+    # docstring for why a construction-time snapshot would go stale).
+    split_model(model)
+
+    # The kernel/chunked/naive decision depends on the hidden dtype and row count `n`,
+    # neither of which is known until a real batch has flowed through the trunk -- so
+    # it is resolved on the first call to `loss_fn` and cached here for every later
+    # step (the decision itself does not vary across steps of the same training run).
+    resolved_impl: Literal["kernel", "chunked", "naive"] | None = None
+
+    def loss_fn(
+        model_arg: Any, batch: mx.array, lengths: mx.array
+    ) -> tuple[mx.array, mx.array]:
+        nonlocal resolved_impl
+        trunk, head = split_model(model_arg)
+        inputs = batch[:, :-1]
+        targets = batch[:, 1:]
+        hidden = trunk(inputs)
+        if resolved_impl is None:
+            n = hidden.shape[0] * hidden.shape[1]
+            resolved_impl = resolve_impl(
+                head=head, dtype=hidden.dtype, n=n, impl=impl,
+                allow_unverified_mlx=allow_unverified_mlx,
+            ).impl
+        # validate_targets=False: mlx_lm's trainer wraps this step in mx.compile, which
+        # forbids the range check's host sync; the trainer feeds in-range tokenizer ids, so
+        # the check is both unusable here and unnecessary. This is what lets `ours` run
+        # through the real compiled train() step, on equal footing with stock.
+        nll = linear_cross_entropy(hidden, head, targets, impl=resolved_impl,
+                                   reduction="none", validate_targets=False)
+        steps = mx.arange(1, targets.shape[1] + 1)
+        mask = (steps >= lengths[:, 0:1]) & (steps <= lengths[:, 1:])
+        ntoks = mask.sum()
+        loss = (nll * mask).astype(mx.float32).sum() / ntoks
+        return loss, ntoks
+
+    return loss_fn

mlx_train_perf/bench/artifacts.py

@@ -0,0 +1,153 @@
+"""Bench artifact conventions: identity-keyed JSON results with resume integrity.
+
+Same shape as the committed `scripts/bench_quant_thresholds.py` house pattern: a run's
+identity captures everything a stale-vs-fresh decision depends on, results are written
+atomically (`.tmp` + rename), and
+freshness requires an EXACT identity match plus `status == "ok"`.
+
+Identity carries a `code_sha` -- the same SHA-256-over-name+bytes pattern
+`scripts/bench_quant_thresholds.py._code_sha` uses -- over a declared list of files the
+measured path actually depends on (the bench worker plus the loss/kernel/guard modules
+it calls into). This is the field `result_is_fresh` actually needs: an installed
+package's `importlib.metadata.version` is hatch-vcs tag-derived and only refreshes on an
+explicit reinstall, NOT on every `uv sync` -- a plain checkout can therefore have edited
+measured-path files while `version("mlx-train-perf")` still reports last release's
+string, which would let a stale artifact be served as fresh (the exact failure this
+harness exists to prevent). `package_version` is kept in the identity too, but purely as
+informational provenance, not as the staleness signal.
+"""
+import hashlib
+import json
+import platform
+import uuid
+from importlib.metadata import PackageNotFoundError, version
+from pathlib import Path
+
+from mlx_train_perf._compat import _installed_mlx_version
+from mlx_train_perf.errors import BenchInputError
+
+SCHEMA_VERSION = 1
+
+_PACKAGE_ROOT = Path(__file__).resolve().parent.parent  # .../src/mlx_train_perf
+
+# Every file whose bytes change what a `loss_layer` (or later `train_step`) condition
+# actually measures -- the bench worker itself, plus everything on the loss-computation
+# path it calls into. Deliberately explicit rather than "every .py in the repo": a
+# docs-only or CLI-only edit must NOT invalidate every bench artifact.
+CODE_SHA_DEPS: tuple[Path, ...] = tuple(
+    _PACKAGE_ROOT / rel for rel in (
+        "bench/worker.py",
+        "core/loss.py",
+        "core/chunked.py",
+        "core/naive.py",
+        "core/kernel/launch.py",
+        "core/kernel/dispatch.py",
+        "core/kernel/source.py",
+        "core/guards.py",
+        "adapters/mlx_lm.py",
+    )
+)
+
+_RESERVED_PARAM_KEYS = ("kind", "session_id")
+
+
+def _code_sha(deps: tuple[Path, ...]) -> str:
+    """SHA-256 over name+bytes of each dep file, in the given order -- any edit to a
+    measured-path file changes this, which is what `result_is_fresh` uses to invalidate
+    a prior artifact (same convention as `scripts/bench_quant_thresholds.py._code_sha`).
+    Recomputed fresh on every call (no caching): identity is meant to reflect the
+    ON-DISK state of these files at the moment it's built, not a snapshot from import
+    time."""
+    h = hashlib.sha256()
+    for p in deps:
+        h.update(p.name.encode())
+        h.update(p.read_bytes())
+    return h.hexdigest()[:16]
+
+
+def _installed_mlx_lm_version() -> str | None:
+    """`mlx-lm` is an optional extra -- absent installs must not fail identity
+    construction, they just record `None` (still a stable, comparable identity value)."""
+    try:
+        return version("mlx-lm")
+    except PackageNotFoundError:
+        return None
+
+
+def new_session_id() -> str:
+    """A fresh identity token for one bench invocation -- distinguishes artifacts from
+    different runs so `report`'s ratio logic can refuse to compare across them."""
+    return uuid.uuid4().hex
+
+
+def run_identity(**kw: object) -> dict[str, object]:
+    """Everything a result's freshness depends on: mlx/mlx-lm versions, machine, macos,
+    a `code_sha` over the measured-path dependency files (see `CODE_SHA_DEPS` -- the
+    staleness signal `result_is_fresh` actually relies on), this package's own installed
+    version (informational only), plus whatever identity-relevant kwargs the caller
+    supplies (condition kind, grid point, dtype, impl, tile/chunk, session_id, ...). Two
+    calls returning equal dicts describe the SAME run in every way that matters for
+    reuse; any difference is what `result_is_fresh` treats as staleness.
+
+    A caller kwarg that happens to reuse one of THIS function's own internal field names
+    (e.g. a condition param literally named `code_sha`) would otherwise silently hijack
+    that field via `{**internal, **kw}` -- caught here by checking for overlap against
+    `internal`'s own keys directly, so the guard can never drift out of sync with the
+    field list (unlike a separately maintained reserved-name constant)."""
+    internal: dict[str, object] = {
+        "schema_version": SCHEMA_VERSION,
+        "mlx_version": _installed_mlx_version(),
+        "mlx_lm_version": _installed_mlx_lm_version(),
+        "machine": platform.machine(),
+        "macos": platform.mac_ver()[0],
+        "code_sha": _code_sha(CODE_SHA_DEPS),
+        "package_version": version("mlx-train-perf"),
+    }
+    collision = set(kw) & internal.keys()
+    if collision:
+        raise BenchInputError(
+            f"condition params must not use reserved identity key(s) {sorted(collision)} "
+            "-- computed internally by run_identity"
+        )
+    return {**internal, **kw}
+
+
+def condition_identity(
+    *, kind: str, session_id: str, params: dict[str, object],
+) -> dict[str, object]:
+    """The single call site both `runner.run_conditions` and `worker.main` use to build
+    one condition's identity. `kind`/`session_id` are supplied separately by the
+    caller -- a `params` dict that happens to reuse either name would otherwise reach
+    `run_identity(kind=kind, session_id=session_id, **params)` and fail with a raw
+    `TypeError: got multiple values for keyword argument`; this raises a clean, named
+    error instead."""
+    for key in _RESERVED_PARAM_KEYS:
+        if key in params:
+            raise BenchInputError(
+                f"condition params must not use the reserved key {key!r} -- it is "
+                "supplied separately by the bench runner/worker"
+            )
+    return run_identity(kind=kind, session_id=session_id, **params)
+
+
+def write_result(path: Path, identity: dict[str, object], status: str, **fields: object) -> None:
+    """Atomic write (tmp + rename) -- an interrupted worker leaves either the PRIOR
+    artifact or nothing at `path`, never a half-written JSON `result_is_fresh` could
+    misparse as fresh."""
+    path.parent.mkdir(parents=True, exist_ok=True)
+    tmp = path.with_suffix(".tmp")
+    tmp.write_text(json.dumps({"identity": identity, "status": status, **fields}, indent=2))
+    tmp.rename(path)
+
+
+def result_is_fresh(path: Path, identity: dict[str, object]) -> bool:
+    """Fresh = parses, `status == "ok"`, identity matches EXACTLY. A missing file, a
+    corrupt one, a recorded error/refusal, or ANY identity field drift (including one the
+    caller didn't think to check) all trigger a recompute rather than a stale reuse."""
+    if not path.exists():
+        return False
+    try:
+        data = json.loads(path.read_text())
+    except (json.JSONDecodeError, OSError):
+        return False
+    return bool(data.get("status") == "ok" and data.get("identity") == identity)

mlx_train_perf/bench/runner.py

@@ -0,0 +1,161 @@
+"""Subprocess-per-condition bench runner: resume-safe, same-session ratio reporting.
+
+Each `Condition` gets its OWN Python process (`python -m mlx_train_perf.bench.worker`) --
+the spike-proven isolation pattern (MLX's lazy allocator otherwise holds buffers across
+runs within one process) -- and its own artifact, written the instant it finishes. A
+condition whose artifact is already fresh is skipped entirely, never spawned; a worker
+that exits nonzero (crashes) gets its failure recorded as a `status="error"` result here,
+on the CALLER's side, so one bad condition never aborts the rest of the sweep.
+"""
+import json
+import subprocess
+import sys
+import tempfile
+from collections import defaultdict
+from dataclasses import dataclass
+from pathlib import Path
+from typing import cast
+
+from mlx_train_perf.bench.artifacts import condition_identity, result_is_fresh, write_result
+
+_STDERR_TAIL_CHARS = 4000  # enough to see the failing assertion/traceback, not a full dump
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class Condition:
+    name: str
+    kind: str
+    params: dict[str, object]
+
+
+def _spawn_worker(config_path: Path) -> subprocess.CompletedProcess[str]:
+    return subprocess.run(
+        [sys.executable, "-m", "mlx_train_perf.bench.worker", "--config", str(config_path)],
+        capture_output=True, text=True, check=False,
+    )
+
+
+def run_conditions(
+    conditions: list[Condition], out_dir: Path, *, session_id: str,
+) -> list[Path]:
+    out_dir.mkdir(parents=True, exist_ok=True)
+    paths: list[Path] = []
+    for condition in conditions:
+        out_path = out_dir / f"{condition.name}.json"
+        ident = condition_identity(
+            kind=condition.kind, session_id=session_id, params=condition.params,
+        )
+        paths.append(out_path)
+        if result_is_fresh(out_path, ident):
+            continue
+        # By definition stale (missing, corrupt, an old error/refusal, or an identity
+        # mismatch) -- remove it BEFORE spawning so `out_path.exists()` after the worker
+        # returns means exactly "THIS worker wrote it". Without this, a worker that fails
+        # silently (exits 0, writes nothing) would leave the stale artifact in place and
+        # the sweep would report someone else's old "ok" result as this run's truth.
+        out_path.unlink(missing_ok=True)
+
+        config = {
+            "kind": condition.kind, "params": condition.params, "session_id": session_id,
+            "out": str(out_path),
+        }
+        # The config lives in the SYSTEM temp dir, deliberately never inside `out_dir` --
+        # an interrupted run must not leave a stray `.json` there for a later glob over
+        # the artifact directory to misread as a result.
+        with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f:
+            json.dump(config, f)
+            config_path = Path(f.name)
+        try:
+            proc = _spawn_worker(config_path)
+            if proc.returncode != 0:
+                # The worker itself wrote nothing (it crashed before/without reaching its
+                # own `write_result`) -- this IS the sweep-level failure envelope, keyed
+                # by the SAME identity the worker would have used, so a later resume run
+                # (after the underlying bug is fixed) still sees it as stale and retries.
+                stderr_tail = (proc.stderr or proc.stdout or "")[-_STDERR_TAIL_CHARS:]
+                write_result(
+                    out_path, ident, "error", error_type="WorkerCrashed",
+                    error_msg=stderr_tail, returncode=proc.returncode,
+                )
+            elif not out_path.exists():
+                # A clean exit that wrote nothing is still a sweep-level failure (e.g. a
+                # worker that swallowed its own crash) -- recorded the same way, so a
+                # later resume still sees this condition as stale and retries it.
+                write_result(
+                    out_path, ident, "error", error_type="WorkerExitedWithoutArtifact",
+                    error_msg="worker exited 0 without writing an artifact", returncode=0,
+                )
+        finally:
+            config_path.unlink(missing_ok=True)
+    return paths
+
+
+def _identity_of(entry: dict[str, object]) -> dict[str, object]:
+    return cast(dict[str, object], entry["identity"])
+
+
+def _group_key(identity: dict[str, object]) -> tuple[tuple[str, object], ...]:
+    """The "same experimental grid point" key: every identity field EXCEPT `impl` (the
+    dimension being compared) and `session_id` (checked separately, as a gate on whether
+    a ratio may be emitted at all -- see `report`)."""
+    return tuple(sorted((k, v) for k, v in identity.items() if k not in ("impl", "session_id")))
+
+
+def _ratio_label_and_value(
+    impl_a: object, impl_b: object, wall_a: object, wall_b: object,
+) -> tuple[str, float] | None:
+    """Direction is by MEASURED speed (`f"{slower_impl}/{faster_impl}"`, value = slower
+    `wall_s` / faster `wall_s`) -- never by the impls' alphabetical name order, which is
+    an accident of spelling and not a proxy for which one actually ran slower. `None`
+    when either wall time is missing or non-numeric (can't rank), or one is zero (can't
+    divide by it)."""
+    if not (isinstance(wall_a, int | float) and isinstance(wall_b, int | float)):
+        return None
+    if not wall_a or not wall_b:
+        return None
+    if wall_a >= wall_b:
+        slower_impl, slower_wall, faster_impl, faster_wall = impl_a, wall_a, impl_b, wall_b
+    else:
+        slower_impl, slower_wall, faster_impl, faster_wall = impl_b, wall_b, impl_a, wall_a
+    return f"{slower_impl}/{faster_impl}", slower_wall / faster_wall
+
+
+def report(paths: list[Path]) -> dict[str, object]:
+    """Aggregate a set of artifacts into pairwise `impl` ratios at each shared grid
+    point (direction/magnitude: see `_ratio_label_and_value`). A pair that shares a grid
+    point but NOT a `session_id` is a cross-machine/cross-run comparison -- refused, and
+    named in `cross_session_excluded`, rather than silently blended into `ratios`."""
+    entries: list[dict[str, object]] = []
+    for p in paths:
+        try:
+            data = json.loads(p.read_text())
+        except (json.JSONDecodeError, OSError):
+            continue
+        if data.get("status") != "ok":
+            continue
+        entries.append(data)
+
+    groups: dict[tuple[tuple[str, object], ...], list[dict[str, object]]] = defaultdict(list)
+    for data in entries:
+        groups[_group_key(_identity_of(data))].append(data)
+
+    ratios: dict[str, float] = {}
+    cross_session_excluded: list[dict[str, object]] = []
+    for members in groups.values():
+        for i, a in enumerate(members):
+            for b in members[i + 1:]:
+                ident_a, ident_b = _identity_of(a), _identity_of(b)
+                impl_a, impl_b = ident_a.get("impl"), ident_b.get("impl")
+                if impl_a == impl_b:
+                    continue  # a genuine re-run of the same impl, not a comparison
+                if ident_a.get("session_id") != ident_b.get("session_id"):
+                    cross_session_excluded.append({
+                        "impl_a": impl_a, "session_a": ident_a.get("session_id"),
+                        "impl_b": impl_b, "session_b": ident_b.get("session_id"),
+                    })
+                    continue
+                result = _ratio_label_and_value(impl_a, impl_b, a.get("wall_s"), b.get("wall_s"))
+                if result is not None:
+                    label, value = result
+                    ratios[label] = value
+    return {"ratios": ratios, "cross_session_excluded": cross_session_excluded}