inferencekit-cli 0.1.0__py3-none-any.whl

inferencekit/quality/video_metrics.py

@@ -0,0 +1,304 @@
+"""Decode video clips and score them frame by frame, inside the video pipeline's own Python.
+
+Usage: python video_metrics.py CONFIG.json
+Prints exactly one line ``INFERKIT_RESULT {...}`` on stdout; the full results go to the file named by
+``out`` in the config; logs go to stderr. Standard library at import time; the work needs PyAV and
+numpy; LPIPS also needs torch and the ``lpips`` package (PSNR, SSIM and MAD are numpy only). A video
+pipeline's venv has all of these; the inferencekit harness has none of them, which is why this runs as a worker (like
+engines/vllm_worker.py) and never imports inferencekit.
+
+Config: {"task": "probe" | "compare", "out": path,
+         "files": [{"key", "path"}]                        probe: what decoding the file really gives
+         "pairs": [{"key", "a", "b"}]                      compare: a = the reference side, b = the other
+         "metrics": ["psnr", "ssim", "lpips", "mad"], "device": "auto" | "cpu" | "cuda",
+         "frame_stride": 1, "psnr_cap_db": 100.0, "lpips_net": "alex", "ssim_backend": "auto" | "numpy" | "torch"}
+
+probe, per file (decoded, not read from the container header):
+  width/height of the decoded frames, the decoded frame count, fps from the decoded timestamps
+  ((n - 1) / (t_last - t_first)), audio samples decoded / sample rate; the header's claims beside them.
+compare, per pair, per frame (frames decoded in step, never all held in memory):
+  psnr  10 log10(255^2 / MSE) over the RGB24 pixels, capped at psnr_cap_db (identical frames)
+  ssim  SSIM of the BT.601 luma, 11x11 Gaussian window sigma 1.5, K1 0.01, K2 0.03, valid region
+        (Wang et al. 2004; skimage structural_similarity with gaussian_weights=True,
+        use_sample_covariance=False and data_range=255 computes the same)
+  lpips LPIPS v0.1 (Zhang et al. 2018) with the given backbone, RGB scaled to [-1, 1]
+  mad   mean absolute difference of the RGB24 pixels (0-255 levels)
+Frames are compared in order up to the shorter clip; both counts are reported. Frames of different
+sizes are not compared (the pair gets an error).
+"""
+
+import json
+import math
+import os
+import sys
+import time
+
+# Run by path, this file's directory comes first on sys.path; its neighbours (metrics.py, client.py...)
+# must not shadow a top-level module that torch or PyAV imports.
+_HERE = os.path.dirname(os.path.abspath(__file__))
+sys.path[:] = [p for p in sys.path if os.path.abspath(p or os.curdir) != _HERE]
+
+MARK = "INFERKIT_RESULT "
+METRICS = ("psnr", "ssim", "lpips", "mad")
+LUMA = (0.299, 0.587, 0.114)  # BT.601
+SSIM_SIGMA, SSIM_WIN, SSIM_K1, SSIM_K2 = 1.5, 11, 0.01, 0.03
+
+
+def log(msg):
+    print(msg, file=sys.stderr, flush=True)
+
+
+# --------------------------------------------------------------------------- probe
+
+def probe(path):
+    """What decoding ``path`` gives, beside what its header claims."""
+    import av
+
+    out = {"decoder": "pyav", "video": None, "audio": None, "header": {}}
+    with av.open(path) as c:
+        out["header"]["format"] = c.format.name
+        out["header"]["duration_s"] = c.duration / 1e6 if c.duration else None
+        if c.streams.video:
+            vs = c.streams.video[0]
+            out["header"]["video"] = {"codec": vs.codec_context.name, "width": vs.codec_context.width,
+                                      "height": vs.codec_context.height, "frames": vs.frames or None,
+                                      "fps": float(vs.average_rate) if vs.average_rate else None}
+            sizes, times, n = set(), [], 0
+            for f in c.decode(vs):
+                n += 1
+                sizes.add((f.width, f.height))
+                if f.time is not None:
+                    times.append(float(f.time))
+            times.sort()
+            fps = (len(times) - 1) / (times[-1] - times[0]) if len(times) >= 2 and times[-1] > times[0] else None
+            w, h = (sorted(sizes)[0] if sizes else (None, None))
+            out["video"] = {"frames": n, "width": w, "height": h, "sizes": sorted(sizes), "fps": fps,
+                            "first_s": times[0] if times else None, "last_s": times[-1] if times else None,
+                            "duration_s": n / fps if fps else None}
+    with av.open(path) as c:
+        if c.streams.audio:
+            st = c.streams.audio[0]
+            out["header"]["audio"] = {"codec": st.codec_context.name, "sample_rate": st.codec_context.sample_rate,
+                                      "channels": st.codec_context.channels if hasattr(st.codec_context, "channels")
+                                      else None,
+                                      "duration_s": float(st.duration * st.time_base) if st.duration else None}
+            samples, rate = 0, None
+            for f in c.decode(st):
+                samples += f.samples
+                rate = rate or f.sample_rate
+            out["audio"] = {"samples": samples, "sample_rate": rate,
+                            "duration_s": samples / rate if rate else None}
+    return out
+
+
+# --------------------------------------------------------------------------- compare
+
+class Scorer:
+    """Per-frame metrics. PSNR and MAD run in numpy. SSIM runs in numpy, or in torch (same float64 maths) when
+    torch is loaded anyway for LPIPS and the device is a GPU (``ssim_backend`` "auto"), or when asked ("torch"):
+    at 1920x1088 numpy SSIM takes ~0.7 s a frame on one CPU core, which adds up over a clip set."""
+
+    def __init__(self, metrics, device, psnr_cap_db, lpips_net, ssim_backend="auto"):
+        import numpy as np
+
+        self.np = np
+        self.metrics = [m for m in metrics if m in METRICS]
+        self.cap = float(psnr_cap_db)
+        g = np.arange(SSIM_WIN, dtype=np.float64) - (SSIM_WIN - 1) / 2
+        g = np.exp(-(g * g) / (2 * SSIM_SIGMA ** 2))
+        self.window = g / g.sum()
+        self.torch = self.dev = self.lpips = self.lpips_error = None
+        self.ssim_backend = "numpy"
+        if "lpips" in self.metrics or ("ssim" in self.metrics and ssim_backend == "torch"):
+            try:
+                import torch
+
+                self.torch = torch
+                if device == "auto":
+                    device = "cuda" if torch.cuda.is_available() else "cpu"
+                self.dev = torch.device(device)
+            except ImportError as e:
+                if ssim_backend == "torch":
+                    raise
+                self.lpips_error = f"torch is not installed ({e})"
+        if "ssim" in self.metrics and self.torch is not None and (
+                ssim_backend == "torch" or (ssim_backend == "auto" and self.dev.type == "cuda")):
+            self.ssim_backend = "torch"
+            self.twin = self.torch.tensor(self.window, dtype=self.torch.float64, device=self.dev)
+        if "lpips" in self.metrics and self.torch is not None:
+            try:
+                import lpips
+
+                self.lpips = lpips.LPIPS(net=lpips_net, verbose=False).to(self.dev).eval()
+            except Exception as e:  # no package, or the backbone weights cannot be fetched
+                self.lpips_error = f"{type(e).__name__}: {e}"[:500]
+        if self.lpips_error:
+            log(f"LPIPS unavailable: {self.lpips_error}")
+        self.device = str(self.dev) if self.lpips is not None or self.ssim_backend == "torch" else "cpu"
+
+    def _blur(self, x):
+        """Separable Gaussian filter over the valid region (no padding): a sum of shifted slices."""
+        k, n = self.window, SSIM_WIN
+        h, w = x.shape
+        rows = sum(k[i] * x[i:h - n + 1 + i, :] for i in range(n))
+        return sum(k[j] * rows[:, j:w - n + 1 + j] for j in range(n))
+
+    def _blur_torch(self, x):
+        f = self.torch.nn.functional
+        return f.conv2d(f.conv2d(x, self.twin.view(1, 1, -1, 1)), self.twin.view(1, 1, 1, -1))
+
+    def ssim(self, a, b):
+        if min(a.shape[0], a.shape[1]) < SSIM_WIN:
+            return None
+        c1, c2 = (SSIM_K1 * 255) ** 2, (SSIM_K2 * 255) ** 2
+        if self.ssim_backend == "torch":
+            torch = self.torch
+            luma = torch.tensor(LUMA, dtype=torch.float64, device=self.dev)
+            ya = (torch.from_numpy(a).to(self.dev, torch.float64) @ luma)[None, None]
+            yb = (torch.from_numpy(b).to(self.dev, torch.float64) @ luma)[None, None]
+            blur = self._blur_torch
+        else:
+            np = self.np
+            luma = np.asarray(LUMA, dtype=np.float64)
+            ya, yb = a.astype(np.float64) @ luma, b.astype(np.float64) @ luma
+            blur = self._blur
+        mu_a, mu_b = blur(ya), blur(yb)
+        va = blur(ya * ya) - mu_a * mu_a
+        vb = blur(yb * yb) - mu_b * mu_b
+        cov = blur(ya * yb) - mu_a * mu_b
+        s = ((2 * mu_a * mu_b + c1) * (2 * cov + c2)) / ((mu_a * mu_a + mu_b * mu_b + c1) * (va + vb + c2))
+        return float(s.mean())
+
+    def lpips_of(self, a, b):
+        torch = self.torch
+
+        def t(x):
+            return torch.from_numpy(x).to(self.dev).permute(2, 0, 1)[None].float() / 127.5 - 1.0
+
+        with torch.no_grad():
+            return float(self.lpips(t(a), t(b)).item())
+
+    def frame(self, a, b):
+        np = self.np
+        d = a.astype(np.float64) - b.astype(np.float64)
+        mse = float(np.mean(d * d))
+        row = {"identical": mse == 0.0}
+        if "psnr" in self.metrics:
+            row["psnr"] = self.cap if mse == 0.0 else min(self.cap, 10.0 * math.log10(255.0 ** 2 / mse))
+        if "mad" in self.metrics:
+            row["mad"] = float(np.mean(np.abs(d)))
+        if "ssim" in self.metrics:
+            row["ssim"] = 1.0 if mse == 0.0 else self.ssim(a, b)
+        if "lpips" in self.metrics and self.lpips is not None:
+            row["lpips"] = 0.0 if mse == 0.0 else self.lpips_of(a, b)
+        return row
+
+
+def _decoder(path):
+    import av
+
+    c = av.open(path)
+    s = c.streams.video[0]
+    s.thread_type = "AUTO"
+    return c, c.decode(s)
+
+
+def compare(scorer, pa, pb, stride):
+    out = {"frames_a": 0, "frames_b": 0, "frames_compared": 0, "size_a": None, "size_b": None,
+           "identical_frames": 0, "per_frame": {m: [] for m in scorer.metrics}}
+    ca, fa = _decoder(pa)
+    cb, fb = _decoder(pb)
+    try:
+        i = 0
+        while True:
+            a = next(fa, None)
+            b = next(fb, None)
+            if a is not None:
+                out["frames_a"] += 1
+                out["size_a"] = out["size_a"] or [a.width, a.height]
+            if b is not None:
+                out["frames_b"] += 1
+                out["size_b"] = out["size_b"] or [b.width, b.height]
+            if a is None or b is None:
+                if a is None and b is None:
+                    break
+                continue  # count the rest of the longer clip
+            if (a.width, a.height) != (b.width, b.height):
+                out["error"] = f"frame size differs: {a.width}x{a.height} vs {b.width}x{b.height} (frame {i})"
+                break
+            if i % stride == 0:
+                row = scorer.frame(a.to_ndarray(format="rgb24"), b.to_ndarray(format="rgb24"))
+                out["frames_compared"] += 1
+                out["identical_frames"] += 1 if row.pop("identical") else 0
+                for m, v in row.items():
+                    out["per_frame"][m].append(v)
+            i += 1
+    finally:
+        ca.close()
+        cb.close()
+    out["mean"] = {m: (sum(v for v in vs if v is not None) / len([v for v in vs if v is not None])
+                       if any(v is not None for v in vs) else None) for m, vs in out["per_frame"].items()}
+    out["frame_count_differs"] = out["frames_a"] != out["frames_b"]
+    return out
+
+
+# --------------------------------------------------------------------------- main
+
+def versions():
+    v = {"python": sys.version.split()[0]}
+    for name in ("av", "numpy", "torch", "lpips"):
+        mod = sys.modules.get(name)
+        if mod is not None:
+            v[name] = getattr(mod, "__version__", None)
+    return v
+
+
+def main(argv):
+    if len(argv) != 2:
+        print(__doc__, file=sys.stderr)
+        return 2
+    with open(argv[1]) as f:
+        cfg = json.load(f)
+    t0 = time.time()
+    items, extra = [], {}
+    try:
+        import av  # noqa: F401  (both tasks need PyAV and numpy: a missing one fails the run, not each clip)
+        import numpy  # noqa: F401
+
+        if cfg["task"] == "probe":
+            for it in cfg.get("files", []):
+                try:
+                    items.append({"key": it["key"], "ok": True, **probe(it["path"])})
+                except Exception as e:
+                    items.append({"key": it["key"], "ok": False, "error": f"{type(e).__name__}: {e}"[:500]})
+        elif cfg["task"] == "compare":
+            scorer = Scorer(cfg.get("metrics") or ["psnr", "ssim", "lpips"], cfg.get("device") or "auto",
+                            cfg.get("psnr_cap_db", 100.0), cfg.get("lpips_net") or "alex",
+                            cfg.get("ssim_backend") or "auto")
+            stride = max(1, int(cfg.get("frame_stride") or 1))
+            extra = {"device": scorer.device, "metrics": scorer.metrics, "lpips_error": scorer.lpips_error,
+                     "ssim_backend": scorer.ssim_backend, "frame_stride": stride, "psnr_cap_db": scorer.cap}
+            for it in cfg.get("pairs", []):
+                try:
+                    r = compare(scorer, it["a"], it["b"], stride)
+                    items.append({"key": it["key"], "ok": "error" not in r, **r})
+                except Exception as e:
+                    items.append({"key": it["key"], "ok": False, "error": f"{type(e).__name__}: {e}"[:500]})
+        else:
+            raise ValueError(f"unknown task {cfg['task']!r}")
+    except ImportError as e:  # the venv lacks PyAV / numpy / torch: say which, so the caller can point elsewhere
+        print(MARK + json.dumps({"ok": False, "error": f"missing module: {e}", "versions": versions()}), flush=True)
+        return 1
+    except Exception as e:
+        print(MARK + json.dumps({"ok": False, "error": f"{type(e).__name__}: {e}"[:500], "versions": versions()}),
+              flush=True)
+        return 1
+    with open(cfg["out"], "w") as f:
+        json.dump({"items": items, "versions": versions(), **extra}, f)
+    print(MARK + json.dumps({"ok": True, "n": len(items), "seconds": round(time.time() - t0, 3),
+                             "versions": versions(), **extra}), flush=True)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main(sys.argv))

inferencekit/registry.py

@@ -0,0 +1,206 @@
+"""The operation registry: the single source of truth for what inferencekit can do.
+
+Each operation is a plain Python function decorated with :func:`operation`. The
+decorator records its name (the MCP tool name), its CLI path (``("nsys", "analyze")``
+becomes ``inferencekit nsys analyze``), a one-line summary and an argument list. The
+CLI (``cli.py``) and the MCP server (``mcp_server.py``) are generated from this
+registry, so the two interfaces cannot drift apart.
+
+Handlers are called as ``handler(call, **validated_args)`` and return a JSON-able
+dict (usually an evidence record). ``call`` carries what the evidence layer needs
+to write provenance: which operation ran, with which arguments, and when.
+"""
+
+from __future__ import annotations
+
+import copy
+import datetime as _dt
+from dataclasses import dataclass, field
+from typing import Any, Callable
+
+from inferencekit.canonical import digest
+
+# JSON-schema type names used for arguments.
+_TYPES = {"string", "integer", "number", "boolean", "array"}
+
+
+class OpError(Exception):
+    """A user-facing failure (bad input, missing tool). Printed without a traceback."""
+
+
+@dataclass(frozen=True)
+class Arg:
+    """One argument of an operation.
+
+    ``positional`` only affects the CLI (``inferencekit compare A B`` instead of flags).
+    ``sensitive`` marks command lines and similar free text: such values are used,
+    but only their digest is ever written to an evidence record.
+    """
+
+    name: str
+    type: str
+    help: str
+    required: bool = False
+    default: Any = None
+    choices: tuple | None = None
+    positional: bool = False
+    items: str = "string"  # element type when type == "array"
+    sensitive: bool = False
+
+    def __post_init__(self):
+        if self.type not in _TYPES:
+            raise ValueError(f"arg {self.name}: unknown type {self.type}")
+
+    def json_schema(self) -> dict:
+        if self.type == "array":
+            schema: dict = {"type": "array", "items": {"type": self.items}}
+            if self.choices:
+                schema["items"]["enum"] = list(self.choices)
+        else:
+            schema = {"type": self.type}
+            if self.choices:
+                schema["enum"] = list(self.choices)
+        schema["description"] = self.help
+        if self.default is not None:
+            schema["default"] = self.default
+        return schema
+
+
+@dataclass(frozen=True)
+class Operation:
+    name: str
+    cli: tuple[str, ...]
+    summary: str
+    args: tuple[Arg, ...]
+    handler: Callable[..., dict]
+    passthrough: bool = False  # True: takes a command after "--" as the "argv" argument
+
+    def input_schema(self) -> dict:
+        """JSON Schema for the MCP tool's input (same names and defaults as the CLI)."""
+        return {
+            "type": "object",
+            "properties": {a.name: a.json_schema() for a in self.args},
+            "required": [a.name for a in self.args if a.required],
+            "additionalProperties": False,
+        }
+
+    def validate(self, raw: dict | None) -> dict:
+        """Check types, fill defaults, reject unknown names. Same rules for CLI and MCP."""
+        raw = dict(raw or {})
+        known = {a.name for a in self.args}
+        unknown = sorted(set(raw) - known)
+        if unknown:
+            raise OpError(f"{self.name}: unknown argument(s) {unknown}; expected {sorted(known)}")
+        out = {}
+        for a in self.args:
+            value = raw.get(a.name)
+            if value is None:
+                if a.required:
+                    raise OpError(f"{self.name}: missing required argument '{a.name}'")
+                value = copy.deepcopy(a.default)
+            else:
+                value = _coerce(self.name, a, value)
+            if a.choices and value is not None:
+                for v in value if isinstance(value, list) else [value]:
+                    if v not in a.choices:
+                        raise OpError(f"{self.name}: {a.name}={v!r} is not one of {list(a.choices)}")
+            out[a.name] = value
+        return out
+
+
+def _coerce(op: str, a: Arg, value: Any) -> Any:
+    """Convert CLI strings (and lenient MCP inputs) to the declared type."""
+    try:
+        if a.type == "integer":
+            if isinstance(value, bool) or (isinstance(value, float) and not value.is_integer()):
+                raise ValueError
+            return int(value)
+        if a.type == "number":
+            if isinstance(value, bool):
+                raise ValueError
+            return float(value)
+        if a.type == "boolean":
+            if isinstance(value, bool):
+                return value
+            if str(value).lower() in ("1", "true", "yes", "on"):
+                return True
+            if str(value).lower() in ("0", "false", "no", "off"):
+                return False
+            raise ValueError
+        if a.type == "array":
+            if isinstance(value, (str, int, float)):
+                value = [value]
+            sub = Arg(a.name, a.items, a.help)
+            return [_coerce(op, sub, v) for v in value]
+        return str(value)
+    except (TypeError, ValueError):
+        raise OpError(f"{op}: argument '{a.name}' expects {a.type}, got {value!r}") from None
+
+
+@dataclass
+class Call:
+    """One invocation of an operation; handed to the handler for provenance."""
+
+    op: Operation
+    args: dict
+    started: str = field(default_factory=lambda: utc_now())
+
+    def argv_digest(self) -> str:
+        # Digest of the full invocation (operation path + every argument, including
+        # sensitive ones). Lets you check "was this the same command?" without storing it.
+        return digest(["inferencekit", *self.op.cli, self.args])
+
+    def request(self) -> dict:
+        """The invocation with sensitive values replaced by digests (safe to store)."""
+        safe = {}
+        for a in self.op.args:
+            v = self.args.get(a.name)
+            if a.sensitive and v not in (None, [], ""):
+                v = {"digest": digest(v)}
+            safe[a.name] = v
+        return {"op": self.op.name, "cli": "inferencekit " + " ".join(self.op.cli), "args": safe}
+
+
+def utc_now() -> str:
+    return _dt.datetime.now(_dt.timezone.utc).isoformat(timespec="milliseconds").replace("+00:00", "Z")
+
+
+REGISTRY: dict[str, Operation] = {}
+
+
+def operation(name: str, cli: tuple[str, ...], summary: str, args: list[Arg] | tuple[Arg, ...] = (),
+              passthrough: bool = False):
+    """Decorator: register ``fn`` as operation ``name``."""
+
+    def register(fn: Callable[..., dict]) -> Callable[..., dict]:
+        if name in REGISTRY:
+            raise ValueError(f"operation {name} registered twice")
+        all_args = tuple(args)
+        if passthrough:
+            all_args += (Arg("argv", "array", "Command to run (after '--' on the CLI).", required=True,
+                             sensitive=True),)
+        REGISTRY[name] = Operation(name, tuple(cli), summary, all_args, fn, passthrough)
+        return fn
+
+    return register
+
+
+def load_all() -> dict[str, Operation]:
+    """Import every module in inferencekit.ops so their decorators run."""
+    import inferencekit.ops  # noqa: F401  (the package __init__ imports each op module)
+
+    return REGISTRY
+
+
+def get(name: str) -> Operation:
+    load_all()
+    if name not in REGISTRY:
+        raise OpError(f"unknown operation '{name}'; known: {sorted(REGISTRY)}")
+    return REGISTRY[name]
+
+
+def call(name: str, raw_args: dict | None = None) -> dict:
+    """Validate arguments and run an operation. Both front ends end up here."""
+    op = get(name)
+    args = op.validate(raw_args)
+    return op.handler(Call(op, args), **args)

inferencekit/review/__init__.py

@@ -0,0 +1,6 @@
+"""The review page: every engine's outputs side by side, per input bucket, in one offline HTML file.
+
+``rundir`` reads ``runs/<run-id>/``; ``diff`` compares texts (first divergence, word
+diffs, word errors); ``page`` renders the HTML. The ``review build`` operation
+(``inferencekit.ops.review``) ties them together.
+"""

inferencekit/review/diff.py

@@ -0,0 +1,163 @@
+"""Text comparison for the review page: first divergence, word diffs and word-error alignment.
+
+Three questions, three functions:
+  * ``first_token_divergence``: where do two greedy generations stop agreeing, in tokens?
+  * ``word_diff``: which words of a candidate text differ from the reference text, for
+    highlighting (inserted, deleted and replaced runs of words, whitespace preserved)?
+  * ``align_words``: which words of a hypothesis transcript are substitutions,
+    insertions or deletions against a ground truth, and the word error rate.
+
+Everything is plain Python (``difflib`` plus a small Levenshtein table), so the page can
+be built anywhere without extra dependencies.
+"""
+
+from __future__ import annotations
+
+import difflib
+import re
+import unicodedata
+from dataclasses import dataclass, field
+
+_WORD_SPACE = re.compile(r"(\S+)(\s*)")
+_LEAD_SPACE = re.compile(r"\s*")
+# Exact Levenshtein alignment up to this many table cells; above it, difflib anchors the
+# alignment on long matching runs and the table only fills the gaps between them.
+EXACT_CELLS = 250_000
+
+
+def first_token_divergence(a: list | None, b: list | None) -> int | None:
+    """Index of the first differing token id; None if the sequences are identical (length included)."""
+    a, b = list(a or []), list(b or [])
+    for i, (x, y) in enumerate(zip(a, b)):
+        if x != y:
+            return i
+    return None if len(a) == len(b) else min(len(a), len(b))
+
+
+def words(text: str) -> tuple[str, list[tuple[str, str]]]:
+    """(leading whitespace, [(word, whitespace after it)]): the text, losslessly, as words."""
+    text = text or ""
+    lead = _LEAD_SPACE.match(text).group(0)
+    return lead, _WORD_SPACE.findall(text[len(lead):])
+
+
+@dataclass
+class TextDiff:
+    """A candidate text as segments against a reference text.
+
+    ``segments`` holds ``(tag, reference_text, candidate_text)`` with tag one of
+    ``equal``, ``replace``, ``delete`` (only in the reference) or ``insert`` (only in the
+    candidate). Joining every candidate_text gives back the candidate exactly.
+    """
+
+    lead: str
+    segments: list[tuple[str, str, str]] = field(default_factory=list)
+    first_word: int | None = None  # index of the first differing word in the candidate; None = identical
+    ref_words: int = 0
+    cand_words: int = 0
+
+    @property
+    def identical(self) -> bool:
+        return self.first_word is None
+
+
+def word_diff(reference: str, candidate: str) -> TextDiff:
+    """Word-level diff of ``candidate`` against ``reference`` (whitespace kept for display)."""
+    _, ref = words(reference)
+    lead, cand = words(candidate)
+    sm = difflib.SequenceMatcher(a=[w for w, _ in ref], b=[w for w, _ in cand], autojunk=False)
+    out = TextDiff(lead=lead, ref_words=len(ref), cand_words=len(cand))
+    for tag, i1, i2, j1, j2 in sm.get_opcodes():
+        r = "".join(w + s for w, s in ref[i1:i2])
+        c = "".join(w + s for w, s in cand[j1:j2])
+        if tag != "equal" and out.first_word is None:
+            out.first_word = j1
+        out.segments.append((tag, r, c))
+    return out
+
+
+# --------------------------------------------------------------------------- word errors
+
+def normalize_word(word: str) -> str:
+    """Lowercase, Unicode NFKC, punctuation removed: the usual WER normalisation for one word."""
+    w = unicodedata.normalize("NFKC", word).lower()
+    return "".join(ch for ch in w if not unicodedata.category(ch).startswith("P"))
+
+
+@dataclass
+class Alignment:
+    """Word-level alignment of a hypothesis against a ground truth.
+
+    ``ops`` holds ``(op, reference_word, hypothesis_word)`` with op one of ``ok``, ``sub``,
+    ``del`` (a ground-truth word the hypothesis missed) or ``ins`` (an extra hypothesis
+    word). Words are shown in their original spelling; matching uses ``normalize_word``.
+    """
+
+    ops: list[tuple[str, str | None, str | None]]
+    ref_len: int
+    exact: bool  # True: minimal edit distance; False: anchored on long matches (very long texts)
+
+    def count(self, op: str) -> int:
+        return sum(1 for o, _, _ in self.ops if o == op)
+
+    @property
+    def wer(self) -> float | None:
+        if self.ref_len == 0:
+            return None
+        return (self.count("sub") + self.count("del") + self.count("ins")) / self.ref_len
+
+
+def _tokens(text: str) -> list[tuple[str, str]]:
+    """(normalised, original) for each word that is not pure punctuation."""
+    out = []
+    for w in (text or "").split():
+        n = normalize_word(w)
+        if n:
+            out.append((n, w))
+    return out
+
+
+def _levenshtein(ref: list[tuple[str, str]], hyp: list[tuple[str, str]]) -> list[tuple[str, str | None, str | None]]:
+    """Minimal-edit alignment (substitution, insertion, deletion all cost 1), by dynamic programming."""
+    n, m = len(ref), len(hyp)
+    # dist[i][j]: edits to turn ref[:i] into hyp[:j].
+    dist = [[0] * (m + 1) for _ in range(n + 1)]
+    for i in range(n + 1):
+        dist[i][0] = i
+    for j in range(m + 1):
+        dist[0][j] = j
+    for i in range(1, n + 1):
+        ri, row, prev = ref[i - 1][0], dist[i], dist[i - 1]
+        for j in range(1, m + 1):
+            cost = 0 if ri == hyp[j - 1][0] else 1
+            row[j] = min(prev[j - 1] + cost, prev[j] + 1, row[j - 1] + 1)
+    ops = []
+    i, j = n, m
+    while i > 0 or j > 0:  # walk back from the corner; prefer match/substitution, then deletion
+        if i > 0 and j > 0 and dist[i][j] == dist[i - 1][j - 1] + (0 if ref[i - 1][0] == hyp[j - 1][0] else 1):
+            ops.append(("ok" if ref[i - 1][0] == hyp[j - 1][0] else "sub", ref[i - 1][1], hyp[j - 1][1]))
+            i, j = i - 1, j - 1
+        elif i > 0 and dist[i][j] == dist[i - 1][j] + 1:
+            ops.append(("del", ref[i - 1][1], None))
+            i -= 1
+        else:
+            ops.append(("ins", None, hyp[j - 1][1]))
+            j -= 1
+    ops.reverse()
+    return ops
+
+
+def align_words(reference: str, hypothesis: str, exact_cells: int = EXACT_CELLS) -> Alignment:
+    """Align a hypothesis transcript to the ground truth: substitutions, insertions, deletions."""
+    ref, hyp = _tokens(reference), _tokens(hypothesis)
+    if (len(ref) + 1) * (len(hyp) + 1) <= exact_cells:
+        return Alignment(_levenshtein(ref, hyp), len(ref), exact=True)
+    # Long transcripts (minutes of audio): anchor on matching runs, align each gap exactly.
+    sm = difflib.SequenceMatcher(a=[r for r, _ in ref], b=[h for h, _ in hyp], autojunk=False)
+    ops: list[tuple[str, str | None, str | None]] = []
+    i = j = 0
+    for block in sm.get_matching_blocks():
+        ops += _levenshtein(ref[i:block.a], hyp[j:block.b])
+        ops += [("ok", ref[block.a + k][1], hyp[block.b + k][1]) for k in range(block.size)]
+        i, j = block.a + block.size, block.b + block.size
+    return Alignment(ops, len(ref), exact=False)