cozy-eval 0.1.0__py3-none-any.whl

cozy_eval/__init__.py

@@ -0,0 +1,106 @@
+"""cozy-eval — quality benchmarking for generative image and video models,
+with the validity rules built into the API.
+
+The measurements here are ordinary. The opinion is not:
+
+* **Two lanes, enforced.** Reference metrics (PSNR/SSIM/LPIPS/VMAF) are valid only
+  when the compared change leaves the sampling trajectory intact. Ask for them on
+  a quantization or compile comparison and the library raises.
+* **Protocol stamping.** Every result carries resolution, frame count, steps,
+  seeds, prompt set, execution lane and hardware. A number without those is not
+  a result.
+* **Population semantics.** Trajectory-perturbing comparisons are gated over
+  n>=8 paired prompts, because a single prompt measures the take, not the recipe.
+* **Thresholds with provenance.** Every budget names the known-good and
+  known-bad populations that fixed it, and its separation margin.
+* **Null controls.** A budget only holds where a zero-change arm (same weights,
+  different seeds) sits inside it. ``measure_null_control`` measures that per
+  family; budgets its control trips are disregarded, and a run with disregarded
+  budgets cannot rise above INDETERMINATE.
+* **No signal is not a failure.** An arm of constant frames returns DEGENERATE,
+  never a confident FAIL with an imaging index of 0.0.
+
+Quick start::
+
+    from cozy_eval import (
+        ChangeKind, Protocol, score_pairs, run_population_gate,
+    )
+
+    proto = Protocol(
+        family="ltx-2.3-distilled",
+        reference_arm="bf16 compiled", candidate_arm="w8a8-pcs compiled",
+        changes=(ChangeKind.WEIGHT_QUANTIZATION, ChangeKind.ACTIVATION_QUANTIZATION),
+        width=1280, height=704, frames=121, steps=8,
+        seeds=(8,), prompts=("forge", "chef", "market", "portrait",
+                             "fabric", "street", "water", "forest"),
+        execution_lane="compiled", hardware="NVIDIA H100 80GB HBM3",
+    )
+    pairs = score_pairs(reference_clips, candidate_clips)
+    print(run_population_gate(pairs, proto).summary())
+"""
+
+from .backends.signal import ClipScore
+from .backends.signal import score as score_clip
+from .benchmarks import (
+    MIN_PROMPTS,
+    REFERENCE_BUDGETS,
+    VIDEO_BUDGETS,
+    BenchmarkResult,
+    Budget,
+    distributional,
+    imaging,
+    temporal,
+)
+from .control import NullControl, measure_null_control
+from .frames import iter_frames, probe
+from .gate import (
+    GateReport,
+    run_population_gate,
+    run_reference_gate,
+    score_pairs,
+)
+from .image import IMAGE_BUDGETS, run_image_population_gate, score_image
+from .protocol import (
+    ChangeKind,
+    Lane,
+    Protocol,
+    ProtocolError,
+    TrajectoryPerturbingError,
+    Verdict,
+    classify_change,
+    require_reference_lane,
+)
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "IMAGE_BUDGETS",
+    "MIN_PROMPTS",
+    "REFERENCE_BUDGETS",
+    "VIDEO_BUDGETS",
+    "BenchmarkResult",
+    "Budget",
+    "ChangeKind",
+    "ClipScore",
+    "GateReport",
+    "Lane",
+    "NullControl",
+    "Protocol",
+    "ProtocolError",
+    "TrajectoryPerturbingError",
+    "Verdict",
+    "classify_change",
+    "distributional",
+    "imaging",
+    "iter_frames",
+    "measure_null_control",
+    "probe",
+    "require_reference_lane",
+    "run_image_population_gate",
+    "run_population_gate",
+    "run_reference_gate",
+    "score_clip",
+    "score_image",
+    "score_pairs",
+    "temporal",
+]

cozy_eval/__main__.py

@@ -0,0 +1,3 @@
+from .cli import main
+
+raise SystemExit(main())

cozy_eval/_stats.py

@@ -0,0 +1,99 @@
+"""The two statistics the population gate needs, with no SciPy in the core.
+
+A paired t-test and Holm-Bonferroni. SciPy would do both, but pulling a 40 MB
+compiled dependency into the base install for one continued fraction is not a
+trade worth making — the base install has to be importable inside a conversion
+worker image that already carries torch and diffusers.
+"""
+
+from __future__ import annotations
+
+import math
+
+
+def _betacf(a: float, b: float, x: float) -> float:
+    """Continued fraction for the incomplete beta function (modified Lentz)."""
+    tiny = 1e-300
+    qab, qap, qam = a + b, a + 1.0, a - 1.0
+    c = 1.0
+    d = 1.0 - qab * x / qap
+    if abs(d) < tiny:
+        d = tiny
+    d = 1.0 / d
+    h = d
+    for m in range(1, 300):
+        m2 = 2 * m
+        aa = m * (b - m) * x / ((qam + m2) * (a + m2))
+        d = 1.0 + aa * d
+        c = 1.0 + aa / c
+        if abs(d) < tiny:
+            d = tiny
+        if abs(c) < tiny:
+            c = tiny
+        d = 1.0 / d
+        h *= d * c
+        aa = -(a + m) * (qab + m) * x / ((a + m2) * (qap + m2))
+        d = 1.0 + aa * d
+        c = 1.0 + aa / c
+        if abs(d) < tiny:
+            d = tiny
+        if abs(c) < tiny:
+            c = tiny
+        d = 1.0 / d
+        delta = d * c
+        h *= delta
+        if abs(delta - 1.0) < 3e-16:
+            break
+    return h
+
+
+def betainc(a: float, b: float, x: float) -> float:
+    """Regularized incomplete beta I_x(a, b)."""
+    if x <= 0.0:
+        return 0.0
+    if x >= 1.0:
+        return 1.0
+    lbeta = (math.lgamma(a + b) - math.lgamma(a) - math.lgamma(b)
+             + a * math.log(x) + b * math.log1p(-x))
+    front = math.exp(lbeta)
+    if x < (a + 1.0) / (a + b + 2.0):
+        return front * _betacf(a, b, x) / a
+    return 1.0 - math.exp(
+        math.lgamma(a + b) - math.lgamma(a) - math.lgamma(b)
+        + b * math.log1p(-x) + a * math.log(x)
+    ) * _betacf(b, a, 1.0 - x) / b
+
+
+def t_two_sided_p(t: float, df: float) -> float:
+    """Two-sided p-value of a t statistic."""
+    if df <= 0:
+        return 1.0
+    if not math.isfinite(t):
+        return 0.0
+    return float(betainc(df / 2.0, 0.5, df / (df + t * t)))
+
+
+def paired_t(diffs) -> tuple[float, float]:
+    """(t, two-sided p) for a paired sample of differences."""
+    n = len(diffs)
+    if n < 2:
+        return 0.0, 1.0
+    mean = sum(diffs) / n
+    var = sum((d - mean) ** 2 for d in diffs) / (n - 1)
+    se = math.sqrt(var / n)
+    if se <= 0.0:
+        return (0.0, 1.0) if mean == 0.0 else (math.inf, 0.0)
+    t = mean / se
+    return t, t_two_sided_p(t, n - 1)
+
+
+def holm(pvalues: dict[str, float]) -> dict[str, float]:
+    """Holm-Bonferroni step-down adjusted p-values."""
+    order = sorted(pvalues, key=lambda k: pvalues[k])
+    k = len(order)
+    out: dict[str, float] = {}
+    running = 0.0
+    for i, name in enumerate(order):
+        running = max(running, min(1.0, pvalues[name] * (k - i)))
+        out[name] = running
+    return out

cozy_eval/backends/__init__.py

@@ -0,0 +1,24 @@
+"""Measurement backends.
+
+The library's opinion is about *protocol*, not about who computes the numbers.
+Backends are therefore pluggable and, wherever a maintained package already owns
+a metric, we call it rather than reimplement it:
+
+* :mod:`.signal` — the always-available core. Elementary signal statistics
+  (Laplacian variance, spectral high-frequency ratio, histogram entropy, first-
+  and second-order temporal differences). These are primitives, not somebody's
+  library; the core keeps them so the base install is numpy + ffmpeg.
+* :mod:`.reference` — reference metrics for the same-trajectory lane. Uses
+  ``libvmaf`` through ffmpeg when present and scikit-image's SSIM when present,
+  falling back to the built-in windowed SSIM.
+* :mod:`.perceptual` — ``pip install cozy-eval[perceptual]``. Wraps
+  ``pyiqa`` (MUSIQ, CLIP-IQA, MANIQA, NIQE, BRISQUE) and, for video, DOVER-class
+  scorers. Never reimplemented here.
+* :mod:`.distributional` — ``pip install cozy-eval[distributional]``.
+  Deep-feature Fréchet distances (I3D-FVD, V-JEPA/JEDi). Opt-in and explicitly
+  NOT validated at the sample sizes this library's gate runs at; see GATE.md.
+"""
+
+from . import reference, signal
+
+__all__ = ["reference", "signal"]

cozy_eval/backends/distributional.py

@@ -0,0 +1,95 @@
+"""Optional deep-feature Fréchet distances — ``pip install cozy-eval[distributional]``.
+
+**Read this before using it.** FVD-class distances are population statistics with
+a well-known small-sample bias: published FVD is computed over thousands of
+clips, and at the n=8-16 a per-artifact quality gate can afford, the estimator's
+bias dominates the effect you are trying to measure. This library's *gate* does
+not use a deep-feature Fréchet distance for that reason — its distributional
+benchmark is a paired test over the no-reference score distributions, which is
+valid at n=8 because the prompt set is identical between arms.
+
+This module exists so a lane that CAN afford n in the hundreds gets the stronger
+statistic without leaving the protocol machinery behind. It reports the sample
+size next to every number and refuses below ``min_samples``.
+"""
+
+from __future__ import annotations
+
+import numpy as np
+
+
+class SampleSizeError(RuntimeError):
+    """Raised when a Fréchet-class statistic is requested at an unusable n."""
+
+
+def frechet(feat_a: np.ndarray, feat_b: np.ndarray, *, min_samples: int = 64) -> float:
+    """Fréchet distance between two Gaussian-fitted feature populations.
+
+    Full covariance; requires ``n >= min_samples`` on both sides. Uses SciPy's
+    matrix square root when available and an eigendecomposition otherwise.
+    """
+    a, b = np.asarray(feat_a, np.float64), np.asarray(feat_b, np.float64)
+    if min(len(a), len(b)) < min_samples:
+        raise SampleSizeError(
+            f"Frechet distance over {min(len(a), len(b))} samples is dominated by "
+            f"estimator bias (need >= {min_samples}). Use the paired population "
+            "benchmark instead — see GATE.md."
+        )
+    mu_a, mu_b = a.mean(0), b.mean(0)
+    ca = np.cov(a, rowvar=False)
+    cb = np.cov(b, rowvar=False)
+    diff = mu_a - mu_b
+    try:
+        from scipy.linalg import sqrtm
+        covmean = sqrtm(ca @ cb)
+        if np.iscomplexobj(covmean):
+            covmean = covmean.real
+    except ImportError:                                        # pragma: no cover
+        w, v = np.linalg.eigh(ca)
+        root = v @ np.diag(np.sqrt(np.clip(w, 0, None))) @ v.T
+        w2, v2 = np.linalg.eigh(root @ cb @ root)
+        covmean = v2 @ np.diag(np.sqrt(np.clip(w2, 0, None))) @ v2.T
+    return float(diff @ diff + np.trace(ca + cb - 2.0 * covmean))
+
+
+def fvd(real_dir, generated_dir, *, model: str = "i3d", device: str = "cuda") -> float:
+    """Content-debiased FVD via ``cd-fvd`` (MIT). Not reimplemented here.
+
+    Vendoring yet another I3D checkpoint loader is how FVD numbers stopped being
+    comparable between papers in the first place.
+
+    Note (verified 2026-07-26): cd-fvd's PyPI release points ``model="videomae"``
+    at a dead weights URL; the HuggingFace fix landed on ``main`` after the last
+    upload. Use ``model="i3d"`` from PyPI, or install cd-fvd from git for
+    VideoMAEv2 features.
+    """
+    try:
+        from cdfvd import fvd as _fvd
+    except ImportError as exc:                                 # pragma: no cover
+        raise RuntimeError(
+            "needs cd-fvd: pip install 'cozy-eval[distributional]'"
+        ) from exc
+    ev = _fvd.cdfvd(model, device=device)
+    ev.compute_real_stats(ev.load_videos(str(real_dir), data_type="video_folder"))
+    ev.compute_fake_stats(ev.load_videos(str(generated_dir), data_type="video_folder"))
+    return float(ev.compute_fvd_from_stats())
+
+
+def fvmd(real_dir, generated_dir, log_dir) -> float:           # pragma: no cover
+    """Fréchet Video Motion Distance via ``fvmd`` (Apache-2.0). Not reimplemented.
+
+    Motion-statistics distance over PIPs++ keypoint tracks. Reported to correlate
+    with human judgement better than FVD or VBench on motion artifacts, which is
+    the failure mode this library's temporal benchmark also targets — run both if
+    you can afford the keypoint tracker.
+
+    NOT declared as a dependency: fvmd 1.0.0 pins ``scipy==1.10.1``, which is
+    unsatisfiable alongside anything modern. Install it into its own environment.
+    """
+    try:
+        from fvmd import fvmd as _fvmd
+    except ImportError as exc:
+        raise RuntimeError(
+            "needs fvmd: pip install 'cozy-eval[distributional]'"
+        ) from exc
+    return float(_fvmd(str(log_dir), str(generated_dir), str(real_dir)))

cozy_eval/backends/perceptual.py

@@ -0,0 +1,76 @@
+"""Optional learned no-reference backends — ``pip install cozy-eval[perceptual]``.
+
+We do not implement MUSIQ, CLIP-IQA, MANIQA, NIQE or BRISQUE; ``pyiqa`` already
+maintains all of them behind one factory, so this module is a ~40-line adapter
+that (a) makes them frame sources like every other backend, (b) aggregates a
+per-frame score into a clip score, and (c) records which model and weights were
+used inside the protocol stamp — an unstamped MUSIQ number is as unusable as an
+unstamped LPIPS number.
+
+**Licence, verified 2026-07-26.** ``pyiqa`` relicensed Apache-2.0 ->
+PolyForm-Noncommercial-1.0.0 at **0.1.16** (released 2026-07-08). The extra pins
+``<0.1.16`` to stay on the Apache-2.0 line; raising that ceiling is a licensing
+decision, not a version bump. DOVER and FAST-VQA are deliberately absent: both
+are S-Lab-1.0 (non-commercial) despite ``setup.py`` files that still declare
+MIT/Apache.
+
+These models download weights on first use and want a GPU to be quick, which is
+why they are an extra rather than the core. The gate's calibrated thresholds are
+on the signal backend; a perceptual score joins the report as an additional,
+uncalibrated observation until somebody banks a clean/degraded population for it.
+"""
+
+from __future__ import annotations
+
+import functools
+
+import numpy as np
+
+from ..frames import iter_frames
+
+#: Models known to work as a per-frame no-reference score through pyiqa.
+FRAME_MODELS = ("musiq", "clipiqa", "maniqa", "niqe", "brisque", "topiq_nr")
+
+
+def available() -> bool:
+    try:
+        import pyiqa  # noqa: F401
+        return True
+    except ImportError:
+        return False
+
+
+@functools.cache
+def _metric(name: str, device: str):
+    import pyiqa
+    return pyiqa.create_metric(name, device=device)
+
+
+def score_frames(source, *, model: str = "musiq", device: str = "cpu",
+                 stride: int = 8) -> dict[str, float]:
+    """Aggregate a pyiqa per-frame no-reference score over a clip.
+
+    ``stride`` subsamples frames: these models are two orders of magnitude more
+    expensive than the signal backend and per-frame scores are highly correlated
+    between neighbours.
+    """
+    if not available():                                        # pragma: no cover
+        raise RuntimeError(
+            "perceptual backend needs pyiqa: pip install 'cozy-eval[perceptual]'"
+        )
+    import torch
+    m = _metric(model, device)
+    vals: list[float] = []
+    for i, f in enumerate(iter_frames(source)):
+        if i % stride:
+            continue
+        t = torch.from_numpy(f.transpose(2, 0, 1)[None]).float().to(device)
+        with torch.no_grad():
+            vals.append(float(m(t).item()))
+    if not vals:
+        raise ValueError("no frames scored")
+    return {
+        f"{model}_mean": float(np.mean(vals)),
+        f"{model}_p10": float(np.percentile(vals, 10)),
+        f"{model}_frames": float(len(vals)),
+    }

cozy_eval/backends/reference.py

@@ -0,0 +1,171 @@
+"""Reference metrics for the same-trajectory lane.
+
+Composed, not reimplemented, wherever a maintained implementation exists:
+
+* **VMAF** via ``ffmpeg_quality_metrics`` (MIT, actively maintained) when it is
+  installed, falling back to a direct ``ffmpeg -lavfi libvmaf`` call. Netflix's
+  model is the right instrument for post-decode changes and there is no reason to
+  have an opinion of our own about it.
+* **ColorVideoVDP** via ``cvvdp`` (MIT) when installed — a colour- and HDR-aware
+  spatiotemporal full-reference video metric, strictly a better instrument than
+  VMAF for this lane. Optional because it is heavier.
+* **SSIM** via ``skimage.metrics.structural_similarity`` when scikit-image is
+  installed; otherwise a built-in 8x8 block-windowed SSIM (global SSIM is far too
+  insensitive on video — it stays at 0.998 through a visibly blocky re-encode).
+* **LPIPS** via the ``lpips`` package when installed.
+* **PSNR** is one line of numpy and has no library worth depending on.
+
+Nothing in this module may be called without passing the
+:func:`~cozy_eval.protocol.require_reference_lane` guard first; the gate
+does that for you.
+"""
+
+from __future__ import annotations
+
+import functools
+import json
+import shutil
+import subprocess
+import tempfile
+from pathlib import Path
+
+import numpy as np
+
+from ..frames import LUMA, iter_frames
+
+
+def psnr(a: np.ndarray, b: np.ndarray) -> float:
+    mse = float(np.mean((a - b) ** 2))
+    return 99.0 if mse < 1e-12 else float(10.0 * np.log10(1.0 / mse))
+
+
+def _ssim_blocks(a: np.ndarray, b: np.ndarray, block: int = 8) -> float:
+    la, lb = a @ LUMA, b @ LUMA
+    h, w = la.shape
+    h, w = h - h % block, w - w % block
+
+    def tiles(x):
+        x = x[:h, :w].reshape(h // block, block, w // block, block)
+        return x.transpose(0, 2, 1, 3).reshape(-1, block * block)
+
+    ta, tb = tiles(la), tiles(lb)
+    m1, m2 = ta.mean(1), tb.mean(1)
+    v1, v2 = ta.var(1), tb.var(1)
+    cov = ((ta - m1[:, None]) * (tb - m2[:, None])).mean(1)
+    c1, c2 = 0.01 ** 2, 0.03 ** 2
+    s = ((2 * m1 * m2 + c1) * (2 * cov + c2)) / ((m1 ** 2 + m2 ** 2 + c1) * (v1 + v2 + c2))
+    return float(s.mean())
+
+
+@functools.cache
+def _have_skimage() -> bool:
+    try:
+        import skimage.metrics  # noqa: F401
+        return True
+    except ImportError:
+        return False
+
+
+def ssim(a: np.ndarray, b: np.ndarray) -> float:
+    if _have_skimage():
+        from skimage.metrics import structural_similarity
+        return float(structural_similarity(a @ LUMA, b @ LUMA, data_range=1.0))
+    return _ssim_blocks(a, b)
+
+
+@functools.cache
+def have_vmaf() -> bool:
+    """Is the local ffmpeg built with libvmaf?"""
+    if not shutil.which("ffmpeg"):
+        return False
+    out = subprocess.run(["ffmpeg", "-hide_banner", "-filters"],
+                         capture_output=True, text=True, check=False).stdout
+    return " libvmaf " in out
+
+
+def vmaf(reference: str | Path, candidate: str | Path) -> dict[str, float]:
+    """Netflix VMAF. Prefers ffmpeg_quality_metrics; falls back to raw ffmpeg."""
+    try:
+        from ffmpeg_quality_metrics import FfmpegQualityMetrics
+    except ImportError:
+        pass
+    else:
+        res = FfmpegQualityMetrics(str(reference), str(candidate)).calculate(["vmaf"])
+        vals = [row["vmaf"] for row in res["vmaf"]]
+        return {"vmaf_mean": float(np.mean(vals)), "vmaf_min": float(np.min(vals))}
+    if not have_vmaf():
+        raise RuntimeError(
+            "this ffmpeg has no libvmaf filter; install an ffmpeg built with "
+            "--enable-libvmaf, or omit VMAF from the reference metric set"
+        )
+    with tempfile.NamedTemporaryFile("r", suffix=".json") as out:
+        subprocess.run(
+            ["ffmpeg", "-v", "error", "-i", str(candidate), "-i", str(reference),
+             "-lavfi", f"libvmaf=log_fmt=json:log_path={out.name}", "-f", "null", "-"],
+            check=True, capture_output=True,
+        )
+        data = json.loads(Path(out.name).read_text())
+    pooled = data["pooled_metrics"]["vmaf"]
+    return {"vmaf_mean": float(pooled["mean"]), "vmaf_min": float(pooled["min"])}
+
+
+@functools.cache
+def _lpips_model():
+    import lpips as _l
+    return _l.LPIPS(net="alex")
+
+
+def lpips_pair(a: np.ndarray, b: np.ndarray) -> float:
+    """Per-frame LPIPS via the ``lpips`` package (optional dependency)."""
+    import torch
+    model = _lpips_model()
+    def t(x):
+        return torch.from_numpy(x.transpose(2, 0, 1)[None] * 2.0 - 1.0).float()
+    with torch.no_grad():
+        return float(model(t(a), t(b)).item())
+
+
+def cvvdp(reference: str | Path, candidate: str | Path,
+          display: str = "standard_4k") -> dict[str, float]:
+    """ColorVideoVDP (JOD units, higher is better). Needs ``[reference]`` extra."""
+    try:
+        import pycvvdp
+    except ImportError as exc:                                 # pragma: no cover
+        raise RuntimeError(
+            "cvvdp not installed: pip install 'cozy-eval[reference]'"
+        ) from exc
+    metric = pycvvdp.cvvdp(display_name=display)
+    jod, _ = metric.predict(str(candidate), str(reference), dim_order="FHWC")
+    return {"cvvdp_jod": float(jod)}
+
+
+def compare(reference, candidate, *, with_lpips: bool = False) -> dict[str, float]:
+    """Per-frame reference metrics between two frame sources of the same take."""
+    ps: list[float] = []
+    ss: list[float] = []
+    lp: list[float] = []
+    n = 0
+    for a, b in zip(iter_frames(reference), iter_frames(candidate), strict=True):
+        if a.shape != b.shape:
+            raise ValueError(
+                f"frame shape mismatch {a.shape} vs {b.shape}: a resize is itself a "
+                "change, declare ChangeKind.OUTPUT_RESIZE and compare at one size"
+            )
+        n += 1
+        ps.append(psnr(a, b))
+        ss.append(ssim(a, b))
+        if with_lpips:
+            lp.append(lpips_pair(a, b))
+    if not n:
+        raise ValueError("no overlapping frames")
+    out = {
+        "frames": float(n),
+        "psnr_mean": float(np.mean(ps)),
+        "psnr_worst": float(np.min(ps)),
+        "ssim_mean": float(np.mean(ss)),
+        "ssim_worst": float(np.min(ss)),
+    }
+    if lp:
+        out["lpips_mean"] = float(np.mean(lp))
+        out["lpips_worst"] = float(np.max(lp))
+    return out