open-interpretability 0.0.20__py3-none-any.whl

openinterp/harness/__init__.py

@@ -0,0 +1,37 @@
+"""The measurement harness: hooks, scans, patching and readouts on the model itself."""
+
+from openinterp.harness.attribution import attribution_patching, top_attributed_neurons
+from openinterp.harness.hooks import (
+    NeuronHookRunner,
+    activation_histogram,
+    isolate_pole,
+    pole_strength,
+    replace_pole,
+)
+from openinterp.harness.lookahead import measure_lookahead, output_token_projection
+from openinterp.harness.patcher import (
+    AblationResult,
+    ActivationPatcher,
+    PatchResult,
+    logit_diff_metric,
+    token_logit_metric,
+)
+from openinterp.harness.scan import top_activating_examples
+
+__all__ = [
+    "AblationResult",
+    "ActivationPatcher",
+    "NeuronHookRunner",
+    "PatchResult",
+    "activation_histogram",
+    "attribution_patching",
+    "isolate_pole",
+    "logit_diff_metric",
+    "measure_lookahead",
+    "output_token_projection",
+    "pole_strength",
+    "replace_pole",
+    "token_logit_metric",
+    "top_activating_examples",
+    "top_attributed_neurons",
+]

openinterp/harness/ane/__init__.py

@@ -0,0 +1,9 @@
+"""Running a scan's forward pass on Apple's Neural Engine (ANE), through Core ML.
+
+- ``forward.py``: the forward pass, written so the ANE computes it well.
+- ``convert.py``: conversion to a compiled Core ML model (coremltools, on Apple-silicon Macs).
+- ``cache.py``: compiled models on disk, built once per machine.
+- ``backend.py``: :class:`~openinterp.harness.ane.backend.AneScan`, the scan's pass.
+
+docs/analyses/ane-feasibility.md has the measurements behind it.
+"""

openinterp/harness/ane/backend.py

@@ -0,0 +1,324 @@
+"""A scan's forward pass on Apple's Neural Engine (ANE), through Core ML.
+
+:class:`AneScan` gives the same :class:`~openinterp.harness.scan.Chunk` objects as the
+torch pass. It reads the same batches (``batches`` in ``harness/scan.py``), so the
+tokens, padding and counted tokens are the same. The ANE needs input shapes that do not
+change. So each batch goes in the smallest of :data:`SEQS` tokens that holds its longest
+piece, with ``2,048 / seq`` rows. With causal attention, padding on the right does not
+change the real tokens' activations. If a piece is longer than the largest shape, the
+scan runs on torch.
+
+The ANE computes in fp16, so its activations are about 1% from fp32. The scan therefore
+measures its top candidates again on the torch device (``TopExamples.rescore`` in
+``harness/scan.py``). Then the numbers that ``open-interpretability reproduce`` checks are exact.
+"""
+
+from __future__ import annotations
+
+import os
+import shutil
+import tempfile
+import threading
+from collections.abc import Iterator, Sequence
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+import torch
+
+from openinterp.harness.ane import chunks, convert
+from openinterp.harness.ane.cache import (
+    MAX_ERROR,
+    META,
+    MODEL,
+    ConversionCache,
+    ConversionRejected,
+    fields,
+)
+from openinterp.harness.ane.forward import ToLayer, calibration_ids, shape_inputs, supported
+from openinterp.harness.ane.qualified import qualified, too_deep
+
+if TYPE_CHECKING:
+    from openinterp.harness.hooks import NeuronHookRunner
+    from openinterp.harness.scan import Chunk, Piece
+
+NAME = "coreml-ane"  # recorded as the scan's backend
+SEQS = (128, 256, 512, 1024, 2048)
+SHAPE_TOKENS = 2048  # rows × seq of each shape, as BATCH_TOKENS
+PREPARE_ENV = "OPENINTERP_ANE_PREPARE_MINUTES"
+# The longest that a scan waits for its models to convert and compile, before it runs on
+# torch. A first compile can take minutes for each shape (Qwen2.5-0.5B: about 1-3 min).
+DEFAULT_PREPARE_MINUTES = 20.0
+# Peak memory of a conversion, as a multiple of the fp32 weights it converts. Measured:
+# 4.2 (GPT-2 L11), 3.1 (Qwen2.5-0.5B L23) and 2.5 (Qwen2.5-1.5B L14), with a margin.
+CONVERSION_OVERHEAD = 4.5
+
+
+def shape_for(length: int, longest: int) -> tuple[int, int] | None:
+    """``(rows, seq)`` of the smallest shape that holds ``length`` tokens, or None."""
+    for seq in SEQS:
+        if length <= seq <= longest:
+            return SHAPE_TOKENS // seq, seq
+    return None
+
+
+class AneScan:
+    """Runs a scan's forward pass on the ANE, for one model."""
+
+    name = NAME
+
+    def __init__(self, model, model_name: str, revision: str | None, cache) -> None:
+        self.model = model
+        self.model_name = model_name
+        self.revision = revision
+        self.cache: ConversionCache | None = cache
+        self._loaded: dict[tuple[int, int, int], list] = {}
+        self.paths: dict[tuple[int, int, int], list[Path]] = {}  # each pass's chunk models
+        self._scratch: list[Path] = []
+        self.used = False  # True once a scan ran on it
+        self.noted: set[str] = set()  # reasons already shown, so each shows once
+        self.broken: str | None = None  # why it may not run again in this process
+        self._decided: dict[tuple, str | None] = {}
+
+    @classmethod
+    def for_model(cls, model, model_name: str, revision: str | None) -> AneScan | None:
+        """An :class:`AneScan` for ``model``, or None if it can't or may not use the ANE.
+
+        It needs coremltools, a supported architecture, and a model that passed
+        qualification (``harness/ane/qualified.py``).
+        """
+        if not convert.available() or not supported(model) or not qualified(model_name):
+            return None
+        return cls(model, model_name, revision, ConversionCache.default())
+
+    def longest(self) -> int:
+        """The longest piece it can run: the largest shape, or the model's context."""
+        cfg = self.model.config
+        context = getattr(cfg, "n_positions", None) or getattr(cfg, "max_position_embeddings", None)
+        return min(SEQS[-1], context or SEQS[-1])
+
+    def prepare(self, lengths: Sequence[int], layer: int, batch_tokens: int) -> str | None:
+        """Make ready every model a scan of pieces of these token lengths needs.
+
+        Return why the ANE can't run that scan, or None if it can. The answer is kept,
+        so every part of one scan (its cache key, its pass, its rescoring) agrees.
+        """
+        if self.broken:
+            return self.broken
+        key = (layer, batch_tokens, tuple(lengths))
+        if key not in self._decided:
+            self._decided[key] = self._prepare(lengths, layer, batch_tokens)
+        return self._decided[key]
+
+    def _prepare(self, lengths: Sequence[int], layer: int, batch_tokens: int) -> str | None:
+        if deep := too_deep(self.model_name, layer):
+            return deep
+        shapes = needed_shapes(lengths, batch_tokens, self.longest())
+        if shapes is None:
+            too_long = max(lengths, default=0)
+            return f"a piece has {too_long} tokens, more than the ANE's {self.longest()}"
+        missing = any(not self._cached(layer, *shape) for shape in shapes)
+        if missing and (short := self._memory_short(layer)):
+            return short
+        return self._ready_in_time(layer, sorted(shapes))
+
+    def _ready_in_time(self, layer: int, shapes: list) -> str | None:
+        """Load (and if needed convert) ``shapes`` within the time limit; else say why not.
+
+        The work runs on a background thread, because a call into Core ML can't be
+        stopped. If it takes longer than the limit, the ANE is not used again in this
+        process, and the scan runs on torch.
+        """
+        done, failed = threading.Event(), []
+
+        def work() -> None:
+            try:
+                for shape in shapes:
+                    self.compiled(layer, *shape)
+            except Exception as exc:  # any failure to convert: the scan runs on torch
+                failed.append(exc)
+            finally:
+                done.set()
+
+        threading.Thread(target=work, daemon=True, name="ane-prepare").start()
+        minutes = prepare_minutes()
+        if not done.wait(timeout=minutes * 60):
+            self.broken = (
+                f"converting and compiling for the ANE took more than {minutes:g} minutes "
+                f"(${PREPARE_ENV}); a later run tries again"
+            )
+            return self.broken
+        if failed:
+            return f"its conversion failed ({type(failed[0]).__name__}: {failed[0]})"
+        return None
+
+    def _cached(self, layer: int, rows: int, seq: int) -> bool:
+        if (layer, rows, seq) in self._loaded:
+            return True
+        return all(self._entry(link, rows, seq) for link in chunks.chain(self.model, layer))
+
+    def _entry(self, link: tuple[int, int, bool], rows: int, seq: int) -> Path | None:
+        """The cached compiled model of one chunk, or None if the cache has none."""
+        if self.cache is None:
+            return None
+        start, last, final = link
+        key_fields = fields(self.model_name, self.revision, last, rows, seq, start, not final)
+        entry = self.cache.path(key_fields)
+        return entry / MODEL if (entry / META).exists() else None
+
+    def _memory_short(self, layer: int) -> str | None:
+        """Why converting the chunks of a pass to ``layer`` would not fit in memory, or None.
+
+        Chunks are converted one at a time, so the largest chunk sets the need.
+        """
+        from openinterp.harness.memory import available_bytes
+
+        links = chunks.chain(self.model, layer)
+        largest = max(chunks.chunk_bytes(self.model, start, last) for start, last, _ in links)
+        need = int(largest * CONVERSION_OVERHEAD)
+        have = available_bytes("cpu")
+        if have is None or need <= have:
+            return None
+        return f"converting it needs about {need / 2**30:.1f} GB and {have / 2**30:.1f} GB is free"
+
+    def chunks(
+        self, runner: NeuronHookRunner, pieces: Sequence[Piece], layer: int, batch_tokens: int
+    ) -> Iterator[Chunk]:
+        """``layer``'s activations over ``pieces``, batch by batch, from the ANE."""
+        from openinterp.harness.scan import Chunk, batches
+
+        for batch in batches(runner, pieces, batch_tokens):
+            acts = self.batch_acts(batch, layer)
+            mask = batch.mask
+            yield Chunk(
+                pieces=batch.pieces,
+                lengths=mask.sum(1).tolist(),
+                ids=batch.ids[mask],
+                positions=batch.positions[mask],
+                counted=batch.counted[mask],
+                acts=acts[mask],
+            )
+
+    def batch_acts(self, batch, layer: int) -> torch.Tensor:
+        """``layer``'s activations on one scan batch, ``[pieces, width, d_mlp]`` in fp32.
+
+        The batch goes in the smallest shape that holds it, in parts of that shape's rows.
+        Each part goes through the chain of chunks (``chunks.py``).
+        """
+        width = batch.ids.shape[1]
+        rows, seq = shape_for(width, self.longest()) or (0, 0)
+        if not rows:
+            raise ValueError(f"no ANE shape holds {width} tokens")
+        from openinterp.harness.scan_backend import AneError
+
+        try:
+            models = self.compiled(layer, rows, seq)
+            parts = []
+            for start in range(0, len(batch.pieces), rows):
+                x = torch.zeros((rows, seq), dtype=torch.long)
+                part = batch.ids[start : start + rows]
+                x[: len(part), :width] = part
+                for model, extra in models:
+                    x = convert.predict(model, x, extra)
+                parts.append(x[: len(part), :width])
+        except Exception as exc:  # Core ML's errors have no common type
+            raise AneError(f"{type(exc).__name__}: {exc}") from exc
+        return torch.cat(parts)
+
+    def compiled(self, layer: int, rows: int, seq: int) -> list:
+        """The loaded Core ML models of a pass to ``layer`` at ``(rows, seq)``, chunk by chunk.
+
+        Each comes with its other inputs (``forward.shape_inputs``), ready for ``predict``.
+        """
+        key = (layer, rows, seq)
+        if key not in self._loaded:
+            links = chunks.chain(self.model, layer)
+            paths = [self._entry(link, rows, seq) for link in links]
+            if not all(paths):
+                paths = self._build(links, rows, seq)
+            self.paths[key] = paths
+            cfg = self.model.config
+            self._loaded[key] = [
+                (convert.load(path), convert.feed(shape_inputs(cfg, seq, start)))
+                for path, (start, _, _) in zip(paths, links, strict=True)
+            ]
+        return self._loaded[key]
+
+    def _build(self, links: list, rows: int, seq: int) -> list[Path]:
+        """Convert and check each chunk that the cache doesn't hold, in order.
+
+        Each chunk's fp32 output is the next chunk's input. On the calibration tokens it
+        sets the next chunk's scales, the same for every shape. On random tokens of the
+        shape, it checks the next chunk. Only one chunk's weights are in memory at a time.
+        """
+        cfg = self.model.config
+        x = torch.randint(
+            0, cfg.vocab_size, (rows, seq), generator=torch.Generator().manual_seed(2)
+        )
+        calibration = calibration_ids(cfg)
+        paths = []
+        for start, last, final in links:
+            source = chunks.part(self.model, self.model_name, self.revision, start, last)
+            net = ToLayer(source, last, rows, seq, start, not final, sample=calibration).eval()
+            with torch.no_grad():
+                ref = net(x)
+                with net.shaped(*calibration.shape[:2]):
+                    calibrated = net(calibration)
+            key_fields = fields(self.model_name, self.revision, last, rows, seq, start, not final)
+            paths.append(self._get(key_fields, net, x, ref))
+            x, calibration = ref, calibrated
+            del source, net
+        return paths
+
+    def _get(self, key_fields: dict, net: ToLayer, x: torch.Tensor, ref: torch.Tensor) -> Path:
+        """One chunk's compiled model, from the cache or built now, and checked if built."""
+
+        def build(dest: Path) -> None:
+            convert.compile_to(net, x, dest)
+
+        def check(path: Path) -> float:
+            got = convert.predict(convert.load(path), x, convert.feed(net.extra))
+            return float((ref - got).pow(2).mean().sqrt() / ref.pow(2).mean().sqrt())
+
+        if self.cache is None:
+            return self._uncached(build, check)
+        return self.cache.get(key_fields, build, check)
+
+    def _uncached(self, build, check) -> Path:
+        """Build into a temporary directory, for a run with the cache turned off."""
+        work = Path(tempfile.mkdtemp(prefix="openinterp-ane-"))
+        self._scratch.append(work)
+        path = work / "model.mlmodelc"
+        build(path)
+        error = check(path)
+        if not error <= MAX_ERROR:
+            raise ConversionRejected(f"relative error {error:.4f} is more than {MAX_ERROR}")
+        return path
+
+    def close(self) -> None:
+        """Delete the temporary models of a run with the cache turned off."""
+        for work in self._scratch:
+            shutil.rmtree(work, ignore_errors=True)
+        self._scratch.clear()
+
+
+def needed_shapes(lengths: Sequence[int], batch_tokens: int, longest: int) -> set | None:
+    """The ``(rows, seq)`` shapes that a scan's batches need, or None if one has none.
+
+    The batches are those of ``batches`` in ``harness/scan.py``: each is as wide as its
+    longest piece.
+    """
+    from openinterp.harness.scan import plan_batches
+
+    shapes, start = set(), 0
+    for size in plan_batches(list(lengths), batch_tokens):
+        shape = shape_for(max(lengths[start : start + size]), longest)
+        if shape is None:
+            return None
+        shapes.add(shape)
+        start += size
+    return shapes
+
+
+def prepare_minutes() -> float:
+    """``$OPENINTERP_ANE_PREPARE_MINUTES``, else :data:`DEFAULT_PREPARE_MINUTES`."""
+    return float(os.environ.get(PREPARE_ENV) or DEFAULT_PREPARE_MINUTES)

openinterp/harness/ane/cache.py

@@ -0,0 +1,276 @@
+"""On-disk cache of compiled Core ML models, so a machine converts each model only once.
+
+Converting a model to one layer takes 5–50 s. The ANE also compiles a model when it first
+loads it, which takes 10–35 s. macOS keeps that compile, keyed by the model's path. A
+later load from the same path takes 0.2 s, but a copy at a new path compiles again. So
+an entry is built in a temporary directory and renamed to its final path. Then it is
+loaded there once, which warms the ANE's compile, and checked (:data:`MAX_ERROR`).
+
+An entry is keyed by everything that changes the compiled model:
+
+- the model, its revision, the chunk's layers and the input shape;
+- a hash of the source of the conversion code (``forward.py`` and ``convert.py``),
+  so any change to that code makes new entries;
+- the coremltools and torch versions;
+- the macOS version and the chip, because the ANE's compiler comes with macOS.
+
+An entry is ``<root>/<model>/L<layer>/<key>/`` with ``model.mlmodelc`` and
+``meta.json``. It is complete only when ``meta.json`` exists, which is written last.
+A lock per key stops two processes from building one entry at the same time. Once the
+cache is larger than its cap, the least recently used entries go.
+
+The weights of one chunk are the same file for every input shape
+(``forward.shape_inputs``). So each weight file is kept once, in ``<root>/.blobs/``
+under its SHA-256, and each entry holds a hard link to it. A blob that no entry links to
+goes when the cache next evicts. The cache's size counts each file once.
+"""
+
+from __future__ import annotations
+
+import fcntl
+import hashlib
+import json
+import os
+import platform
+import secrets
+import shutil
+import subprocess
+from collections.abc import Callable, Iterator
+from contextlib import contextmanager, suppress
+from datetime import UTC, datetime
+from pathlib import Path
+
+import torch
+
+from openinterp.core.cache_dirs import cache_root
+from openinterp.harness.ane import convert
+
+FORMAT = 3  # 2: chunks of layers; 3: shape tables as inputs, one calibration sample
+ENV = "OPENINTERP_COREML_CACHE"  # cache directory, or "off"
+SIZE_ENV = "OPENINTERP_COREML_CACHE_GB"
+DEFAULT_GB = 20.0
+MAX_ERROR = 0.03  # largest relative RMS error against fp32 (measured: 0.003 to 0.011)
+SOURCES = ("forward.py", "convert.py", "chunks.py")  # the code that sets what an entry holds
+MODEL = "model.mlmodelc"
+META = "meta.json"
+BLOBS = ".blobs"
+
+
+class ConversionRejected(RuntimeError):
+    """A converted model did not agree with fp32 closely enough to use."""
+
+
+def code_digest() -> str:
+    """A hash of the source of the conversion code."""
+    digest = hashlib.sha256()
+    for name in SOURCES:
+        digest.update((Path(__file__).parent / name).read_bytes())
+    return digest.hexdigest()[:16]
+
+
+def machine() -> dict:
+    """The macOS version and the chip, which set what the ANE's compiler makes."""
+    chip = None
+    with suppress(OSError, subprocess.SubprocessError):
+        chip = subprocess.run(
+            ["sysctl", "-n", "machdep.cpu.brand_string"], capture_output=True, text=True, check=True
+        ).stdout.strip()
+    return {"macos": platform.mac_ver()[0] or None, "chip": chip or platform.machine()}
+
+
+def fields(
+    model: str,
+    revision: str | None,
+    layer: int,
+    rows: int,
+    seq: int,
+    start: int = 0,
+    resid_out=False,
+) -> dict:
+    """What an entry is keyed by: one chunk, layers ``start..layer`` (``chunks.py``)."""
+    return {
+        "format": FORMAT,
+        "model": model,
+        "revision": revision,
+        "layer": layer,
+        "start": start,
+        "resid_out": resid_out,
+        "shape": [rows, seq],
+        "code": code_digest(),
+        "coremltools": convert.version(),
+        "torch": torch.__version__,
+        **machine(),
+    }
+
+
+class ConversionCache:
+    """Compiled Core ML models on disk, built once per machine and key."""
+
+    def __init__(self, root: Path, max_bytes: int | None = None) -> None:
+        self.root = root
+        self.max_bytes = max_bytes
+
+    @classmethod
+    def default(cls) -> ConversionCache | None:
+        """The cache at ``$OPENINTERP_COREML_CACHE`` (default ``~/.cache/openinterp/coreml``)."""
+        root = cache_root(ENV, "coreml")
+        if root is None:
+            return None
+        gb = float(os.environ.get(SIZE_ENV) or DEFAULT_GB)
+        return cls(root, int(gb * 2**30))
+
+    def path(self, key_fields: dict) -> Path:
+        """The entry directory for ``key_fields``."""
+        key = hashlib.sha256(json.dumps(key_fields, sort_keys=True).encode()).hexdigest()[:20]
+        model = key_fields["model"].replace("/", "--")
+        return self.root / model / f"L{key_fields['layer']}" / key
+
+    def get(
+        self,
+        key_fields: dict,
+        build: Callable[[Path], None],
+        check: Callable[[Path], float],
+    ) -> Path:
+        """The compiled model for ``key_fields``; build and check it if the cache has none.
+
+        ``build`` writes a ``.mlmodelc`` to the path it gets. ``check`` loads the model
+        at its final path and returns its relative error against fp32. An entry with an
+        error above :data:`MAX_ERROR` is deleted, and this raises :class:`ConversionRejected`.
+        """
+        entry = self.path(key_fields)
+        if (entry / META).exists():
+            self._touch(entry)
+            return entry / MODEL
+        with self._lock(entry):
+            if not (entry / META).exists():
+                self._build(entry, key_fields, build, check)
+        self.evict(keep=entry)
+        return entry / MODEL
+
+    def _build(self, entry: Path, key_fields: dict, build, check) -> None:
+        shutil.rmtree(entry, ignore_errors=True)  # left by a build that stopped
+        tmp = entry.parent / f".tmp-{entry.name}-{secrets.token_hex(4)}"
+        try:
+            tmp.mkdir(parents=True)
+            build(tmp / MODEL)
+            os.rename(tmp, entry)
+        finally:
+            shutil.rmtree(tmp, ignore_errors=True)
+        error = check(entry / MODEL)  # at the final path, so macOS keeps the ANE's compile
+        if not error <= MAX_ERROR:
+            shutil.rmtree(entry, ignore_errors=True)
+            raise ConversionRejected(f"relative error {error:.4f} is more than {MAX_ERROR}")
+        self._share(entry / MODEL)
+        meta = {"fields": key_fields, "error": error, "built": _now(), "last_used": _now()}
+        _write_json(entry / META, meta)
+
+    def _share(self, model: Path) -> None:
+        """Replace each weight file in ``model`` by a hard link to the blob with its contents."""
+        blobs = self.root / BLOBS
+        blobs.mkdir(parents=True, exist_ok=True)
+        for weights in model.glob("weights/*.bin"):
+            blob = blobs / f"{_sha256(weights)}.bin"
+            if not blob.exists():
+                with suppress(FileExistsError):  # another process stored it first
+                    os.link(weights, blob)
+                continue
+            if os.path.samefile(blob, weights):
+                continue
+            tmp = weights.with_name(f".{weights.name}.{secrets.token_hex(4)}")
+            os.link(blob, tmp)
+            os.replace(tmp, weights)  # atomic: the file is whole at every moment
+
+    def size(self) -> int:
+        """Bytes on disk: each file once, however many entries link to it."""
+        seen, total = set(), 0
+        for path in self.root.rglob("*"):
+            with suppress(OSError):
+                info = path.stat()
+                if path.is_file() and (info.st_dev, info.st_ino) not in seen:
+                    seen.add((info.st_dev, info.st_ino))
+                    total += info.st_size
+        return total
+
+    def _drop_old_formats(self) -> list[Path]:
+        """Delete the entries of another :data:`FORMAT`. No key can find them again."""
+        gone = []
+        for meta in self.root.glob(f"*/L*/*/{META}"):
+            with suppress(OSError, ValueError):
+                if json.loads(meta.read_text())["fields"].get("format") != FORMAT:
+                    shutil.rmtree(meta.parent, ignore_errors=True)
+                    gone.append(meta.parent)
+        if gone:
+            self._sweep()
+        return gone
+
+    def _sweep(self) -> None:
+        """Delete each blob that no entry links to."""
+        for blob in (self.root / BLOBS).glob("*.bin"):
+            with suppress(OSError):
+                if blob.stat().st_nlink == 1:
+                    blob.unlink()
+
+    @contextmanager
+    def _lock(self, entry: Path) -> Iterator[None]:
+        locks = self.root / ".locks"
+        locks.mkdir(parents=True, exist_ok=True)
+        with open(locks / f"{entry.name}.lock", "w") as handle:
+            fcntl.flock(handle, fcntl.LOCK_EX)
+            try:
+                yield
+            finally:
+                fcntl.flock(handle, fcntl.LOCK_UN)
+
+    def _touch(self, entry: Path) -> None:
+        with suppress(OSError, ValueError):
+            meta = json.loads((entry / META).read_text())
+            meta["last_used"] = _now()
+            _write_json(entry / META, meta)
+
+    def entries(self) -> list[tuple[Path, int, str]]:
+        """Each complete entry, its size in bytes, and when it was last used."""
+        found = []
+        for meta in self.root.glob(f"*/L*/*/{META}"):
+            with suppress(OSError, ValueError):
+                used = json.loads(meta.read_text()).get("last_used", "")
+                size = sum(f.stat().st_size for f in meta.parent.rglob("*") if f.is_file())
+                found.append((meta.parent, size, used))
+        return found
+
+    def evict(self, keep: Path | None = None) -> list[Path]:
+        """Delete old entries, then the least recently used, until the cache is inside its cap.
+
+        Old entries are those of another :data:`FORMAT`. An entry frees its weights only
+        if no other entry links to them.
+        """
+        gone = self._drop_old_formats()
+        if self.max_bytes is None:
+            return gone
+        self._sweep()
+        for path, _, _ in sorted(self.entries(), key=lambda e: e[2]):
+            if self.size() <= self.max_bytes:
+                break
+            if path == keep:
+                continue
+            shutil.rmtree(path, ignore_errors=True)
+            self._sweep()
+            gone.append(path)
+        return gone
+
+
+def _sha256(path: Path) -> str:
+    digest = hashlib.sha256()
+    with open(path, "rb") as handle:
+        for block in iter(lambda: handle.read(1 << 24), b""):
+            digest.update(block)
+    return digest.hexdigest()
+
+
+def _now() -> str:
+    return datetime.now(UTC).isoformat()
+
+
+def _write_json(path: Path, data: dict) -> None:
+    tmp = path.with_name(f".{path.name}.{secrets.token_hex(4)}")
+    tmp.write_text(json.dumps(data, indent=1, sort_keys=True))
+    os.replace(tmp, path)