idun 0.1.0__py3-none-any.whl

idun/__init__.py

@@ -0,0 +1,11 @@
+"""Idun: tabular learners on the Apple silicon GPU (Metal), with CPU reference backends.
+
+The learners are `idun.gbm` (idun_gbm), `idun.tabm` (idun_tabm) and `idun.realmlp` (idun_realmlp); `Dataset` holds
+tabular data binned for them. The public interface is documented in docs/api/python-api.md.
+"""
+
+from idun._version import __version__
+from idun.dataset import Dataset
+from idun.models import load
+
+__all__ = ["Dataset", "__version__", "load"]

idun/_jit.py

@@ -0,0 +1,33 @@
+"""The one place idun compiles Python with numba.
+
+Every jitted function goes through :func:`njit`, so the options that bit-exact parity with the GPU backends depends
+on are set once. ``fastmath`` stays off: with it, LLVM contracts ``a * b + c`` into a fused multiply-add and the CPU
+results stop matching the Metal kernels, which are compiled with contraction off (docs/architecture/determinism.md).
+Compiled code is cached next to the source. Under ``NUMBA_DISABLE_JIT=1`` (the coverage run) numba hands back the
+Python function itself.
+"""
+
+from collections.abc import Callable
+from typing import Any, cast
+
+import numba
+
+
+def njit[F: Callable[..., Any]](*, parallel: bool = False) -> Callable[[F], F]:
+    """Return numba's ``njit`` decorator with idun's fixed options.
+
+    Parameters
+    ----------
+    parallel
+        Compile ``numba.prange`` loops as parallel loops.
+
+    Returns
+    -------
+    Callable
+        A decorator that compiles a function in nopython mode.
+    """
+
+    def decorate(func: F) -> F:
+        return cast("F", numba.njit(cache=True, fastmath=False, nogil=True, parallel=parallel)(func))
+
+    return decorate

idun/_version.py

@@ -0,0 +1,3 @@
+"""The package version: a leaf module, so that any module can name it without importing the package."""
+
+__version__ = "0.1.0"

idun/base.py

@@ -0,0 +1,127 @@
+"""What every idun estimator shares: scikit-learn's conventions without importing it.
+
+`Estimator` holds the constructor parameters (`get_params`, `set_params`), the checks of inputs that scikit-learn's
+``validate_data`` would make (a missing target, a column count other than the fitted one, an unfitted estimator) and
+scikit-learn's tags. `Regressor` and `Classifier` add the task and ``score`` (R² and accuracy, as scikit-learn's mixins
+define them).
+"""
+
+import inspect
+from typing import Any, ClassVar, Self
+
+import numpy as np
+from numpy.typing import NDArray
+
+from idun.dataset import n_columns
+from idun.errors import NotFittedError
+
+
+class Estimator:
+    """Constructor parameters as attributes; `get_params` and `set_params` as scikit-learn defines them.
+
+    Subclasses name their task in ``_estimator_type`` (``"classifier"`` or ``"regressor"``), which scikit-learn reads
+    through `__sklearn_tags__`.
+    """
+
+    _estimator_type: ClassVar[str]
+
+    @classmethod
+    def _param_names(cls) -> list[str]:
+        return [name for name in inspect.signature(cls.__init__).parameters if name != "self"]
+
+    def get_params(self, deep: bool = True) -> dict[str, Any]:  # noqa: ARG002, FBT001, FBT002 - scikit-learn's signature
+        """Return the constructor parameters (scikit-learn's convention)."""
+        return {name: getattr(self, name) for name in self._param_names()}
+
+    def set_params(self, **params: Any) -> Self:
+        """Set constructor parameters; an unknown name raises ``ValueError``."""
+        unknown = sorted(set(params) - set(self._param_names()))
+        if unknown:
+            raise ValueError(f"unknown parameters {unknown}")
+        for name, value in params.items():
+            setattr(self, name, value)
+        return self
+
+    def _check_fitted(self, attribute: str) -> None:
+        if not hasattr(self, attribute):
+            raise NotFittedError(f"this {type(self).__name__} is not fitted: call fit first")
+
+    def _require_y(self, y: Any, hint: str = "") -> None:
+        if y is None:
+            raise ValueError(f"{type(self).__name__} requires y to be passed, but the target y is None{hint}")
+
+    def _check_n_features(self, X: Any) -> None:  # noqa: N803 - scikit-learn's name
+        n = n_columns(X)
+        if n != self.n_features_in_:
+            raise ValueError(
+                f"X has {n} features, but {type(self).__name__} is expecting {self.n_features_in_} features as input."
+            )
+
+    n_features_in_: int
+
+    def __sklearn_tags__(self) -> Any:
+        """Return scikit-learn's tags for this estimator.
+
+        Only scikit-learn calls this, so it is imported here and is never a dependency of idun. Missing values are
+        allowed (idun_gbm bins them apart; the networks read them as 0).
+        """
+        from sklearn.utils import ClassifierTags, InputTags, RegressorTags, Tags, TargetTags  # noqa: PLC0415, TID251
+
+        classifier = self._estimator_type == "classifier"
+        return Tags(
+            estimator_type=self._estimator_type,
+            target_tags=TargetTags(required=True),
+            classifier_tags=ClassifierTags() if classifier else None,
+            regressor_tags=None if classifier else RegressorTags(),
+            input_tags=InputTags(allow_nan=True),
+        )
+
+    def __repr__(self) -> str:
+        defaults = {n: p.default for n, p in inspect.signature(type(self).__init__).parameters.items() if n != "self"}
+        changed = ", ".join(f"{n}={v!r}" for n, v in self.get_params().items() if v != defaults.get(n))
+        return f"{type(self).__name__}({changed})"
+
+
+def _weights(sample_weight: Any, n: int) -> NDArray[np.float64]:
+    if sample_weight is None:
+        return np.ones(n, np.float64)
+    weights = np.asarray(sample_weight, np.float64)
+    if weights.shape != (n,):
+        raise ValueError(f"sample_weight must have one value per row ({n}), got shape {weights.shape}")
+    return weights
+
+
+class Regressor(Estimator):
+    """A regression estimator: ``score`` is the coefficient of determination R² of ``predict``."""
+
+    _estimator_type = "regressor"
+
+    def predict(self, X: Any) -> NDArray[Any]:  # noqa: N803 - scikit-learn's name
+        """Return the prediction of each row of ``X``."""
+        raise NotImplementedError
+
+    def score(self, X: Any, y: Any, sample_weight: Any = None) -> float:  # noqa: N803 - scikit-learn's name
+        """Return R² = 1 − Σ w (y − ŷ)² / Σ w (y − ȳ)², with ȳ the weighted mean of ``y`` (1.0 when y is constant)."""
+        target = np.asarray(y, np.float64)
+        weights = _weights(sample_weight, target.size)
+        residual = float(np.sum(weights * (target - np.asarray(self.predict(X), np.float64)) ** 2))
+        total = float(np.sum(weights * (target - np.average(target, weights=weights)) ** 2))
+        if total == 0:
+            return 1.0 if residual == 0 else 0.0
+        return 1.0 - residual / total
+
+
+class Classifier(Estimator):
+    """A classification estimator: ``score`` is the (weighted) share of rows whose ``predict`` equals ``y``."""
+
+    _estimator_type = "classifier"
+
+    def predict(self, X: Any) -> NDArray[Any]:  # noqa: N803 - scikit-learn's name
+        """Return the predicted label of each row of ``X``."""
+        raise NotImplementedError
+
+    def score(self, X: Any, y: Any, sample_weight: Any = None) -> float:  # noqa: N803 - scikit-learn's name
+        """Return the accuracy of ``predict(X)`` against the labels ``y``."""
+        labels = np.asarray(y)
+        weights = _weights(sample_weight, labels.size)
+        return float(np.average(np.asarray(self.predict(X)) == labels, weights=weights))

idun/data/__init__.py

@@ -0,0 +1,12 @@
+"""The packed on-disk dataset: its writer and its memory-mapped reader.
+
+A packed dataset is a date-sorted panel stored column-major (4-bit codes, raw numeric values and their bins per
+window, one categorical column, a target), wrapped by Metal without a copy. Format: docs/architecture/data-layout.md.
+API: docs/api/python-api.md. Numerai's builder is `idun_numerai.data.build_cache`.
+"""
+
+from idun.data.dataset import PackedDataset
+from idun.data.refits import Refit
+from idun.data.writer import PackedWriter
+
+__all__ = ["PackedDataset", "PackedWriter", "Refit"]

idun/data/binning.py

@@ -0,0 +1,74 @@
+"""Quantile bins for the 22 named features (docs/architecture/data-layout.md §2).
+
+Each refit gets its own binning, fitted on a sample of its own training window so later rows never leak into the
+edges (LightGBM: ``bin_construct_sample_cnt = 200000``, ``max_bin = 255``):
+
+1. `sample_rows`: 200,000 row indices drawn without replacement from the window by ``PCG64([seed, refit])``.
+2. `quantile_edges`: per feature, the distinct finite sample values. With at most 255 of them every value gets its own
+   bin; otherwise cuts sit at the ``i / 255`` quantiles (by count) of the sample. Each upper edge is the float32
+   midpoint between the two distinct values it separates, and never rounds onto the upper one.
+3. `apply_bins`: a value goes to the first bin whose upper edge is ≥ the value; NaN goes to bin 255.
+
+This is plain quantile binning, not LightGBM's ``BinMapper``.
+"""
+
+from collections.abc import Sequence
+
+import numpy as np
+from numpy.typing import NDArray
+
+from idun.data.layout import MAX_NAMED_BINS, MISSING_BIN
+
+SAMPLE_ROWS = 200_000
+
+
+def sample_rows(window_rows: int, size: int, *, seed: int, refit: int) -> NDArray[np.int64]:
+    """Sorted, distinct row indices in ``[0, window_rows)``: all of them when ``size >= window_rows``."""
+    if window_rows < 1 or size < 1:
+        raise ValueError(f"window_rows and size must be positive, got {window_rows} and {size}")
+    if size >= window_rows:
+        return np.arange(window_rows, dtype=np.int64)
+    generator = np.random.Generator(np.random.PCG64([seed, refit]))
+    return np.sort(generator.choice(window_rows, size=size, replace=False)).astype(np.int64)
+
+
+def quantile_edges(values: NDArray[np.float32], max_bins: int = MAX_NAMED_BINS) -> NDArray[np.float32]:
+    """Strictly increasing float32 upper edges (``n_bins - 1`` of them, ``n_bins <= max_bins``) for a sample.
+
+    Non-finite sample values are ignored. With no finite value there is a single bin and no edge.
+    """
+    if values.dtype != np.float32:
+        raise TypeError(f"values must be float32, got {values.dtype}")
+    if max_bins < 1:
+        raise ValueError(f"max_bins must be positive, got {max_bins}")
+    distinct, counts = np.unique(values[np.isfinite(values)], return_counts=True)
+    if distinct.size <= max_bins:
+        cuts = np.arange(max(distinct.size - 1, 0))
+    else:
+        cumulative = np.cumsum(counts)
+        total = int(cumulative[-1])
+        quantiles = (total * np.arange(1, max_bins, dtype=np.int64) + max_bins - 1) // max_bins  # ceil(total * i / max)
+        cuts = np.unique(np.searchsorted(cumulative, quantiles, side="left"))
+        cuts = cuts[cuts < distinct.size - 1]
+    lower, upper = distinct[cuts], distinct[cuts + 1]
+    middle = ((lower.astype(np.float64) + upper.astype(np.float64)) / 2).astype(np.float32)
+    return np.where(middle < upper, middle, lower)
+
+
+def edge_table(edges: Sequence[NDArray[np.float32]], max_bins: int = MAX_NAMED_BINS) -> NDArray[np.float32]:
+    """Stack per-feature edges into a ``(features, max_bins)`` table padded with +inf (so +inf ends every row)."""
+    table = np.full((len(edges), max_bins), np.inf, np.float32)
+    for row, feature_edges in enumerate(edges):
+        if feature_edges.size >= max_bins:
+            raise ValueError(f"feature {row} has {feature_edges.size} edges; at most {max_bins - 1} fit")
+        table[row, : feature_edges.size] = feature_edges
+    return table
+
+
+def apply_bins(values: NDArray[np.float32], edges_row: NDArray[np.float32]) -> NDArray[np.uint8]:
+    """Bin of each value under one row of an `edge_table`: first edge ≥ value; NaN → `MISSING_BIN`."""
+    if edges_row.shape != (MAX_NAMED_BINS,) or not np.isposinf(edges_row[-1]):
+        raise ValueError("edges_row must be one +inf-padded row of an edge_table")
+    bins = np.searchsorted(edges_row, values, side="left").astype(np.uint8)
+    bins[np.isnan(values)] = MISSING_BIN
+    return bins

idun/data/dataset.py

@@ -0,0 +1,130 @@
+"""`PackedDataset`: the read-only, memory-mapped reader of the packed cache (docs/architecture/data-layout.md §2).
+
+Opening a cache checks its format version, that no source file changed since it was built, and that every array file
+has its recorded size. ``verify=True`` (or `PackedDataset.verify`) also recomputes every checksum. Nothing is copied
+into memory: each array is an ``np.memmap`` view, read-only, and columns are contiguous.
+"""
+
+import json
+import os
+from functools import cached_property
+from pathlib import Path
+from typing import Any, Self
+
+import numpy as np
+from numpy.typing import NDArray
+
+from idun.data import layout
+from idun.data.manifest import check_fresh, read_manifest, sha256_file
+from idun.data.refits import Refit
+from idun.errors import CacheError
+
+COPY_ENV = "IDUN_CACHE_COPY"  # "1": a copied cache, checked by its checksums instead of its source files
+
+
+class PackedDataset:
+    """One packed cache with one named-feature binning selected. Build with `idun.data.build_cache`.
+
+    Rows are in date order (on Numerai's cache, every row with a ``target_jupiter_60``). Features are in the order named
+    (sorted), obfuscated (sorted), country.
+
+    Attributes
+    ----------
+    n_rows : int
+        Rows in the panel.
+    binning : str
+        The selected named-feature binning: ``r<k>``, fitted on refit ``k``'s training window.
+    target, era, ticker, country : NDArray
+        Per row: float32 target, int32 days since 1970-01-01, int32 index into `tickers`, uint8 index into `countries`.
+    obfuscated : NDArray[np.uint8]
+        ``(n_obfuscated, ceil(n_rows / 2))``: two rows per byte (`idun.data.layout.unpack_nibbles`).
+    named_values, named_bins : NDArray
+        ``(n_named, n_rows)``: raw float32 values (NaN kept), and their bins under `binning` (255 = missing).
+    named_edges : NDArray[np.float32]
+        ``(n_named, 255)`` upper bin edges under `binning`, +inf padded; `named_n_bins` real bins per feature.
+    """
+
+    def __init__(self, path: Path, manifest: dict[str, Any], binning: str) -> None:
+        if binning not in manifest["binnings"]:
+            raise CacheError(f"{path} has no binning {binning!r}; it has {sorted(manifest['binnings'])}")
+        self.path = path
+        self.binning = binning
+        self.n_rows = n = int(manifest["n_rows"])
+        features = manifest["features"]
+        self.named_features: tuple[str, ...] = tuple(features["named"])
+        self.obfuscated_features: tuple[str, ...] = tuple(features["obfuscated"])
+        self.feature_names = (*self.named_features, *self.obfuscated_features, features["country"])
+        self.countries: tuple[str, ...] = tuple(manifest["countries"])
+        self.refits = tuple(Refit(**refit) for refit in manifest["refits"])
+        self.seed = int(manifest["seed"])
+        self._manifest = manifest
+        strides = manifest["strides"]
+        n_named, n_obfuscated = len(self.named_features), len(self.obfuscated_features)
+        self.target: NDArray[np.float32] = _map(path / layout.TARGET_FILE, np.float32, (n,))
+        self.era: NDArray[np.int32] = _map(path / layout.ERA_FILE, np.int32, (n,))
+        self.ticker: NDArray[np.int32] = _map(path / layout.TICKER_FILE, np.int32, (n,))
+        self.country: NDArray[np.uint8] = _map(path / layout.COUNTRY_FILE, np.uint8, (n,))
+        # idun_gbm reads the country as the one categorical column (BinnedData): a zero-stride view, nothing copied
+        self.categories: tuple[tuple[str, ...], ...] = (self.countries,)
+        self.categorical: NDArray[np.uint8] = self.country[np.newaxis, :]
+        obfuscated = _map(path / layout.OBFUSCATED_FILE, np.uint8, (n_obfuscated, strides["obfuscated"]))
+        self.obfuscated: NDArray[np.uint8] = obfuscated[:, : layout.packed_bytes(n)]
+        values = _map(path / layout.NAMED_VALUES_FILE, np.float32, (n_named, strides["named_values"] // 4))
+        self.named_values: NDArray[np.float32] = values[:, :n]
+        bins = _map(path / layout.named_bins_file(binning), np.uint8, (n_named, strides["named_bins"]))
+        self.named_bins: NDArray[np.uint8] = bins[:, :n]
+        edges = np.fromfile(path / layout.named_edges_file(binning), np.float32)
+        self.named_edges = edges.reshape(n_named, layout.MAX_NAMED_BINS)
+        self.named_n_bins: tuple[int, ...] = tuple(manifest["binnings"][binning]["n_bins"])
+
+    @classmethod
+    def open(cls, path: str | Path, *, binning: str, verify: bool = False) -> Self:
+        """Open the cache at ``path`` with ``binning`` selected (``"r0"``, ``"r1"`` or ``"r2"`` for the v6 protocol).
+
+        Raises `CacheError` when the cache is missing, of another format version, stale (a source file changed),
+        truncated, or (with ``verify``) when a checksum differs. With the environment variable ``IDUN_CACHE_COPY=1``
+        the cache is a copy on a machine without the source files (a second Mac): the source check is replaced by
+        the checksum check of every array file, so a copy is never opened unchecked.
+        """
+        path = Path(path)
+        manifest = read_manifest(path)
+        copy = os.environ.get(COPY_ENV) == "1"
+        try:
+            if not copy:
+                check_fresh(manifest)
+            for name, array in manifest["arrays"].items():
+                file = path / name
+                if not file.is_file() or file.stat().st_size != array["bytes"]:
+                    raise CacheError(f"{file} is missing or truncated; rebuild the cache")
+            dataset = cls(path, manifest, binning)
+        except KeyError as missing:
+            raise CacheError(f"{path / layout.MANIFEST} lacks {missing}; rebuild the cache") from None
+        if verify or copy:
+            dataset.verify()
+        return dataset
+
+    def verify(self) -> None:
+        """Recompute the sha256 of every array file and raise `CacheError` on any mismatch."""
+        for name, array in self._manifest["arrays"].items():
+            if sha256_file(self.path / name) != array["sha256"]:
+                raise CacheError(f"{self.path / name}: checksum mismatch; the cache is corrupt, rebuild it")
+
+    @cached_property
+    def tickers(self) -> tuple[str, ...]:
+        """Sorted ticker names; `ticker` indexes into them."""
+        names: list[str] = json.loads((self.path / layout.TICKERS_FILE).read_text())
+        return tuple(names)
+
+    def obfuscated_codes(self, feature: int, lo: int = 0, hi: int | None = None) -> NDArray[np.uint8]:
+        """Unpacked codes (0..4) of obfuscated feature ``feature`` for rows ``[lo, hi)``; all rows by default."""
+        hi = self.n_rows if hi is None else hi
+        if hi > self.n_rows:
+            raise ValueError(f"row {hi} is past the panel's {self.n_rows} rows")
+        return layout.unpack_nibbles(self.obfuscated[feature], lo, hi)
+
+    def __repr__(self) -> str:
+        return f"PackedDataset({str(self.path)!r}, binning={self.binning!r}, n_rows={self.n_rows})"
+
+
+def _map(path: Path, dtype: type[np.generic], shape: tuple[int, ...]) -> NDArray[Any]:
+    return np.memmap(path, dtype=dtype, mode="r", shape=shape)

idun/data/layout.py

@@ -0,0 +1,86 @@
+"""On-disk layout of the packed cache: file names, column strides, 4-bit packing (docs/architecture/data-layout.md §2).
+
+Every multi-column array is column-major: column ``c`` starts at byte ``c * stride`` of its file, and the stride is the
+column's byte length rounded up to a 16 KiB page. A whole file therefore maps page-aligned (Metal wraps it without a
+copy), and every column starts on a 16-byte boundary for vector loads.
+
+Obfuscated features pack two **rows** per byte: row ``2i`` in the low nibble of byte ``i``, row ``2i + 1`` in the high
+nibble. A column with an odd row count has a zero high nibble in its last byte.
+"""
+
+import numpy as np
+from numpy.typing import NDArray
+
+FORMAT = "idun-packed"  # the packed format's id, stamped into every manifest
+FORMAT_VERSION = 1
+PAGE_BYTES = 16 * 1024
+
+
+OBFUSCATED_LEVELS = 5  # obfuscated values are exactly 0..4: they are their own bins
+MAX_NAMED_BINS = 255  # real bins 0..254 for a named feature
+MISSING_BIN = 255  # NaN
+
+MANIFEST = "manifest.json"
+OBFUSCATED_FILE = "obf.u4"
+NAMED_VALUES_FILE = "named.f32"
+COUNTRY_FILE = "country.u8"
+TARGET_FILE = "y.f32"
+ERA_FILE = "era.i32"
+TICKER_FILE = "ticker.i32"
+TICKERS_FILE = "tickers.json"
+
+_LOW = np.uint8(0x0F)
+_FOUR = np.uint8(4)
+
+
+def binning_name(refit: int) -> str:
+    """Name of the named-feature binning fitted on refit ``refit``'s training window: ``r0``, ``r1``, ..."""
+    return f"r{refit}"
+
+
+def named_bins_file(binning: str) -> str:
+    """File holding every row's named-feature bins under ``binning``."""
+    return f"named.{binning}.u8"
+
+
+def named_edges_file(binning: str) -> str:
+    """File holding the upper bin edges of ``binning``: float32, one row of `MAX_NAMED_BINS` per named feature."""
+    return f"named.{binning}.edges.f32"
+
+
+def column_stride(column_bytes: int) -> int:
+    """Byte distance between consecutive columns: ``column_bytes`` rounded up to a whole page (at least one page)."""
+    if column_bytes < 1:
+        raise ValueError(f"column_bytes must be positive, got {column_bytes}")
+    return -(-column_bytes // PAGE_BYTES) * PAGE_BYTES
+
+
+def packed_bytes(n_rows: int) -> int:
+    """Bytes of one 4-bit packed column of ``n_rows`` rows."""
+    if n_rows < 0:
+        raise ValueError(f"n_rows must be non-negative, got {n_rows}")
+    return (n_rows + 1) // 2
+
+
+def pack_nibbles(codes: NDArray[np.uint8]) -> NDArray[np.uint8]:
+    """Pack 4-bit codes two rows per byte along the last axis (low nibble = even row); an odd tail is zero-padded."""
+    if codes.dtype != np.uint8:
+        raise TypeError(f"codes must be uint8, got {codes.dtype}")
+    if codes.size and int(codes.max()) > 0x0F:
+        raise ValueError(f"codes must fit in 4 bits, got {int(codes.max())}")
+    if codes.shape[-1] % 2:
+        codes = np.concatenate([codes, np.zeros((*codes.shape[:-1], 1), np.uint8)], axis=-1)
+    return np.ascontiguousarray(codes[..., 0::2] | (codes[..., 1::2] << _FOUR))
+
+
+def unpack_nibbles(packed: NDArray[np.uint8], lo: int, hi: int) -> NDArray[np.uint8]:
+    """Codes of rows ``[lo, hi)`` from columns packed by `pack_nibbles` (last axis = packed bytes)."""
+    if not 0 <= lo <= hi <= 2 * packed.shape[-1]:
+        raise ValueError(f"rows [{lo}, {hi}) are outside the {2 * packed.shape[-1]} rows these bytes hold")
+    first, stop = lo // 2, (hi + 1) // 2
+    chunk = packed[..., first:stop]
+    codes = np.empty((*chunk.shape[:-1], 2 * chunk.shape[-1]), np.uint8)
+    codes[..., 0::2] = chunk & _LOW
+    codes[..., 1::2] = chunk >> _FOUR
+    start = lo - 2 * first
+    return codes[..., start : start + hi - lo]

idun/data/manifest.py

@@ -0,0 +1,68 @@
+"""The cache manifest: what the cache was built from, how, and the checksum of every array file.
+
+The manifest is the cache's single source of metadata. The reader trusts nothing else: it compares the source
+fingerprints against the files on disk (a stale cache fails loudly) and the array sizes against the files it maps.
+"""
+
+import hashlib
+import json
+from collections.abc import Sequence
+from pathlib import Path
+from typing import Any
+
+from idun.data.layout import FORMAT, FORMAT_VERSION, MANIFEST
+from idun.errors import CacheError
+
+_HASH_CHUNK = 8 * 1024 * 1024
+
+
+def fingerprint_sources(source_dir: Path, names: Sequence[str]) -> list[dict[str, Any]]:
+    """Name, size and modification time of each source file ``names``; a change to any of them makes a cache stale."""
+    prints = []
+    for name in names:
+        path = source_dir / name
+        if not path.is_file():
+            raise CacheError(f"source file {path} is missing")
+        stat = path.stat()
+        prints.append({"name": name, "bytes": stat.st_size, "mtime_ns": stat.st_mtime_ns})
+    return prints
+
+
+def sha256_file(path: Path) -> str:
+    """Hex sha256 of a file, read in chunks."""
+    digest = hashlib.sha256()
+    with path.open("rb") as handle:
+        while chunk := handle.read(_HASH_CHUNK):
+            digest.update(chunk)
+    return digest.hexdigest()
+
+
+def write_manifest(cache_dir: Path, manifest: dict[str, Any]) -> None:
+    """Write ``manifest.json`` (format and version are stamped here)."""
+    stamped = {"format": FORMAT, "format_version": FORMAT_VERSION, **manifest}
+    (cache_dir / MANIFEST).write_text(json.dumps(stamped, indent=1) + "\n")
+
+
+def read_manifest(cache_dir: Path) -> dict[str, Any]:
+    """Read ``manifest.json`` and check its format and version."""
+    path = cache_dir / MANIFEST
+    if not path.is_file():
+        raise CacheError(f"no cache at {cache_dir}: {MANIFEST} is missing (build it with `make cache`)")
+    manifest: dict[str, Any] = json.loads(path.read_text())
+    if manifest.get("format") != FORMAT or manifest.get("format_version") != FORMAT_VERSION:
+        raise CacheError(
+            f"{path}: format {manifest.get('format')!r} version {manifest.get('format_version')!r}; this idun reads "
+            f"{FORMAT!r} version {FORMAT_VERSION}. Rebuild the cache."
+        )
+    return manifest
+
+
+def check_fresh(manifest: dict[str, Any]) -> None:
+    """Raise `CacheError` when a source file changed since the cache was built."""
+    built = {source["name"]: source for source in manifest["sources"]}
+    for now in fingerprint_sources(Path(manifest["source_dir"]), list(built)):
+        if built.get(now["name"]) != now:
+            raise CacheError(
+                f"stale cache: {now['name']} changed since the cache was built "
+                f"({built.get(now['name'])} → {now}). Rebuild the cache."
+            )

idun/data/refits.py

@@ -0,0 +1,17 @@
+"""`Refit`: one walk-forward window of a date-sorted packed dataset.
+
+A packed dataset's manifest lists its windows, and each binning is fitted on one window's training rows. Which dates
+the windows start on is the protocol's choice: Numerai's is `idun_numerai.refits`.
+"""
+
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True)
+class Refit:
+    """One refit: train on rows ``[0, train_hi)``, predict rows ``[pred_lo, pred_hi)`` of the date-sorted panel."""
+
+    date: str
+    train_hi: int
+    pred_lo: int
+    pred_hi: int