islkit 0.1.0__py3-none-any.whl

islkit/labels.py

@@ -0,0 +1,213 @@
+"""Gloss <-> index mapping, dataset assembly, and prediction decoding.
+
+Split from features.py: that module owns the geometry,
+this owns everything in gloss space. One-way dependency, features.py never
+imports from here.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+import numpy as np
+
+from islkit.features import RawFrame, encode_clip
+
+if TYPE_CHECKING:
+    from islkit.data import IncludeData
+
+# --------------------------------------------------------------------------
+# Labels
+# --------------------------------------------------------------------------
+
+
+class LabelMap:
+    """Gloss <-> index, frozen at training time and shipped with the weights.
+
+    Rebuilding this from a directory listing at deploy time is the single
+    most common silent-failure bug in this pipeline: add one sign and every
+    index above it shifts, so the model predicts confidently and wrongly.
+    """
+
+    def __init__(self, glosses: list[str]):
+        self.glosses = list(glosses)
+        self._to_idx = {g: i for i, g in enumerate(self.glosses)}
+
+    @classmethod
+    def from_directory(cls, root: str | Path) -> LabelMap:
+        root = Path(root)
+        glosses = sorted(
+            p.name for p in root.iterdir() if p.is_dir() and not p.name.startswith(".")
+        )
+        return cls(glosses)
+
+    def encode(self, gloss: str) -> int:
+        return self._to_idx[gloss]
+
+    def decode(self, index: int) -> str:
+        return self.glosses[index]
+
+    def save(self, path: str | Path) -> None:
+        Path(path).write_text(
+            json.dumps({"glosses": self.glosses, "n_classes": len(self.glosses)}, indent=2)
+        )
+
+    @classmethod
+    def load(cls, path: str | Path) -> LabelMap:
+        return cls(json.loads(Path(path).read_text())["glosses"])
+
+    def __len__(self) -> int:
+        return len(self.glosses)
+
+
+def read_take(path: str | Path) -> list[RawFrame]:
+    """One take_NN.npz -> list[RawFrame], the layout `ClipStore.save` writes.
+
+    Split out of `build_dataset` so a leave-one-session-out eval can read a
+    single held-out take's raw frames straight through `SignRecogniser`, the
+    exact path live inference uses, instead of the pre-encoded arrays
+    `build_dataset` returns.
+    """
+    d = np.load(Path(path), allow_pickle=True)
+    n = len(d["pose"])
+    return [
+        RawFrame(
+            pose=d["pose"][i] if d["pose"][i] is not None else None,
+            face=d["face"][i] if d["face"][i] is not None else None,
+            hand_left=d["hand_left"][i],
+            hand_right=d["hand_right"][i],
+        )
+        for i in range(n)
+    ]
+
+
+def build_dataset(root: str | Path, T: int = 48, dominant: str = "right"):
+    """Read data/<GLOSS>/<session>/take_NN.npz -> X, y, sessions, label_map.
+
+    Each .npz holds raw landmark arrays saved at capture time, never the
+    encoded features — normalisation will change several times and
+    re-recording is not an option.
+    """
+    root = Path(root)
+    label_map = LabelMap.from_directory(root)
+    X, y, sessions = [], [], []
+
+    for gloss in label_map.glosses:
+        for take in sorted((root / gloss).rglob("*.npz")):
+            frames = read_take(take)
+            X.append(encode_clip(frames, T=T, dominant=dominant))
+            y.append(label_map.encode(gloss))
+            # Session id drives leave-one-session-out splitting. A random
+            # split leaks near-duplicate frames and inflates accuracy by
+            # 10-15 points.
+            sessions.append(take.parent.name)
+
+    return (np.stack(X), np.array(y, np.int64), np.array(sessions), label_map)
+
+
+# --------------------------------------------------------------------------
+# Fine-tuning data assembly
+# --------------------------------------------------------------------------
+
+
+def remap_labels(y: np.ndarray, source_map: LabelMap, target_map: LabelMap) -> np.ndarray:
+    """Re-encode `y` from `source_map`'s class order onto `target_map`'s.
+
+    Two label maps only agree on an index by coincidence — each is built by
+    sorting whatever glosses its own source happens to contain, so index 3 in
+    one is not index 3 in the other. Every remap in this pipeline has to go
+    through the gloss string, never the position (an unfrozen label map's failure mode is a
+    silent index shift; this is the same bug one hop removed from a directory
+    listing).
+
+    Raises if `y` contains a class `target_map` does not have: a class that
+    quietly has no target slot must not train as index noise.
+    """
+    glosses = [source_map.decode(int(c)) for c in y]
+    unknown = sorted(set(glosses) - set(target_map.glosses))
+    if unknown:
+        raise ValueError(f"labels outside the target vocabulary {target_map.glosses}: {unknown}")
+    return np.array([target_map.encode(g) for g in glosses], dtype=np.int64)
+
+
+def pool_finetune_data(
+    include: IncludeData,
+    own_X: np.ndarray,
+    own_y: np.ndarray,
+    own_label_map: LabelMap,
+    target_label_map: LabelMap,
+) -> tuple[np.ndarray, np.ndarray]:
+    """INCLUDE clips (filtered to `target_label_map`) plus own recordings, all
+    on `target_label_map`'s class order. Stage 1's training set.
+
+    Safe to pool INCLUDE here specifically because stage 1 freezes the
+    backbone: a
+    frozen backbone cannot re-anchor to INCLUDE's signer mix, it only gives
+    the new head more label-aligned examples in the pretrained feature space.
+    Stage 2 must NOT reuse this — it fine-tunes on own recordings alone.
+    """
+    own_target_y = remap_labels(own_y, own_label_map, target_label_map)
+
+    target_set = set(target_label_map.glosses)
+    include_glosses = np.array([include.label_map.decode(int(c)) for c in include.y])
+    keep = np.array([g in target_set for g in include_glosses])
+    include_target_y = np.array(
+        [target_label_map.encode(g) for g in include_glosses[keep]], dtype=np.int64
+    )
+
+    X = np.concatenate([include.X[keep], own_X], axis=0)
+    y = np.concatenate([include_target_y, own_target_y], axis=0)
+    return X, y
+
+
+# --------------------------------------------------------------------------
+# Inference
+# --------------------------------------------------------------------------
+
+
+def decode_prediction(
+    probs: np.ndarray, label_map: LabelMap, threshold: float = 0.6
+) -> tuple[str | None, float]:
+    """Softmax vector -> (gloss, confidence). None below threshold.
+
+    Returning None is a feature. A device that admits uncertainty is more
+    trustworthy than one that guesses at a stranger on the user's behalf.
+    """
+    idx = int(np.argmax(probs))
+    conf = float(probs[idx])
+    return (label_map.decode(idx) if conf >= threshold else None), conf
+
+
+class VoteBuffer:
+    """2-of-3 agreement before emitting. Kills flicker for ~100 ms latency."""
+
+    def __init__(self, window: int = 3, agree: int = 2):
+        self.window, self.agree, self.buf = window, agree, []
+
+    def push(self, gloss: str | None) -> str | None:
+        self.buf.append(gloss)
+        if len(self.buf) > self.window:
+            self.buf.pop(0)
+        for g in set(self.buf):
+            if g is not None and self.buf.count(g) >= self.agree:
+                self.buf.clear()
+                return g
+        return None
+
+
+def normalise_gloss(text: str) -> str | None:
+    """Fold a typed label to the dump's convention: lowercase, alphanumeric only.
+
+    Folder names become class labels, so a label typed with a space or a capital
+    is a different class from the one intended — "Good morning" is not
+    "goodmorning", and the mistake produces no error at all. It costs twice: the
+    INCLUDE clips for that sign can no longer be pooled with the new recordings,
+    and two people spelling it differently produce two classes for one sign.
+
+    Returns None when nothing survives, so an empty prompt is an absent label
+    rather than a class named "".
+    """
+    cleaned = "".join(ch for ch in text.lower() if ch.isalnum())
+    return cleaned or None

islkit/metrics.py

@@ -0,0 +1,81 @@
+"""Classification metrics, written out rather than imported from sklearn.
+
+Aggregate accuracy hides a class that never works. Per-class F1 is the metric
+that tells you the truth, so it lives here and gets printed by default.
+"""
+
+import numpy as np
+
+
+def _as_numpy(a) -> np.ndarray:
+    """Accept torch tensors (on any device) or anything array-like."""
+    if hasattr(a, "detach"):
+        a = a.detach().cpu()
+    return np.asarray(a)
+
+
+def accuracy(pred, true) -> float:
+    pred, true = _as_numpy(pred), _as_numpy(true)
+    if len(true) == 0:
+        return 0.0
+    return float((pred == true).mean())
+
+
+def per_class_f1(pred, true, n_classes: int) -> list[float]:
+    """F1 for each class index in range(n_classes).
+
+    A class with no predictions and no true labels scores 0.0 rather than
+    raising — that case shows up whenever a split misses a rare class.
+    """
+    pred, true = _as_numpy(pred), _as_numpy(true)
+    scores = []
+    for c in range(n_classes):
+        tp = int(np.sum((pred == c) & (true == c)))
+        fp = int(np.sum((pred == c) & (true != c)))
+        fn = int(np.sum((pred != c) & (true == c)))
+        denom = 2 * tp + fp + fn
+        scores.append(2 * tp / denom if denom else 0.0)
+    return scores
+
+
+def format_per_class_f1(pred, true, n_classes: int, names: list[str] | None = None) -> str:
+    scores = per_class_f1(pred, true, n_classes)
+    labels = names or [f"class {c}" for c in range(n_classes)]
+    width = max(len(label) for label in labels)
+    return "\n".join(
+        f"  {label:<{width}} : {s:.3f}" for label, s in zip(labels, scores, strict=True)
+    )
+
+
+def confusion_matrix(pred, true, n_classes: int) -> np.ndarray:
+    """(n_classes, n_classes) counts, rows = true class, columns = predicted."""
+    pred, true = _as_numpy(pred).astype(int), _as_numpy(true).astype(int)
+    m = np.zeros((n_classes, n_classes), dtype=np.int64)
+    np.add.at(m, (true, pred), 1)
+    return m
+
+
+def top_confusions(
+    cm: np.ndarray, names: list[str] | None = None, k: int = 25
+) -> list[tuple[str, str, int, float]]:
+    """The k worst off-diagonal cells: (true, predicted, count, share of true).
+
+    A 262x262 confusion image is a texture, not a finding. This is the part of
+    it anyone can act on — which specific pairs the model cannot tell apart,
+    which is also how the demo vocabulary gets pruned of near-collisions.
+    """
+    cm = np.asarray(cm)
+    labels = names or [f"class {c}" for c in range(len(cm))]
+    off = cm.copy()
+    np.fill_diagonal(off, 0)
+    support = cm.sum(axis=1)
+
+    flat = np.argsort(off, axis=None)[::-1][:k]
+    out = []
+    for f in flat:
+        t, p = int(f // len(cm)), int(f % len(cm))
+        if off[t, p] == 0:
+            break
+        share = float(off[t, p] / support[t]) if support[t] else 0.0
+        out.append((labels[t], labels[p], int(off[t, p]), share))
+    return out