islkit 0.1.0__py3-none-any.whl

islkit/data.py

@@ -0,0 +1,320 @@
+"""Ingest the Kaggle INCLUDE landmark dump into encoded feature arrays.
+
+The dump is long-format parquet, one row per landmark per frame:
+
+    frame | row_id | type | landmark_index | x | y | z
+
+with 553 rows per frame — face(478) + left_hand(21) + pose(33) + right_hand(21),
+in that order.
+
+Three things about this source differ from what features.py was written against,
+and each is handled here rather than by changing the encoder:
+
+  1. **Missing parts are NaN, not zero.** Other public 1662-vector dumps
+     zero-fill missing parts. This one uses NaN, which is strictly better
+     — no ambiguity with a landmark that genuinely sits at the origin. Either way the
+     part becomes None and the validity mask carries it.
+  2. **Face has 478 landmarks, not 468.** That is refine_landmarks=True: 468 canonical
+     mesh points plus 10 iris points appended at 468..477. Indices 0..467 are unchanged,
+     and FACE_IDX only reaches 454, so the block is truncated to [:468] and the four
+     non-manual scalars are computed exactly as they are on live capture.
+  3. **There is no visibility column.** encode_frame writes pose visibility straight
+     into the validity mask, so it is synthesised here as 1.0 present / 0.0 NaN. This
+     is a real pretrain/deploy difference: 11 of the 183 dims are hard 0/1 here and
+     continuous in [0, 1] from live MediaPipe. Flagged deliberately — do not
+     paper over it by "fixing" the encoder.
+
+**There is no signer identifier in this dump.** Not in filenames, folders, parquet
+metadata, or any manifest — the archive contains only keypoints/ and label_map.json.
+`load_include` therefore returns `signer=None`, deliberately, so that a caller reaching
+for a group-held-out split fails loudly instead of silently splitting at random.
+A random frame-level split over this data inflates accuracy by 10-15 points.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from dataclasses import dataclass
+from pathlib import Path
+
+import numpy as np
+
+from islkit import features
+from islkit.features import RawFrame, encode_clip
+from islkit.labels import LabelMap
+
+# --------------------------------------------------------------------------
+# Source layout
+# --------------------------------------------------------------------------
+
+N_FACE_RAW = 478  # 468 canonical mesh + 10 iris (refine_landmarks=True)
+N_FACE_USE = 468  # what FACE_IDX indexes into
+N_HAND = 21
+N_POSE = 33
+
+# Row order within one frame. Verified per file before the fast reshape path.
+BLOCKS = (("face", N_FACE_RAW), ("left_hand", N_HAND), ("pose", N_POSE), ("right_hand", N_HAND))
+ROWS_PER_FRAME = sum(n for _, n in BLOCKS)  # 553
+
+_OFFSETS = {}
+_acc = 0
+for _name, _n in BLOCKS:
+    _OFFSETS[_name] = (_acc, _acc + _n)
+    _acc += _n
+
+# Expected landmark_index sequence for one frame. Reading this int32 column is
+# cheap and pins the block order without materialising the string `type` column.
+_EXPECTED_INDEX = np.concatenate([np.arange(n, dtype=np.int32) for _, n in BLOCKS])
+
+# Bump when the reading rules above change in a way that alters the output
+# arrays. Feeds the cache fingerprint alongside the hash of features.py.
+READER_VERSION = "1"
+
+
+class LayoutError(ValueError):
+    """A parquet file does not match the layout this module was written for."""
+
+
+# --------------------------------------------------------------------------
+# Reading one clip
+# --------------------------------------------------------------------------
+
+
+def read_clip(path: str | Path) -> list[RawFrame]:
+    """One INCLUDE parquet -> list of RawFrame, one per source frame.
+
+    Raises LayoutError rather than guessing if the row layout is not the
+    expected 553-row canonical block order. Assuming a layout and being wrong
+    trains fine and predicts nonsense, which is the whole reason this module inspects
+    before it ingests.
+    """
+    import pyarrow.parquet as pq  # noqa: PLC0415 — [train]-only, deferred so bare islkit imports
+
+    path = Path(path)
+    tbl = pq.read_table(path, columns=["landmark_index", "x", "y", "z"])
+    n_rows = tbl.num_rows
+
+    if n_rows == 0 or n_rows % ROWS_PER_FRAME:
+        raise LayoutError(f"{path.name}: {n_rows} rows is not a multiple of {ROWS_PER_FRAME}")
+    n_frames = n_rows // ROWS_PER_FRAME
+
+    idx = tbl.column("landmark_index").to_numpy(zero_copy_only=False)
+    idx = idx.reshape(n_frames, ROWS_PER_FRAME)
+    if not np.array_equal(idx[0], _EXPECTED_INDEX) or not (idx == idx[0]).all():
+        raise LayoutError(f"{path.name}: landmark rows are not in the canonical block order")
+
+    xyz = np.empty((n_rows, 3), np.float32)
+    for j, col in enumerate(("x", "y", "z")):
+        xyz[:, j] = tbl.column(col).to_numpy(zero_copy_only=False)
+    xyz = xyz.reshape(n_frames, ROWS_PER_FRAME, 3)
+
+    def block(name: str) -> np.ndarray:
+        a, b = _OFFSETS[name]
+        return xyz[:, a:b, :]
+
+    face_all, lh_all = block("face"), block("left_hand")
+    pose_all, rh_all = block("pose"), block("right_hand")
+
+    # A part is absent when the tracker lost it, which this dump marks as NaN
+    # across the whole block. Test with .any() rather than .all() so a partially
+    # NaN block is treated as lost instead of silently encoding NaN into the
+    # feature vector, where it would propagate through every downstream sum.
+    face_lost = np.isnan(face_all).any(axis=(1, 2))
+    lh_lost = np.isnan(lh_all).any(axis=(1, 2))
+    rh_lost = np.isnan(rh_all).any(axis=(1, 2))
+    pose_lost = np.isnan(pose_all).any(axis=(1, 2))
+
+    frames: list[RawFrame] = []
+    for t in range(n_frames):
+        pose = None
+        if not pose_lost[t]:
+            # encode_frame reads column 3 as visibility and writes it into the
+            # validity mask. The dump has no such column, so presence stands in.
+            pose = np.concatenate([pose_all[t], np.ones((N_POSE, 1), np.float32)], axis=1)
+        frames.append(
+            RawFrame(
+                pose=pose,
+                face=None if face_lost[t] else face_all[t, :N_FACE_USE],
+                hand_left=None if lh_lost[t] else lh_all[t],
+                hand_right=None if rh_lost[t] else rh_all[t],
+            )
+        )
+    return frames
+
+
+def both_hands_fraction(frames: list[RawFrame]) -> float:
+    """Share of source frames where the tracker held both hands at once."""
+    if not frames:
+        return 0.0
+    both = sum(f.hand_left is not None and f.hand_right is not None for f in frames)
+    return both / len(frames)
+
+
+# --------------------------------------------------------------------------
+# Cache identity
+# --------------------------------------------------------------------------
+
+
+def encoder_fingerprint(T: int, dominant: str, with_velocity: bool) -> str:
+    """Short hash identifying exactly which features these arrays are.
+
+    Hashes the *source* of features.py, not a hand-maintained version constant:
+    a cache that survives an encoder edit is worse than no cache, and a version
+    number someone forgets to bump is precisely how that happens.
+    """
+    h = hashlib.sha256()
+    h.update(Path(features.__file__).read_bytes())
+    h.update(READER_VERSION.encode())
+    h.update(f"T={T};dominant={dominant};velocity={with_velocity}".encode())
+    return h.hexdigest()[:12]
+
+
+def cache_path(cache_dir: str | Path, T: int, dominant: str, with_velocity: bool) -> Path:
+    fp = encoder_fingerprint(T, dominant, with_velocity)
+    return Path(cache_dir) / f"include_{features.DIM_FRAME}_{fp}.npz"
+
+
+# --------------------------------------------------------------------------
+# The dataset
+# --------------------------------------------------------------------------
+
+
+@dataclass
+class IncludeData:
+    """Encoded INCLUDE clips plus everything needed to split them honestly.
+
+    `signer` is None because the dump carries no signer identifier. It is kept
+    in the signature so the absence is visible at every call site rather than
+    being something a reader has to know.
+    """
+
+    X: np.ndarray  # (N, T, 352) float32
+    y: np.ndarray  # (N,) int64
+    signer: np.ndarray | None  # None for this source — see module docstring
+    label_map: LabelMap
+    clips: np.ndarray  # (N,) source path relative to the dataset root
+    category: np.ndarray  # (N,) INCLUDE's own topic folder, e.g. "adjectives"
+    both_hands: np.ndarray  # (N,) fraction of source frames with both hands
+    n_frames: np.ndarray  # (N,) source frame count, before resampling to T
+
+    def __len__(self) -> int:
+        return len(self.y)
+
+
+def _clip_files(root: Path) -> list[Path]:
+    return sorted((root / "keypoints").rglob("*.parquet"))
+
+
+def load_include(
+    root: str | Path,
+    T: int = 48,
+    dominant: str = "right",
+    with_velocity: bool = True,
+    cache_dir: str | Path = "data/cache",
+    rebuild: bool = False,
+    limit: int | None = None,
+    verbose: bool = True,
+) -> IncludeData:
+    """Encode the INCLUDE dump to (N, T, 352), caching the result.
+
+    The cache filename carries a hash of features.py, so editing the encoder
+    invalidates it automatically and the next call re-encodes from the raw
+    parquet. Raw landmarks are never cached in encoded form anywhere else —
+    normalisation will change, the recordings will not.
+
+    `limit` truncates to the first N clips for a smoke run and bypasses the
+    cache entirely, so a partial run can never be mistaken for a full one.
+    """
+    root = Path(root)
+    path = cache_path(cache_dir, T, dominant, with_velocity)
+
+    if limit is None and not rebuild and path.exists():
+        if verbose:
+            print(f"cache hit: {path}")
+        return _from_cache(path)
+
+    files = _clip_files(root)
+    if not files:
+        raise FileNotFoundError(f"no parquet clips under {root / 'keypoints'}")
+    if limit is not None:
+        files = files[:limit]
+
+    # Label map is built from the directories that actually hold clips, then
+    # frozen and shipped beside the weights. The dump's own label_map.json
+    # lists one class with no data, so trusting it would leave a dead output
+    # unit and shift every index above it.
+    glosses = sorted({p.parent.name for p in files})
+    label_map = LabelMap(glosses)
+
+    X, y, clips, cats, both, lens = [], [], [], [], [], []
+    for i, p in enumerate(files):
+        if verbose and i % 250 == 0:
+            print(f"  encoding {i}/{len(files)}")
+        frames = read_clip(p)
+        X.append(encode_clip(frames, T=T, dominant=dominant, with_velocity=with_velocity))
+        y.append(label_map.encode(p.parent.name))
+        clips.append(str(p.relative_to(root)))
+        cats.append(p.parent.parent.name)
+        both.append(both_hands_fraction(frames))
+        lens.append(len(frames))
+
+    data = IncludeData(
+        X=np.stack(X).astype(np.float32),
+        y=np.array(y, np.int64),
+        signer=None,
+        label_map=label_map,
+        clips=np.array(clips),
+        category=np.array(cats),
+        both_hands=np.array(both, np.float32),
+        n_frames=np.array(lens, np.int32),
+    )
+
+    if limit is None:
+        _to_cache(path, data, root, T, dominant, with_velocity)
+        if verbose:
+            print(f"cached: {path} ({path.stat().st_size / 1e6:.0f} MB)")
+    return data
+
+
+def _to_cache(path: Path, d: IncludeData, root: Path, T, dominant, with_velocity) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    meta = {
+        "source_root": str(root),
+        "reader_version": READER_VERSION,
+        "encoder_fingerprint": encoder_fingerprint(T, dominant, with_velocity),
+        "T": T,
+        "dominant": dominant,
+        "with_velocity": with_velocity,
+        "signer": None,
+        "signer_note": "INCLUDE dump carries no signer identifier",
+    }
+    # Uncompressed: dense float32 barely compresses and training will reload this
+    # on every run. Gitignored either way.
+    np.savez(
+        path,
+        X=d.X,
+        y=d.y,
+        clips=d.clips,
+        category=d.category,
+        both_hands=d.both_hands,
+        n_frames=d.n_frames,
+        glosses=np.array(d.label_map.glosses),
+        meta=json.dumps(meta),
+    )
+    label_json = path.with_name(path.stem + "_labels.json")
+    d.label_map.save(label_json)
+
+
+def _from_cache(path: Path) -> IncludeData:
+    z = np.load(path, allow_pickle=False)
+    return IncludeData(
+        X=z["X"],
+        y=z["y"],
+        signer=None,
+        label_map=LabelMap(list(z["glosses"])),
+        clips=z["clips"],
+        category=z["category"],
+        both_hands=z["both_hands"],
+        n_frames=z["n_frames"],
+    )

islkit/device.py

@@ -0,0 +1,38 @@
+"""Where the tensors go.
+
+On this machine that is almost always MPS (Apple Silicon GPU). The fallback
+order is deliberate: MPS, then CUDA, then CPU.
+"""
+
+import torch
+
+
+def pick_device(prefer: str | None = None) -> torch.device:
+    """Return the best available device.
+
+    Pass `prefer` ("cpu", "mps", "cuda") to override — useful when you suspect
+    an MPS-specific bug and want to check the same code on CPU.
+    """
+    if prefer is not None:
+        return torch.device(prefer)
+    if torch.backends.mps.is_available():
+        return torch.device("mps")
+    if torch.cuda.is_available():
+        return torch.device("cuda")
+    return torch.device("cpu")
+
+
+def describe_device() -> str:
+    """A one-block summary of the compute situation, for sanity checks."""
+    lines = [
+        f"torch             : {torch.__version__}",
+        f"selected device   : {pick_device()}",
+        f"mps available     : {torch.backends.mps.is_available()}",
+        f"mps built         : {torch.backends.mps.is_built()}",
+        f"cuda available    : {torch.cuda.is_available()}",
+    ]
+    return "\n".join(lines)
+
+
+if __name__ == "__main__":
+    print(describe_device())

islkit/domain.py

@@ -0,0 +1,187 @@
+"""Make an INCLUDE clip look like something the device's camera produced.
+
+The pretrained head is accurate on INCLUDE and wrong on the board, and the gap
+is not the encoder — INCLUDE clips score 100% top-1
+through the full inference path. It is that INCLUDE clips and board takes are
+drawn from different distributions in four measurable ways. Every constant here
+is measured off the board (2026-09-07, 47 takes), not chosen:
+
+| property        | INCLUDE            | board                       |
+|-----------------|--------------------|-----------------------------|
+| frame rate      | ~25 fps native     | 2.1 - 4.7, median ~3        |
+| hands tracked   | near always        | 43% of frames (0.22 - 0.87) |
+| dropout runs    | --                 | 1-5 frames, mode 2          |
+| hips            | in shot            | below the frame             |
+| rest frames     | ~half the clip     | close framing loses them    |
+
+Training on the raw dump and serving this is the train/serve skew that feature design and
+masking exist to prevent, one level up: not a wrong feature, a wrong
+*distribution*. So rather than demanding the signer stand where INCLUDE's
+signers stood — which is not available on a handheld device — the training
+clips are pushed into the device's domain and the model learns that instead.
+
+This augments RAW frames and never encoded ones, for the same reason recordings
+store raw landmarks: the encoder will change again, and an augmentation baked
+into encoded features could not be re-derived.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, replace
+
+import numpy as np
+
+from islkit.features import RawFrame
+
+# Body-frame wrist height above which a wrist counts as resting. Imported
+# rather than redefined so this and the segmenter can never disagree about
+# what "at rest" means.
+from islkit.infer import REST_Y, _wrist_height
+
+# BlazePose's lower body. On a handheld unit framing head and torso these are
+# never in shot, and MediaPipe extrapolates them anyway — the fabricated hips
+# land ~2.4 shoulder-widths down against INCLUDE's 1.53, which cost 100% ->
+# 62.5% top-1 until `to_include_convention` started masking them.
+LOWER_BODY = [23, 24, 25, 26, 27, 28, 29, 30, 31, 32]
+
+INCLUDE_FPS = 25.0  # the dump's native rate
+
+
+@dataclass(frozen=True)
+class DeviceProfile:
+    """The board's camera, as measured. Defaults are measured on an Arduino UNO Q."""
+
+    fps_low: float = 2.1
+    fps_high: float = 4.7
+    source_fps: float = INCLUDE_FPS
+    # 1 - hand_fraction, median 0.43 across 47 takes
+    hand_dropout: float = 0.45
+    # Consecutive frames with no dominant wrist, measured {1:15, 2:61, 3:27, 5:2}
+    run_lengths: tuple[int, ...] = (1, 2, 3, 5)
+    run_weights: tuple[float, ...] = (0.143, 0.581, 0.257, 0.019)
+    hide_lower_body: bool = True
+    crop_rest: bool = True
+
+    def __post_init__(self) -> None:
+        if len(self.run_lengths) != len(self.run_weights):
+            raise ValueError("run_lengths and run_weights must be the same length")
+        if not 0.0 <= self.hand_dropout < 1.0:
+            raise ValueError(f"hand_dropout must be in [0, 1), got {self.hand_dropout}")
+        if self.fps_low <= 0 or self.fps_high < self.fps_low:
+            raise ValueError("need 0 < fps_low <= fps_high")
+
+
+def subsample_to_rate(
+    frames: list[RawFrame], target_fps: float, source_fps: float, rng: np.random.Generator
+) -> list[RawFrame]:
+    """Keep every Nth frame, so a 25 fps clip becomes a 3 fps one.
+
+    This is the augmentation that matters most. `encode_clip` resamples to T
+    frames rather than padding, so a sign the camera sampled five times reaches
+    the model as a piecewise-linear trajectory with five knots where training
+    had forty — and the TCN keys on exactly the fine temporal texture that
+    removes. That is why top-1 falls off a cliff below
+    5 fps. Training on sparse trajectories puts the model's expectations where
+    the device actually operates instead of demanding hardware that cannot
+    exist on a Cortex-A53.
+
+    The phase is random so the same clip yields different sparse views.
+    """
+    step = max(1, int(round(source_fps / max(target_fps, 0.1))))
+    if step == 1:
+        return list(frames)
+    offset = int(rng.integers(0, step))
+    return frames[offset::step] or frames[:1]
+
+
+def drop_hands(
+    frames: list[RawFrame], profile: DeviceProfile, rng: np.random.Generator
+) -> list[RawFrame]:
+    """Lose the hands in runs, the way MediaPipe actually loses them.
+
+    Dropping independent frames at 45% would be the wrong noise: the board's
+    losses come in runs of 1-5 (mode 2) because the tracker needs a few frames
+    to re-acquire. A model trained against independent noise would never see
+    the gap structure it has to survive at inference.
+    """
+    if profile.hand_dropout <= 0:
+        return list(frames)
+
+    out = list(frames)
+    weights = np.asarray(profile.run_weights, float)
+    weights = weights / weights.sum()
+    target = int(round(profile.hand_dropout * len(out)))
+    dropped, guard = 0, 0
+    while dropped < target and guard < 100:
+        guard += 1
+        run = int(rng.choice(profile.run_lengths, p=weights))
+        start = int(rng.integers(0, max(1, len(out))))
+        for i in range(start, min(start + run, len(out))):
+            if out[i].hand_left is None and out[i].hand_right is None:
+                continue
+            out[i] = replace(out[i], hand_left=None, hand_right=None)
+            dropped += 1
+    return out
+
+
+def hide_lower_body(frames: list[RawFrame]) -> list[RawFrame]:
+    """Push the hips and legs out of shot, as a head-and-torso framing does.
+
+    Moved outside [0, 1] rather than deleted, because that is what the device
+    hands the encoder: MediaPipe still reports a position for an off-camera
+    landmark, and `to_include_convention` is what marks it absent. Simulating
+    the absence any other way would train against a mask the live path never
+    produces.
+    """
+    out = []
+    for frame in frames:
+        if frame.pose is None:
+            out.append(frame)
+            continue
+        pose = frame.pose.copy()
+        pose[LOWER_BODY, 1] = np.maximum(pose[LOWER_BODY, 1], 1.05)
+        out.append(replace(frame, pose=pose))
+    return out
+
+
+def crop_to_signing(frames: list[RawFrame], dominant: str = "right") -> list[RawFrame]:
+    """Drop the leading and trailing rest, keeping the excursion.
+
+    A close-framed take has no rest frames because the signer's hands leave the
+    bottom of the picture when lowered — measured on the board, the wrist was
+    above REST_Y in 2 of 92 frames across three takes. INCLUDE clips are
+    roughly half rest, so a model trained on them expects a shape the device
+    cannot deliver.
+
+    Returns the clip unchanged when there is no rest to remove, so a clip that
+    is already all signing is not truncated to nothing.
+    """
+    heights = [_wrist_height(f, dominant) for f in frames]
+    active = [i for i, y in enumerate(heights) if y is not None and y <= REST_Y]
+    if not active:
+        return list(frames)
+    return frames[active[0] : active[-1] + 1]
+
+
+def to_device_domain(
+    frames: list[RawFrame],
+    profile: DeviceProfile | None = None,
+    rng: np.random.Generator | None = None,
+) -> list[RawFrame]:
+    """One INCLUDE clip -> one plausible board take.
+
+    Order matters. Cropping happens first, while the wrist track is still dense
+    enough to locate the excursion; the hands are dropped last, so the dropout
+    rate applies to the frames that actually survive into the take.
+    """
+    profile = profile or DeviceProfile()
+    rng = rng or np.random.default_rng()
+
+    out = list(frames)
+    if profile.crop_rest:
+        out = crop_to_signing(out)
+    target_fps = float(rng.uniform(profile.fps_low, profile.fps_high))
+    out = subsample_to_rate(out, target_fps, profile.source_fps, rng)
+    if profile.hide_lower_body:
+        out = hide_lower_body(out)
+    return drop_hands(out, profile, rng)