islkit 0.1.0__py3-none-any.whl

islkit/infer.py

@@ -0,0 +1,911 @@
+"""Live inference: camera frames in, a gloss or an honest silence out.
+
+This is the first code in the repo that feeds the model something MediaPipe
+produced here rather than something the INCLUDE dump produced elsewhere, so it
+is also the first place the train/serve gap can bite. Two corrections live in
+`to_include_convention` and both are about making live input look like what the
+model was actually trained on; see the tests for why each one matters.
+
+`mediapipe` is imported inside `HolisticExtractor.__init__` rather than at module
+scope, so `SignRecogniser` and `ClipStore` can be imported — and tested — without
+paying for it or requiring it.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import time
+from dataclasses import dataclass
+from datetime import datetime
+from pathlib import Path
+from typing import Any
+
+import numpy as np
+
+from islkit.features import RawFrame, from_holistic
+
+log = logging.getLogger("islkit.infer")
+
+INCLUDE_FACE_POINTS = 468  # what load_include truncates to, and FACE_IDX indexes
+HOLISTIC_FACE_POINTS = 478  # 468 mesh + 10 iris, with refine_face_landmarks=True
+
+
+def to_include_convention(raw: RawFrame) -> RawFrame:
+    """Make a live frame look like one that came out of the INCLUDE dump.
+
+    Two differences, both silent if left alone:
+
+    **Visibility.** The Kaggle dump has no visibility column, so `load_include`
+    writes 1.0 for all 33 pose landmarks whenever the block is present. That
+    value lands in 11 of the 14 validity-mask dims, and the mask is
+    multiplied through the network — so the model has never seen a pose mask
+    that was not exactly 1.0. Live Holistic returns real values across [0, 1],
+    which would attenuate geometry the model expects at full strength.
+
+    **Face length.** `refine_face_landmarks=True` appends 10 iris points.
+    `load_include` truncates to 468. Indices 0..467 are identical so this is
+    cosmetic, but the length check underneath it is not: it turns a version
+    that quietly drops iris refinement into a loud failure instead of four
+    slightly-wrong non-manual scalars.
+    """
+    pose = raw.pose
+    if pose is not None:
+        pose = pose.copy()
+        # 1.0 for what the camera actually saw. MediaPipe reports a position for
+        # every landmark whether or not it is in shot, extrapolating the ones
+        # that are not — and those guesses are bad: hips invented below the frame
+        # land ~2.4 shoulder-widths down against INCLUDE's 1.53, which costs
+        # 100% -> 62.5% top-1. Marking them absent instead restores 100.0%, so
+        # the validity mask is what makes a head-and-torso framing work
+        # at all. This matters for the device, which will not see anyone's hips.
+        xy = pose[:, :2]
+        seen = np.all((xy >= 0.0) & (xy <= 1.0), axis=1)
+        pose[:, 3] = seen.astype(np.float32)
+
+    face = raw.face
+    if face is not None:
+        if face.shape[0] not in (INCLUDE_FACE_POINTS, HOLISTIC_FACE_POINTS):
+            raise ValueError(
+                f"face has {face.shape[0]} landmarks; expected 468 or 478. "
+                "The encoder's FACE_IDX is calibrated against the canonical mesh."
+            )
+        # .copy(), like the pose branch: a slice is a view onto the caller's
+        # array, and the two branches should not differ in whether the returned
+        # frame still aliases the recorded one.
+        face = face[:INCLUDE_FACE_POINTS].copy()
+
+    return RawFrame(pose=pose, face=face, hand_left=raw.hand_left, hand_right=raw.hand_right)
+
+
+# --------------------------------------------------------------------------
+# XNNPACK threading
+# --------------------------------------------------------------------------
+#
+# MediaPipe compiles the Holistic graph with every InferenceCalculator set to
+# `delegate { xnnpack {} }` — an empty block, which XNNPACK reads as ONE
+# thread. That default is invisible and it is expensive: measured on the UNO Q
+# (4 x Kryo-V2 @ 2.0 GHz) stock Holistic uses 0.99 of four cores and delivers
+# ~2.2 fps with a signer in shot, leaving three cores idle.
+#
+# That number decides whether the device works at all. Top-1 against effective
+# frame rate, measured on the pretrained checkpoint:
+#
+#     fps    27    13     9     5     3     2
+#     top-1  100%  100%  100%  100%  52.5%  2.5%
+#
+# encode_clip resamples a take to T frames rather than padding it, so a sign
+# sampled five times is fed to the model as a piecewise-linear trajectory with
+# five knots where training had forty. The TCN keys on exactly the temporal
+# texture that removes, so it degrades to noise — and
+# the model is overconfident out of distribution (0.9978 on random landmarks),
+# so noise arrives as a confident wrong gloss rather than as a decline.
+#
+# Raising the thread count is therefore not tuning. On the board it measured
+# 2.32 -> 3.95 fps at num_threads=2, which is the difference between sitting
+# under the cliff and sitting on it.
+
+_HOLISTIC_BINARYPB = "modules/holistic_landmark/holistic_landmark_cpu.binarypb"
+
+# The two confidence knobs `Holistic.__init__` sets through calculator_params,
+# by their canonical node names once the graph is expanded. Named here because
+# they are long enough to wrap, and a typo in one is silent — SolutionBase
+# raises on an unknown node, but a valid node with the wrong field is not
+# something it can catch.
+_DETECTION_THRESHOLD_PARAM = (
+    "poselandmarkcpu__posedetectioncpu__TensorsToDetectionsCalculator.min_score_thresh"
+)
+_TRACKING_THRESHOLD_PARAM = (
+    "poselandmarkcpu__poselandmarkbyroicpu__tensorstoposelandmarksandsegmentation"
+    "__ThresholdingCalculator.threshold"
+)
+
+
+def _expanded_holistic_config():
+    """The Holistic graph with its subgraphs expanded into concrete nodes.
+
+    The shipped .binarypb holds four subgraph references, not the inference
+    nodes themselves, so the thread setting has nowhere to land until MediaPipe
+    expands it. `CalculatorGraph.binary_config` is that expansion — the only
+    accessor the Python package exposes for it.
+    """
+    import mediapipe as mp
+    from mediapipe.framework import calculator_pb2
+    from mediapipe.python import CalculatorGraph
+
+    shipped = calculator_pb2.CalculatorGraphConfig()
+    shipped.ParseFromString((Path(mp.__file__).parent / _HOLISTIC_BINARYPB).read_bytes())
+
+    expanded = calculator_pb2.CalculatorGraphConfig()
+    expanded.ParseFromString(CalculatorGraph(graph_config=shipped).binary_config)
+    return expanded
+
+
+def xnnpack_threaded_config(num_threads: int):
+    """The Holistic graph with XNNPACK told how many threads it may use.
+
+    Returns `(config, n_patched)`. `n_patched` is reported rather than assumed
+    because the failure this guards against is silent: a patch that matches no
+    node yields a graph that validates, runs, and is still single-threaded.
+
+    Nodes with no options block are skipped deliberately. One inference node in
+    this graph takes its model as a side packet, and materialising an empty
+    options block there gives it an empty `model_path`, which trips
+    InferenceCalculatorCpu's "either model as side packet or model path in
+    options" check and the graph then fails to validate at all.
+    """
+    if num_threads < 1:
+        raise ValueError(f"num_threads must be at least 1, got {num_threads}")
+
+    from mediapipe.calculators.tensor import inference_calculator_pb2
+
+    ext = inference_calculator_pb2.InferenceCalculatorOptions.ext
+    config = _expanded_holistic_config()
+
+    patched = 0
+    for node in config.node:
+        if "Inference" not in node.calculator or not node.options.HasExtension(ext):
+            continue
+        node.options.Extensions[ext].delegate.xnnpack.num_threads = num_threads
+        patched += 1
+
+    if not patched:
+        raise RuntimeError(
+            "found no InferenceCalculator to thread in the Holistic graph. "
+            "mediapipe is pinned to 0.10.18 for the board's sake; if that pin "
+            "has moved, the node layout has moved with it and this needs "
+            "re-deriving rather than silently running at one thread."
+        )
+    return config, patched
+
+
+def _threaded_holistic(
+    num_threads: int,
+    model_complexity: int,
+    refine_face: bool,
+    min_detection_confidence: float,
+    min_tracking_confidence: float,
+):
+    """`mp.solutions.Holistic` rebuilt on a thread-patched graph.
+
+    Holistic itself takes a `binary_graph_path` and offers no way in, but the
+    SolutionBase underneath it accepts a whole `graph_config`. Its
+    `calculator_params` hook cannot reach this setting: it splits a key on '.'
+    and requires exactly two parts, so the three-level
+    `delegate.xnnpack.num_threads` is not expressible there.
+
+    Everything except the graph mirrors `Holistic.__init__` exactly. Diverging
+    would change what the encoder sees, which is the one thing threading must
+    not do.
+    """
+    from mediapipe.python import solution_base
+    from mediapipe.python.solutions.holistic import _download_oss_pose_landmark_model
+
+    # Holistic does this first: complexity 0 and 2 use lite/heavy weights that
+    # are fetched on demand rather than shipped in the wheel.
+    _download_oss_pose_landmark_model(model_complexity)
+
+    config, patched = xnnpack_threaded_config(num_threads)
+
+    class _ThreadedHolistic(solution_base.SolutionBase):
+        def process(self, image):
+            results = super().process(input_data={"image": image})
+            # Holistic.process strips `presence` from both pose lists. Nothing
+            # downstream reads it, but a divergence here is exactly the kind of
+            # quiet difference this whole class exists to avoid.
+            for landmarks in (results.pose_landmarks, results.pose_world_landmarks):
+                if landmarks:
+                    for landmark in landmarks.landmark:
+                        landmark.ClearField("presence")
+            return results
+
+    solution = _ThreadedHolistic(
+        graph_config=config,
+        side_inputs={
+            "model_complexity": model_complexity,
+            # static_image_mode is False throughout this package: these are video
+            # frames from one continuous take, and the tracker's frame-to-frame
+            # state is what keeps hands attached through a sign.
+            "smooth_landmarks": True,
+            "enable_segmentation": False,
+            "smooth_segmentation": False,
+            "refine_face_landmarks": refine_face,
+            "use_prev_landmarks": True,
+        },
+        calculator_params={
+            _DETECTION_THRESHOLD_PARAM: min_detection_confidence,
+            _TRACKING_THRESHOLD_PARAM: min_tracking_confidence,
+        },
+        outputs=[
+            "pose_landmarks",
+            "pose_world_landmarks",
+            "left_hand_landmarks",
+            "right_hand_landmarks",
+            "face_landmarks",
+            "segmentation_mask",
+        ],
+    )
+    log.info(
+        "Holistic graph patched for %d XNNPACK threads across %d inference nodes",
+        num_threads,
+        patched,
+    )
+    return solution
+
+
+class HolisticExtractor:
+    """Owns MediaPipe. Does not mirror — that used to be wrong here, on purpose.
+
+    A sibling project flipped the frame before inference because it ran MediaPipe
+    *Hands*, whose handedness classifier assumes a selfie-mirrored image. This
+    package never reads that classifier's label: `encode_frame` assigns hand slots
+    geometrically, by proximity to the dominant-side shoulder, precisely
+    because the label is unreliable. Carrying the flip over corrected a label
+    this codebase ignores while corrupting the geometry it depends on — a
+    mirrored right-handed signer is geometrically left-handed, so `dominant`
+    then selects the wrong hand for slot 0.
+
+    Measured on 60 random INCLUDE clips through the real 262-class checkpoint:
+    unmirrored **100.0%** top-1, mirrored **28.3%** top-1. Do not reintroduce the
+    flip here. A mirror is a legitimate ergonomic choice for what the signer
+    *sees*, but that is a display concern and belongs to the caller, applied to
+    the frame only after inference and landmark drawing are done with it.
+    """
+
+    def __init__(
+        self,
+        model_complexity: int = 1,
+        refine_face: bool = True,
+        min_detection_confidence: float = 0.5,
+        min_tracking_confidence: float = 0.5,
+        num_threads: int | None = None,
+    ):
+        """`num_threads` is how many XNNPACK threads each inference node may use.
+
+        None keeps MediaPipe's own graph untouched, which means one thread and,
+        on the UNO Q, ~2.2 fps — under the rate at which this model stops
+        working (see the section above). Set it on any machine where the frame
+        rate is what limits recognition; leave it alone on a laptop that is
+        already fast enough, since the threaded path rebuilds a compiled graph
+        and there is no reason to take that on for nothing.
+        """
+        import cv2
+        from mediapipe.python.solutions import drawing_utils, holistic
+
+        self._cv2 = cv2
+        self._mp_holistic = holistic
+        self._drawing = drawing_utils
+        self.num_threads = num_threads
+        if num_threads is None:
+            # NB: the kwarg is refine_face_landmarks. `refine_landmarks` is
+            # FaceMesh's spelling and raises TypeError here.
+            self._holistic = holistic.Holistic(
+                static_image_mode=False,
+                model_complexity=model_complexity,
+                refine_face_landmarks=refine_face,
+                min_detection_confidence=min_detection_confidence,
+                min_tracking_confidence=min_tracking_confidence,
+            )
+        else:
+            self._holistic = _threaded_holistic(
+                num_threads=num_threads,
+                model_complexity=model_complexity,
+                refine_face=refine_face,
+                min_detection_confidence=min_detection_confidence,
+                min_tracking_confidence=min_tracking_confidence,
+            )
+
+    def process(self, frame_bgr: np.ndarray) -> tuple[np.ndarray, RawFrame, Any]:
+        """One camera frame -> (the same frame, unmirrored, raw RawFrame, raw MediaPipe results).
+
+        No flip. INCLUDE is ordinary unmirrored video, so MediaPipe runs on the
+        frame exactly as the camera produced it — that is what makes the
+        landmark geometry line up with training. See the class docstring for the
+        100.0% vs 28.3% measurement this is pinned against. The returned frame is
+        `frame_bgr` itself, handed back so the caller can draw landmarks on it
+        before doing anything else to it (a display-only mirror included).
+
+        The RawFrame is exactly what MediaPipe produced: real visibility, all 478
+        face points. `to_include_convention` is deliberately *not* applied here,
+        because the caller records these frames raw and a correction made at
+        capture time would be baked into data that outlives the decision to make
+        it. The correction belongs to inference, and lives in `SignRecogniser`.
+        """
+        results = self._holistic.process(self._cv2.cvtColor(frame_bgr, self._cv2.COLOR_BGR2RGB))
+        return frame_bgr, from_holistic(results), results
+
+    def draw(self, frame_bgr: np.ndarray, results: Any) -> np.ndarray:
+        """Draw pose and hand landmarks onto the frame `process` returned.
+
+        Call this before any display-only mirror: the landmark coordinates in
+        `results` are relative to the unmirrored frame, so drawing has to happen
+        while frame and coordinates still agree.
+        """
+        h = self._mp_holistic
+        spec = self._drawing.DrawingSpec(thickness=1, circle_radius=1)
+        for landmarks, connections in (
+            (results.pose_landmarks, h.POSE_CONNECTIONS),
+            (results.left_hand_landmarks, h.HAND_CONNECTIONS),
+            (results.right_hand_landmarks, h.HAND_CONNECTIONS),
+        ):
+            if landmarks is not None:
+                self._drawing.draw_landmarks(frame_bgr, landmarks, connections, spec, spec)
+        return frame_bgr
+
+    def close(self) -> None:
+        self._holistic.close()
+
+    def __enter__(self) -> HolisticExtractor:
+        return self
+
+    def __exit__(self, *exc: object) -> None:
+        self.close()
+
+
+class ClipTooShort(ValueError):
+    """Too few frames to mean anything. Refuse rather than resample noise."""
+
+
+class EncoderMismatch(RuntimeError):
+    """The checkpoint was fitted with a different encoder than the one installed.
+
+    Not a warning. The model would run happily and be confidently wrong, which
+    is the one outcome this project refuses (declining is about the device saying
+    "I don't know"; this is about it never saying something it cannot mean).
+    """
+
+
+@dataclass(frozen=True)
+class Prediction:
+    """One classification attempt, including the parts that explain it.
+
+    `gloss` is None when the model declined (low confidence). `usable_frames` exists so
+    a bad answer can be attributed: a clip the tracker never saw is a different
+    failure from a clip the model got wrong, and they look identical in a gloss.
+    """
+
+    gloss: str | None
+    confidence: float
+    top3: list[tuple[str, float]]
+    encode_ms: float
+    forward_ms: float
+    usable_frames: int
+    n_frames: int
+
+
+class SignRecogniser:
+    """list[RawFrame] -> Prediction. No camera, no cv2, no window.
+
+    Deliberately camera-free: it keeps the classification path testable without
+    hardware, and it is the piece the board app imports unchanged later.
+
+    The encoder settings come out of the checkpoint rather than the constructor.
+    The label map is frozen because a rebuilt class order predicts
+    confidently and wrongly; `T` and `dominant` fail the same way, so they are
+    not left to a default that can drift.
+
+    `to_include_convention` runs here rather than in the extractor because it is
+    a train/serve alignment concern, not a capture one: it exists only to make
+    live frames resemble what the head was fitted on. Keeping it on this side
+    leaves recordings raw, and puts the correction on the exact code
+    path the board app imports.
+    """
+
+    def __init__(
+        self,
+        classifier_path: str | Path,
+        labels_path: str | Path | None = None,
+        device=None,
+        threshold: float = 0.6,
+        min_frames: int = 8,
+        match_include: bool = True,
+        allow_encoder_mismatch: bool = False,
+    ):
+        from islkit.labels import LabelMap
+        from islkit.model import load_classifier
+
+        classifier_path = Path(classifier_path)
+        self.model, self.meta = load_classifier(classifier_path, device)
+        self.device = device
+        self.threshold = threshold
+        self.min_frames = min_frames
+        self.match_include = match_include
+
+        encoder = self.meta["encoder"]
+        self.T: int = encoder["T"]
+        self.dominant: str = encoder["dominant"]
+        self.with_velocity: bool = encoder["with_velocity"]
+
+        # The label map ships beside the weights, so the checkpoint
+        # records a filename and we resolve it next to the checkpoint.
+        self.labels_path = Path(labels_path or classifier_path.parent / self.meta["labels"])
+        self.label_map = LabelMap.load(self.labels_path)
+        if len(self.label_map) != self.meta["n_classes"]:
+            raise ValueError(
+                f"{self.labels_path.name} has {len(self.label_map)} glosses but the head "
+                f"is {self.meta['n_classes']} wide. One of them is from a different run."
+            )
+
+        # Import inside, not at module scope: this check runs at construction,
+        # and features is already imported by then via the encoder settings above.
+        from islkit.features import encoder_fingerprint, encoder_signature, signatures_match
+
+        trained_with = self.meta.get("encoder_fingerprint")
+        installed = encoder_fingerprint()
+
+        # Prefer the numeric signature when the checkpoint carries one. The hash
+        # is exact and machines are not: the same features.py on macOS/arm64 and
+        # on the board's linux/aarch64 produces probe clips that differ by up to
+        # 2.4e-07, so hash equality fails a checkpoint that is perfectly good.
+        # Left unfixed, that trains people to pass allow_encoder_mismatch, which
+        # is how a guard against silent confident wrongness becomes decoration.
+        trained_signature = self.meta.get("encoder_signature")
+        if trained_signature is not None:
+            if not signatures_match(trained_signature, encoder_signature()):
+                if not allow_encoder_mismatch:
+                    raise EncoderMismatch(
+                        f"{classifier_path.name} was fitted with an encoder whose "
+                        f"behaviour differs from the installed one (fingerprints "
+                        f"{trained_with} vs {installed}), by more than float noise. "
+                        f"Serving it would feed the model input it has never seen — "
+                        f"silently, and confidently. Retrain, install the matching "
+                        f"islkit, or pass allow_encoder_mismatch=True if you "
+                        f"genuinely mean it."
+                    )
+                log.warning(
+                    "%s was fitted with a materially different encoder. Loading "
+                    "anyway because allow_encoder_mismatch=True — predictions from "
+                    "this model are not trustworthy.",
+                    classifier_path.name,
+                )
+        elif trained_with is None:
+            log.warning(
+                "%s carries no encoder fingerprint, so it cannot be checked against "
+                "the installed encoder (%s). Checkpoints trained before the guard "
+                "existed look like this; anything trained since should not.",
+                classifier_path.name,
+                installed,
+            )
+        elif trained_with != installed:
+            if not allow_encoder_mismatch:
+                raise EncoderMismatch(
+                    f"{classifier_path.name} was fitted with encoder {trained_with}, but "
+                    f"the installed islkit encodes as {installed}. Serving it would feed "
+                    f"the model input it has never seen — silently, and confidently. "
+                    f"Retrain, install the matching islkit, or pass "
+                    f"allow_encoder_mismatch=True if you genuinely mean it."
+                )
+            log.warning(
+                "%s was fitted with encoder %s but the installed islkit encodes as %s. "
+                "Loading anyway because allow_encoder_mismatch=True — predictions from "
+                "this model are not trustworthy.",
+                classifier_path.name,
+                trained_with,
+                installed,
+            )
+
+    def classify(self, frames: list[RawFrame]) -> Prediction:
+        from islkit.features import encode_clip, encode_frame
+        from islkit.labels import decode_prediction
+        from islkit.model import predict_proba
+
+        if len(frames) < self.min_frames:
+            raise ClipTooShort(
+                f"{len(frames)} frames, need {self.min_frames}. encode_clip resamples "
+                "rather than pads, so a shorter clip would classify without complaint."
+            )
+
+        t0 = time.perf_counter()
+        # Once, at the top, and everything downstream sees the corrected frames.
+        # The caller's list is left untouched: it is what gets recorded.
+        if self.match_include:
+            frames = [to_include_convention(f) for f in frames]
+        usable = sum(1 for f in frames if encode_frame(f, self.dominant).any())
+        clip = encode_clip(
+            frames, T=self.T, dominant=self.dominant, with_velocity=self.with_velocity
+        )
+        t1 = time.perf_counter()
+        probs = predict_proba(self.model, clip[None], device=self.device)[0]
+        t2 = time.perf_counter()
+
+        gloss, confidence = decode_prediction(probs, self.label_map, self.threshold)
+        order = np.argsort(probs)[::-1][:3]
+        return Prediction(
+            gloss=gloss,
+            confidence=confidence,
+            top3=[(self.label_map.decode(int(i)), float(probs[i])) for i in order],
+            encode_ms=(t1 - t0) * 1000,
+            forward_ms=(t2 - t1) * 1000,
+            usable_frames=usable,
+            n_frames=len(frames),
+        )
+
+
+class ClipStore:
+    """Every classification attempt, as raw landmarks fine-tuning can read back.
+
+    The layout is `<root>/<gloss>/<session>/take_NN.npz`, which is exactly what
+    `build_dataset` globs and exactly where it reads the session id from — so
+    fine-tuning consumes these recordings with no adapter and no migration.
+
+    Unlabelled attempts go to a **sibling root**, not to a placeholder directory
+    inside the gloss tree, because `LabelMap.from_directory` would otherwise
+    enrol the placeholder as a class and shift every index after it.
+
+    Raw landmarks only, raw visibility included: normalisation will
+    change again, and the corrections in `to_include_convention` belong to
+    inference, not to the data.
+    """
+
+    def __init__(
+        self,
+        root: str | Path = "data/live",
+        unlabelled_root: str | Path = "data/live_unlabelled",
+        session: str | None = None,
+    ):
+        self.root = Path(root)
+        self.unlabelled_root = Path(unlabelled_root)
+        self.session = session or datetime.now().strftime("%Y-%m-%d-%H%M")
+
+    def _next_take(self, directory: Path) -> Path:
+        directory.mkdir(parents=True, exist_ok=True)
+        used = [
+            int(p.stem.split("_")[1])
+            for p in directory.glob("take_*.npz")
+            if p.stem.split("_")[1].isdigit()
+        ]
+        return directory / f"take_{max(used, default=0) + 1:02d}.npz"
+
+    def save(
+        self,
+        frames: list[RawFrame],
+        gloss: str | None,
+        pred: Prediction | None,
+        threshold: float,
+        meta: dict | None = None,
+    ) -> Path:
+        """Write one attempt. Returns the npz path; the sidecar sits beside it."""
+        directory = (
+            self.unlabelled_root / self.session
+            if gloss is None
+            else self.root / gloss / self.session
+        )
+        path = self._next_take(directory)
+
+        # Allocate the object arrays explicitly. np.array(list, dtype=object)
+        # over equal-shaped arrays collapses to a 3-D object array, and
+        # build_dataset would then hand encode_frame an object-dtype slice that
+        # np.linalg.norm cannot take a norm of — at training time, not here.
+        blocks: dict[str, np.ndarray] = {}
+        for key in ("pose", "face", "hand_left", "hand_right"):
+            column = np.empty(len(frames), dtype=object)
+            for i, frame in enumerate(frames):
+                column[i] = getattr(frame, key)
+            blocks[key] = column
+        np.savez_compressed(path, **blocks)
+
+        # meta comes first so it can only add keys, never overwrite them: caller
+        # metadata is supplementary, while the structural fields below — above
+        # all the threshold in force — are what make a take interpretable later,
+        # and a collision must not be able to silently rewrite them.
+        sidecar = {
+            **(meta or {}),
+            "declared_gloss": gloss,
+            "session": self.session,
+            "n_frames": len(frames),
+            "threshold": threshold,
+            "saved_at": datetime.now().isoformat(timespec="seconds"),
+            "predicted_gloss": pred.gloss if pred else None,
+            "confidence": pred.confidence if pred else None,
+            "top3": pred.top3 if pred else None,
+            "usable_frames": pred.usable_frames if pred else None,
+            "encode_ms": pred.encode_ms if pred else None,
+            "forward_ms": pred.forward_ms if pred else None,
+        }
+        path.with_suffix(".json").write_text(json.dumps(sidecar, indent=2))
+        return path
+
+
+# --------------------------------------------------------------------------
+# Framing coach
+# --------------------------------------------------------------------------
+#
+# The first ten field takes were all wrong and none of it was a code defect. INCLUDE
+# clips are roughly half rest frames, arms hanging down, with the sign as a
+# brief excursion; the model learned that whole trajectory. Recorded close-up
+# with the hands already raised, a take sits outside the range of *every* class
+# at once — stripping the rest frames out of INCLUDE's own clips reproduces the
+# failure exactly, turning 12/12 correct into 4/12, confidently wrong.
+#
+# The thresholds below are measured from INCLUDE, not chosen. Shoulder width as
+# a fraction of image width runs p5 0.121 / median 0.140 / p95 0.165, and the
+# dominant wrist sits above y=1.2 in body frame for 52.5% of frames (rest) and
+# below y=0.5 for 40% (signing). The bounds are widened past those percentiles
+# so the coach nudges rather than nags.
+
+# The encoder is provably scale-invariant, so how big you are in frame does not
+# by itself matter. What matters is whether the camera can SEE your hands once
+# they hang at your sides — if it cannot, the take has no rest frames and lands
+# outside every class at once. So the check is geometric headroom, not size:
+# project where a hanging wrist would fall and ask whether it is still in shot.
+# INCLUDE lands at 0.42 of frame height (p95 0.49); 0.85 leaves real margin.
+# Two different quantities, easy to conflate. REST_Y is the threshold above
+# which a wrist counts as resting; SIGNING_Y the one below which it counts as
+# raised. They describe a take's shape, which is recorded as a diagnostic but is
+# NOT a requirement — see the tracking section below for why.
+REST_Y, SIGNING_Y = 1.2, 0.5
+
+# A landmark this close to the border is truncated by it, and the geometry it
+# feeds is quietly wrong rather than missing.
+EDGE_MARGIN = 0.02
+
+_WRIST_Y = 127  # slot-0 wrist height within the encoded frame vector
+_SLOT0_PRESENT = 169
+
+
+def _wrist_height(raw: RawFrame, dominant: str) -> float | None:
+    """Dominant-hand wrist height in body frame, or None if no hand is tracked.
+
+    Read back out of `encode_frame` rather than recomputed, so this and the model
+    always agree about which hand is the dominant one — that choice is geometric
+    and non-obvious, and a second implementation of it would drift.
+    """
+    from islkit.features import encode_frame
+
+    v = encode_frame(raw, dominant)
+    return float(v[_WRIST_Y]) if v[_SLOT0_PRESENT] > 0.5 else None
+
+
+@dataclass(frozen=True)
+class Tracking:
+    """Whether the camera can actually see what the encoder needs."""
+
+    pose: bool
+    hands: int
+    clipped: bool
+    hips_visible: bool
+    ok: bool
+    status: str
+    hint: str
+
+
+def check_tracking(raw: RawFrame, dominant: str = "right") -> Tracking:
+    """Live verdict on one frame: is this trackable?
+
+    Deliberately not a check that the signer matches INCLUDE's framing. The
+    encoder is scale- and translation-invariant, and the pretrained features stay
+    separable with no rest frames at all (leave-one-out 1-NN over 12 classes:
+    100.0% on whole clips, 98.9% on the raised portion alone). Enforcing
+    INCLUDE's setup would train the model for conditions nobody deploys in. What
+    genuinely matters is that the hands are visible and un-truncated.
+    """
+    if raw.pose is None:
+        return Tracking(
+            False, 0, False, False, False, "no pose", "Step into frame — no body detected"
+        )
+
+    hands = [h for h in (raw.hand_left, raw.hand_right) if h is not None]
+
+    # Only the shoulders and the hands. MediaPipe routinely extrapolates hips
+    # and legs beyond the visible frame, which is normal at close framing and
+    # would make an all-landmark check fire on essentially every real frame.
+    # These are the landmarks the geometry actually rests on: the shoulders fix
+    # the body frame, and a truncated hand is a wrong handshape rather than a
+    # missing one.
+    clipped = False
+    for xy in (raw.pose[[11, 12], :2], *(h[:, :2] for h in hands)):
+        if xy.size and (xy.min() < EDGE_MARGIN or xy.max() > 1.0 - EDGE_MARGIN):
+            clipped = True
+            break
+
+    # Reported, deliberately NOT a failure. The device this feeds will be a small
+    # unit framing head and torso, so hips are off-camera by design. Trusting
+    # MediaPipe's extrapolated hips would cost 100% -> 62.5% top-1, but
+    # `to_include_convention` marks them absent instead and that restores 100.0%.
+    # Feature separability in full device conditions — no hips, no rest frames —
+    # measures 98.3% against 100.0% in the lab. So this is worth recording and
+    # not worth refusing over.
+    hips_visible = bool(np.all((raw.pose[[23, 24], 1] >= 0.0) & (raw.pose[[23, 24], 1] <= 1.0)))
+
+    if clipped:
+        return Tracking(
+            True,
+            len(hands),
+            True,
+            hips_visible,
+            False,
+            "clipped",
+            "Part of you is at the edge of frame — move back or centre yourself",
+        )
+    if not hands:
+        # Normal between signs, so reported without being called an error.
+        return Tracking(True, 0, False, hips_visible, True, "no hands", "Hands not visible")
+    return Tracking(True, len(hands), False, hips_visible, True, "ok", "Tracking")
+
+
+@dataclass(frozen=True)
+class TakeQuality:
+    """Whether a recorded take was seen well enough to be worth keeping."""
+
+    frames: int
+    pose_fraction: float
+    hand_fraction: float
+    clipped_fraction: float
+    usable: bool
+    hint: str
+
+
+def take_quality(frames: list[RawFrame], dominant: str = "right") -> TakeQuality:
+    """Post-hoc tracking check. Reported, never enforced — the clip is saved regardless."""
+    if not frames:
+        return TakeQuality(0, 0.0, 0.0, 0.0, False, "Empty take")
+
+    checks = [check_tracking(f, dominant) for f in frames]
+    n = len(checks)
+    pose_f = sum(c.pose for c in checks) / n
+    hand_f = sum(c.hands > 0 for c in checks) / n
+    clip_f = sum(c.clipped for c in checks) / n
+
+    # Ordered most-fundamental first: a body the camera never found is a
+    # different problem from a body it found but only partly framed, and
+    # reporting the framing issue there would send the signer to fix the wrong
+    # thing.
+    if pose_f < 0.8:
+        return TakeQuality(n, pose_f, hand_f, clip_f, False, "Could not track your body reliably")
+    if hand_f < 0.4:
+        return TakeQuality(n, pose_f, hand_f, clip_f, False, "Hands were tracked in too few frames")
+    if clip_f > 0.3:
+        return TakeQuality(n, pose_f, hand_f, clip_f, False, "You were clipped by the frame edge")
+    return TakeQuality(n, pose_f, hand_f, clip_f, True, "Tracked cleanly")
+
+
+@dataclass(frozen=True)
+class TakeAssessment:
+    """Whether a finished take has the shape INCLUDE clips have."""
+
+    rest_fraction: float
+    signing_fraction: float
+    starts_at_rest: bool
+    ends_at_rest: bool
+    looks_like_include: bool
+    hint: str
+
+
+def assess_take(frames: list[RawFrame], dominant: str = "right") -> TakeAssessment:
+    """Post-hoc check on a recorded clip, so a bad one is caught at take 1.
+
+    Reported rather than enforced: the clip is saved either way (a refused take
+    is still evidence), but the signer is told immediately instead of finding
+    out after two hundred of them.
+    """
+    heights = [_wrist_height(f, dominant) for f in frames]
+    tracked = [h for h in heights if h is not None]
+    if not tracked:
+        return TakeAssessment(0.0, 0.0, False, False, False, "No hand tracked in this take")
+
+    rest = sum(h > REST_Y for h in tracked) / len(tracked)
+    signing = sum(h < SIGNING_Y for h in tracked) / len(tracked)
+    starts = heights[0] is not None and heights[0] > REST_Y
+    ends = heights[-1] is not None and heights[-1] > REST_Y
+    ok = rest >= 0.25 and signing >= 0.15
+
+    if signing < 0.15:
+        hint = "Nothing was signed — the hands never left the rest position"
+    elif rest < 0.25:
+        hint = "Hands stayed up — begin and end each take with them resting down"
+    elif not starts:
+        hint = "Started mid-sign — rest first, then raise"
+    elif not ends:
+        hint = "Ended mid-sign — return to rest before stopping"
+    else:
+        hint = "Good shape — rest, sign, rest"
+    return TakeAssessment(rest, signing, starts, ends, ok, hint)
+
+
+class AutoTake:
+    """Rest → raise → rest, captured without touching the keyboard.
+
+    The signer has to stand far enough back for their hands-at-rest to be in
+    shot, which puts them out of reach of the spacebar. Rest is already the
+    delimiter a usable take depends on, so it serves as the trigger too — and
+    triggering on it makes a badly-shaped take structurally impossible instead
+    of merely detected afterwards.
+
+    The pre-roll is the part that matters. A take that began the instant the
+    hands moved would contain no rest frames at all, which is precisely the
+    shape that made the first ten field recordings unusable. A rolling window of
+    recent frames is seeded into every take so the leading rest is always there.
+
+    Not the auto-segmentation ruled out earlier: that was about carving a continuous
+    stream of signing into signs. This is a recording aid delimited by an
+    explicit, deliberate return to rest, and it still emits exactly one take per
+    gesture.
+    """
+
+    def __init__(
+        self,
+        dominant: str = "right",
+        pre_roll: int = 12,
+        rest_to_arm: int = 6,
+        raised_to_start: int = 6,
+        rest_to_close: int = 8,
+        max_frames: int = 400,
+    ):
+        from collections import deque
+
+        self.dominant = dominant
+        self.rest_to_arm = rest_to_arm
+        self.raised_to_start = raised_to_start
+        self.rest_to_close = rest_to_close
+        self.max_frames = max_frames
+        self._recent: deque = deque(maxlen=pre_roll)
+        self._buf: list[RawFrame] = []
+        self._rest_run = 0
+        self._raised_run = 0
+        self.state = "waiting"
+
+    def disarm(self) -> None:
+        """Drop any take in progress and require rest again before re-arming."""
+        self._buf = []
+        self._recent.clear()
+        self._rest_run = self._raised_run = 0
+        self.state = "waiting"
+
+    @property
+    def n_frames(self) -> int:
+        return len(self._buf)
+
+    def update(self, raw: RawFrame) -> list[RawFrame] | None:
+        """Feed one frame. Returns a finished take, or None.
+
+        An untracked hand counts as rest rather than as unknown: MediaPipe loses
+        hands constantly, especially held still against the body, and treating a
+        dropout as ambiguous would either stall arming or chop a sign in half.
+        """
+        y = _wrist_height(raw, self.dominant)
+        raised = y is not None and y < SIGNING_Y
+        at_rest = y is None or y > REST_Y
+
+        self._rest_run = self._rest_run + 1 if at_rest else 0
+        self._raised_run = self._raised_run + 1 if raised else 0
+        self._recent.append(raw)
+
+        if self.state == "waiting":
+            if self._rest_run >= self.rest_to_arm:
+                self.state = "armed"
+            return None
+
+        if self.state == "armed":
+            if self._raised_run >= self.raised_to_start:
+                self._buf = list(self._recent)
+                self.state = "recording"
+            return None
+
+        # recording
+        self._buf.append(raw)
+        if self._rest_run >= self.rest_to_close or len(self._buf) >= self.max_frames:
+            take, self._buf = self._buf, []
+            self._raised_run = 0
+            self.state = "armed"
+            return take
+        return None