islkit 0.1.0__py3-none-any.whl

islkit/features.py

@@ -0,0 +1,323 @@
+"""MediaPipe landmarks to feature vectors.
+
+Backend-neutral: the RawFrame dataclass is the contract. Legacy
+mp.solutions.holistic and the Tasks API landmarkers both adapt into it,
+so nothing downstream changes if you swap backends.
+
+Feature vector: 183 floats per frame, 352 with velocity.
+    hand-local shapes   2 x 21 x 3 = 126
+    wrist positions     2 x 3      =   6
+    upper-body pose     11 x 3     =  33
+    non-manual scalars              =   4
+    validity mask                   =  14
+"""
+
+from __future__ import annotations
+
+import hashlib
+from dataclasses import dataclass
+
+import numpy as np
+
+# --------------------------------------------------------------------------
+# Landmark index selections
+# --------------------------------------------------------------------------
+
+# BlazePose 33-point model. Legs discarded — ISL signing space is torso-up.
+POSE_SEL = [
+    0,  # nose
+    7,
+    8,  # ears — head orientation
+    11,
+    12,  # shoulders — the body frame
+    13,
+    14,  # elbows
+    15,
+    16,  # wrists
+    23,
+    24,  # hips — torso scale reference
+]
+L_SHOULDER, R_SHOULDER = 11, 12
+
+# Canonical 468-point face mesh. Used only to derive 4 scalars.
+FACE_IDX = {
+    "brow_l": 105,
+    "brow_r": 334,
+    "brow_inner_l": 55,
+    "brow_inner_r": 285,
+    "eye_upper_l": 159,
+    "eye_lower_l": 145,
+    "eye_upper_r": 386,
+    "eye_lower_r": 374,
+    "lip_upper": 13,
+    "lip_lower": 14,
+    "mouth_l": 61,
+    "mouth_r": 291,
+    "forehead": 10,
+    "chin": 152,
+    "cheek_l": 234,
+    "cheek_r": 454,
+}
+
+N_HAND, N_POSE_SEL = 21, len(POSE_SEL)
+DIM_GEOM = 2 * N_HAND * 3 + 2 * 3 + N_POSE_SEL * 3 + 4  # 169
+DIM_MASK = 2 + N_POSE_SEL + 1  # 14
+DIM_FRAME = DIM_GEOM + DIM_MASK  # 183
+
+
+# --------------------------------------------------------------------------
+# Backend-neutral frame contract
+# --------------------------------------------------------------------------
+
+
+@dataclass
+class RawFrame:
+    """One frame of raw landmarks. None means the tracker lost that part."""
+
+    pose: np.ndarray | None  # (33, 4) x, y, z, visibility
+    face: np.ndarray | None  # (468, 3)
+    hand_left: np.ndarray | None  # (21, 3)
+    hand_right: np.ndarray | None  # (21, 3)
+
+
+def _to_array(landmark_list, with_visibility=False):
+    if landmark_list is None:
+        return None
+    lms = landmark_list.landmark
+    if with_visibility:
+        return np.array([[p.x, p.y, p.z, p.visibility] for p in lms], np.float32)
+    return np.array([[p.x, p.y, p.z] for p in lms], np.float32)
+
+
+def from_holistic(results) -> RawFrame:
+    """Adapter for legacy mp.solutions.holistic results."""
+    return RawFrame(
+        pose=_to_array(results.pose_landmarks, with_visibility=True),
+        face=_to_array(results.face_landmarks),
+        hand_left=_to_array(results.left_hand_landmarks),
+        hand_right=_to_array(results.right_hand_landmarks),
+    )
+
+
+def from_tasks(pose_result, hand_result, face_result) -> RawFrame:
+    """Adapter for the Tasks API: PoseLandmarker + HandLandmarker + FaceLandmarker.
+
+    HandLandmarker returns a list of hands plus a parallel handedness list.
+    We only use handedness to fill the two slots; canonicalisation below does
+    not trust it.
+    """
+
+    def pack(lms, vis=False):
+        if not lms:
+            return None
+        if vis:
+            return np.array(
+                [[p.x, p.y, p.z, getattr(p, "visibility", 1.0)] for p in lms], np.float32
+            )
+        return np.array([[p.x, p.y, p.z] for p in lms], np.float32)
+
+    left = right = None
+    if hand_result and hand_result.hand_landmarks:
+        for lms, hd in zip(hand_result.hand_landmarks, hand_result.handedness, strict=False):
+            arr = pack(lms)
+            if hd[0].category_name.lower().startswith("l"):
+                left = arr
+            else:
+                right = arr
+
+    pose = face = None
+    if pose_result and pose_result.pose_landmarks:
+        pose = pack(pose_result.pose_landmarks[0], vis=True)
+    if face_result and face_result.face_landmarks:
+        face = pack(face_result.face_landmarks[0])
+
+    return RawFrame(pose=pose, face=face, hand_left=left, hand_right=right)
+
+
+# --------------------------------------------------------------------------
+# Feature encoding
+# --------------------------------------------------------------------------
+
+
+def _hand_local(hand: np.ndarray) -> np.ndarray:
+    """Wrist-origin, hand-size-scaled. Handshape independent of position."""
+    wrist = hand[0]
+    centred = hand - wrist
+    scale = np.linalg.norm(hand[9] - wrist)  # wrist to middle-finger MCP
+    if scale < 1e-6:
+        return np.zeros_like(centred)
+    return centred / scale
+
+
+def _nonmanual(face: np.ndarray, height: float, width: float) -> np.ndarray:
+    """Four scalars replacing 1404 raw face coordinates."""
+    f = FACE_IDX
+    brow_mid = (face[f["brow_l"]] + face[f["brow_r"]]) / 2
+    eye_mid = (face[f["eye_upper_l"]] + face[f["eye_upper_r"]]) / 2
+    return np.array(
+        [
+            np.linalg.norm(brow_mid - eye_mid) / height,  # raise
+            np.linalg.norm(face[f["brow_inner_l"]] - face[f["brow_inner_r"]]) / width,
+            np.linalg.norm(face[f["lip_upper"]] - face[f["lip_lower"]]) / height,
+            np.linalg.norm(face[f["mouth_l"]] - face[f["mouth_r"]]) / width,
+        ],
+        np.float32,
+    )
+
+
+def encode_frame(raw: RawFrame, dominant: str = "right") -> np.ndarray:
+    """RawFrame -> (183,) float32. All-zero mask means an unusable frame."""
+    out = np.zeros(DIM_FRAME, np.float32)
+
+    # Body frame requires shoulders. Without pose there is no usable geometry.
+    if raw.pose is None:
+        return out
+    pose_xyz = raw.pose[:, :3]
+    origin = (pose_xyz[L_SHOULDER] + pose_xyz[R_SHOULDER]) / 2
+    scale = float(np.linalg.norm(pose_xyz[L_SHOULDER] - pose_xyz[R_SHOULDER]))
+    if scale < 1e-6:
+        return out
+
+    # --- Canonicalise hand slots by geometry, never by the handedness label.
+    # Slot 0 is the dominant hand: the one nearer the dominant-side shoulder.
+    dom_shoulder = pose_xyz[R_SHOULDER if dominant == "right" else L_SHOULDER]
+    present = [
+        (h, np.linalg.norm(h[0] - dom_shoulder))
+        for h in (raw.hand_left, raw.hand_right)
+        if h is not None
+    ]
+    present.sort(key=lambda t: t[1])
+    slots: list[np.ndarray | None] = [None, None]
+    for i, (hand, _) in enumerate(present[:2]):
+        slots[i] = hand
+
+    i = 0
+    for s, hand in enumerate(slots):
+        if hand is not None:
+            out[i : i + 63] = _hand_local(hand).ravel()
+            out[126 + s * 3 : 126 + s * 3 + 3] = (hand[0] - origin) / scale
+            out[DIM_GEOM + s] = 1.0  # hand-present flag
+        i += 63
+
+    # --- Upper-body pose in the body frame
+    out[132 : 132 + N_POSE_SEL * 3] = ((pose_xyz[POSE_SEL] - origin) / scale).ravel()
+    out[DIM_GEOM + 2 : DIM_GEOM + 2 + N_POSE_SEL] = raw.pose[POSE_SEL, 3]  # visibility
+
+    # --- Non-manual scalars
+    if raw.face is not None:
+        f = raw.face
+        h = float(np.linalg.norm(f[FACE_IDX["forehead"]] - f[FACE_IDX["chin"]]))
+        w = float(np.linalg.norm(f[FACE_IDX["cheek_l"]] - f[FACE_IDX["cheek_r"]]))
+        if h > 1e-6 and w > 1e-6:
+            out[165:169] = _nonmanual(f, h, w)
+            out[DIM_FRAME - 1] = 1.0  # face-present flag
+
+    return out
+
+
+def encode_clip(
+    frames: list[RawFrame], T: int = 48, dominant: str = "right", with_velocity: bool = True
+) -> np.ndarray:
+    """Variable-length clip -> (T, 352). Resampled, not padded."""
+    if not frames:
+        return np.zeros(
+            (T, DIM_FRAME + DIM_GEOM if with_velocity else DIM_FRAME),
+            np.float32,
+        )
+
+    seq = np.stack([encode_frame(f, dominant) for f in frames])
+
+    # Linear resample to fixed T. Duration becomes an augmentation axis,
+    # not a confound the model has to learn around.
+    src = np.linspace(0, len(seq) - 1, len(seq))
+    dst = np.linspace(0, len(seq) - 1, T)
+    seq = np.stack([np.interp(dst, src, seq[:, d]) for d in range(seq.shape[1])], axis=1).astype(
+        np.float32
+    )
+
+    if not with_velocity:
+        return seq
+    vel = np.zeros((T, DIM_GEOM), np.float32)
+    vel[1:] = seq[1:, :DIM_GEOM] - seq[:-1, :DIM_GEOM]
+    return np.concatenate([seq, vel], axis=1)
+
+
+# --------------------------------------------------------------------------
+# Train/serve identity
+# --------------------------------------------------------------------------
+
+# A fixed probe clip, encoded and hashed. The point is to detect a change in
+# what the encoder DOES, which no config field can catch: change how the body
+# frame is scaled and every number moves while T, dominant and with_velocity
+# stay identical. That failure is silent and confident, and this project has
+# been bitten by its shape three times (the mirror, the visibility skew, the
+# face block length).
+#
+# Hashing the source instead would change on every docstring edit, and an alarm
+# that fires on cosmetic changes is one people learn to ignore.
+FINGERPRINT_T = 16
+FINGERPRINT_DECIMALS = 6  # absorbs float differences between x86 and aarch64
+_FINGERPRINT_SEED = 20260906
+
+
+def _fingerprint_clip(n_frames: int = 24) -> list[RawFrame]:
+    """Deterministic nonsense with the right shape. Never touches a camera.
+
+    Every third frame drops the left hand so the validity mask and the
+    geometric slot assignment are both exercised, not just the dense path.
+    """
+    rng = np.random.default_rng(_FINGERPRINT_SEED)
+    frames = []
+    for i in range(n_frames):
+        pose = np.zeros((33, 4), np.float32)
+        pose[:, :3] = rng.random((33, 3), dtype=np.float32)
+        pose[:, 3] = 1.0
+        frames.append(
+            RawFrame(
+                pose=pose,
+                face=rng.random((468, 3), dtype=np.float32),  # FACE_IDX's canonical mesh
+                hand_left=rng.random((21, 3), dtype=np.float32) if i % 3 else None,
+                hand_right=rng.random((21, 3), dtype=np.float32),
+            )
+        )
+    return frames
+
+
+# The hash above is exact, and exactness is the wrong test across machines.
+# Measured 2026-09-07, same features.py and same numpy 1.26.4 on both ends:
+# the probe clip differs between macOS/arm64 and the board's linux/aarch64 by
+# up to 2.384e-07 in 1165 of 5632 values — float32 epsilon from libm and SIMD
+# ordering, not a change in what the encoder does. Rounding cannot fix that
+# reliably: a value sitting within the noise of a rounding boundary still flips,
+# and at 4 dp roughly 27 of 5632 values are expected to sit there.
+#
+# So the guard compares numerically, with a tolerance two orders above the
+# observed noise and orders below any real encoder change (the visibility skew
+# and the mirror both moved values by O(0.1)). The hash stays, for logs and for
+# reading a checkpoint's identity at a glance.
+FINGERPRINT_ATOL = 1e-5
+
+
+def encoder_signature() -> np.ndarray:
+    """The probe clip itself, for a comparison that tolerates float noise."""
+    return encode_clip(
+        _fingerprint_clip(), T=FINGERPRINT_T, dominant="right", with_velocity=True
+    )
+
+
+def signatures_match(a, b, atol: float = FINGERPRINT_ATOL) -> bool:
+    """Whether two encoder signatures describe the same encoder behaviour."""
+    a, b = np.asarray(a), np.asarray(b)
+    return a.shape == b.shape and bool(np.allclose(a, b, atol=atol, rtol=0.0))
+
+
+def encoder_fingerprint() -> str:
+    """16 hex chars identifying this encoder's behaviour.
+
+    Written into every checkpoint at training time and checked when one is
+    loaded, so a model can never be served by an encoder it was not fitted on.
+    """
+    clip = encode_clip(_fingerprint_clip(), T=FINGERPRINT_T, dominant="right", with_velocity=True)
+    # +0.0 normalises -0.0, which hashes differently from 0.0 for no useful reason.
+    quantised = np.round(clip.astype(np.float64), FINGERPRINT_DECIMALS) + 0.0
+    return hashlib.sha256(quantised.tobytes()).hexdigest()[:16]