mctrl 0.1.0__py3-none-any.whl

mindcontrol/filters.py

@@ -0,0 +1,95 @@
+"""Signal smoothing.
+
+Landmark streams are noisy at a level you can see as cursor jitter, but a plain
+low-pass trades that jitter for lag you can feel. The one-euro filter adapts:
+heavy smoothing while the hand is still, light smoothing while it moves, so the
+pointer is both steady when parked and responsive when thrown.
+
+Reference: Casiez, Roussel & Vogel, "1 e filter" (CHI 2012).
+"""
+
+from __future__ import annotations
+
+import math
+
+
+def _alpha(cutoff: float, dt: float) -> float:
+    tau = 1.0 / (2.0 * math.pi * max(cutoff, 1e-6))
+    return 1.0 / (1.0 + tau / max(dt, 1e-6))
+
+
+class _LowPass:
+    __slots__ = ("_value",)
+
+    def __init__(self) -> None:
+        self._value: float | None = None
+
+    def __call__(self, sample: float, alpha: float) -> float:
+        self._value = (
+            sample if self._value is None else alpha * sample + (1.0 - alpha) * self._value
+        )
+        return self._value
+
+    @property
+    def value(self) -> float | None:
+        return self._value
+
+    def reset(self) -> None:
+        self._value = None
+
+
+class OneEuroFilter:
+    """Adaptive low-pass filter for a scalar stream."""
+
+    def __init__(self, fc_min: float = 1.0, beta: float = 0.01, dc_cutoff: float = 1.0) -> None:
+        self.fc_min = fc_min
+        self.beta = beta
+        self.dc_cutoff = dc_cutoff
+        self._x = _LowPass()
+        self._dx = _LowPass()
+
+    def __call__(self, sample: float, dt: float) -> float:
+        previous = self._x.value
+        rate = 0.0 if previous is None else (sample - previous) / max(dt, 1e-6)
+        edge = self._dx(rate, _alpha(self.dc_cutoff, dt))
+        cutoff = self.fc_min + self.beta * abs(edge)
+        return self._x(sample, _alpha(cutoff, dt))
+
+    def reset(self) -> None:
+        self._x.reset()
+        self._dx.reset()
+
+
+class OneEuroFilter2D:
+    """Two-axis one-euro filter that shares one adaptive cutoff.
+
+    Both axes are driven by the *combined* speed rather than their own, so a
+    diagonal move is smoothed evenly instead of bending toward whichever axis
+    happened to be quieter.
+    """
+
+    def __init__(self, fc_min: float = 1.0, beta: float = 0.01, dc_cutoff: float = 1.0) -> None:
+        self.fc_min = fc_min
+        self.beta = beta
+        self.dc_cutoff = dc_cutoff
+        self._x = _LowPass()
+        self._y = _LowPass()
+        self._speed = _LowPass()
+
+    def __call__(self, x: float, y: float, dt: float) -> tuple[float, float]:
+        px, py = self._x.value, self._y.value
+        stale = px is None or py is None
+        rate = 0.0 if stale else math.hypot(x - px, y - py) / max(dt, 1e-6)
+        edge = self._speed(rate, _alpha(self.dc_cutoff, dt))
+        alpha = _alpha(self.fc_min + self.beta * abs(edge), dt)
+        return self._x(x, alpha), self._y(y, alpha)
+
+    @property
+    def speed(self) -> float:
+        """Smoothed magnitude of recent motion, in input units per second."""
+        return abs(self._speed.value or 0.0)
+
+    def reset(self) -> None:
+        self._x.reset()
+        self._y.reset()
+        self._speed.reset()

mindcontrol/fusion.py

@@ -0,0 +1,240 @@
+"""Meshing several cameras into one view of your hands.
+
+The two halves of a hand observation are merged differently, because they mean
+different things across viewpoints.
+
+*Shape* -- pinch distances, which fingers are out -- is scale invariant and
+comparable between cameras, so it is combined by confidence-weighted vote. This
+is the real payoff of a second camera: a pinch hidden behind your palm from the
+laptop is plainly visible from the side, and either camera can carry the gesture.
+
+*Position* is not comparable. Each camera has its own viewpoint, so the same hand
+sits at different normalised coordinates in each. Averaging them would invent a
+location belonging to no camera and lurch whenever one dropped out. Instead one
+camera *leads* for position, chosen by confidence and held onto until it is
+clearly beaten, and a change of leader is reported so the pointer can rebase
+instead of jumping.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Iterator
+from dataclasses import dataclass, replace
+from typing import TYPE_CHECKING
+
+from .config import GestureConfig, TrackingConfig
+from .geometry import HandFeatures, classify
+from .tracking.gaze import GazeObservation
+
+if TYPE_CHECKING:
+    from .session import RecordedFrame, Session
+
+
+
+@dataclass(frozen=True)
+class Observation:
+    """One camera's hands, with the age of the frame they came from."""
+
+    camera_id: int
+    hands: list[HandFeatures]
+    age_ms: float
+
+
+@dataclass(frozen=True)
+class FusedHand:
+    features: HandFeatures
+    camera_id: int
+    cameras: tuple[int, ...]
+    rebased: bool
+
+    @property
+    def merged(self) -> bool:
+        return len(self.cameras) > 1
+
+
+class HandFusion:
+    """Combines per-camera hand observations into one hand per side."""
+
+    def __init__(self, tracking: TrackingConfig, gestures: GestureConfig) -> None:
+        self._tracking = tracking
+        self._gestures = gestures
+        self._leader: dict[str, int] = {}
+        # State for stitching position across a change of leading camera. Per side:
+        # the offset currently mapping the leader's coordinates into the continuous
+        # track, the last anchor emitted, and the last anchor each camera reported.
+        self._offset: dict[str, tuple[float, float]] = {}
+        self._emitted: dict[str, tuple[float, float]] = {}
+        self._seen: dict[tuple[str, int], tuple[float, float]] = {}
+
+    def fuse(self, observations: list[Observation]) -> list[FusedHand]:
+        """Merge observations, newest-first, dropping stale frames."""
+        by_side: dict[str, list[tuple[int, HandFeatures]]] = {}
+        for observation in observations:
+            if observation.age_ms > self._tracking.stale_after_ms:
+                continue
+            for hand in observation.hands:
+                by_side.setdefault(hand.handedness, []).append((observation.camera_id, hand))
+
+        return [self._fuse_side(side, entries) for side, entries in by_side.items()]
+
+    def _fuse_side(self, side: str, entries: list[tuple[int, HandFeatures]]) -> FusedHand:
+        cameras = tuple(sorted(camera_id for camera_id, _ in entries))
+        leader_id, leader = self._pick_leader(side, entries)
+        rebased = self._stitch(side, leader_id, leader.anchor)
+        self._leader[side] = leader_id
+
+        features = leader if len(entries) == 1 else self._blend(entries, leader)
+        anchor, offset = features.anchor, self._offset[side]
+        if offset != (0.0, 0.0):
+            features = replace(features, anchor=(anchor[0] + offset[0], anchor[1] + offset[1]))
+
+        self._emitted[side] = features.anchor
+        for camera_id, hand in entries:
+            self._seen[side, camera_id] = hand.anchor
+        return FusedHand(features, leader_id, cameras, rebased)
+
+    def _stitch(self, side: str, leader_id: int, anchor: tuple[float, float]) -> bool:
+        """Absorb a change of leader into an offset, and say whether that failed.
+
+        Two cameras looking at one hand disagree about where it is, so handing the
+        lead over moves the anchor by the parallax between them. That used to be
+        dealt with by telling the pointer to forget its baseline, which stops the
+        cursor flinging but throws away the frame -- and a fast gesture crossing
+        between views spends a third of its frames doing exactly that, which is how
+        a real sweep changed leader 27 times and registered one swipe out of four.
+
+        The new leader has usually been watching all along, so its *own* movement
+        since the previous frame is known and is a faithful measure of the hand's.
+        Choosing an offset that continues the track from there keeps position
+        continuous while spending none of the motion:
+
+            emitted = leader.anchor + offset,  offset = last_emitted - leader.previous
+
+        The offset then stays put until the next handover, so between them the
+        motion is exactly the leader's own. Only a leader that was not in the
+        previous frame -- one that just appeared -- has nothing to continue from,
+        and that alone still needs a rebase.
+        """
+        previous = self._leader.get(side)
+        if previous == leader_id:
+            return False
+
+        here = self._seen.get((side, leader_id))
+        emitted = self._emitted.get(side)
+        if previous is None or here is None or emitted is None:
+            self._offset[side] = (0.0, 0.0)
+            # A first sighting is not a jump; there is no baseline to invalidate.
+            return previous is not None
+
+        self._offset[side] = (emitted[0] - here[0], emitted[1] - here[1])
+        return False
+
+    def _pick_leader(
+        self, side: str, entries: list[tuple[int, HandFeatures]]
+    ) -> tuple[int, HandFeatures]:
+        """Best camera for position, with hysteresis so it does not flip-flop.
+
+        The margin used to carry more weight than it should have, because every
+        handover discarded a frame of motion; `_stitch` now absorbs them, so this
+        only keeps the lead from flitting between views of near-equal confidence.
+        """
+        best_id, best = max(entries, key=lambda item: item[1].score)
+        held = self._leader.get(side)
+        if held is None:
+            return best_id, best
+        margin = self._tracking.leader_margin
+        for camera_id, hand in entries:
+            if camera_id == held and hand.score + margin >= best.score:
+                return camera_id, hand
+        return best_id, best
+
+    def _blend(self, entries: list[tuple[int, HandFeatures]], leader: HandFeatures) -> HandFeatures:
+        """Average shape across cameras, keeping the leader's position."""
+        weights = [max(hand.score, 1e-3) for _, hand in entries]
+        total = sum(weights)
+        hands = [hand for _, hand in entries]
+
+        def weighted(pick) -> float:
+            pairs = zip(weights, hands, strict=True)
+            return sum(weight * pick(hand) for weight, hand in pairs) / total
+
+        # A finger counts as extended when the cameras that can see it agree by
+        # weight; occlusion in one view is outvoted rather than trusted.
+        flags = tuple(
+            sum(weight for weight, hand in zip(weights, hands, strict=True) if hand.extended[index])
+            > total / 2.0
+            for index in range(5)
+        )
+        spread = weighted(lambda h: h.spread)
+        facing = weighted(lambda h: h.facing)
+
+        return replace(
+            leader,
+            pinch_index=weighted(lambda h: h.pinch_index),
+            pinch_middle=weighted(lambda h: h.pinch_middle),
+            extended=flags,  # type: ignore[arg-type]
+            spread=spread,
+            facing=facing,
+            score=max(hand.score for hand in hands),
+            pose=classify(flags, spread, facing, self._gestures),
+        )
+
+    def reset(self) -> None:
+        """Forget everything, including the stitched track.
+
+        Clearing the offset here is what bounds its drift: it only accumulates
+        while one hand stays continuously in view, and each camera measures motion
+        in its own field of view, so the scales are not identical.
+        """
+        self._leader.clear()
+        self._offset.clear()
+        self._emitted.clear()
+        self._seen.clear()
+
+
+def fuse_session(
+    session: Session, gestures: GestureConfig, tracking: TrackingConfig | None = None
+) -> Iterator[tuple[RecordedFrame, list[FusedHand]]]:
+    """Push a recording through fusion, yielding each frame's merged hands.
+
+    Tuning and reporting both want the hand the *engine* sees. Reading the raw
+    per-camera views instead skews everything downstream, and not by a little:
+    taking the lowest value across three viewpoints for a cluster meant to be low,
+    and the highest for one meant to be high, manufactures a separation that no
+    single camera ever saw. A threshold fitted to that invented gap lands in the
+    space between the cameras, where no real measurement falls.
+
+    Frames are walked in order because fusion carries state -- which camera
+    currently leads position, and the hysteresis holding it there.
+    """
+    fusion = HandFusion(tracking or TrackingConfig(), gestures)
+    for frame in session.frames:
+        yield (
+            frame,
+            fusion.fuse(
+                [
+                    Observation(
+                        camera_id=view.camera_id,
+                        hands=[hand.remeasure(gestures) for hand in view.hands],
+                        age_ms=view.age_ms,
+                    )
+                    for view in frame.views
+                ]
+            ),
+        )
+
+
+def fuse_gaze(observations: dict[int, GazeObservation], primary: int) -> GazeObservation:
+    """Prefer the camera designated for gaze; fall back to any that sees a face.
+
+    Gaze normally runs on one camera only -- it is the expensive model, and only
+    a camera near the screen you look at can produce a useful answer -- but the
+    fallback keeps gaze alive if that camera is unplugged.
+    """
+    preferred = observations.get(primary)
+    if preferred is not None and preferred.usable:
+        return preferred
+    for observation in observations.values():
+        if observation.usable:
+            return observation
+    return preferred or GazeObservation(present=False)

mindcontrol/geometry.py

@@ -0,0 +1,200 @@
+"""Hand-shape features and pose classification.
+
+Everything here is scale invariant: raw landmark distances shrink as you lean
+back from the camera, so each measurement is divided by the palm span
+(wrist to middle knuckle). A pinch is then "0.3 palms" whether you are at the
+keyboard or across the room.
+
+Landmark ordering is MediaPipe's 21-point hand model.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from enum import Enum
+from itertools import pairwise
+
+import numpy as np
+
+WRIST = 0
+THUMB_MCP, THUMB_IP, THUMB_TIP = 2, 3, 4
+INDEX_MCP, INDEX_PIP, INDEX_TIP = 5, 6, 8
+MIDDLE_MCP, MIDDLE_PIP, MIDDLE_TIP = 9, 10, 12
+RING_MCP, RING_PIP, RING_TIP = 13, 14, 16
+PINKY_MCP, PINKY_PIP, PINKY_TIP = 17, 18, 20
+
+PALM_POINTS = (WRIST, INDEX_MCP, MIDDLE_MCP, RING_MCP, PINKY_MCP)
+# (tip, pip) pairs for the four non-thumb fingers, in finger order.
+FINGERS = (
+    (INDEX_TIP, INDEX_PIP),
+    (MIDDLE_TIP, MIDDLE_PIP),
+    (RING_TIP, RING_PIP),
+    (PINKY_TIP, PINKY_PIP),
+)
+TIPS = (INDEX_TIP, MIDDLE_TIP, RING_TIP, PINKY_TIP)
+
+# Drawing skeleton: pairs of landmark indices connected by a bone, one row per
+# finger, then the strap across the base of the palm.
+# fmt: off
+SKELETON = (
+    (0, 1), (1, 2), (2, 3), (3, 4),
+    (0, 5), (5, 6), (6, 7), (7, 8),
+    (5, 9), (9, 10), (10, 11), (11, 12),
+    (9, 13), (13, 14), (14, 15), (15, 16),
+    (13, 17), (17, 18), (18, 19), (19, 20),
+    (0, 17),
+)
+# fmt: on
+
+
+class Pose(Enum):
+    """Coarse hand shape, evaluated fresh on every frame."""
+
+    NONE = "none"
+    OTHER = "other"
+    READY = "ready"
+    FIST = "fist"
+    OPEN_PALM = "open_palm"
+    TELEPHONE = "telephone"
+
+
+@dataclass(frozen=True)
+class HandFeatures:
+    """Scale-invariant measurements of one hand, plus its classified pose."""
+
+    handedness: str
+    score: float
+    pose: Pose
+    anchor: tuple[float, float]
+    palm_size: float
+    pinch_index: float
+    pinch_middle: float
+    extended: tuple[bool, bool, bool, bool, bool]
+    spread: float
+    facing: float
+    landmarks: np.ndarray
+    # The metric landmarks every measurement above was derived from. Kept so a
+    # recorded session can be re-measured under different thresholds; without it
+    # a recording would only ever prove the thresholds it was captured with.
+    world: np.ndarray
+    seen_handedness: str = ""
+
+    @property
+    def pinch(self) -> float:
+        """Distance of whichever pinch is closest to closing."""
+        return min(self.pinch_index, self.pinch_middle)
+
+    @property
+    def pinch_is_middle(self) -> bool:
+        """True when the thumb is meeting the middle finger, not the index."""
+        return self.pinch_middle < self.pinch_index
+
+
+def _norm(vector: np.ndarray) -> float:
+    return float(np.linalg.norm(vector))
+
+
+def palm_span(points: np.ndarray) -> float:
+    """Wrist-to-middle-knuckle distance: the unit every other measure divides by."""
+    return max(_norm(points[MIDDLE_MCP] - points[WRIST]), 1e-6)
+
+
+def palm_normal(points: np.ndarray, handedness: str) -> float:
+    """Signed z of the palm normal; positive means the palm faces the camera.
+
+    MediaPipe's z axis grows away from the camera, and the cross product's sign
+    flips between hands, so both are corrected here.
+    """
+    edge_a = points[INDEX_MCP] - points[WRIST]
+    edge_b = points[PINKY_MCP] - points[WRIST]
+    normal = np.cross(edge_a, edge_b)
+    magnitude = _norm(normal)
+    if magnitude < 1e-9:
+        return 0.0
+    chirality = 1.0 if handedness.lower().startswith("r") else -1.0
+    return float(-normal[2] / magnitude) * chirality
+
+
+def measure(
+    world: np.ndarray,
+    image_points: np.ndarray,
+    handedness: str,
+    seen_handedness: str,
+    score: float,
+    thresholds: object,
+) -> HandFeatures:
+    """Turn 21 landmarks into features and a pose label.
+
+    Shape is measured from ``world`` (MediaPipe's metric landmarks) because those
+    axes share one scale; normalised image coordinates do not, and a pinch judged
+    in them would drift with the frame's aspect ratio. Position comes from
+    ``image_points``, which is what the pointer actually needs.
+
+    ``thresholds`` is a ``GestureConfig``, passed in rather than imported so the
+    classifier stays tunable from ``config.toml`` at runtime.
+    """
+    points = world
+    span = palm_span(points)
+    wrist = points[WRIST]
+
+    # A finger is extended when its tip sits farther from the wrist than its
+    # middle joint. Comparing against the joint instead of a fixed length keeps
+    # the test honest for the short pinky and the long middle finger alike.
+    extended: list[bool] = []
+    for tip, pip in FINGERS:
+        pip_reach = max(_norm(points[pip] - wrist), 1e-6)
+        extended.append(_norm(points[tip] - wrist) / pip_reach > thresholds.finger_extended)
+
+    # The thumb rotates rather than curls, so it is measured by how far its tip
+    # has swung away from the far edge of the palm.
+    thumb_out = _norm(points[THUMB_TIP] - points[PINKY_MCP]) / span > thresholds.thumb_extended
+    flags = (thumb_out, *extended)
+
+    spread = float(np.mean([_norm(points[a] - points[b]) for a, b in pairwise(TIPS)]) / span)
+    # Chirality follows the hand as the camera sees it, which is what the cross
+    # product is computed from; a mirrored frame flips that but not the physical
+    # label carried in `handedness`.
+    facing = palm_normal(points, seen_handedness)
+    # The palm centre is the anchor rather than a fingertip: it barely moves when
+    # you pinch, so clicking does not shove the cursor off target.
+    anchor_point = image_points[list(PALM_POINTS)].mean(axis=0)
+
+    return HandFeatures(
+        handedness=handedness,
+        score=score,
+        pose=classify(flags, spread, facing, thresholds),
+        anchor=(float(anchor_point[0]), float(anchor_point[1])),
+        palm_size=span,
+        pinch_index=_norm(points[THUMB_TIP] - points[INDEX_TIP]) / span,
+        pinch_middle=_norm(points[THUMB_TIP] - points[MIDDLE_TIP]) / span,
+        extended=flags,
+        spread=spread,
+        facing=facing,
+        landmarks=image_points,
+        world=world,
+        seen_handedness=seen_handedness,
+    )
+
+
+def classify(flags: tuple[bool, ...], spread: float, facing: float, thresholds: object) -> Pose:
+    """Label a hand shape. Order matters: the specific poses are tested first."""
+    thumb, index, middle, ring, pinky = flags
+
+    if index and middle and ring and pinky:
+        # An open hand only means "open palm" when it is shown to the camera;
+        # otherwise it is just a hand that happens to be relaxed.
+        if spread >= thresholds.palm_spread and facing >= thresholds.palm_facing:
+            return Pose.OPEN_PALM
+        return Pose.OTHER
+
+    if thumb and pinky and not index and not middle and not ring:
+        return Pose.TELEPHONE
+
+    if not any((thumb, index, middle, ring, pinky)):
+        return Pose.FIST
+
+    # Thumb and index available to pinch, with the hand otherwise relaxed.
+    if index and thumb and not (ring and pinky):
+        return Pose.READY
+
+    return Pose.OTHER

mindcontrol/gestures/__init__.py

@@ -0,0 +1 @@
+"""Gesture recognition: hand shapes and motion turned into intents."""