mctrl 0.1.0__py3-none-any.whl

mindcontrol/session.py

@@ -0,0 +1,328 @@
+"""Recorded gesture sessions.
+
+A session is a labelled capture of real hands: for every frame, the raw landmarks
+plus the pose the user was *asked* to hold. That pairing is what makes the file
+useful twice over.
+
+Raw landmarks rather than finished measurements, because a recording of derived
+numbers could only ever validate the thresholds it was captured with. Storing the
+landmarks means any threshold can be re-evaluated against the same hands later --
+which is exactly what `autotune` and the replay tests do.
+
+The prompted label is the supervision signal. Knowing that these 150 frames were
+meant to be a fist is what turns "here is a distribution of finger curl" into
+"here is where the boundary between curled and extended belongs".
+
+Format is JSON Lines: one self-describing header, then one object per frame, so a
+session can be appended to as it is captured and read back in a streaming pass.
+
+A frame holds one *view* per camera rather than a flat list of hands, mirroring
+the `Observation` list the live pipeline fuses. Recording each camera separately
+is what lets a replay exercise the fusion path; flattening them here would throw
+away the disagreement between viewpoints that fusion exists to resolve.
+
+Older single-camera recordings (format 1) are read as one view, because a
+recording costs somebody a minute of holding poses and should not be invalidated
+by a change to how the file is arranged.
+"""
+
+from __future__ import annotations
+
+import json
+import time
+from collections.abc import Iterator
+from dataclasses import dataclass, field
+from pathlib import Path
+
+import numpy as np
+
+from .config import STATE_DIR
+from .geometry import HandFeatures, measure
+
+SESSIONS_DIR = STATE_DIR / "sessions"
+FORMAT_VERSION = 2
+
+
+@dataclass(frozen=True)
+class RecordedHand:
+    """One hand in one frame, as landmarks."""
+
+    handedness: str
+    seen_handedness: str
+    score: float
+    world: np.ndarray
+    image: np.ndarray
+
+    def to_json(self) -> dict:
+        return {
+            "handedness": self.handedness,
+            "seen": self.seen_handedness,
+            "score": round(self.score, 4),
+            # Rounded to keep sessions readable and small; far finer than the
+            # tracker's own noise floor, so nothing measurable is lost.
+            "world": np.round(self.world, 5).tolist(),
+            "image": np.round(self.image, 5).tolist(),
+        }
+
+    @classmethod
+    def from_json(cls, data: dict) -> RecordedHand:
+        return cls(
+            handedness=data["handedness"],
+            seen_handedness=data.get("seen", data["handedness"]),
+            score=float(data["score"]),
+            world=np.array(data["world"], dtype=np.float32),
+            image=np.array(data["image"], dtype=np.float32),
+        )
+
+    def remeasure(self, thresholds) -> HandFeatures:
+        """Re-derive features under a given `GestureConfig`."""
+        return measure(
+            world=self.world,
+            image_points=self.image,
+            handedness=self.handedness,
+            seen_handedness=self.seen_handedness,
+            score=self.score,
+            thresholds=thresholds,
+        )
+
+
+@dataclass(frozen=True)
+class RecordedView:
+    """What one camera saw at one instant.
+
+    ``age_ms`` is how far behind the newest camera this view was, which is what
+    fusion uses to decide how much to trust it.
+    """
+
+    camera_id: int
+    hands: list[RecordedHand] = field(default_factory=list)
+    age_ms: float = 0.0
+
+    def to_json(self) -> dict:
+        return {
+            "camera": self.camera_id,
+            "age": round(self.age_ms, 2),
+            "hands": [hand.to_json() for hand in self.hands],
+        }
+
+    @classmethod
+    def from_json(cls, data: dict) -> RecordedView:
+        return cls(
+            camera_id=int(data["camera"]),
+            hands=[RecordedHand.from_json(h) for h in data.get("hands", [])],
+            age_ms=float(data.get("age", 0.0)),
+        )
+
+
+@dataclass(frozen=True)
+class RecordedFrame:
+    """One instant, across every camera, with the pose that was being asked for."""
+
+    time: float
+    label: str
+    views: list[RecordedView] = field(default_factory=list)
+
+    def to_json(self) -> dict:
+        return {
+            "t": round(self.time, 4),
+            "label": self.label,
+            "views": [view.to_json() for view in self.views],
+        }
+
+    @classmethod
+    def from_json(cls, data: dict) -> RecordedFrame:
+        if "views" in data:
+            views = [RecordedView.from_json(v) for v in data["views"]]
+        else:
+            # Format 1 stored a single camera's hands directly on the frame.
+            # Reading it as one view keeps existing recordings usable, which
+            # matters because a recording is a minute of somebody's time.
+            views = [
+                RecordedView(
+                    camera_id=int(data.get("camera", 0)),
+                    hands=[RecordedHand.from_json(h) for h in data.get("hands", [])],
+                )
+            ]
+        return cls(time=float(data["t"]), label=data["label"], views=views)
+
+    @property
+    def hands(self) -> list[RecordedHand]:
+        """Every hand seen by every camera, for analysis that ignores viewpoint."""
+        return [hand for view in self.views for hand in view.hands]
+
+    @property
+    def cameras(self) -> tuple[int, ...]:
+        return tuple(view.camera_id for view in self.views)
+
+
+class SessionWriter:
+    """Appends frames to a session file as they are captured."""
+
+    def __init__(self, path: Path, note: str = "") -> None:
+        self.path = path
+        path.parent.mkdir(parents=True, exist_ok=True)
+        self._handle = path.open("w")
+        self._count = 0
+        self._write(
+            {
+                "format": FORMAT_VERSION,
+                "kind": "mindcontrol-session",
+                "created": time.strftime("%Y-%m-%dT%H:%M:%S"),
+                "note": note,
+            }
+        )
+
+    def _write(self, payload: dict) -> None:
+        self._handle.write(json.dumps(payload) + "\n")
+
+    def add(self, frame: RecordedFrame) -> None:
+        self._write(frame.to_json())
+        self._count += 1
+
+    @property
+    def frames(self) -> int:
+        return self._count
+
+    def close(self) -> None:
+        self._handle.flush()
+        self._handle.close()
+
+    def __enter__(self) -> SessionWriter:
+        return self
+
+    def __exit__(self, *_exc) -> None:
+        self.close()
+
+
+@dataclass
+class Session:
+    """A whole recording, in memory."""
+
+    frames: list[RecordedFrame]
+    header: dict = field(default_factory=dict)
+    path: Path | None = None
+
+    @classmethod
+    def load(cls, path: Path) -> Session:
+        header: dict = {}
+        frames: list[RecordedFrame] = []
+        with path.open() as handle:
+            for line in handle:
+                line = line.strip()
+                if not line:
+                    continue
+                data = json.loads(line)
+                if data.get("kind") == "mindcontrol-session":
+                    header = data
+                    continue
+                frames.append(RecordedFrame.from_json(data))
+        version = int(header.get("format", FORMAT_VERSION))
+        if version > FORMAT_VERSION:
+            raise ValueError(
+                f"{path} is format {version}, newer than this build understands "
+                f"({FORMAT_VERSION}); upgrade mindcontrol"
+            )
+        return cls(frames=frames, header=header, path=path)
+
+    def labels(self) -> list[str]:
+        """Distinct labels, in the order they were recorded."""
+        seen: dict[str, None] = {}
+        for frame in self.frames:
+            seen.setdefault(frame.label, None)
+        return list(seen)
+
+    def segment(self, *labels: str) -> Iterator[RecordedFrame]:
+        """Frames recorded under any of the given labels."""
+        wanted = set(labels)
+        for frame in self.frames:
+            if frame.label in wanted:
+                yield frame
+
+    def hands(self, *labels: str) -> Iterator[RecordedHand]:
+        """Every hand recorded under the given labels, ignoring frame grouping."""
+        for frame in self.segment(*labels):
+            yield from frame.hands
+
+    @property
+    def cameras(self) -> tuple[int, ...]:
+        """Every camera that contributed to this recording."""
+        seen: dict[int, None] = {}
+        for frame in self.frames:
+            for camera_id in frame.cameras:
+                seen.setdefault(camera_id, None)
+        return tuple(sorted(seen))
+
+    def problems(self) -> list[str]:
+        """Ways this recording cannot answer the questions asked of it.
+
+        Worth reporting loudly. A threshold fitted from a prompt where the other
+        hand was also in shot is not a measurement of anything, but it looks
+        exactly like one -- so the recording has to be able to say when it is not
+        fit to be fitted.
+        """
+        found: list[str] = []
+        cameras = max(len(self.cameras), 1)
+
+        for label in self.labels():
+            frames = list(self.segment(label))
+            if not frames:
+                continue
+
+            with_hands = [f for f in frames if f.hands]
+            if label == "none":
+                share = len(with_hands) / len(frames)
+                if share > 0.1:
+                    found.append(
+                        f"'{label}': a hand was visible in {share:.0%} of frames; "
+                        "the baseline needs both hands right out of shot"
+                    )
+                continue
+
+            if len(with_hands) < len(frames) * 0.5:
+                found.append(
+                    f"'{label}': a hand was found in only "
+                    f"{len(with_hands) / len(frames):.0%} of frames"
+                )
+                continue
+
+            # More hands than cameras means a second hand was in shot, and its
+            # shape is being recorded under this prompt's label.
+            crowded = [f for f in with_hands if len(f.hands) > cameras]
+            if len(crowded) > len(with_hands) * 0.25:
+                found.append(
+                    f"'{label}': two hands were in shot for "
+                    f"{len(crowded) / len(with_hands):.0%} of frames; only the "
+                    "performing hand should be visible"
+                )
+
+        return found
+
+    def summary(self) -> str:
+        counts: dict[str, int] = {}
+        hands = 0
+        for frame in self.frames:
+            counts[frame.label] = counts.get(frame.label, 0) + 1
+            hands += len(frame.hands)
+        parts = ", ".join(f"{label} {count}" for label, count in counts.items())
+        rig = f"{len(self.cameras)} camera(s) {list(self.cameras)}"
+        return f"{len(self.frames)} frames ({hands} hand samples, {rig}): {parts}"
+
+
+def latest_session() -> Path | None:
+    """Most recently recorded session, if any."""
+    if not SESSIONS_DIR.is_dir():
+        return None
+    files = sorted(SESSIONS_DIR.glob("*.jsonl"), key=lambda p: p.stat().st_mtime)
+    return files[-1] if files else None
+
+
+def resolve(path: Path | None) -> Path:
+    """Pick an explicit session, or fall back to the newest one."""
+    if path is not None:
+        return path
+    found = latest_session()
+    if found is None:
+        raise FileNotFoundError(
+            f"no recordings in {SESSIONS_DIR}; run 'mindcontrol record' first"
+        )
+    return found

mindcontrol/tracking/__init__.py

@@ -0,0 +1 @@
+"""Perception wrappers around the MediaPipe task models."""

mindcontrol/tracking/gaze.py

@@ -0,0 +1,272 @@
+"""Eye tracking and the gaze-to-screen model.
+
+Where you are looking is not directly observable from a webcam, so it is
+*learned*: calibration shows you nine dots, records what your eyes and head look
+like for each, and fits a small regression from those features to screen
+coordinates. Inference then runs that regression per frame.
+
+The feature vector is built in one place, `feature_vector`, and used by both
+calibration and inference. If the two ever built it differently the model would
+silently predict nonsense, so there is deliberately only one definition.
+"""
+
+from __future__ import annotations
+
+import json
+import math
+import time
+from collections import deque
+from dataclasses import dataclass
+from pathlib import Path
+
+import cv2
+import mediapipe as mp
+import numpy as np
+from mediapipe.tasks import python as mp_tasks
+from mediapipe.tasks.python import vision
+
+from .. import models
+from ..capture import Frame
+from ..config import TrackingConfig
+
+# Indices into MediaPipe's 478-point face mesh (the last ten are the irises).
+RIGHT_EYE_OUTER, RIGHT_EYE_INNER = 33, 133
+LEFT_EYE_INNER, LEFT_EYE_OUTER = 362, 263
+RIGHT_IRIS, LEFT_IRIS = 468, 473
+RIGHT_LID_TOP, RIGHT_LID_BOTTOM = 159, 145
+LEFT_LID_TOP, LEFT_LID_BOTTOM = 386, 374
+IRIS_MESH_POINTS = 478
+
+FEATURE_NAMES = (
+    "bias",
+    "iris_x",
+    "iris_y",
+    "head_x",
+    "head_y",
+    "iris_x^2",
+    "iris_y^2",
+    "iris_x*iris_y",
+    "iris_x*head_x",
+    "iris_y*head_y",
+)
+FEATURE_COUNT = len(FEATURE_NAMES)
+
+
+def feature_vector(iris: tuple[float, float], head: tuple[float, float]) -> np.ndarray:
+    """Build the regression input from eye offset and head rotation.
+
+    Quadratic and cross terms are included because the mapping is not linear:
+    the same eye offset lands somewhere different depending on how your head is
+    turned, and screen edges compress relative to the centre.
+    """
+    ix, iy = iris
+    hx, hy = head
+    return np.array(
+        [1.0, ix, iy, hx, hy, ix * ix, iy * iy, ix * iy, ix * hx, iy * hy],
+        dtype=np.float64,
+    )
+
+
+@dataclass(frozen=True)
+class GazeObservation:
+    """What one frame tells us about the eyes."""
+
+    present: bool
+    features: np.ndarray | None = None
+    openness: float = 0.0
+    iris_points: tuple[tuple[float, float], ...] = ()
+
+    @property
+    def usable(self) -> bool:
+        return self.present and self.features is not None
+
+
+class GazeTracker:
+    """Extracts gaze features from the camera trusted for gaze."""
+
+    def __init__(self, cfg: TrackingConfig) -> None:
+        self._last_timestamp = -1
+        self._warned_no_iris = False
+        options = vision.FaceLandmarkerOptions(
+            base_options=mp_tasks.BaseOptions(
+                model_asset_path=str(models.ensure("face_landmarker.task"))
+            ),
+            running_mode=vision.RunningMode.VIDEO,
+            num_faces=1,
+            min_face_detection_confidence=cfg.face_detection_confidence,
+            min_tracking_confidence=cfg.hand_tracking_confidence,
+            # The head's own rotation is half of where you are looking.
+            output_facial_transformation_matrixes=True,
+        )
+        self._landmarker = vision.FaceLandmarker.create_from_options(options)
+
+    def process(self, frame: Frame) -> GazeObservation:
+        image = mp.Image(
+            image_format=mp.ImageFormat.SRGB,
+            data=cv2.cvtColor(frame.image, cv2.COLOR_BGR2RGB),
+        )
+        timestamp = max(frame.timestamp_ms, self._last_timestamp + 1)
+        self._last_timestamp = timestamp
+        result = self._landmarker.detect_for_video(image, timestamp)
+        if not result.face_landmarks:
+            return GazeObservation(present=False)
+
+        landmarks = result.face_landmarks[0]
+        points = np.array([[p.x, p.y] for p in landmarks], dtype=np.float64)
+        has_iris = len(landmarks) >= IRIS_MESH_POINTS
+        if not has_iris and not self._warned_no_iris:
+            print("[gaze] face model returned no iris points; falling back to head-only gaze")
+            self._warned_no_iris = True
+
+        right = _eye_offset(points, RIGHT_EYE_OUTER, RIGHT_EYE_INNER, RIGHT_IRIS, has_iris)
+        left = _eye_offset(points, LEFT_EYE_OUTER, LEFT_EYE_INNER, LEFT_IRIS, has_iris)
+        iris = ((right[0] + left[0]) / 2.0, (right[1] + left[1]) / 2.0)
+
+        head = _head_rotation(result.facial_transformation_matrixes)
+        openness = min(
+            _aspect(points, RIGHT_LID_TOP, RIGHT_LID_BOTTOM, RIGHT_EYE_OUTER, RIGHT_EYE_INNER),
+            _aspect(points, LEFT_LID_TOP, LEFT_LID_BOTTOM, LEFT_EYE_OUTER, LEFT_EYE_INNER),
+        )
+        iris_points = (tuple(points[RIGHT_IRIS]), tuple(points[LEFT_IRIS])) if has_iris else ()
+        return GazeObservation(
+            present=True,
+            features=feature_vector(iris, head),
+            openness=openness,
+            iris_points=iris_points,  # type: ignore[arg-type]
+        )
+
+    def close(self) -> None:
+        self._landmarker.close()
+
+
+def _eye_offset(
+    points: np.ndarray, outer: int, inner: int, iris: int, has_iris: bool
+) -> tuple[float, float]:
+    """Iris displacement from the eye's centre, in units of eye width.
+
+    Dividing by eye width is what makes this survive leaning toward or away from
+    the camera: the eye shrinks in the image but the ratio holds.
+    """
+    corner_a, corner_b = points[outer], points[inner]
+    width = max(float(np.linalg.norm(corner_a - corner_b)), 1e-6)
+    centre = (corner_a + corner_b) / 2.0
+    if not has_iris:
+        return 0.0, 0.0
+    offset = (points[iris] - centre) / width
+    return float(offset[0]), float(offset[1])
+
+
+def _head_rotation(matrices: list) -> tuple[float, float]:
+    """Head yaw and pitch in radians, from the face transformation matrix."""
+    if not matrices:
+        return 0.0, 0.0
+    rotation = np.asarray(matrices[0])[:3, :3]
+    magnitude = math.hypot(rotation[0, 0], rotation[1, 0])
+    if magnitude < 1e-6:
+        return 0.0, 0.0
+    yaw = math.atan2(-rotation[2, 0], magnitude)
+    pitch = math.atan2(rotation[2, 1], rotation[2, 2])
+    return yaw, pitch
+
+
+def _aspect(points: np.ndarray, top: int, bottom: int, outer: int, inner: int) -> float:
+    """Eye aspect ratio: lid separation over eye width. Collapses toward 0 on a blink."""
+    width = max(float(np.linalg.norm(points[outer] - points[inner])), 1e-6)
+    return float(np.linalg.norm(points[top] - points[bottom])) / width
+
+
+class GazeModel:
+    """Ridge regression from gaze features to a point on screen."""
+
+    def __init__(self, weights: np.ndarray | None = None, quality: float = 0.0) -> None:
+        self.weights = weights
+        self.quality = quality
+
+    @property
+    def ready(self) -> bool:
+        return self.weights is not None
+
+    @classmethod
+    def fit(cls, features: np.ndarray, targets: np.ndarray, ridge: float = 1e-3) -> GazeModel:
+        """Solve for the feature-to-screen mapping.
+
+        Ridge rather than plain least squares: nine calibration points give
+        nearly collinear features, and unregularised weights would blow up and
+        fling the cursor off screen.
+        """
+        gram = features.T @ features + ridge * np.eye(features.shape[1])
+        weights = np.linalg.solve(gram, features.T @ targets)
+        residual = features @ weights - targets
+        error = float(np.sqrt(np.mean(np.sum(residual**2, axis=1))))
+        return cls(weights=weights, quality=error)
+
+    def predict(self, features: np.ndarray) -> tuple[float, float]:
+        """Screen position as fractions of width and height, clamped on screen."""
+        if self.weights is None:
+            raise RuntimeError("gaze model is not calibrated")
+        point = features @ self.weights
+        return float(np.clip(point[0], 0.0, 1.0)), float(np.clip(point[1], 0.0, 1.0))
+
+    def save(self, path: Path) -> None:
+        path.parent.mkdir(parents=True, exist_ok=True)
+        assert self.weights is not None
+        path.write_text(
+            json.dumps(
+                {
+                    "version": 1,
+                    "features": list(FEATURE_NAMES),
+                    "weights": self.weights.tolist(),
+                    "rms_error": self.quality,
+                    "created": time.strftime("%Y-%m-%dT%H:%M:%S"),
+                },
+                indent=2,
+            )
+        )
+
+    @classmethod
+    def load(cls, path: Path) -> GazeModel:
+        """Load a saved model, or an uncalibrated one if it is missing or stale."""
+        if not path.is_file():
+            return cls()
+        try:
+            data = json.loads(path.read_text())
+            weights = np.array(data["weights"], dtype=np.float64)
+        except (json.JSONDecodeError, KeyError, ValueError) as exc:
+            print(f"[gaze] ignoring unreadable calibration at {path}: {exc}")
+            return cls()
+        if weights.shape != (FEATURE_COUNT, 2):
+            print("[gaze] calibration was built for a different feature set; recalibrate")
+            return cls()
+        return cls(weights=weights, quality=float(data.get("rms_error", 0.0)))
+
+
+class FixationDetector:
+    """Reports where gaze has settled, ignoring the constant flicker of saccades.
+
+    A cursor that chased raw gaze would be unusable, since the eye never truly
+    holds still. Only once the recent samples all fall inside a small radius is
+    the gaze treated as a deliberate target.
+    """
+
+    def __init__(self, window_ms: float, radius: float) -> None:
+        self._window_s = window_ms / 1000.0
+        self._radius = radius
+        self._samples: deque[tuple[float, float, float]] = deque()
+
+    def update(self, x: float, y: float, now: float | None = None) -> tuple[float, float] | None:
+        now = time.monotonic() if now is None else now
+        self._samples.append((now, x, y))
+        cutoff = now - self._window_s
+        while self._samples and self._samples[0][0] < cutoff:
+            self._samples.popleft()
+        if len(self._samples) < 3 or self._samples[0][0] > cutoff + self._window_s:
+            return None
+
+        points = np.array([[sx, sy] for _, sx, sy in self._samples])
+        centre = points.mean(axis=0)
+        if float(np.max(np.linalg.norm(points - centre, axis=1))) > self._radius:
+            return None
+        return float(centre[0]), float(centre[1])
+
+    def reset(self) -> None:
+        self._samples.clear()