tperm-visor 1.0.1

package/backend/gesture_engine.py

@@ -0,0 +1,106 @@
+"""
+gesture_engine.py
+Stateless gesture classifiers + stateful detectors (throw, absent).
+Pattern mirrors Gesture-Media-control baseline: deque buffers, EMA, cooldowns.
+"""
+
+import numpy as np
+
+from hand_tracker import HandData
+
+
+# ── Distance helpers ──────────────────────────────────────────────────────────
+
+def _norm_dist(lms, i: int, j: int) -> float:
+    """Euclidean distance between two normalized landmarks."""
+    a, b = lms[i], lms[j]
+    return float(np.sqrt((a.x - b.x)**2 + (a.y - b.y)**2 + (a.z - b.z)**2))
+
+
+# ── Spawn / grab ──────────────────────────────────────────────────────────────
+
+def get_spawn_distance(h1: HandData, h2: HandData) -> float:
+    return float(np.linalg.norm(h1.palm_center - h2.palm_center))
+
+
+def is_cube_spawn_ready(hands: list[HandData], frame_w: int) -> bool:
+    if len(hands) != 2:
+        return False
+    dist = get_spawn_distance(hands[0], hands[1])
+    return dist <= frame_w * 0.40
+
+
+def midpoint(h1: HandData, h2: HandData) -> np.ndarray:
+    return (h1.palm_center + h2.palm_center) / 2.0
+
+
+# ── Fist / Pinch ──────────────────────────────────────────────────────────────
+
+def is_fist(hand: HandData) -> bool:
+    """True if most fingers are tightly curled (fist)."""
+    curled_count = 0
+    # For each finger, if tip is closer to wrist than its MCP joint, it is curled.
+    for tip, mcp in [(8, 5), (12, 9), (16, 13), (20, 17)]:
+        d_tip = _norm_dist(hand.landmarks, tip, 0)
+        d_mcp = _norm_dist(hand.landmarks, mcp, 0)
+        if d_tip < d_mcp:
+            curled_count += 1
+    return curled_count >= 3
+
+
+def is_open_palm(hand: HandData) -> bool:
+    """True if all four fingers are extended (showing 5 / stop hand).
+    Used as a 'lock' gesture — cube won't rotate while this hand is up."""
+    extended = 0
+    for tip, mcp in [(8, 5), (12, 9), (16, 13), (20, 17)]:
+        d_tip = _norm_dist(hand.landmarks, tip, 0)
+        d_mcp = _norm_dist(hand.landmarks, mcp, 0)
+        if d_tip > d_mcp:
+            extended += 1
+    # Also require thumb to be somewhat spread (not pinching)
+    thumb_idx_dist = _norm_dist(hand.landmarks, 4, 8)
+    return extended >= 4 and thumb_idx_dist > 0.08
+
+
+def is_pinch(hand: HandData) -> bool:
+    """Thumb tip ↔ index tip closer than 0.06 normalised units."""
+    d = _norm_dist(hand.landmarks, 4, 8)
+    return d < 0.06
+
+
+# ── Palm orientation ──────────────────────────────────────────────────────────
+
+def is_palm_facing_down(hand: HandData) -> bool:
+    if is_fist(hand) or is_pinch(hand):
+        return False
+    # Right hand palm down -> +Y, Left hand palm down -> -Y
+    return hand.palm_normal[1] > 0.6 if hand.label == 'Right' else hand.palm_normal[1] < -0.6
+
+def is_palm_facing_up(hand: HandData) -> bool:
+    if is_fist(hand) or is_pinch(hand):
+        return False
+    # Right hand palm up -> -Y, Left hand palm up -> +Y
+    return hand.palm_normal[1] < -0.6 if hand.label == 'Right' else hand.palm_normal[1] > 0.6
+
+
+# ── Hands-absent detection ────────────────────────────────────────────────────
+
+class AbsentDetector:
+    """
+    Frame-counter based. threshold_frames = 48 ≈ 0.8s @ 60fps.
+    Same philosophy as baseline's gesture cooldown but inverted (counting absence).
+    """
+
+    def __init__(self, threshold_frames: int = 48):
+        self.threshold = threshold_frames
+        self.counter = 0
+
+    def update(self, hands: list[HandData]) -> bool:
+        if len(hands) == 0:
+            self.counter += 1
+        else:
+            self.counter = 0
+        return self.counter >= self.threshold
+
+    def reset(self):
+        self.counter = 0

package/backend/hand_tracker.py

@@ -0,0 +1,199 @@
+"""
+hand_tracker.py
+MediaPipe Hand Landmarker setup + skeleton draw + HandData extraction.
+Threading model and CONNECTIONS list copied from Gesture-Media-control baseline.
+"""
+
+import time
+import threading
+from dataclasses import dataclass
+
+import cv2
+import mediapipe as mp
+import numpy as np
+from mediapipe.tasks import python
+from mediapipe.tasks.python import vision
+
+from utils.transforms import compute_palm_normal, compute_finger_direction
+
+# ── Exact CONNECTIONS from baseline (pranavpant9916-ctrl/Gesture-Media-control) ──
+CONNECTIONS = [
+    (0, 1), (1, 2), (2, 3), (3, 4),
+    (0, 5), (5, 6), (6, 7), (7, 8),
+    (5, 9), (9, 10), (10, 11), (11, 12),
+    (9, 13), (13, 14), (14, 15), (15, 16),
+    (13, 17), (17, 18), (18, 19), (19, 20),
+    (0, 17)
+]
+
+# Landmark roles
+IDX_WRIST      = 0
+IDX_THUMB_TIP  = 4
+IDX_INDEX_TIP  = 8
+
+PALM_LANDMARKS = [0, 5, 9, 13, 17]  # for palm center average
+
+
+# ── Threaded webcam stream ────────────────────────────────────────────────────
+
+class WebcamStream:
+    """Runs the webcam on a separate thread to achieve 60 fps without lag."""
+
+    def __init__(self, src: int = 0):
+        self.stream = cv2.VideoCapture(src, cv2.CAP_DSHOW)
+        if not self.stream.isOpened():
+            self.stream = cv2.VideoCapture(src)
+        # Request 720p 60fps
+        self.stream.set(cv2.CAP_PROP_FRAME_WIDTH, 1280)
+        self.stream.set(cv2.CAP_PROP_FRAME_HEIGHT, 720)
+        self.stream.set(cv2.CAP_PROP_FPS, 60)
+        self.stream.set(cv2.CAP_PROP_AUTOFOCUS, 1)
+        # Keep only the newest frame in the driver queue. Without this the camera
+        # buffers a backlog and read() hands frames back in order, so whenever
+        # this thread is briefly starved the queue fills, never drains, and the
+        # feed sits seconds behind reality. Ignored harmlessly by some backends.
+        self.stream.set(cv2.CAP_PROP_BUFFERSIZE, 1)
+        self.grabbed, self.frame = self.stream.read()
+        self.stopped = False
+        self._lock = threading.Lock()
+        self._thread = None  # FIX: keep reference so we can join() on shutdown
+
+    def start(self):
+        self._thread = threading.Thread(target=self._update, daemon=True)
+        self._thread.start()
+        return self
+
+    def _update(self):
+        while not self.stopped:
+            grabbed, frame = self.stream.read()
+            # FIX: if camera returns bad frames (e.g. after release is called),
+            # sleep briefly instead of spinning at 100% CPU hammering a dead stream
+            if not grabbed or frame is None:
+                time.sleep(0.01)
+                continue
+            with self._lock:
+                self.grabbed = grabbed
+                self.frame = frame
+
+    def read(self):
+        with self._lock:
+            return self.grabbed, self.frame.copy() if self.frame is not None else None
+
+    def isOpened(self) -> bool:
+        return self.stream.isOpened()
+
+    def release(self):
+        self.stopped = True
+        # FIX: wait for the _update thread to finish its current stream.read() call
+        # before releasing the camera handle.
+        #
+        # Without this join(), stream.release() fires while the thread is still
+        # inside stream.read(). On Windows DirectShow (CAP_DSHOW), the camera
+        # handle is reference-counted at the driver level — it stays open until
+        # ALL threads exit their read() calls. A new process that opens the same
+        # camera then gets a contested, half-initialised handle → lag and jitter.
+        #
+        # join(timeout=2.0) gives the thread up to 2s to finish its current read
+        # (one frame at 60fps = ~16ms, so 2s is extremely conservative).
+        if self._thread is not None and self._thread.is_alive():
+            self._thread.join(timeout=2.0)
+        self.stream.release()
+
+
+# ── HandData dataclass ────────────────────────────────────────────────────────
+
+@dataclass
+class HandData:
+    label: str              # "Left" or "Right"
+    landmarks: list         # all 21 raw landmark objects (normalized .x .y .z)
+    palm_center: np.ndarray # pixel (x, y)
+    palm_normal: np.ndarray # 3-D unit vector
+    finger_direction: np.ndarray  # 3-D unit vector (wrist → middle MCP)
+
+
+# ── HandTracker ───────────────────────────────────────────────────────────────
+
+class HandTracker:
+    def __init__(self, model_path: str = 'hand_landmarker.task',
+                 result_callback=None, num_hands: int = 2):
+        """result_callback: callable(result, image, timestamp_ms), for LIVE_STREAM.
+
+        num_hands is the single biggest performance lever here. Whenever fewer
+        hands are visible than requested, MediaPipe re-runs its expensive palm
+        detector on EVERY frame looking for the ones it hasn't found. Measured on
+        a real frame with one hand up: num_hands=2 costs 143ms per detection,
+        num_hands=1 costs 66ms - a 2.18x difference. Input resolution, by
+        contrast, changes nothing (640x360 and 213x120 both land within 2ms),
+        because MediaPipe rescales to the model's own input size regardless.
+        """
+        self.num_hands = num_hands
+        options = vision.HandLandmarkerOptions(
+            base_options=python.BaseOptions(model_asset_path=model_path),
+            running_mode=vision.RunningMode.LIVE_STREAM,
+            num_hands=num_hands,
+            min_hand_detection_confidence=0.4,
+            min_hand_presence_confidence=0.4,
+            min_tracking_confidence=0.4,
+            result_callback=result_callback,
+        )
+        self.detector = vision.HandLandmarker.create_from_options(options)
+
+    def detect_async(self, rgb_frame: np.ndarray, timestamp_ms: int):
+        """Non-blocking detect — LIVE_STREAM mode. Result arrives via callback."""
+        mp_image = mp.Image(image_format=mp.ImageFormat.SRGB, data=rgb_frame)
+        self.detector.detect_async(mp_image, timestamp_ms)
+
+    def extract_hands(self, result, frame_w: int, frame_h: int) -> list[HandData]:
+        hands = []
+        if not result.hand_landmarks:
+            return hands
+
+        for lms, handedness in zip(result.hand_landmarks, result.handedness):
+            palm_pixels = np.array(
+                [[lms[i].x * frame_w, lms[i].y * frame_h] for i in PALM_LANDMARKS]
+            )
+            hands.append(HandData(
+                label=handedness[0].category_name,  # "Left" or "Right"
+                landmarks=lms,
+                palm_center=palm_pixels.mean(axis=0),
+                palm_normal=compute_palm_normal(lms),
+                finger_direction=compute_finger_direction(lms),
+            ))
+        return hands
+
+    def draw_skeleton(self, frame: np.ndarray, result) -> None:
+        """
+        Draw the full MediaPipe skeleton overlay.
+        Stays visible in EVERY state as long as hands are in frame.
+        - Bones: cyan, thickness 2
+        - Joints: white, radius 4
+        - Index tip (8) & thumb tip (4): red, radius 7
+        - Wrist (0): amber, radius 8
+        Landmarks are normalised 0-1, so this scales correctly onto the full-size
+        frame even though detection ran on a 1/3-scale copy.
+        """
+        if not result.hand_landmarks:
+            return
+
+        h, w = frame.shape[:2]
+        for lms in result.hand_landmarks:
+            # Bones
+            for s, e in CONNECTIONS:
+                sp = (int(lms[s].x * w), int(lms[s].y * h))
+                ep = (int(lms[e].x * w), int(lms[e].y * h))
+                cv2.line(frame, sp, ep, (0, 255, 255), 2, cv2.LINE_AA)
+
+            # Joints
+            for idx, lm in enumerate(lms):
+                px = (int(lm.x * w), int(lm.y * h))
+                if idx == IDX_WRIST:
+                    cv2.circle(frame, px, 8, (0, 220, 255), -1, cv2.LINE_AA)
+                    cv2.circle(frame, px, 9, (0, 0, 0), 1, cv2.LINE_AA)
+                elif idx in (IDX_THUMB_TIP, IDX_INDEX_TIP):
+                    cv2.circle(frame, px, 7, (0, 0, 255), -1, cv2.LINE_AA)
+                    cv2.circle(frame, px, 8, (0, 0, 0), 1, cv2.LINE_AA)
+                else:
+                    cv2.circle(frame, px, 4, (255, 255, 255), -1, cv2.LINE_AA)
+
+    def close(self):
+        self.detector.close()

package/backend/hud.py

@@ -0,0 +1,136 @@
+"""
+hud.py
+All HUD overlays — glassmorphism pill style copied from Gesture-Media-control baseline.
+Confetti, state label, solved/unsolved banner, spawn ring.
+"""
+
+import random
+
+import cv2
+import numpy as np
+
+
+# ── Glassmorphism helpers (from baseline) ─────────────────────────────────────
+
+def _glass_pill(frame: np.ndarray, cx: int, cy: int, text: str,
+                font_scale: float = 0.8, thickness: int = 2,
+                alpha: float = 1.0, text_color=(255, 255, 255)):
+    """Lightweight pill overlay — no blur, no full-frame copy."""
+    if alpha < 0.01:
+        return
+    h, w = frame.shape[:2]
+    font = cv2.FONT_HERSHEY_SIMPLEX
+    text_size, _ = cv2.getTextSize(text, font, font_scale, thickness)
+    px, py = 30, 15
+    pill_w = text_size[0] + px * 2
+    pill_h = text_size[1] + py * 2
+    x1 = cx - pill_w // 2
+    y1 = cy - pill_h // 2
+    x2, y2 = x1 + pill_w, y1 + pill_h
+    if x1 < 0 or y1 < 0 or x2 > w or y2 > h:
+        return
+
+    # Draw a dark semi-transparent rounded rectangle (ROI-only, no full copy)
+    overlay = frame[y1:y2, x1:x2].copy()
+    dark = np.full_like(overlay, (18, 20, 24))
+    blended = cv2.addWeighted(dark, 0.55 * alpha, overlay, 1.0 - 0.55 * alpha, 0)
+    frame[y1:y2, x1:x2] = blended
+
+    # Border
+    cv2.rectangle(frame, (x1, y1), (x2, y2), (180, 180, 180), 1)
+
+    tx = cx - text_size[0] // 2
+    ty = cy + text_size[1] // 2
+    cv2.putText(frame, text, (tx + 1, ty + 1), font, font_scale, (0, 0, 0), thickness + 1)
+    cv2.putText(frame, text, (tx, ty), font, font_scale, text_color, thickness)
+
+
+# ── State label (bottom centre) ───────────────────────────────────────────────
+
+STATE_LABELS = {
+    'IDLE':             'Show both hands to spawn cube',
+    'SPAWN_READY':      'Spawning...',
+    'HOLDING':          'Twist wrist to rotate • Pinch to turn face',
+    'DRAGGING_CUBE':    'Moving cube...',
+    'DRAGGING_SLICE':   'Turning layer...',
+    'COMPLETION_CHECK': 'Checking...',
+}
+
+def draw_state_label(frame: np.ndarray, state: str, alpha: float = 1.0):
+    h, w = frame.shape[:2]
+    label = STATE_LABELS.get(state, state)
+    _glass_pill(frame, w // 2, h - 55, label, font_scale=0.6, alpha=alpha)
+
+
+# ── Spawn pulsing ring ────────────────────────────────────────────────────────
+
+def draw_spawn_ring(frame: np.ndarray, cx: int, cy: int, progress: float):
+    """progress 0→1 drives scale-in animation."""
+    r = int(60 * progress)
+    if r < 2:
+        return
+    alpha_ring = max(0.0, 1.0 - progress * 0.5)
+    color = (0, int(255 * progress), 255)
+    overlay = frame.copy()
+    cv2.circle(overlay, (cx, cy), r, color, 2, cv2.LINE_AA)
+    cv2.addWeighted(overlay, alpha_ring, frame, 1 - alpha_ring, 0, frame)
+
+
+# ── Solved / unsolved banner ──────────────────────────────────────────────────
+
+def draw_solved_banner(frame: np.ndarray, solved: bool, alpha: float = 1.0):
+    h, w = frame.shape[:2]
+    if solved:
+        text, color = 'SOLVED!', (0, 255, 100)
+    else:
+        text, color = 'NOT YET — KEEP GOING', (80, 80, 255)
+    _glass_pill(frame, w // 2, h // 2, text,
+                font_scale=1.2, thickness=3, alpha=alpha, text_color=color)
+
+
+# ── Red border flash (unsolved) ───────────────────────────────────────────────
+
+def draw_fail_border(frame: np.ndarray, alpha: float = 0.6):
+    h, w = frame.shape[:2]
+    overlay = frame.copy()
+    cv2.rectangle(overlay, (0, 0), (w - 1, h - 1), (0, 0, 220), 12)
+    cv2.addWeighted(overlay, alpha, frame, 1 - alpha, 0, frame)
+
+
+# ── Confetti ──────────────────────────────────────────────────────────────────
+
+_confetti_particles = []
+
+def reset_confetti(frame_w: int, n: int = 120):
+    """Seed particles just above the top edge; they fall into view from there."""
+    global _confetti_particles
+    _confetti_particles = []
+    colors = [(255, 80, 80), (80, 255, 80), (80, 80, 255),
+              (255, 255, 80), (255, 80, 255), (80, 255, 255)]
+    for _ in range(n):
+        _confetti_particles.append({
+            'x': random.randint(0, frame_w),
+            'y': random.randint(-60, 0),
+            'vx': random.uniform(-2, 2),
+            'vy': random.uniform(3, 9),
+            'color': random.choice(colors),
+            'size': random.randint(6, 16),
+            'rot': random.uniform(0, 360),
+            'rot_vel': random.uniform(-5, 5),
+        })
+
+
+def draw_confetti(frame: np.ndarray):
+    h, w = frame.shape[:2]
+    alive = []
+    for p in _confetti_particles:
+        p['x'] += p['vx']
+        p['y'] += p['vy']
+        p['rot'] += p['rot_vel']
+        if p['y'] < h + 20:
+            alive.append(p)
+            s = p['size']
+            cx, cy = int(p['x']), int(p['y'])
+            cv2.rectangle(frame, (cx - s//2, cy - s//4),
+                          (cx + s//2, cy + s//4), p['color'], -1)
+    _confetti_particles[:] = alive

package/backend/requirements.txt

@@ -0,0 +1,9 @@
+opencv-python>=4.9.0
+mediapipe>=0.10.14
+numpy>=1.26.0
+scipy>=1.12.0
+pyglet>=2.0.18
+PyOpenGL>=3.1.7
+PyOpenGL-accelerate>=3.1.7
+flask>=3.0.0
+flask-cors>=4.0.0