tperm-visor 1.0.2 → 1.0.4

package/backend/cube/renderer.py

@@ -101,6 +101,8 @@ _BOUND_CORNERS = [(x, y, z) for x in (-2.8, 2.8)
                             for y in (-2.8, 2.8)
                             for z in (-2.8, 2.8)]
 
+_PROBE_DEG = 0.5     # probe angle for measuring slice screen direction
+
 _POINTERS_3D = []
 for _x in [-1.6, 0.0, 1.6]:
     for _y in [-1.6, 0.0, 1.6]:
@@ -156,6 +158,7 @@ class CubeRenderer:
         self._ready = False
         self._body_list = None      # display list: one cubie body
         self._sticker_base = None   # 54 consecutive display lists, one per sticker
+        self._slice_dirs = {}       # face_key -> unit screen dir of the grabbed pointer
 
     def init_gl(self):
         self._fbo = glGenFramebuffers(1)
@@ -329,9 +332,9 @@ class CubeRenderer:
 
         for p3d in _POINTERS_3D:
             try:
-                # Need to manually project with slice rotation applied to get accurate 2D pointers
-                # But it's easier to just use the un-rotated pointer for the interaction hit-test
-                # Since when you drag, the slice rotates, but your finger stays near the pointer.
+                # Project the UN-rotated pointer: while you drag, the slice turns
+                # but your finger stays near where you grabbed, so the un-rotated
+                # position is the right hit-test target.
                 win_x, win_y, win_z = gluProject(p3d[0], p3d[1], p3d[2], modelview, projection, viewport)
                 if 0 <= win_z <= 1:
                     cv_y = h - win_y
@@ -339,6 +342,34 @@ class CubeRenderer:
             except Exception:
                 pass
 
+        # Screen direction each pointer travels for a small POSITIVE turn of each
+        # layer it belongs to. Measured through the very matrices used to draw
+        # this frame, rather than re-deriving the transform by hand - an analytic
+        # version disagreed with the renderer on 88% of random orientations.
+        self._slice_dirs = {}
+        if highlighted_pointer is not None:
+            try:
+                base = gluProject(highlighted_pointer[0], highlighted_pointer[1],
+                                  highlighted_pointer[2], modelview, projection, viewport)
+                p = np.asarray(highlighted_pointer, dtype=float)
+                for face_key in ('U', 'D', 'E', 'R', 'L', 'M'):
+                    if not _is_in_layer(face_key, *highlighted_pointer):
+                        continue
+                    axis = np.asarray(_get_rotation_axis(face_key), dtype=float)
+                    # render() applies glRotatef(-angle, axis), so probe with -eps
+                    theta = np.radians(-_PROBE_DEG)
+                    c, s_ = np.cos(theta), np.sin(theta)
+                    moved = (p * c + np.cross(axis, p) * s_
+                             + axis * np.dot(axis, p) * (1.0 - c))
+                    wx, wy, _wz = gluProject(moved[0], moved[1], moved[2],
+                                             modelview, projection, viewport)
+                    d = np.array([wx - base[0], (h - wy) - (h - base[1])])
+                    n = np.linalg.norm(d)
+                    if n > 1e-9:
+                        self._slice_dirs[face_key] = d / n
+            except Exception:
+                self._slice_dirs = {}
+
         # Read back only the cube's bounding box, not the whole frame. A full 720p
         # RGBA readback is 3.7 MB across the bus every tick and dominates the
         # frame; the cube usually covers a few percent of the screen.
@@ -379,6 +410,14 @@ class CubeRenderer:
 
         return cv_frame, pointers_2d
 
+    def slice_directions(self):
+        """Screen direction each candidate layer pushes the grabbed pointer.
+
+        Populated during render() for whichever pointer is highlighted, i.e. the
+        one currently grabbed. Empty when nothing is grabbed.
+        """
+        return self._slice_dirs
+
     def cleanup(self):
         if self._body_list:
             glDeleteLists(self._body_list, 1)

package/backend/gesture_engine.py

@@ -62,10 +62,27 @@ def is_open_palm(hand: HandData) -> bool:
     return extended >= 4 and thumb_idx_dist > 0.08
 
 
-def is_pinch(hand: HandData) -> bool:
-    """Thumb tip ↔ index tip closer than 0.06 normalised units."""
-    d = _norm_dist(hand.landmarks, 4, 8)
-    return d < 0.06
+# Hysteresis band for the pinch. A single threshold makes the gesture drop out
+# whenever landmark noise nudges the distance across it, which ends a drag
+# mid-swipe. Requiring a firm pinch to START but a clearly-open hand to RELEASE
+# means jitter around the boundary cannot break contact.
+PINCH_CLOSE = 0.055   # must get at least this close to begin pinching
+PINCH_OPEN  = 0.085   # must open at least this wide to stop
+
+
+def pinch_distance(hand: HandData) -> float:
+    """Thumb tip to index tip, in normalised units."""
+    return _norm_dist(hand.landmarks, 4, 8)
+
+
+def is_pinch(hand: HandData, was_pinching: bool = False) -> bool:
+    """True while the thumb and index finger are pinched together.
+
+    Pass the previous frame's state to get hysteresis; without it this is a
+    plain threshold at PINCH_CLOSE.
+    """
+    d = pinch_distance(hand)
+    return d < (PINCH_OPEN if was_pinching else PINCH_CLOSE)
 
 
 # ── Palm orientation ──────────────────────────────────────────────────────────

package/backend/server.py

@@ -28,8 +28,8 @@ from gesture_engine import (
 from cube.rubiks import solved_state, scramble, is_solved, apply_move
 from cube.renderer import CubeRenderer, LAYER_TURN_MOVE
 from utils.smoothing import EMA, QuatEMA
-from utils.transforms import (quat_multiply, quat_conjugate, cube_axes_on_screen,
-                              snap_to_nearest_90, hand_orientation_quat)
+from utils.transforms import (quat_multiply, quat_conjugate, snap_to_nearest_90,
+                              hand_orientation_quat)
 import hud
 
 
@@ -224,7 +224,7 @@ class AREngine:
         snapping = False
         active_pointer_3d = None
         drag_start_pos = None
-        drag_direction = None
+        drag_sign = 1.0
         DRAG_LOCK_THRESHOLD = 15
         pinch_released = True
 
@@ -493,43 +493,45 @@ class AREngine:
                         prev_hands = {h.label: h for h in hands}
 
                     elif state == State.DRAGGING_SLICE:
-                        pinching_hand = next((h for h in hands if is_pinch(h)), None)
+                        # Already pinching: use the wide release threshold so a
+                        # noisy frame cannot break the drag half-way through.
+                        pinching_hand = next(
+                            (h for h in hands if is_pinch(h, was_pinching=True)), None)
                         if pinching_hand and not snapping:
                             px = pinching_hand.landmarks[8].x * frame_w
                             py = pinching_hand.landmarks[8].y * frame_h
                             dx = px - drag_start_pos[0]
                             dy = py - drag_start_pos[1]
 
-                            if drag_direction is None:
-                                smooth_q = rot_ema.update(cube_rotation)
-                                if abs(dx) > DRAG_LOCK_THRESHOLD or abs(dy) > DRAG_LOCK_THRESHOLD:
-                                    screen_x, screen_y = cube_axes_on_screen(smooth_q)
-                                    swipe = np.array([dx, dy])
-                                    proj_x = abs(np.dot(swipe, screen_x))
-                                    proj_y = abs(np.dot(swipe, screen_y))
-                                    if proj_x > proj_y:
-                                        drag_direction = 'ROW'
-                                        if active_pointer_3d[1] > 0.1: face_rot_face = 'U'
-                                        elif active_pointer_3d[1] < -0.1: face_rot_face = 'D'
-                                        else: face_rot_face = 'E'
-                                    else:
-                                        drag_direction = 'COL'
-                                        if active_pointer_3d[0] > 0.1: face_rot_face = 'R'
-                                        elif active_pointer_3d[0] < -0.1: face_rot_face = 'L'
-                                        else: face_rot_face = 'M'
-
-                            if drag_direction is not None and face_rot_face is not None:
-                                smooth_q = rot_ema.update(cube_rotation)
-                                screen_x, screen_y = cube_axes_on_screen(smooth_q)
-                                swipe = np.array([dx, dy])
-                                if drag_direction == 'ROW':
-                                    proj = np.dot(swipe, screen_x)
-                                    sign = -1.0 if face_rot_face in ('U', 'E') else 1.0
-                                    face_rot_angle = sign * proj / 2.0
-                                elif drag_direction == 'COL':
-                                    proj = np.dot(swipe, screen_y)
-                                    sign = -1.0 if face_rot_face in ('R', 'M') else 1.0
-                                    face_rot_angle = sign * proj / 2.0
+                            # Both the layer and the direction come from
+                            # renderer._slice_dirs: the screen direction each
+                            # candidate layer would push the grabbed cubie,
+                            # measured through the same matrices that drew the
+                            # frame. Deriving this analytically did not match the
+                            # renderer on 88% of random orientations, so it is
+                            # measured rather than computed.
+                            swipe = np.array([dx, dy], dtype=float)
+                            swipe_len = float(np.linalg.norm(swipe))
+
+                            if face_rot_face is None and swipe_len > DRAG_LOCK_THRESHOLD:
+                                best_align = 0.0
+                                for key, direction in renderer.slice_directions().items():
+                                    align = float(np.dot(direction, swipe / swipe_len))
+                                    if abs(align) > abs(best_align):
+                                        best_align = align
+                                        face_rot_face = key
+                                        drag_sign = 1.0 if align > 0 else -1.0
+                                if face_rot_face is not None and abs(best_align) < 0.1:
+                                    face_rot_face = None      # too ambiguous, keep waiting
+
+                            if face_rot_face is not None:
+                                direction = renderer.slice_directions().get(face_rot_face)
+                                if direction is not None:
+                                    # Project the drag onto the direction that layer
+                                    # actually travels, so the turn keeps following
+                                    # the finger for the whole gesture.
+                                    face_rot_angle = drag_sign * float(np.dot(swipe, direction)) / 2.0
+
                         elif not snapping:
                             snap_target_angle = snap_to_nearest_90(face_rot_angle)
                             snap_start_angle = face_rot_angle
@@ -580,7 +582,6 @@ class AREngine:
                         face_rot_angle = 0.0
                         snapping = False
                         active_pointer_3d = None
-                        drag_direction = None
                         state = State.HOLDING
 
                 # ── Always render the cube at current state ──

package/backend/utils/transforms.py

@@ -109,40 +109,3 @@ def quat_multiply(q1: np.ndarray, q2: np.ndarray) -> np.ndarray:
 def snap_to_nearest_90(angle_degrees: float) -> float:
     """Round to nearest multiple of 90°."""
     return round(angle_degrees / 90.0) * 90.0
-
-
-def cube_axes_on_screen(q: np.ndarray) -> tuple[np.ndarray, np.ndarray]:
-    """
-    Given cube rotation quaternion, return the cube's local X and Y
-    axes projected to 2D screen space (as unit vectors in pixel-direction).
-    
-    Uses the SAME quaternion convention as the renderer: [x, y, z, w].
-
-    Returns (screen_x, screen_y) — each is a 2D numpy array.
-    """
-    x, y, z, w = q
-    
-    # Rotation matrix columns (local axes in world space)
-    # Same formula as renderer.py's quat_to_matrix
-    local_x = np.array([
-        1 - 2*y*y - 2*z*z,
-        2*x*y + 2*z*w,
-        2*x*z - 2*y*w
-    ])
-    local_y = np.array([
-        2*x*y - 2*z*w,
-        1 - 2*x*x - 2*z*z,
-        2*y*z + 2*x*w
-    ])
-    
-    # Project to screen: take x,y components, flip y for pixel coords
-    sx = np.array([local_x[0], -local_x[1]])
-    sy = np.array([local_y[0], -local_y[1]])
-    
-    # Normalize
-    nx = np.linalg.norm(sx)
-    ny = np.linalg.norm(sy)
-    if nx > 1e-6: sx /= nx
-    if ny > 1e-6: sy /= ny
-    
-    return sx, sy

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tperm-visor",
-  "version": "1.0.2",
+  "version": "1.0.4",
   "description": "T-PERM - gesture-driven AR Rubik's Cube in your browser. Webcam hand tracking via MediaPipe, 3D cube rendered with OpenGL. Requires Python 3.9+.",
   "bin": {
     "tperm-visor": "bin/t-perm.js"