hand-tracking-teleop 0.1.0__py3-none-any.whl

hand_tracking/visualizer_combined.py

@@ -0,0 +1,221 @@
+"""Combined 2D + 3D visualization module for hand tracking."""
+
+import numpy as np
+import cv2
+import matplotlib.pyplot as plt
+from matplotlib.animation import FuncAnimation
+from mpl_toolkits.mplot3d import Axes3D
+from matplotlib.gridspec import GridSpec
+import matplotlib
+
+
+class CombinedVisualizer:
+    """Combined 2D camera + 3D hand visualization."""
+    
+    HAND_CONNECTIONS = [
+        (0, 1), (1, 2), (2, 3), (3, 4),
+        (0, 5), (5, 6), (6, 7), (7, 8),
+        (0, 9), (9, 10), (10, 11), (11, 12),
+        (0, 13), (13, 14), (14, 15), (15, 16),
+        (0, 17), (17, 18), (18, 19), (19, 20),
+        (5, 9), (9, 13), (13, 17)
+    ]
+    
+    def __init__(self, detector):
+        """Initialize with hand detector."""
+        self.detector = detector
+        self.is_running = False
+        
+        # Create figure
+        self.fig = plt.figure(figsize=(18, 7))
+        gs = GridSpec(2, 3, figure=self.fig, width_ratios=[2, 1, 1], height_ratios=[1, 1])
+        
+        # Camera view
+        self.ax_camera = self.fig.add_subplot(gs[:, 0])
+        self.ax_camera.set_title('Camera View', fontsize=14, fontweight='bold')
+        self.ax_camera.axis('off')
+        
+        # 3D plots
+        self.ax_left = self.fig.add_subplot(gs[0, 1], projection='3d')
+        self.ax_left.set_title('Left Hand', fontsize=12, fontweight='bold')
+        
+        self.ax_right = self.fig.add_subplot(gs[0, 2], projection='3d')
+        self.ax_right.set_title('Right Hand', fontsize=12, fontweight='bold')
+        
+        # Info panel
+        self.ax_info = self.fig.add_subplot(gs[1, 1:])
+        self.ax_info.axis('off')
+        
+        # Setup 3D axes
+        for ax in [self.ax_left, self.ax_right]:
+            self._setup_3d_axis(ax)
+        
+        # Initialize plots
+        self.camera_img = None
+        self.left_hand_plots = self._init_hand_plots(self.ax_left)
+        self.right_hand_plots = self._init_hand_plots(self.ax_right)
+        self.info_text = self.ax_info.text(0.1, 0.5, '', fontsize=10, family='monospace')
+        
+        plt.tight_layout()
+    
+    def _setup_3d_axis(self, ax):
+        """Setup 3D axis with metric units (meters).
+
+        Axis limits accommodate hand movement across the frame:
+        - X/Y: ±0.3m (hand can be ~300px from center, 300 * ~0.0005 scale ≈ 0.15m)
+        - Z: ±0.15m (depth range for hand distance variations)
+        """
+        ax.set_xlim([-0.30, 0.30])
+        ax.set_ylim([-0.30, 0.30])
+        ax.set_zlim([-0.15, 0.15])
+        ax.set_xlabel('X (m)', fontsize=8)
+        ax.set_ylabel('Y (m)', fontsize=8)
+        ax.set_zlabel('Z (m)', fontsize=8)
+        ax.view_init(elev=15, azim=45)
+        ax.grid(True, alpha=0.3)
+    
+    def _init_hand_plots(self, ax):
+        """Initialize plot elements."""
+        plots = {}
+        plots['landmarks'] = ax.scatter([], [], [], c='red', s=30, alpha=0.8)
+        plots['connections'] = []
+        for _ in self.HAND_CONNECTIONS:
+            line, = ax.plot([], [], [], 'b-', linewidth=1.5, alpha=0.6)
+            plots['connections'].append(line)
+        
+        # Coordinate axes
+        plots['axes'] = []
+        key_landmarks = [0, 4, 8, 12, 16, 20]
+        for _ in key_landmarks:
+            x_axis, = ax.plot([], [], [], 'r-', linewidth=1, alpha=0.6)
+            y_axis, = ax.plot([], [], [], 'g-', linewidth=1, alpha=0.6)
+            z_axis, = ax.plot([], [], [], 'b-', linewidth=1, alpha=0.6)
+            plots['axes'].append([x_axis, y_axis, z_axis])
+        
+        return plots
+    
+    def _get_connection_color(self, conn_idx):
+        """Get color for connection."""
+        start, end = self.HAND_CONNECTIONS[conn_idx]
+        
+        if max(start, end) <= 4:
+            return '#FF6B6B'
+        elif max(start, end) <= 8:
+            return '#4ECDC4'
+        elif max(start, end) <= 12:
+            return '#45B7D1'
+        elif max(start, end) <= 16:
+            return '#96CEB4'
+        elif max(start, end) <= 20:
+            return '#FFEAA7'
+        else:
+            return '#DFE6E9'
+    
+    def _update_camera_view(self, frame, hands):
+        """Update 2D camera view."""
+        if frame is None:
+            return
+        
+        frame_with_landmarks = self.detector.draw_landmarks(frame.copy(), hands)
+        frame_rgb = cv2.cvtColor(frame_with_landmarks, cv2.COLOR_BGR2RGB)
+        
+        if self.camera_img is None:
+            self.camera_img = self.ax_camera.imshow(frame_rgb)
+        else:
+            self.camera_img.set_data(frame_rgb)
+    
+    def _update_3d_plot(self, ax, plots, hand_data):
+        """Update 3D plot."""
+        if hand_data is None:
+            plots['landmarks']._offsets3d = ([], [], [])
+            for line in plots['connections']:
+                line.set_data([], [])
+                line.set_3d_properties([])
+            for axis_lines in plots['axes']:
+                for line in axis_lines:
+                    line.set_data([], [])
+                    line.set_3d_properties([])
+            return
+        
+        # Get normalized landmarks in meters (palm width ≈ 0.08m)
+        # Pass calibration params for depth estimation from palm size
+        normalized = hand_data.get_normalized_landmarks(
+            palm_size_meters=0.08,
+            reference_distance_m=self.detector.reference_distance_m,
+            reference_palm_width_px=self.detector.calibrated_palm_width_px
+        )
+        
+        # Update landmarks
+        plots['landmarks']._offsets3d = (normalized[:, 0], normalized[:, 1], normalized[:, 2])
+        
+        # Update connections
+        for idx, (start, end) in enumerate(self.HAND_CONNECTIONS):
+            x_data = [normalized[start, 0], normalized[end, 0]]
+            y_data = [normalized[start, 1], normalized[end, 1]]
+            z_data = [normalized[start, 2], normalized[end, 2]]
+            plots['connections'][idx].set_data(x_data, y_data)
+            plots['connections'][idx].set_3d_properties(z_data)
+            plots['connections'][idx].set_color(self._get_connection_color(idx))
+        
+        # Update axes
+        key_landmarks = [0, 4, 8, 12, 16, 20]
+        axis_length = 0.01  # 1cm axis lines in meters
+        
+        for i, landmark_idx in enumerate(key_landmarks):
+            point = normalized[landmark_idx]
+            plots['axes'][i][0].set_data([point[0], point[0] + axis_length], [point[1], point[1]])
+            plots['axes'][i][0].set_3d_properties([point[2], point[2]])
+            plots['axes'][i][1].set_data([point[0], point[0]], [point[1], point[1] + axis_length])
+            plots['axes'][i][1].set_3d_properties([point[2], point[2]])
+            plots['axes'][i][2].set_data([point[0], point[0]], [point[1], point[1]])
+            plots['axes'][i][2].set_3d_properties([point[2], point[2] + axis_length])
+    
+    def _update_info_display(self, hands):
+        """Update info panel."""
+        info = [f"Frame: {self.detector.frame_count}", f"Hands: {len(hands)}", ""]
+        for i, hand in enumerate(hands):
+            info.append(f"Hand {i+1}: {hand.handedness} (Conf: {hand.confidence:.3f})")
+        if not hands:
+            info.append("No hands detected")
+        self.info_text.set_text('\n'.join(info))
+    
+    def update(self, frame_num):
+        """Animation update."""
+        frame = self.detector.get_frame()
+        if frame is None:
+            return []
+        
+        hands = self.detector.detect_hands(frame)
+        
+        left_hand = next((h for h in hands if h.handedness == 'Left'), None)
+        right_hand = next((h for h in hands if h.handedness == 'Right'), None)
+        
+        self._update_camera_view(frame, hands)
+        self._update_3d_plot(self.ax_left, self.left_hand_plots, left_hand)
+        self._update_3d_plot(self.ax_right, self.right_hand_plots, right_hand)
+        self._update_info_display(hands)
+        
+        self.fig.suptitle('Hand Tracking - 2D + 3D View', fontsize=16, fontweight='bold')
+        
+        self.detector.frame_count += 1
+        return []
+    
+    def run(self):
+        """Run visualization."""
+        print("Starting combined 2D + 3D visualization...")
+        print("Controls: Rotate 3D (drag), Zoom (scroll), Close window to exit")
+        
+        self.is_running = True
+        
+        try:
+            anim = FuncAnimation(self.fig, self.update, interval=33, blit=False)
+            plt.show()
+        except KeyboardInterrupt:
+            print("\nVisualization stopped")
+        finally:
+            self.cleanup()
+    
+    def cleanup(self):
+        """Cleanup."""
+        self.is_running = False
+        plt.close('all')

hand_tracking/visualizer_rerun.py

@@ -0,0 +1,255 @@
+"""Rerun-based visualization module for hand tracking.
+
+This module provides real-time hand visualization using the Rerun SDK,
+offering improved performance and better 3D interaction compared to matplotlib.
+"""
+
+import numpy as np
+import cv2
+from typing import Optional, List
+
+try:
+    import rerun as rr
+    import rerun.blueprint as rrb
+    RERUN_AVAILABLE = True
+except ImportError:
+    RERUN_AVAILABLE = False
+
+
+class RerunVisualizer:
+    """Real-time hand visualization using Rerun SDK."""
+
+    # MediaPipe hand connections grouped by finger
+    FINGER_CONNECTIONS = {
+        'thumb': [(0, 1), (1, 2), (2, 3), (3, 4)],
+        'index': [(0, 5), (5, 6), (6, 7), (7, 8)],
+        'middle': [(0, 9), (9, 10), (10, 11), (11, 12)],
+        'ring': [(0, 13), (13, 14), (14, 15), (15, 16)],
+        'pinky': [(0, 17), (17, 18), (18, 19), (19, 20)],
+        'palm': [(5, 9), (9, 13), (13, 17)],
+    }
+
+    # Colors for each finger (RGB format)
+    FINGER_COLORS = {
+        'thumb': [255, 107, 107],   # Red
+        'index': [78, 205, 196],    # Teal
+        'middle': [69, 183, 209],   # Blue
+        'ring': [150, 206, 180],    # Green
+        'pinky': [255, 234, 167],   # Yellow
+        'palm': [223, 230, 233],    # Gray
+    }
+
+    # Landmark indices for each finger
+    FINGER_LANDMARKS = {
+        'thumb': [1, 2, 3, 4],
+        'index': [5, 6, 7, 8],
+        'middle': [9, 10, 11, 12],
+        'ring': [13, 14, 15, 16],
+        'pinky': [17, 18, 19, 20],
+        'palm': [0],  # Wrist
+    }
+
+    def __init__(self, detector):
+        """Initialize with hand detector.
+
+        Args:
+            detector: HandDetector instance for accessing camera and detection.
+        """
+        if not RERUN_AVAILABLE:
+            raise ImportError(
+                "Rerun SDK is required for this visualization mode.\n"
+                "Install with: pip install rerun-sdk"
+            )
+
+        self.detector = detector
+        self.is_running = False
+        self._setup_rerun()
+
+    def _setup_rerun(self):
+        """Initialize Rerun recording and send blueprint."""
+        # Initialize Rerun with application name
+        rr.init("hand_tracking", spawn=True)
+
+        # Create blueprint for layout
+        blueprint = rrb.Horizontal(
+            rrb.Spatial2DView(
+                name="Camera",
+                origin="world/camera",
+            ),
+            rrb.Spatial3DView(
+                name="3D Hands",
+                origin="world/hands_3d",
+            ),
+            column_shares=[1, 1],
+        )
+
+        # Send the blueprint
+        rr.send_blueprint(blueprint)
+
+        # Log initial coordinate system info
+        rr.log("world/hands_3d", rr.ViewCoordinates.RIGHT_HAND_Y_DOWN, static=True)
+
+    def _get_landmark_color(self, landmark_idx: int) -> List[int]:
+        """Get color for a specific landmark index.
+
+        Args:
+            landmark_idx: Index of the landmark (0-20).
+
+        Returns:
+            RGB color as list of 3 integers.
+        """
+        for finger, indices in self.FINGER_LANDMARKS.items():
+            if landmark_idx in indices:
+                return self.FINGER_COLORS[finger]
+        return self.FINGER_COLORS['palm']
+
+    def _log_hand_3d(self, hand_data, handedness: str):
+        """Log 3D hand landmarks and connections to Rerun.
+
+        Args:
+            hand_data: HandLandmarks object with detection data.
+            handedness: 'left' or 'right'.
+        """
+        if hand_data is None:
+            # Clear the hand data if no detection
+            rr.log(f"world/hands_3d/{handedness}/landmarks", rr.Clear(recursive=False))
+            for finger in self.FINGER_CONNECTIONS:
+                rr.log(f"world/hands_3d/{handedness}/{finger}", rr.Clear(recursive=False))
+            return
+
+        # Get normalized landmarks in meters (palm width ≈ 0.08m)
+        # Pass calibration params for depth estimation from palm size
+        normalized = hand_data.get_normalized_landmarks(
+            palm_size_meters=0.08,
+            reference_distance_m=self.detector.reference_distance_m,
+            reference_palm_width_px=self.detector.calibrated_palm_width_px
+        )
+
+        # Prepare landmark data with colors
+        # Radii in meters (coordinates are now metric)
+        positions = normalized.tolist()
+        colors = [self._get_landmark_color(i) for i in range(21)]
+        radii = [0.008 if i == 0 else 0.005 for i in range(21)]  # ~8mm wrist, ~5mm joints
+
+        # Log landmarks as 3D points
+        rr.log(
+            f"world/hands_3d/{handedness}/landmarks",
+            rr.Points3D(
+                positions=positions,
+                colors=colors,
+                radii=radii,
+            )
+        )
+
+        # Log connections as line strips for each finger
+        for finger, connections in self.FINGER_CONNECTIONS.items():
+            strips = []
+            for start_idx, end_idx in connections:
+                strips.append([
+                    normalized[start_idx].tolist(),
+                    normalized[end_idx].tolist()
+                ])
+
+            rr.log(
+                f"world/hands_3d/{handedness}/{finger}",
+                rr.LineStrips3D(
+                    strips=strips,
+                    colors=[self.FINGER_COLORS[finger]],
+                    radii=[0.002],  # ~2mm line thickness
+                )
+            )
+
+        # Log metadata as text (positions now in meters)
+        wrist_pos = normalized[0]
+        rr.log(
+            f"world/hands_3d/{handedness}/info",
+            rr.TextLog(
+                f"{hand_data.handedness} Hand | "
+                f"Conf: {hand_data.confidence:.2f} | "
+                f"Wrist: ({wrist_pos[0]:.3f}m, {wrist_pos[1]:.3f}m, {wrist_pos[2]:.3f}m)"
+            )
+        )
+
+    def _log_camera_frame(self, frame: np.ndarray, hands: List):
+        """Log camera frame with optional overlay to Rerun.
+
+        Args:
+            frame: BGR image from camera.
+            hands: List of detected HandLandmarks.
+        """
+        if frame is None:
+            return
+
+        # Draw landmarks on frame
+        frame_with_landmarks = self.detector.draw_landmarks(frame.copy(), hands)
+
+        # Convert BGR to RGB for Rerun
+        frame_rgb = cv2.cvtColor(frame_with_landmarks, cv2.COLOR_BGR2RGB)
+
+        # Log the image
+        rr.log("world/camera/image", rr.Image(frame_rgb))
+
+        # Log frame metadata
+        rr.log(
+            "world/camera/info",
+            rr.TextLog(f"Frame: {self.detector.frame_count} | Hands: {len(hands)}")
+        )
+
+    def update(self) -> bool:
+        """Process one frame and log to Rerun.
+
+        Returns:
+            True if frame was processed successfully, False otherwise.
+        """
+        frame = self.detector.get_frame()
+        if frame is None:
+            return False
+
+        # Detect hands
+        hands = self.detector.detect_hands(frame)
+
+        # Log camera frame with overlays
+        self._log_camera_frame(frame, hands)
+
+        # Find left and right hands
+        left_hand = next((h for h in hands if h.handedness == 'Left'), None)
+        right_hand = next((h for h in hands if h.handedness == 'Right'), None)
+
+        # Log 3D hand data
+        self._log_hand_3d(left_hand, 'left')
+        self._log_hand_3d(right_hand, 'right')
+
+        self.detector.frame_count += 1
+        return True
+
+    def run(self):
+        """Run the main visualization loop."""
+        print("Starting Rerun visualization...")
+        print("The Rerun viewer should open automatically.")
+        print("Features:")
+        print("  - 2D camera view with landmark overlay")
+        print("  - 3D hand visualization with color-coded fingers")
+        print("  - Timeline scrubbing and playback controls")
+        print("Press Ctrl+C to stop.")
+
+        self.is_running = True
+
+        try:
+            while self.is_running:
+                if not self.update():
+                    print("Failed to get frame, stopping...")
+                    break
+
+                # Check for OpenCV window quit (optional backup)
+                if cv2.waitKey(1) & 0xFF == ord('q'):
+                    break
+
+        except KeyboardInterrupt:
+            print("\nVisualization stopped by user")
+        finally:
+            self.cleanup()
+
+    def cleanup(self):
+        """Clean up resources."""
+        self.is_running = False
+        print("Rerun visualization cleanup complete.")