pyfaceau 1.0.3__cp310-cp310-win_amd64.whl

pyfaceau/alignment/face_aligner.py

@@ -0,0 +1,352 @@
+#!/usr/bin/env python3
+"""
+OpenFace 2.2 Face Alignment - Pure Python Implementation
+
+This module provides face alignment to a canonical 112×112 reference frame
+using similarity transform (scale + rotation + translation) based on 68 facial landmarks.
+
+Replicates the OpenFace 2.2 C++ alignment algorithm from:
+- Face_utils.cpp::AlignFace() (lines 109-146)
+- RotationHelpers.h::AlignShapesWithScale() (lines 195-242)
+- RotationHelpers.h::AlignShapesKabsch2D() (lines 168-191)
+
+Usage:
+    aligner = OpenFace22FaceAligner("pdm_68_multi_pie.txt")
+    aligned_face = aligner.align_face(image, landmarks_68, pose_tx, pose_ty)
+"""
+
+import numpy as np
+import cv2
+from pathlib import Path
+from typing import Tuple, Optional
+from pyfaceau.features.pdm import PDMParser
+
+
+class OpenFace22FaceAligner:
+    """
+    Pure Python implementation of OpenFace 2.2 face alignment
+
+    Aligns faces from 68 landmarks to 112×112 canonical reference frame
+    using similarity transform (scale + rotation + translation).
+    """
+
+    # Rigid landmark indices (0-indexed) from OpenFace C++
+    # These correspond to rigid facial structures: forehead, nose bridge, eye corners
+    # Excludes soft tissue like lips and cheeks
+    # NOTE: Includes 8 eye landmarks (36,39,40,41,42,45,46,47) which affect rotation
+    # Testing shows removing eyes improves STABILITY but ruins MAGNITUDE (31° vs 5°)
+    RIGID_INDICES = [1, 2, 3, 4, 12, 13, 14, 15, 27, 28, 29, 31, 32, 33, 34, 35, 36, 39, 40, 41, 42, 45, 46, 47]
+
+    def __init__(self, pdm_file: str, sim_scale: float = 0.7, output_size: Tuple[int, int] = (112, 112)):
+        """
+        Initialize face aligner with PDM reference shape
+
+        Args:
+            pdm_file: Path to PDM model file (e.g., "pdm_68_multi_pie.txt")
+            sim_scale: Scaling factor for reference shape (default: 0.7 for AU analysis)
+            output_size: Output aligned face size in pixels (default: 112×112)
+        """
+        self.sim_scale = sim_scale
+        self.output_width, self.output_height = output_size
+
+        # Load PDM and extract mean shape
+        print(f"Loading PDM from: {pdm_file}")
+        pdm = PDMParser(pdm_file)
+
+        # Preprocess mean shape: 204 values (68 landmarks × 3D) → 68 landmarks × 2D
+        # OpenFace C++ logic (Face_utils.cpp:112-119):
+        # 1. Scale mean shape by sim_scale
+        # 2. Discard Z component (take first 136 values = all X,Y coords)
+        # 3. Reshape to (68, 2) format
+        #
+        # CRITICAL: PDM stores as: [x0, y0, x1, y1, ..., x67, y67, z0, z1, ..., z67]
+        # NOT as: [x0, y0, z0, x1, y1, z1, ...]
+        # So we must: take first 136 values (all X,Y), then reshape
+        mean_shape_scaled = pdm.mean_shape * sim_scale  # (204, 1)
+        mean_shape_2d = mean_shape_scaled[:136]  # First 136 = all X,Y values
+        self.reference_shape = mean_shape_2d.reshape(68, 2)  # (68, 2)
+
+        print(f"Face aligner initialized")
+        print(f"  Sim scale: {sim_scale}")
+        print(f"  Output size: {output_size}")
+        print(f"  Reference shape: {self.reference_shape.shape}")
+        print(f"  Rigid points: {len(self.RIGID_INDICES)}")
+
+    def align_face(self, image: np.ndarray, landmarks_68: np.ndarray,
+                   pose_tx: float, pose_ty: float, p_rz: float = 0.0,
+                   apply_mask: bool = False, triangulation=None) -> np.ndarray:
+        """
+        Align face to canonical 112×112 reference frame
+
+        Args:
+            image: Input BGR image (any size)
+            landmarks_68: 68 facial landmarks as (68, 2) array or (136,) flat array
+            pose_tx: Pose translation X (from OpenFace pose estimation)
+            pose_ty: Pose translation Y (from OpenFace pose estimation)
+            p_rz: Pose rotation Z in radians (from OpenFace params_global[3])
+            apply_mask: If True, mask out regions outside the face (like OpenFace C++)
+            triangulation: TriangulationParser object (required if apply_mask=True)
+
+        Returns:
+            aligned_face: 112×112 aligned face image (BGR format)
+        """
+        # Ensure landmarks are (68, 2) shape
+        if landmarks_68.shape == (136,):
+            landmarks_68 = landmarks_68.reshape(68, 2)
+        elif landmarks_68.shape != (68, 2):
+            raise ValueError(f"landmarks_68 must be (68, 2) or (136,), got {landmarks_68.shape}")
+
+        # Extract rigid points from both source and destination
+        source_rigid = self._extract_rigid_points(landmarks_68)
+        dest_rigid = self._extract_rigid_points(self.reference_shape)
+
+        # Compute scale (no rotation from Kabsch)
+        scale_identity = self._align_shapes_with_scale(source_rigid, dest_rigid)
+        scale = scale_identity[0, 0]  # Extract scale from identity matrix
+
+        # Apply INVERSE of CSV p_rz rotation
+        # CSV p_rz describes rotation FROM canonical TO tilted
+        # We need rotation FROM tilted TO canonical, which is -p_rz
+        angle = -p_rz
+        cos_a = np.cos(angle)
+        sin_a = np.sin(angle)
+
+        R = np.array([[cos_a, -sin_a],
+                      [sin_a,  cos_a]], dtype=np.float32)
+
+        # Combine scale and rotation
+        scale_rot_matrix = scale * R
+
+        # Build 2×3 affine warp matrix
+        warp_matrix = self._build_warp_matrix(scale_rot_matrix, pose_tx, pose_ty)
+
+        # Apply affine transformation
+        aligned_face = cv2.warpAffine(
+            image,
+            warp_matrix,
+            (self.output_width, self.output_height),
+            flags=cv2.INTER_LINEAR
+        )
+
+        # Apply face mask if requested
+        if apply_mask:
+            if triangulation is None:
+                raise ValueError("triangulation required when apply_mask=True")
+
+            # Transform landmarks to aligned space
+            aligned_landmarks = self._transform_landmarks(landmarks_68, warp_matrix)
+
+            # Adjust eyebrow landmarks upward to include forehead (like C++)
+            # Indices 17-26 are eyebrows, 0 and 16 are jaw corners
+            forehead_offset = (30 / 0.7) * self.sim_scale
+            for idx in [0, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26]:
+                aligned_landmarks[idx, 1] -= forehead_offset
+
+            # Create mask
+            mask = triangulation.create_face_mask(
+                aligned_landmarks,
+                self.output_width,
+                self.output_height
+            )
+
+            # Apply mask to each channel
+            aligned_face = cv2.bitwise_and(aligned_face, aligned_face, mask=mask)
+
+        return aligned_face
+
+    def _transform_landmarks(self, landmarks: np.ndarray, warp_matrix: np.ndarray) -> np.ndarray:
+        """
+        Transform landmarks using affine warp matrix
+
+        Args:
+            landmarks: (68, 2) landmark coordinates in original image
+            warp_matrix: (2, 3) affine transformation matrix
+
+        Returns:
+            (68, 2) transformed landmark coordinates
+        """
+        # Convert to homogeneous coordinates
+        ones = np.ones((landmarks.shape[0], 1))
+        landmarks_hom = np.concatenate([landmarks, ones], axis=1)  # (68, 3)
+
+        # Apply transform
+        transformed = (warp_matrix @ landmarks_hom.T).T  # (68, 2)
+
+        return transformed
+
+    def _compute_scale_only(self, src: np.ndarray, dst: np.ndarray) -> float:
+        """
+        Compute scale factor between two point sets (no rotation)
+
+        Args:
+            src: (N, 2) source points
+            dst: (N, 2) destination points
+
+        Returns:
+            Scale factor
+        """
+        n = src.shape[0]
+
+        # Mean normalize
+        src_centered = src - src.mean(axis=0)
+        dst_centered = dst - dst.mean(axis=0)
+
+        # Compute RMS scale
+        s_src = np.sqrt(np.sum(src_centered ** 2) / n)
+        s_dst = np.sqrt(np.sum(dst_centered ** 2) / n)
+
+        return s_dst / s_src
+
+    def _build_warp_matrix(self, scale_rot: np.ndarray, pose_tx: float, pose_ty: float) -> np.ndarray:
+        """
+        Build 2×3 affine warp matrix from 2×2 scale-rotation matrix and pose translation
+
+        Implementation matches Face_utils.cpp::AlignFace (lines 127-143)
+
+        Critical details:
+        1. Copy scale-rotation matrix directly (NO transpose)
+        2. Transform pose translation THROUGH scale-rotation matrix
+        3. Simple centering with out_width/2, out_height/2 (NO empirical shifts)
+
+        Args:
+            scale_rot: (2, 2) similarity transform matrix (scale × rotation)
+            pose_tx: Pose translation X (from params_global[4])
+            pose_ty: Pose translation Y (from params_global[5])
+
+        Returns:
+            (2, 3) affine warp matrix for cv2.warpAffine
+        """
+        # Initialize 2×3 warp matrix
+        warp_matrix = np.zeros((2, 3), dtype=np.float32)
+
+        # Copy scale-rotation to first 2×2 block (NO transpose)
+        warp_matrix[:2, :2] = scale_rot
+
+        # Transform pose translation through scale-rotation
+        # C++ code (line 138-139):
+        #   cv::Vec2f T(tx, ty);
+        #   T = scale_rot_matrix * T;
+        T = np.array([pose_tx, pose_ty], dtype=np.float32)
+        T_transformed = scale_rot @ T
+
+        # Translation for centering in output image
+        # C++ code (lines 142-143):
+        #   warp_matrix(0,2) = -T(0) + out_width/2;
+        #   warp_matrix(1,2) = -T(1) + out_height/2;
+        # NO empirical shifts (+2, -2) - those were incorrect!
+        warp_matrix[0, 2] = -T_transformed[0] + self.output_width / 2
+        warp_matrix[1, 2] = -T_transformed[1] + self.output_height / 2
+
+        return warp_matrix
+
+    def _extract_rigid_points(self, landmarks: np.ndarray) -> np.ndarray:
+        """
+        Extract 24 rigid points from 68 landmarks
+
+        Rigid points correspond to stable facial structures (forehead, nose, eye corners)
+        that don't deform much during facial expressions.
+
+        Args:
+            landmarks: (68, 2) array of facial landmarks
+
+        Returns:
+            (24, 2) array of rigid landmarks
+        """
+        return landmarks[self.RIGID_INDICES]
+
+    def _align_shapes_kabsch_2d(self, src: np.ndarray, dst: np.ndarray) -> np.ndarray:
+        """
+        Compute 2D rotation matrix using Kabsch algorithm
+
+        This assumes src and dst are already mean-normalized.
+
+        Implementation based on RotationHelpers.h::AlignShapesKabsch2D (lines 168-191)
+
+        Args:
+            src: (N, 2) mean-normalized source points
+            dst: (N, 2) mean-normalized destination points
+
+        Returns:
+            (2, 2) rotation matrix
+        """
+        # SVD decomposition: src^T × dst
+        # OpenFace C++ uses: cv::SVD svd(align_from.t() * align_to)
+        U, S, Vt = np.linalg.svd(src.T @ dst)
+
+        # Check for reflection vs rotation
+        # OpenFace C++ uses determinant check to prevent reflections
+        d = np.linalg.det(Vt.T @ U.T)
+
+        # Correction matrix to prevent reflection
+        corr = np.eye(2)
+        if d > 0:
+            corr[1, 1] = 1
+        else:
+            corr[1, 1] = -1
+
+        # Rotation matrix: R = V^T × corr × U^T
+        # OpenFace C++ uses: R = svd.vt.t() * corr * svd.u.t()
+        R = Vt.T @ corr @ U.T
+
+        return R
+
+    def _align_shapes_with_scale(self, src: np.ndarray, dst: np.ndarray) -> np.ndarray:
+        """
+        Compute similarity transform (scale only, NO rotation) between two point sets
+
+        CRITICAL FIX: Since CSV landmarks are PDM-reconstructed (via CalcShape2D),
+        they are already in canonical orientation. We only need scale + translation,
+        NOT rotation via Kabsch.
+
+        Background: FaceAnalyser.cpp calls CalcParams TWICE:
+        1. On raw landmarks → params_global₁ → CalcShape2D → reconstructed landmarks (CSV output)
+        2. On reconstructed landmarks → params_global₂ → AlignFace
+
+        The second CalcParams produces near-zero rotation because reconstructed landmarks
+        are already canonical. Our Python uses CSV landmarks (already canonical), so we
+        skip rotation computation entirely.
+
+        Algorithm:
+        1. Mean-normalize both src and dst
+        2. Compute RMS scale for each
+        3. Return: (s_dst / s_src) × Identity (scale only, no rotation)
+
+        Args:
+            src: (N, 2) source points (detected landmarks)
+            dst: (N, 2) destination points (reference shape)
+
+        Returns:
+            (2, 2) similarity transform matrix (scale × identity)
+        """
+        n = src.shape[0]
+
+        # 1. Mean normalize both point sets
+        mean_src_x = np.mean(src[:, 0])
+        mean_src_y = np.mean(src[:, 1])
+        mean_dst_x = np.mean(dst[:, 0])
+        mean_dst_y = np.mean(dst[:, 1])
+
+        src_mean_normed = src.copy()
+        src_mean_normed[:, 0] -= mean_src_x
+        src_mean_normed[:, 1] -= mean_src_y
+
+        dst_mean_normed = dst.copy()
+        dst_mean_normed[:, 0] -= mean_dst_x
+        dst_mean_normed[:, 1] -= mean_dst_y
+
+        # 2. Compute RMS scale for each point set
+        # OpenFace C++ uses: sqrt(sum(points^2) / n)
+        src_sq = src_mean_normed ** 2
+        dst_sq = dst_mean_normed ** 2
+
+        s_src = np.sqrt(np.sum(src_sq) / n)
+        s_dst = np.sqrt(np.sum(dst_sq) / n)
+
+        # 3. Normalize by scale
+        src_norm = src_mean_normed / s_src
+        dst_norm = dst_mean_normed / s_dst
+
+        # 3. Return scale only (no rotation computed via Kabsch)
+        # Rotation will be provided externally from CSV p_rz
+        scale = s_dst / s_src
+        return scale * np.eye(2, dtype=np.float32)

pyfaceau/alignment/numba_calcparams_accelerator.py

@@ -0,0 +1,244 @@
+#!/usr/bin/env python3
+"""
+Numba JIT-Accelerated Functions for CalcParams
+
+Provides 2-5x speedup for pose estimation by compiling hot loops to machine code.
+Maintains 100% accuracy (identical math, just compiled).
+
+Performance targets:
+- CalcParams: 42.5ms → 14-21ms (2-3x speedup)
+- Overall FPS: 17.99 → 30-40 FPS
+
+Author: Optimized for Apple Silicon MacBook
+Date: 2025-11-01
+"""
+
+import numpy as np
+import numba
+
+
+@numba.jit(nopython=True, cache=True, fastmath=False)
+def euler_to_rotation_matrix_jit(rx, ry, rz):
+    """
+    JIT-compiled Euler to rotation matrix conversion
+
+    Args:
+        rx, ry, rz: Euler angles in radians
+
+    Returns:
+        3x3 rotation matrix (float32)
+    """
+    s1, s2, s3 = np.sin(rx), np.sin(ry), np.sin(rz)
+    c1, c2, c3 = np.cos(rx), np.cos(ry), np.cos(rz)
+
+    R = np.empty((3, 3), dtype=np.float32)
+    R[0, 0] = c2 * c3
+    R[0, 1] = -c2 * s3
+    R[0, 2] = s2
+    R[1, 0] = c1 * s3 + c3 * s1 * s2
+    R[1, 1] = c1 * c3 - s1 * s2 * s3
+    R[1, 2] = -c2 * s1
+    R[2, 0] = s1 * s3 - c1 * c3 * s2
+    R[2, 1] = c3 * s1 + c1 * s2 * s3
+    R[2, 2] = c1 * c2
+
+    return R
+
+
+@numba.jit(nopython=True, cache=True, fastmath=False)
+def project_shape_to_2d_jit(shape_3d, R, s, tx, ty, n_vis):
+    """
+    JIT-compiled 2D projection of 3D shape
+
+    Replaces the slow Python loop in calc_params (lines 521-525)
+
+    Args:
+        shape_3d: (3, n_vis) 3D coordinates [X, Y, Z] rows
+        R: (3, 3) rotation matrix
+        s: scale
+        tx, ty: translation
+        n_vis: number of visible landmarks
+
+    Returns:
+        (n_vis*2,) 2D projected landmarks [X0...Xn, Y0...Yn]
+    """
+    curr_shape_2d = np.empty(n_vis * 2, dtype=np.float32)
+
+    r11, r12, r13 = R[0, 0], R[0, 1], R[0, 2]
+    r21, r22, r23 = R[1, 0], R[1, 1], R[1, 2]
+
+    for i in range(n_vis):
+        X = shape_3d[0, i]
+        Y = shape_3d[1, i]
+        Z = shape_3d[2, i]
+
+        curr_shape_2d[i] = s * (r11*X + r12*Y + r13*Z) + tx
+        curr_shape_2d[i + n_vis] = s * (r21*X + r22*Y + r23*Z) + ty
+
+    return curr_shape_2d
+
+
+@numba.jit(nopython=True, cache=True, fastmath=False)
+def compute_jacobian_jit(shape_3d, R, s, princ_comp_vis, n_vis, m):
+    """
+    JIT-compiled Jacobian computation - the main bottleneck!
+
+    Replaces compute_jacobian() nested loops (lines 271-313)
+
+    This function computes partial derivatives of 2D projection w.r.t.:
+    - 6 global params: scale, rx, ry, rz, tx, ty
+    - m local params: PCA coefficients (34)
+
+    Args:
+        shape_3d: (3, n_vis) 3D shape [X, Y, Z] rows
+        R: (3, 3) rotation matrix
+        s: scale
+        princ_comp_vis: (n_vis*3, m) principal components for visible landmarks
+        n_vis: number of visible landmarks
+        m: number of PCA modes (34)
+
+    Returns:
+        J: (n_vis*2, 6+m) Jacobian matrix
+    """
+    J = np.zeros((n_vis * 2, 6 + m), dtype=np.float32)
+
+    # Extract rotation matrix elements
+    r11, r12, r13 = R[0, 0], R[0, 1], R[0, 2]
+    r21, r22, r23 = R[1, 0], R[1, 1], R[1, 2]
+    r31, r32, r33 = R[2, 0], R[2, 1], R[2, 2]  # Not used in 2D projection but needed for derivatives
+
+    # Iterate over each landmark
+    for i in range(n_vis):
+        X = shape_3d[0, i]
+        Y = shape_3d[1, i]
+        Z = shape_3d[2, i]
+
+        row_x = i
+        row_y = i + n_vis
+
+        # --- Global parameter derivatives ---
+
+        # Scaling term (column 0)
+        J[row_x, 0] = X * r11 + Y * r12 + Z * r13
+        J[row_y, 0] = X * r21 + Y * r22 + Z * r23
+
+        # Rotation terms (columns 1-3: rx, ry, rz)
+        J[row_x, 1] = s * (Y * r13 - Z * r12)
+        J[row_y, 1] = s * (Y * r23 - Z * r22)
+
+        J[row_x, 2] = -s * (X * r13 - Z * r11)
+        J[row_y, 2] = -s * (X * r23 - Z * r21)
+
+        J[row_x, 3] = s * (X * r12 - Y * r11)
+        J[row_y, 3] = s * (X * r22 - Y * r21)
+
+        # Translation terms (columns 4-5: tx, ty)
+        J[row_x, 4] = 1.0
+        J[row_y, 4] = 0.0
+
+        J[row_x, 5] = 0.0
+        J[row_y, 5] = 1.0
+
+        # --- Local parameter derivatives (columns 6 to 6+m-1) ---
+        for j in range(m):
+            # Get principal component for this mode at this landmark
+            Vx = princ_comp_vis[i, j]
+            Vy = princ_comp_vis[i + n_vis, j]
+            Vz = princ_comp_vis[i + 2*n_vis, j]
+
+            # Derivative of 2D projection w.r.t. local parameter j
+            J[row_x, 6 + j] = s * (r11*Vx + r12*Vy + r13*Vz)
+            J[row_y, 6 + j] = s * (r21*Vx + r22*Vy + r23*Vz)
+
+    return J
+
+
+@numba.jit(nopython=True, cache=True, fastmath=False)
+def apply_jacobian_weighting_jit(J, weight_diag, n_vis):
+    """
+    JIT-compiled Jacobian weighting application
+
+    Args:
+        J: (n_vis*2, 6+m) Jacobian matrix
+        weight_diag: (n_vis*2,) diagonal of weight matrix
+        n_vis: number of visible landmarks
+
+    Returns:
+        J_weighted: (n_vis*2, 6+m) weighted Jacobian
+    """
+    J_weighted = J.copy()
+
+    for i in range(n_vis):
+        w_x = weight_diag[i]
+        w_y = weight_diag[i + n_vis]
+
+        # Weight all columns for this landmark's rows
+        for j in range(J.shape[1]):
+            J_weighted[i, j] *= w_x
+            J_weighted[i + n_vis, j] *= w_y
+
+    return J_weighted
+
+
+def compute_jacobian_accelerated(params_local, params_global, princ_comp_vis, mean_shape_vis, weight_matrix, n_vis, m):
+    """
+    Python wrapper for JIT-compiled Jacobian computation
+
+    This function provides the same interface as CalcParams.compute_jacobian()
+    but uses Numba JIT for 2-5x speedup.
+
+    Args:
+        params_local: (34,) local parameters
+        params_global: (6,) global parameters [scale, rx, ry, rz, tx, ty]
+        princ_comp_vis: (n_vis*3, 34) principal components for visible landmarks
+        mean_shape_vis: (n_vis*3, 1) mean shape for visible landmarks
+        weight_matrix: (n_vis*2, n_vis*2) diagonal weight matrix
+        n_vis: number of visible landmarks (68 or less)
+        m: number of PCA modes (34)
+
+    Returns:
+        J: (n_vis*2, 6+m) Jacobian matrix
+        J_w_t: (6+m, n_vis*2) weighted Jacobian transpose
+    """
+    # Extract global parameters
+    s = params_global[0]
+    euler = params_global[1:4]
+
+    # Get 3D shape from current local parameters
+    params_local_col = params_local.reshape(-1, 1)
+    shape_3d_flat = mean_shape_vis.flatten() + (princ_comp_vis @ params_local_col).flatten()
+    shape_3d = shape_3d_flat.reshape(3, n_vis).astype(np.float32)
+
+    # Get rotation matrix using JIT function
+    R = euler_to_rotation_matrix_jit(euler[0], euler[1], euler[2])
+
+    # Compute Jacobian using JIT function
+    J = compute_jacobian_jit(shape_3d, R, s, princ_comp_vis, n_vis, m)
+
+    # Apply weighting
+    weight_diag = np.diag(weight_matrix).astype(np.float32)
+    if not np.allclose(weight_diag, 1.0):
+        J_weighted = apply_jacobian_weighting_jit(J, weight_diag, n_vis)
+        J_w_t = J_weighted.T
+    else:
+        J_w_t = J.T
+
+    return J, J_w_t
+
+
+# Warmup JIT compilation on import (prevents first-call overhead)
+def _warmup_jit():
+    """Pre-compile JIT functions to avoid first-call overhead"""
+    # Dummy data for compilation
+    dummy_R = euler_to_rotation_matrix_jit(0.1, 0.2, 0.3)
+    dummy_shape = np.random.randn(3, 68).astype(np.float32)
+    dummy_proj = project_shape_to_2d_jit(dummy_shape, dummy_R, 1.0, 0.0, 0.0, 68)
+    dummy_princ = np.random.randn(68*3, 34).astype(np.float32)
+    dummy_J = compute_jacobian_jit(dummy_shape, dummy_R, 1.0, dummy_princ, 68, 34)
+    dummy_weight = np.ones(68*2, dtype=np.float32)
+    dummy_J_weighted = apply_jacobian_weighting_jit(dummy_J, dummy_weight, 68)
+
+
+# Warmup on import
+_warmup_jit()
+print("Numba CalcParams accelerator loaded - targeting 2-5x speedup")