pyclnf 0.1.0__py3-none-any.whl

pyclnf/__init__.py

@@ -0,0 +1,42 @@
+"""
+PyCLNF - Pure Python CLNF (Constrained Local Neural Fields) implementation
+
+A pure Python implementation of OpenFace's CLNF facial landmark detector.
+Uses exported OpenFace models with no C++ dependencies, making it perfect
+for PyInstaller distribution and cross-platform deployment.
+
+Usage:
+    from pyclnf import CLNF
+
+    # Initialize model
+    clnf = CLNF(model_dir="pyclnf/models")
+
+    # Detect landmarks
+    landmarks, info = clnf.fit(image, face_bbox)
+
+Components:
+    - PDM: Point Distribution Model (statistical shape model)
+    - CCNF: Patch experts for landmark detection
+    - NU-RLMS: Optimization algorithm for fitting
+    - CLNF: Complete pipeline
+"""
+
+from .clnf import CLNF
+from .core import PDM, CCNFModel, CCNFPatchExpert, NURLMSOptimizer
+from .utils.retinaface_correction import (
+    apply_retinaface_correction,
+    RetinaFaceCorrectedDetector,
+    get_correction_info
+)
+
+__version__ = "0.1.0"
+__all__ = [
+    'CLNF',
+    'PDM',
+    'CCNFModel',
+    'CCNFPatchExpert',
+    'NURLMSOptimizer',
+    'apply_retinaface_correction',
+    'RetinaFaceCorrectedDetector',
+    'get_correction_info'
+]

pyclnf/clnf.py

@@ -0,0 +1,581 @@
+"""
+CLNF (Constrained Local Neural Fields) - Complete facial landmark detector
+
+This is the main user-facing API that combines:
+- PDM (Point Distribution Model) for shape representation
+- CCNF patch experts for landmark detection
+- NU-RLMS optimizer for parameter fitting
+- Corrected RetinaFace detector (ARM Mac optimized, primary detector)
+
+Usage (PRIMARY - with automatic face detection):
+    from pyclnf import CLNF
+    import cv2
+
+    # Initialize model with corrected RetinaFace detector (default)
+    clnf = CLNF()  # ARM Mac optimized, 8.23px accuracy
+
+    # Detect and fit landmarks automatically
+    image = cv2.imread("face.jpg")
+    landmarks, info = clnf.detect_and_fit(image)
+
+Usage (LEGACY - with manual bbox):
+    from pyclnf import CLNF
+
+    # Initialize model without detector
+    clnf = CLNF(detector=None)
+
+    # Fit landmarks with manual bbox
+    landmarks, info = clnf.fit(image, face_bbox)
+"""
+
+import numpy as np
+from typing import Tuple, Optional, Dict, List
+import cv2
+from pathlib import Path
+
+from .core.pdm import PDM
+from .core.patch_expert import CCNFModel
+from .core.optimizer import NURLMSOptimizer
+from .utils.retinaface_correction import RetinaFaceCorrectedDetector
+
+
+class CLNF:
+    """
+    Complete CLNF facial landmark detector.
+
+    Fits a statistical shape model (PDM) to detected facial features using
+    patch experts and constrained optimization.
+    """
+
+    def __init__(self,
+                 model_dir: str = "pyclnf/models",
+                 scale: float = 0.25,
+                 regularization: float = 25.0,
+                 max_iterations: int = 10,
+                 convergence_threshold: float = 0.005,
+                 sigma: float = 1.5,
+                 weight_multiplier: float = 0.0,
+                 window_sizes: list = None,
+                 detector: str = "retinaface",
+                 detector_model_path: Optional[str] = None,
+                 use_coreml: bool = False):
+        """
+        Initialize CLNF model.
+
+        Args:
+            model_dir: Directory containing exported PDM and CCNF models
+            scale: DEPRECATED - now loads all scales [0.25, 0.35, 0.5]
+            regularization: Shape regularization weight (higher = stricter shape prior)
+                          (OpenFace default: r=25)
+            max_iterations: Maximum optimization iterations TOTAL across all window sizes
+                          (OpenFace default: 5 per window × 4 windows = 20 total)
+            convergence_threshold: Convergence threshold for parameter updates
+                          (OpenFace default: 0.01 for shape change)
+            sigma: Gaussian kernel sigma for KDE mean-shift
+                  (OpenFace uses σ=1.5 for Multi-PIE, σ=2.0 for in-the-wild)
+            weight_multiplier: Weight multiplier w for patch confidences
+                             (OpenFace uses w=7 for Multi-PIE, w=5 for in-the-wild)
+            window_sizes: List of window sizes for hierarchical refinement (default: [11, 9, 7])
+                         Note: Only window sizes with sigma components are supported ([7, 9, 11, 15])
+            detector: Face detector to use ("retinaface" or None). Default: "retinaface"
+            detector_model_path: Path to detector model. If None, uses default path
+            use_coreml: Enable CoreML acceleration for RetinaFace (ARM Mac optimization)
+        """
+        self.model_dir = Path(model_dir)
+        self.regularization = regularization
+        self.sigma = sigma
+        self.weight_multiplier = weight_multiplier
+        # Changed from [11, 9, 7, 5] to [11, 9, 7] because ws=5 has no sigma components
+        self.window_sizes = window_sizes if window_sizes is not None else [11, 9, 7]
+
+        # OpenFace uses multiple patch scales
+        self.patch_scaling = [0.25, 0.35, 0.5]
+
+        # Map window sizes to patch scale indices (coarse-to-fine)
+        # Larger windows use coarser scales, smaller windows use finer scales
+        self.window_to_scale = self._map_windows_to_scales()
+
+        # Load PDM (shape model)
+        pdm_dir = self.model_dir / "exported_pdm"
+        self.pdm = PDM(str(pdm_dir))
+
+        # Load CCNF patch experts for ALL scales
+        self.ccnf = CCNFModel(str(self.model_dir), scales=self.patch_scaling)
+
+        # Initialize optimizer with OpenFace parameters
+        self.optimizer = NURLMSOptimizer(
+            regularization=regularization,
+            max_iterations=max_iterations,
+            convergence_threshold=convergence_threshold,
+            sigma=sigma,
+            weight_multiplier=weight_multiplier  # CRITICAL: Apply weight multiplier
+        )
+
+        # Initialize face detector (PRIMARY: Corrected RetinaFace for ARM Mac optimization)
+        self.detector = None
+        if detector == "retinaface":
+            # Default model path if not specified
+            if detector_model_path is None:
+                detector_model_path = "S1 Face Mirror/weights/retinaface_mobilenet025_coreml.onnx"
+
+            # Initialize corrected RetinaFace detector
+            try:
+                self.detector = RetinaFaceCorrectedDetector(
+                    model_path=detector_model_path,
+                    use_coreml=use_coreml
+                )
+            except Exception as e:
+                print(f"Warning: Could not initialize RetinaFace detector: {e}")
+                print("Detector will not be available. Use fit() with manual bbox instead.")
+
+    def fit(self,
+            image: np.ndarray,
+            face_bbox: Tuple[float, float, float, float],
+            initial_params: Optional[np.ndarray] = None,
+            return_params: bool = False) -> Tuple[np.ndarray, Dict]:
+        """
+        Fit CLNF model to detect facial landmarks.
+
+        Args:
+            image: Input image (grayscale or color, will be converted to grayscale)
+            face_bbox: Face bounding box [x, y, width, height]
+            initial_params: Optional initial parameter guess (default: from bbox)
+            return_params: If True, include optimized parameters in info dict
+
+        Returns:
+            landmarks: Detected 2D landmarks, shape (68, 2)
+            info: Dictionary with fitting information:
+                - converged: bool
+                - iterations: int
+                - final_update: float
+                - params: np.ndarray (if return_params=True)
+        """
+        # Convert to grayscale if needed
+        if len(image.shape) == 3:
+            gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
+        else:
+            gray = image
+
+        # Initialize parameters from bounding box
+        if initial_params is None:
+            params = self.pdm.init_params(face_bbox)
+        else:
+            params = initial_params.copy()
+
+        # Estimate head pose from bbox for view selection
+        # For now, assume frontal view (view 0)
+        # TODO: Implement pose estimation from bbox orientation
+        view_idx = 0
+        pose = np.array([0.0, 0.0, 0.0])  # [pitch, yaw, roll]
+
+        # Hierarchical optimization with multiple window sizes and patch scales
+        # OpenFace optimizes from large to small windows for coarse-to-fine refinement
+
+        # FIX: Distribute max_iterations across window sizes instead of per-window
+        # This ensures total iterations match max_iterations, not max_iterations × num_windows
+        n_windows = len(self.window_sizes)
+        iters_per_window = self.optimizer.max_iterations // n_windows
+        iters_remainder = self.optimizer.max_iterations % n_windows
+
+        # Save original max_iterations to restore later
+        original_max_iterations = self.optimizer.max_iterations
+
+        total_iterations = 0
+        for window_idx, window_size in enumerate(self.window_sizes):
+            # Distribute remainder iterations to early windows
+            # e.g., max_iter=10, 3 windows → [4, 3, 3]
+            window_iters = iters_per_window + (1 if window_idx < iters_remainder else 0)
+
+            # Override optimizer max_iterations for this window
+            self.optimizer.max_iterations = window_iters
+
+            # Get the appropriate patch scale for this window (coarse-to-fine)
+            scale_idx = self.window_to_scale[window_size]
+            patch_scale = self.patch_scaling[scale_idx]
+
+            # Get patch experts for this view and scale
+            patch_experts = self._get_patch_experts(view_idx, patch_scale)
+
+            # Extract patch confidence weights (OpenFace NU-RLMS: Non-Uniform weighting)
+            # For landmarks with patch experts, use their confidence values
+            # For landmarks without patch experts, use default weight=1.0
+            weights = np.ones(self.pdm.n_points)  # Default: uniform weights
+            for landmark_idx, patch_expert in patch_experts.items():
+                if hasattr(patch_expert, 'patch_confidence'):
+                    weights[landmark_idx] = patch_expert.patch_confidence
+
+            # Adjust regularization based on patch scale (OpenFace formula)
+            # Reduce regularization at finer scales
+            # Formula: reg = reg_base - 15 * log2(patch_scale / base_scale)
+            scale_ratio = patch_scale / self.patch_scaling[0]  # Ratio to base scale (0.25)
+            adjusted_reg = self.regularization - 15 * np.log(scale_ratio) / np.log(2)
+            adjusted_reg = max(0.001, adjusted_reg)  # Ensure positive
+
+            # Update optimizer parameters for this scale
+            self.optimizer.regularization = adjusted_reg
+            # NOTE: Don't override optimizer.sigma - respect the value set by user or __init__
+
+            # Run optimization for this window size
+            # Using patch experts trained at patch_scale for this window
+            # CRITICAL: Pass patch_scaling to enable image warping
+            optimized_params, opt_info = self.optimizer.optimize(
+                self.pdm,
+                params,
+                patch_experts,
+                gray,
+                weights=weights,  # CRITICAL: Pass patch confidence weights
+                window_size=window_size,
+                patch_scaling=patch_scale,  # CRITICAL: Enable image warping to reference coordinates
+                sigma_components=self.ccnf.sigma_components  # Enable CCNF spatial correlation modeling
+            )
+
+            # Update params for next iteration
+            params = optimized_params
+            total_iterations += opt_info['iterations']
+
+            # Early stopping if face becomes too small
+            if params[0] < 0.25:  # Scale parameter
+                break
+
+        # Restore original max_iterations
+        self.optimizer.max_iterations = original_max_iterations
+
+        # Extract final landmarks
+        landmarks = self.pdm.params_to_landmarks_2d(optimized_params)
+
+        # Prepare output info
+        info = {
+            'converged': opt_info['converged'],
+            'iterations': total_iterations,
+            'final_update': opt_info['final_update'],
+            'view': view_idx,
+            'pose': pose
+        }
+
+        if return_params:
+            info['params'] = optimized_params
+
+        return landmarks, info
+
+    def detect_and_fit(self,
+                       image: np.ndarray,
+                       return_all_faces: bool = False,
+                       return_params: bool = False) -> Tuple[np.ndarray, Dict]:
+        """
+        Detect faces and fit CLNF landmarks in one call using the built-in detector.
+
+        This is the primary method for using pyCLNF with automatic face detection.
+        Uses corrected RetinaFace as the default detector (ARM Mac optimized).
+
+        Args:
+            image: Input image (grayscale or color, will be converted to grayscale)
+            return_all_faces: If True, return results for all detected faces
+                            If False, return only the first (largest) face
+            return_params: If True, include optimized parameters in info dict
+
+        Returns:
+            If return_all_faces=False (default):
+                landmarks: Detected 2D landmarks for first face, shape (68, 2)
+                info: Dictionary with fitting information including 'bbox'
+            If return_all_faces=True:
+                results: List of (landmarks, info) tuples for each detected face
+
+        Raises:
+            ValueError: If no detector is initialized or no faces detected
+
+        Example:
+            >>> from pyclnf import CLNF
+            >>> clnf = CLNF()  # Initializes with corrected RetinaFace
+            >>> image = cv2.imread("face.jpg")
+            >>> landmarks, info = clnf.detect_and_fit(image)
+            >>> print(f"Detected {len(landmarks)} landmarks")
+        """
+        if self.detector is None:
+            raise ValueError(
+                "No detector initialized. Either:\n"
+                "1. Initialize CLNF with detector='retinaface' (default)\n"
+                "2. Use fit() method with manual bbox instead"
+            )
+
+        # Convert to color for detector (detector needs BGR)
+        if len(image.shape) == 2:
+            # Grayscale -> BGR for detector
+            image_bgr = cv2.cvtColor(image, cv2.COLOR_GRAY2BGR)
+        else:
+            image_bgr = image
+
+        # Detect faces with corrected RetinaFace
+        bboxes = self.detector.detect_and_correct(image_bgr)
+
+        if len(bboxes) == 0:
+            raise ValueError("No faces detected in image")
+
+        # Process all faces if requested
+        if return_all_faces:
+            results = []
+            for bbox in bboxes:
+                landmarks, info = self.fit(image, bbox, return_params=return_params)
+                info['bbox'] = bbox  # Add bbox to info
+                results.append((landmarks, info))
+            return results
+
+        # Process only largest face (matches C++ CLNF behavior)
+        # C++ selects face with largest width (LandmarkDetectorUtils.cpp:809)
+        # Python bboxes are (x, y, width, height), so bbox[2] is width
+        if len(bboxes) == 1:
+            bbox = bboxes[0]
+        else:
+            # Select largest face by width (matching C++ DetectSingleFaceMTCNN)
+            widths = [bbox[2] for bbox in bboxes]
+            largest_idx = np.argmax(widths)
+            bbox = bboxes[largest_idx]
+
+        landmarks, info = self.fit(image, bbox, return_params=return_params)
+        info['bbox'] = bbox  # Add bbox to info
+        return landmarks, info
+
+    def fit_video(self,
+                  video_path: str,
+                  face_detector,
+                  output_path: Optional[str] = None,
+                  visualize: bool = True) -> list:
+        """
+        Fit CLNF to all frames in a video.
+
+        Args:
+            video_path: Path to input video
+            face_detector: Face detector function (image -> bbox or None)
+            output_path: Optional path to save visualization video
+            visualize: If True, draw landmarks on frames
+
+        Returns:
+            results: List of (landmarks, info) tuples for each frame
+        """
+        cap = cv2.VideoCapture(video_path)
+
+        # Get video properties
+        fps = cap.get(cv2.CAP_PROP_FPS)
+        width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
+        height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
+
+        # Setup video writer if output requested
+        writer = None
+        if output_path:
+            fourcc = cv2.VideoWriter_fourcc(*'mp4v')
+            writer = cv2.VideoWriter(output_path, fourcc, fps, (width, height))
+
+        results = []
+        frame_idx = 0
+        prev_params = None  # For temporal consistency
+
+        while True:
+            ret, frame = cap.read()
+            if not ret:
+                break
+
+            # Detect face
+            bbox = face_detector(frame)
+
+            if bbox is not None:
+                # Use previous frame's parameters as initialization for temporal consistency
+                landmarks, info = self.fit(
+                    frame,
+                    bbox,
+                    initial_params=prev_params
+                )
+
+                # Store parameters for next frame
+                if 'params' not in info:
+                    info['params'] = self.pdm.params_to_landmarks_2d.im_func.__self__.params
+                prev_params = info.get('params')
+
+                # Visualize if requested
+                if visualize:
+                    frame = self._draw_landmarks(frame, landmarks)
+
+                results.append((landmarks, info))
+            else:
+                results.append((None, {'converged': False}))
+                prev_params = None  # Reset on detection failure
+
+            # Write frame if output requested
+            if writer:
+                writer.write(frame)
+
+            frame_idx += 1
+
+        cap.release()
+        if writer:
+            writer.release()
+
+        return results
+
+    def _map_windows_to_scales(self) -> Dict[int, int]:
+        """
+        Map window sizes to patch scale indices.
+
+        OpenFace uses coarse-to-fine strategy:
+        - Large windows → coarse patch scale (0.25)
+        - Medium windows → medium patch scale (0.35)
+        - Small windows → fine patch scale (0.5)
+
+        Returns:
+            Dictionary mapping window_size -> scale_index
+        """
+        mapping = {}
+        for i, window_size in enumerate(self.window_sizes):
+            # Coarse-to-fine: first third uses scale 0, second third uses scale 1, final third uses scale 2
+            if i < len(self.window_sizes) // 3:
+                scale_idx = 0  # 0.25
+            elif i < 2 * len(self.window_sizes) // 3:
+                scale_idx = 1  # 0.35
+            else:
+                scale_idx = min(2, len(self.patch_scaling) - 1)  # 0.5
+            mapping[window_size] = scale_idx
+        return mapping
+
+    def _get_patch_experts(self, view_idx: int, scale: float) -> Dict[int, 'CCNFPatchExpert']:
+        """
+        Get patch experts for a specific view and scale.
+
+        Args:
+            view_idx: View index (0-6)
+            scale: Patch scale (0.25, 0.35, or 0.5)
+
+        Returns:
+            Dictionary mapping landmark_idx -> CCNFPatchExpert
+        """
+        patch_experts = {}
+
+        scale_model = self.ccnf.scale_models.get(scale)
+        if scale_model is None:
+            return patch_experts
+
+        view_data = scale_model['views'].get(view_idx)
+        if view_data is None:
+            return patch_experts
+
+        # Get all available patches for this view
+        patch_experts = view_data['patches']
+
+        return patch_experts
+
+    def _draw_landmarks(self,
+                       image: np.ndarray,
+                       landmarks: np.ndarray,
+                       color: Tuple[int, int, int] = (0, 255, 0),
+                       radius: int = 2) -> np.ndarray:
+        """
+        Draw landmarks on image.
+
+        Args:
+            image: Input image
+            landmarks: Landmark positions (n_points, 2)
+            color: Landmark color (B, G, R)
+            radius: Landmark radius in pixels
+
+        Returns:
+            image: Image with landmarks drawn
+        """
+        vis = image.copy()
+
+        for i, (x, y) in enumerate(landmarks):
+            cv2.circle(vis, (int(x), int(y)), radius, color, -1)
+
+        return vis
+
+    def get_info(self) -> Dict:
+        """Get model information."""
+        return {
+            'pdm': self.pdm.get_info(),
+            'ccnf': self.ccnf.get_info(),
+            'optimizer': {
+                'regularization': self.optimizer.regularization,
+                'max_iterations': self.optimizer.max_iterations,
+                'convergence_threshold': self.optimizer.convergence_threshold
+            },
+            'patch_scales': self.patch_scaling
+        }
+
+
+def test_clnf():
+    """Test CLNF complete pipeline."""
+    print("=" * 60)
+    print("Testing Complete CLNF Pipeline")
+    print("=" * 60)
+
+    # Test 1: Initialize CLNF
+    print("\nTest 1: Initialize CLNF")
+    clnf = CLNF(
+        model_dir="pyclnf/models",
+        scale=0.25,
+        max_iterations=5
+    )
+
+    info = clnf.get_info()
+    print(f"  PDM: {info['pdm']['n_points']} landmarks, {info['pdm']['n_params']} params")
+    print(f"  CCNF scales: {info['ccnf']['scales']}")
+    print(f"  CCNF patches at 0.25: {info['ccnf']['scale_models'][0.25]['total_patches']}")
+    print(f"  Optimizer: max_iter={info['optimizer']['max_iterations']}")
+
+    # Test 2: Create test image with face-like features
+    print("\nTest 2: Create test image")
+    test_image = np.random.randint(0, 256, (480, 640), dtype=np.uint8)
+
+    # Add some edge structure to simulate facial features
+    center_y, center_x = 240, 320
+    cv2.circle(test_image, (center_x - 50, center_y - 30), 15, 200, 2)  # Left eye
+    cv2.circle(test_image, (center_x + 50, center_y - 30), 15, 200, 2)  # Right eye
+    cv2.ellipse(test_image, (center_x, center_y + 30), (40, 20), 0, 0, 180, 200, 2)  # Mouth
+
+    print(f"  Test image: {test_image.shape}")
+
+    # Test 3: Fit CLNF to image
+    print("\nTest 3: Fit CLNF to test image")
+    face_bbox = (220, 140, 200, 250)  # [x, y, width, height]
+
+    landmarks, fit_info = clnf.fit(test_image, face_bbox, return_params=True)
+
+    print(f"  Bbox: {face_bbox}")
+    print(f"  Converged: {fit_info['converged']}")
+    print(f"  Iterations: {fit_info['iterations']}")
+    print(f"  Final update: {fit_info['final_update']:.6f}")
+    print(f"  Landmarks shape: {landmarks.shape}")
+    print(f"  Landmark range: x=[{landmarks[:, 0].min():.1f}, {landmarks[:, 0].max():.1f}], y=[{landmarks[:, 1].min():.1f}, {landmarks[:, 1].max():.1f}]")
+
+    # Test 4: Verify landmarks are within expected region
+    print("\nTest 4: Verify landmark positions")
+    bbox_center_x = face_bbox[0] + face_bbox[2] / 2
+    bbox_center_y = face_bbox[1] + face_bbox[3] / 2
+
+    landmark_center_x = landmarks[:, 0].mean()
+    landmark_center_y = landmarks[:, 1].mean()
+
+    center_offset = np.sqrt((landmark_center_x - bbox_center_x)**2 + (landmark_center_y - bbox_center_y)**2)
+
+    print(f"  Bbox center: ({bbox_center_x:.1f}, {bbox_center_y:.1f})")
+    print(f"  Landmark center: ({landmark_center_x:.1f}, {landmark_center_y:.1f})")
+    print(f"  Center offset: {center_offset:.1f} pixels")
+
+    # Test 5: Test with different bbox
+    print("\nTest 5: Fit with different bbox")
+    face_bbox2 = (150, 100, 150, 180)
+    landmarks2, fit_info2 = clnf.fit(test_image, face_bbox2)
+
+    print(f"  Bbox: {face_bbox2}")
+    print(f"  Converged: {fit_info2['converged']}")
+    print(f"  Iterations: {fit_info2['iterations']}")
+    print(f"  Landmark shift from first fit: {np.linalg.norm(landmarks2 - landmarks, axis=1).mean():.1f} pixels")
+
+    print("\n" + "=" * 60)
+    print("✓ Complete CLNF Pipeline Tests Complete!")
+    print("=" * 60)
+    print("\nCLNF is ready to use!")
+    print("  - Pure Python implementation (no C++ dependencies)")
+    print("  - Loads OpenFace trained models")
+    print("  - Ready for PyInstaller distribution")
+    print("  - Can be optimized with CoreML/Cython/CuPy as needed")
+
+
+if __name__ == "__main__":
+    test_clnf()

pyclnf/core/__init__.py

@@ -0,0 +1,9 @@
+"""
+PyCLNF Core - Pure Python CLNF implementation
+"""
+
+from .pdm import PDM
+from .patch_expert import CCNFPatchExpert, CCNFModel
+from .optimizer import NURLMSOptimizer
+
+__all__ = ['PDM', 'CCNFPatchExpert', 'CCNFModel', 'NURLMSOptimizer']