pyfaceau 1.0.6__cp312-cp312-macosx_11_0_arm64.whl

pyfaceau/detectors/pfld.py

@@ -0,0 +1,128 @@
+#!/usr/bin/env python3
+"""Cunjian PFLD landmark detector wrapper for AU extraction pipeline"""
+
+import numpy as np
+import cv2
+import onnxruntime as ort
+
+
+class CunjianPFLDDetector:
+    """
+    Wrapper for cunjian's PFLD 68-point landmark detector.
+
+    Model: PFLD_ExternalData (112×112)
+    Published accuracy: 3.97% NME on 300W Full Set
+    Measured accuracy: 4.37% NME on our validation
+    Size: 2.9MB
+    Speed: 0.01s per face
+    """
+
+    def __init__(self, model_path, use_coreml=True):
+        """Initialize the PFLD detector.
+
+        Args:
+            model_path: Path to the ONNX model file
+            use_coreml: Whether to attempt CoreML acceleration on Apple Silicon (default: True)
+        """
+        # Configure execution providers for Apple Silicon Neural Engine acceleration
+        if use_coreml:
+            providers = [
+                ('CoreMLExecutionProvider', {
+                    'MLComputeUnits': 'ALL',  # Use Neural Engine + GPU + CPU
+                    'ModelFormat': 'MLProgram',  # Use latest CoreML format
+                }),
+                'CPUExecutionProvider'  # Fallback
+            ]
+        else:
+            providers = ['CPUExecutionProvider']
+
+        # Suppress CoreML compilation warnings
+        import warnings
+        with warnings.catch_warnings():
+            warnings.filterwarnings('ignore')
+            self.session = ort.InferenceSession(model_path, providers=providers)
+
+        # Check which provider is active
+        active_providers = self.session.get_providers()
+        if 'CoreMLExecutionProvider' in active_providers:
+            print("PFLD using CoreML Neural Engine acceleration (2-3x speedup)")
+        else:
+            print("Warning: PFLD using CPU execution (CoreML unavailable)")
+
+        self.input_name = self.session.get_inputs()[0].name
+
+        # Model expects 112x112 RGB input normalized to [0, 1]
+        self.input_size = 112
+
+    def detect_landmarks(self, frame, bbox):
+        """Detect 68 facial landmarks.
+
+        Args:
+            frame: BGR image (HxWx3)
+            bbox: Face bounding box [x_min, y_min, x_max, y_max]
+
+        Returns:
+            landmarks: (68, 2) array of (x, y) coordinates in original image space
+            confidence: Dummy confidence (always 1.0 for this model)
+        """
+        x_min, y_min, x_max, y_max = bbox
+
+        # Calculate square bbox with 10% padding (cunjian approach)
+        w = x_max - x_min
+        h = y_max - y_min
+        size = int(max([w, h]) * 1.1)
+        cx = int(x_min + w / 2)
+        cy = int(y_min + h / 2)
+        x1 = cx - size // 2
+        x2 = x1 + size
+        y1 = cy - size // 2
+        y2 = y1 + size
+
+        # Clip to image bounds and add padding if needed
+        height, width = frame.shape[:2]
+        dx = max(0, -x1)
+        dy = max(0, -y1)
+        x1 = max(0, x1)
+        y1 = max(0, y1)
+        edx = max(0, x2 - width)
+        edy = max(0, y2 - height)
+        x2 = min(width, x2)
+        y2 = min(height, y2)
+
+        # Crop face
+        cropped = frame[y1:y2, x1:x2]
+
+        # Add border padding if face was at edge
+        if dx > 0 or dy > 0 or edx > 0 or edy > 0:
+            cropped = cv2.copyMakeBorder(cropped, int(dy), int(edy), int(dx), int(edx),
+                                          cv2.BORDER_CONSTANT, 0)
+
+        # Preprocess: BGR -> RGB, resize to 112x112, normalize to [0, 1]
+        cropped_rgb = cv2.cvtColor(cropped, cv2.COLOR_BGR2RGB)
+        face_resized = cv2.resize(cropped_rgb, (self.input_size, self.input_size))
+        face_normalized = face_resized.astype(np.float32) / 255.0
+        face_input = np.transpose(face_normalized, (2, 0, 1))  # HWC -> CHW
+        face_input = np.expand_dims(face_input, axis=0)  # Add batch dimension
+
+        # Run inference
+        output = self.session.run(None, {self.input_name: face_input})[0]
+
+        # Output is (1, 136) -> reshape to (68, 2) normalized coordinates
+        landmarks = output.reshape(-1, 2)
+
+        # Reproject from normalized [0, 1] space to original image coordinates
+        bbox_w = x2 - x1
+        bbox_h = y2 - y1
+        landmarks_reprojected = np.zeros_like(landmarks)
+        for i, point in enumerate(landmarks):
+            landmarks_reprojected[i, 0] = point[0] * bbox_w + x1
+            landmarks_reprojected[i, 1] = point[1] * bbox_h + y1
+
+        # Return dummy confidence (PFLD doesn't output per-landmark confidence)
+        confidence = 1.0
+
+        return landmarks_reprojected, confidence
+
+    def __repr__(self):
+        return (f"CunjianPFLDDetector(input_size={self.input_size}, "
+                f"landmarks=68, accuracy=4.37% NME)")

pyfaceau/detectors/retinaface.py

@@ -0,0 +1,352 @@
+#!/usr/bin/env python3
+"""
+Optimized RetinaFace detector using ONNX Runtime with CoreML acceleration.
+
+This module provides a drop-in replacement for the PyTorch-based RetinaFace detector,
+optimized for Apple Silicon using the Neural Engine via CoreML execution provider.
+
+Expected performance: 5-10x speedup (from ~191ms to ~20-40ms per detection)
+"""
+
+import numpy as np
+import torch
+import onnxruntime as ort
+from typing import Tuple
+
+# Import RetinaFace post-processing utilities
+from openface.Pytorch_Retinaface.layers.functions.prior_box import PriorBox
+from openface.Pytorch_Retinaface.utils.box_utils import decode, decode_landm
+from openface.Pytorch_Retinaface.utils.nms.py_cpu_nms import py_cpu_nms
+from openface.Pytorch_Retinaface.data import cfg_mnet
+
+# Import performance profiler
+from performance_profiler import get_profiler
+
+
+class ONNXRetinaFaceDetector:
+    """
+    ONNX-accelerated RetinaFace detector for Apple Silicon
+
+    This class provides the same interface as OpenFace 3.0's FaceDetector,
+    but uses ONNX Runtime with CoreML execution provider for massive speedup.
+    """
+
+    def __init__(self, onnx_model_path: str, use_coreml: bool = True,
+                 confidence_threshold: float = 0.02,
+                 nms_threshold: float = 0.4,
+                 vis_threshold: float = 0.5):
+        """
+        Initialize ONNX RetinaFace detector
+
+        Args:
+            onnx_model_path: Path to converted ONNX model
+            use_coreml: Whether to attempt CoreML execution provider (default: True)
+            confidence_threshold: Minimum confidence for face detection (default: 0.02)
+            nms_threshold: NMS threshold for duplicate suppression (default: 0.4)
+            vis_threshold: Visibility threshold for filtering weak detections (default: 0.5)
+        """
+        self.confidence_threshold = confidence_threshold
+        self.nms_threshold = nms_threshold
+        self.vis_threshold = vis_threshold
+        self.cfg = cfg_mnet
+
+        # Configure execution providers
+        # NOTE: CoreML may not fully support all RetinaFace operations
+        # We try CoreML first, but gracefully fall back to optimized CPU execution
+        if use_coreml:
+            providers = [
+                ('CoreMLExecutionProvider', {
+                    'MLComputeUnits': 'ALL',  # Use Neural Engine + GPU + CPU
+                    'ModelFormat': 'MLProgram',  # Use latest CoreML format
+                }),
+                'CPUExecutionProvider'  # Fallback
+            ]
+        else:
+            providers = ['CPUExecutionProvider']
+
+        # Load ONNX model
+        print(f"Loading ONNX RetinaFace model from: {onnx_model_path}")
+
+        if use_coreml:
+            print("")
+            print("=" * 70)
+            print("⏰ CoreML First-Time Compilation Notice:")
+            print("   If this is the first time loading this model with CoreML,")
+            print("   compilation may take 30-60 seconds (one-time only).")
+            print("   Subsequent loads will be instant (model is cached).")
+            print("   Please wait...")
+            print("=" * 70)
+            print("")
+
+        # Configure session options to prevent thread conflicts
+        sess_options = ort.SessionOptions()
+        sess_options.intra_op_num_threads = 1  # Single thread per operator
+        sess_options.inter_op_num_threads = 1  # Sequential operator execution
+        sess_options.execution_mode = ort.ExecutionMode.ORT_SEQUENTIAL
+
+        # Suppress CoreML compilation warnings (they're expected for complex models)
+        import warnings
+        with warnings.catch_warnings():
+            warnings.filterwarnings('ignore')
+            self.session = ort.InferenceSession(onnx_model_path, sess_options=sess_options, providers=providers)
+
+        # Check which providers are actually active
+        active_providers = self.session.get_providers()
+
+        if 'CoreMLExecutionProvider' in active_providers:
+            print("Using CoreML Neural Engine acceleration for face detection")
+            print("  Expected: 5-10x speedup")
+            self.backend = 'coreml'
+        else:
+            print("Using ONNX Runtime with optimized CPU execution for face detection")
+            print("  Expected: 2-4x speedup over PyTorch")
+            print("  (CoreML not available for this model - some operations unsupported)")
+            self.backend = 'onnx_cpu'
+
+    def preprocess_image(self, img_array: np.ndarray, resize: float = 1.0) -> Tuple[np.ndarray, np.ndarray]:
+        """
+        Preprocess image for RetinaFace inference
+
+        Args:
+            img_array: BGR image array (H, W, 3)
+            resize: Resize factor (default: 1.0 for no resize)
+
+        Returns:
+            Tuple of (preprocessed_tensor, original_image)
+        """
+        img_raw = img_array.copy()
+        img = np.float32(img_raw)
+
+        # Resize if needed
+        if resize != 1.0:
+            import cv2
+            img = cv2.resize(img, None, fx=resize, fy=resize, interpolation=cv2.INTER_LINEAR)
+
+        # RetinaFace preprocessing: subtract ImageNet mean
+        img -= np.array([104.0, 117.0, 123.0], dtype=np.float32)
+
+        # Convert to NCHW format (batch, channels, height, width)
+        img = img.transpose(2, 0, 1)
+        img = np.expand_dims(img, axis=0)
+
+        return img, img_raw
+
+    def detect_faces(self, img_array: np.ndarray, resize: float = 1.0):
+        """
+        Detect faces in image array
+
+        Args:
+            img_array: BGR image array (H, W, 3)
+            resize: Resize factor for detection (default: 1.0)
+
+        Returns:
+            Tuple of (detections, original_image)
+            detections format: [x1, y1, x2, y2, confidence, landmark_x1, landmark_y1, ...]
+        """
+        profiler = get_profiler()
+
+        # Preprocess
+        with profiler.time_block("preprocessing", f"RetinaFace_preprocess"):
+            img, img_raw = self.preprocess_image(img_array, resize)
+
+        # Run ONNX inference on Neural Engine (or optimized CPU)
+        with profiler.time_block("model_inference", f"RetinaFace_{self.backend}"):
+            outputs = self.session.run(None, {'input': img})
+
+        # Unpack outputs: loc, conf, landms
+        loc = outputs[0]  # Bounding box predictions
+        conf = outputs[1]  # Confidence scores
+        landms = outputs[2]  # 5-point landmarks
+
+        # Post-processing (same as PyTorch version)
+        # This part stays in Python as it's fast and complex to export
+        with profiler.time_block("postprocessing", f"RetinaFace_postprocess"):
+            im_height, im_width, _ = img_raw.shape
+
+            # Convert outputs to torch tensors for compatibility with existing utilities
+            loc = torch.from_numpy(loc)
+            conf = torch.from_numpy(conf)
+            landms = torch.from_numpy(landms)
+
+            # Create scale tensor
+            scale = torch.Tensor([img.shape[3], img.shape[2], img.shape[3], img.shape[2]])
+
+            # Generate prior boxes for decoding
+            priorbox = PriorBox(self.cfg, image_size=(im_height, im_width))
+            priors = priorbox.forward()
+            prior_data = priors.data
+
+            # Decode boxes
+            boxes = decode(loc.data.squeeze(0), prior_data, self.cfg['variance'])
+            boxes = boxes * scale / resize
+            boxes = boxes.cpu().numpy()
+
+            # Extract scores
+            scores = conf.squeeze(0).data.cpu().numpy()[:, 1]
+
+            # Decode landmarks
+            landms_decoded = decode_landm(landms.data.squeeze(0), prior_data, self.cfg['variance'])
+            scale1 = torch.Tensor([img.shape[3], img.shape[2]] * 5)
+            landms_decoded = landms_decoded * scale1 / resize
+            landms_decoded = landms_decoded.cpu().numpy()
+
+            # Filter by confidence threshold
+            inds = np.where(scores > self.confidence_threshold)[0]
+            boxes, landms_decoded, scores = boxes[inds], landms_decoded[inds], scores[inds]
+
+            # Apply NMS
+            dets = np.hstack((boxes, scores[:, np.newaxis])).astype(np.float32, copy=False)
+            keep = py_cpu_nms(dets, self.nms_threshold)
+            dets = dets[keep]
+            landms_decoded = landms_decoded[keep]
+
+            # Concatenate boxes and landmarks
+            dets = np.concatenate((dets, landms_decoded), axis=1)
+
+        return dets, img_raw
+
+    def get_face(self, img_array: np.ndarray, resize: float = 1.0):
+        """
+        Get the primary face from image
+
+        Args:
+            img_array: BGR image array (H, W, 3)
+            resize: Resize factor (default: 1.0)
+
+        Returns:
+            Tuple of (face_crop, detections) or (None, None) if no face found
+        """
+        dets, img_raw = self.detect_faces(img_array, resize)
+
+        if dets is None or len(dets) == 0:
+            return None, None
+
+        det = dets[0]
+        confidence = det[4]
+
+        if confidence < self.vis_threshold:
+            return None, None
+
+        bbox = det[:4].astype(int)
+        face = img_raw[bbox[1]:bbox[3], bbox[0]:bbox[2]]
+
+        return face, dets
+
+
+class OptimizedFaceDetector:
+    """
+    Wrapper class that automatically selects ONNX or PyTorch implementation
+
+    This class provides seamless fallback from ONNX (fast) to PyTorch (slow)
+    based on model availability.
+    """
+
+    def __init__(self, model_path: str, onnx_model_path: str = None,
+                 device: str = "cpu",
+                 confidence_threshold: float = 0.02,
+                 nms_threshold: float = 0.4,
+                 vis_threshold: float = 0.5):
+        """
+        Initialize face detector with intelligent backend selection.
+
+        Selection logic:
+        - CUDA device: Use PyTorch (optimized for NVIDIA GPUs)
+        - CPU device: Use ONNX (CoreML on Apple Silicon, optimized CPU on Intel)
+
+        Args:
+            model_path: Path to PyTorch model (.pth)
+            onnx_model_path: Path to ONNX model (.onnx), defaults to same directory
+            device: Device ('cpu' or 'cuda')
+            confidence_threshold: Minimum confidence for detection
+            nms_threshold: NMS threshold
+            vis_threshold: Visibility threshold
+        """
+        from pathlib import Path
+
+        # Store configuration
+        self.confidence_threshold = confidence_threshold
+        self.nms_threshold = nms_threshold
+        self.vis_threshold = vis_threshold
+        self.cfg = cfg_mnet
+
+        # Determine ONNX model path
+        if onnx_model_path is None:
+            model_dir = Path(model_path).parent
+            onnx_model_path = model_dir / 'retinaface_mobilenet025_coreml.onnx'
+
+        # CUDA: Use PyTorch directly (best for NVIDIA GPUs)
+        if device == 'cuda':
+            print("Using PyTorch RetinaFace detector (CUDA-accelerated)")
+            from openface.face_detection import FaceDetector
+            self.detector = FaceDetector(
+                model_path=model_path,
+                device=device,
+                confidence_threshold=confidence_threshold,
+                nms_threshold=nms_threshold,
+                vis_threshold=vis_threshold
+            )
+            self.backend = 'pytorch_cuda'
+            self.model = self.detector.model
+            return
+
+        # CPU: Try ONNX first (CoreML on Apple Silicon, optimized CPU on Intel)
+        if Path(onnx_model_path).exists():
+            try:
+                print("Using ONNX-accelerated RetinaFace detector")
+                self.detector = ONNXRetinaFaceDetector(
+                    str(onnx_model_path),
+                    use_coreml=True,
+                    confidence_threshold=confidence_threshold,
+                    nms_threshold=nms_threshold,
+                    vis_threshold=vis_threshold
+                )
+                self.backend = 'onnx'
+
+                # Expose model wrapper for compatibility
+                class ONNXModelWrapper:
+                    def __init__(self, detector):
+                        self.detector = detector
+
+                    def __call__(self, img):
+                        img_np = img.cpu().numpy()
+                        outputs = self.detector.session.run(None, {'input': img_np})
+                        return tuple(torch.from_numpy(o) for o in outputs)
+
+                self.model = ONNXModelWrapper(self.detector)
+                return
+            except Exception as e:
+                print(f"Failed to load ONNX model: {e}")
+                print("Falling back to PyTorch CPU")
+
+        # Fallback: PyTorch CPU
+        print("Using PyTorch RetinaFace detector (CPU)")
+        from openface.face_detection import FaceDetector
+        self.detector = FaceDetector(
+            model_path=model_path,
+            device=device,
+            confidence_threshold=confidence_threshold,
+            nms_threshold=nms_threshold,
+            vis_threshold=vis_threshold
+        )
+        self.backend = 'pytorch_cpu'
+        self.model = self.detector.model
+
+    def detect_faces(self, img_array: np.ndarray, resize: float = 1.0):
+        """Detect faces using the selected backend"""
+        return self.detector.detect_faces(img_array, resize)
+
+    def get_face(self, img_array: np.ndarray, resize: float = 1.0):
+        """Get primary face using the selected backend"""
+        return self.detector.get_face(img_array, resize)
+
+
+if __name__ == '__main__':
+    print("ONNX RetinaFace Detector Module")
+    print("=" * 60)
+    print("This module provides CoreML-accelerated RetinaFace face detection.")
+    print("")
+    print("Usage:")
+    print("  from onnx_retinaface_detector import OptimizedFaceDetector")
+    print("  detector = OptimizedFaceDetector('weights/Alignment_RetinaFace.pth')")
+    print("  dets, img = detector.detect_faces(image_array)")
+    print("=" * 60)

pyfaceau/download_weights.py

@@ -0,0 +1,134 @@
+#!/usr/bin/env python3
+"""
+Download model weights for PyFaceAU
+
+Usage:
+    python -m pyfaceau.download_weights
+"""
+
+import os
+import sys
+import urllib.request
+from pathlib import Path
+from tqdm import tqdm
+
+
+WEIGHTS_BASE_URL = "https://github.com/johnwilsoniv/face-analysis/raw/main/S0%20PyfaceAU/weights/"
+
+REQUIRED_WEIGHTS = {
+    "retinaface_mobilenet025_coreml.onnx": "1.7MB - Face detection model",
+    "pfld_cunjian.onnx": "2.8MB - Landmark detection model",
+    "In-the-wild_aligned_PDM_68.txt": "67KB - PDM parameters",
+    "svr_patches_0.25_general.txt": "1.1MB - CLNF patch experts",
+    "tris_68_full.txt": "1KB - Triangulation data",
+}
+
+AU_PREDICTOR_FILES = [
+    "AU_1_dynamic_intensity_comb.dat",
+    "AU_2_dynamic_intensity_comb.dat",
+    "AU_4_static_intensity_comb.dat",
+    "AU_5_dynamic_intensity_comb.dat",
+    "AU_6_static_intensity_comb.dat",
+    "AU_7_static_intensity_comb.dat",
+    "AU_9_dynamic_intensity_comb.dat",
+    "AU_10_static_intensity_comb.dat",
+    "AU_12_static_intensity_comb.dat",
+    "AU_14_static_intensity_comb.dat",
+    "AU_15_dynamic_intensity_comb.dat",
+    "AU_17_dynamic_intensity_comb.dat",
+    "AU_20_dynamic_intensity_comb.dat",
+    "AU_23_dynamic_intensity_comb.dat",
+    "AU_25_dynamic_intensity_comb.dat",
+    "AU_26_dynamic_intensity_comb.dat",
+    "AU_45_dynamic_intensity_comb.dat",
+]
+
+
+class DownloadProgressBar(tqdm):
+    """Progress bar for downloads"""
+    def update_to(self, b=1, bsize=1, tsize=None):
+        if tsize is not None:
+            self.total = tsize
+        self.update(b * bsize - self.n)
+
+
+def download_file(url, output_path, desc=None):
+    """Download a file with progress bar"""
+    os.makedirs(os.path.dirname(output_path), exist_ok=True)
+
+    with DownloadProgressBar(unit='B', unit_scale=True, miniters=1, desc=desc) as t:
+        urllib.request.urlretrieve(url, filename=output_path, reporthook=t.update_to)
+
+
+def get_weights_dir():
+    """Get or create weights directory"""
+    # First try package installation location
+    try:
+        import pyfaceau
+        pkg_dir = Path(pyfaceau.__file__).parent
+        weights_dir = pkg_dir / "weights"
+    except:
+        # Fall back to current directory
+        weights_dir = Path.cwd() / "weights"
+
+    weights_dir.mkdir(parents=True, exist_ok=True)
+    return weights_dir
+
+
+def main():
+    """Download all required weights"""
+    print("PyFaceAU Weight Downloader")
+    print("=" * 60)
+
+    weights_dir = get_weights_dir()
+    print(f"Downloading weights to: {weights_dir}")
+    print()
+
+    # Download main weights
+    print("Downloading main model weights...")
+    for filename, description in REQUIRED_WEIGHTS.items():
+        output_path = weights_dir / filename
+
+        if output_path.exists():
+            print(f"✓ {filename} (already exists)")
+            continue
+
+        url = WEIGHTS_BASE_URL + filename
+        try:
+            download_file(url, output_path, desc=f"{filename} ({description})")
+            print(f"✓ Downloaded {filename}")
+        except Exception as e:
+            print(f"✗ Failed to download {filename}: {e}")
+            return 1
+
+    # Download AU predictors
+    print("\nDownloading AU predictor models...")
+    au_dir = weights_dir / "AU_predictors"
+    au_dir.mkdir(exist_ok=True)
+
+    for filename in AU_PREDICTOR_FILES:
+        output_path = au_dir / filename
+
+        if output_path.exists():
+            print(f"✓ {filename} (already exists)")
+            continue
+
+        url = WEIGHTS_BASE_URL + "AU_predictors/" + filename
+        try:
+            download_file(url, output_path, desc=filename)
+            print(f"✓ Downloaded {filename}")
+        except Exception as e:
+            print(f"✗ Failed to download {filename}: {e}")
+            return 1
+
+    print("\n" + "=" * 60)
+    print("✓ All weights downloaded successfully!")
+    print(f"Weights location: {weights_dir}")
+    print("\nYou can now use PyFaceAU:")
+    print("  from pyfaceau import FullPythonAUPipeline")
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())