pyfaceau 1.0.3__cp312-cp312-win_amd64.whl

pyfaceau/prediction/running_median_fallback.py

@@ -0,0 +1,200 @@
+#!/usr/bin/env python3
+"""
+Running Median Tracker for Dynamic AU Models
+
+Implements person-specific normalization by tracking the running median
+of feature vectors across video frames. This removes person-specific biases
+in neutral expressions.
+
+Reference: OpenFace 2.2 FaceAnalyser::UpdateRunningMedian()
+"""
+
+import numpy as np
+from collections import deque
+
+
+class RunningMedianTracker:
+    """
+    Tracks running median of feature vectors for person-specific normalization
+
+    Used by dynamic AU models to normalize for individual neutral expressions.
+    """
+
+    def __init__(self, feature_dim: int, window_size: int = 200):
+        """
+        Initialize running median tracker
+
+        Args:
+            feature_dim: Dimensionality of feature vectors
+            window_size: Number of frames to use for median calculation
+                        (OF2.2 uses histogram with bins, we use rolling window)
+        """
+        self.feature_dim = feature_dim
+        self.window_size = window_size
+
+        # Rolling window of feature vectors (stores last N frames)
+        self.history = deque(maxlen=window_size)
+
+        # Current median estimate
+        self.current_median = np.zeros(feature_dim, dtype=np.float64)
+
+        # Frame counter
+        self.frame_count = 0
+
+    def update(self, features: np.ndarray, update_median: bool = True) -> None:
+        """
+        Update tracker with new feature vector
+
+        Args:
+            features: Feature vector (1D array)
+            update_median: Whether to recompute median (OF2.2 does this every 2nd frame)
+        """
+        # Ensure 1D
+        if features.ndim == 2:
+            features = features.flatten()
+
+        assert features.shape[0] == self.feature_dim, \
+            f"Expected {self.feature_dim} features, got {features.shape[0]}"
+
+        # Add to history
+        self.history.append(features.copy())
+        self.frame_count += 1
+
+        # Update median (only if requested and have enough samples)
+        if update_median and len(self.history) >= 10:
+            # Convert history to array (N_frames, N_features)
+            history_array = np.array(self.history)
+
+            # Compute median along frame axis
+            self.current_median = np.median(history_array, axis=0)
+
+    def get_median(self) -> np.ndarray:
+        """
+        Get current running median
+
+        Returns:
+            Median feature vector (1D array)
+        """
+        return self.current_median.copy()
+
+    def reset(self) -> None:
+        """Reset tracker (e.g., for new video)"""
+        self.history.clear()
+        self.current_median = np.zeros(self.feature_dim, dtype=np.float64)
+        self.frame_count = 0
+
+
+class DualMedianTracker:
+    """
+    Manages separate running medians for HOG and geometric features
+
+    OF2.2 tracks HOG and geometric features separately, then concatenates
+    them when computing dynamic model predictions.
+    """
+
+    def __init__(self, hog_dim: int = 4464, geom_dim: int = 238, window_size: int = 200):
+        """
+        Initialize dual median tracker
+
+        Args:
+            hog_dim: HOG feature dimensionality
+            geom_dim: Geometric feature dimensionality
+            window_size: Rolling window size
+        """
+        self.hog_tracker = RunningMedianTracker(hog_dim, window_size)
+        self.geom_tracker = RunningMedianTracker(geom_dim, window_size)
+
+    def update(self, hog_features: np.ndarray, geom_features: np.ndarray,
+               update_median: bool = True) -> None:
+        """
+        Update both trackers
+
+        Args:
+            hog_features: HOG feature vector
+            geom_features: Geometric feature vector
+            update_median: Whether to recompute medians
+        """
+        self.hog_tracker.update(hog_features, update_median)
+        self.geom_tracker.update(geom_features, update_median)
+
+    def get_combined_median(self) -> np.ndarray:
+        """
+        Get concatenated [HOG_median, geom_median]
+
+        Returns:
+            Combined median vector (4702 dims)
+        """
+        hog_median = self.hog_tracker.get_median()
+        geom_median = self.geom_tracker.get_median()
+        return np.concatenate([hog_median, geom_median])
+
+    def get_hog_median(self) -> np.ndarray:
+        """Get HOG median only"""
+        return self.hog_tracker.get_median()
+
+    def get_geom_median(self) -> np.ndarray:
+        """Get geometric median only"""
+        return self.geom_tracker.get_median()
+
+    def reset(self) -> None:
+        """Reset both trackers"""
+        self.hog_tracker.reset()
+        self.geom_tracker.reset()
+
+
+def test_running_median():
+    """Test running median tracker"""
+    print("="*80)
+    print("Running Median Tracker - Test")
+    print("="*80)
+
+    # Create tracker
+    tracker = RunningMedianTracker(feature_dim=10, window_size=100)
+
+    # Generate synthetic data (random walk around mean)
+    np.random.seed(42)
+    mean_features = np.array([1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0])
+
+    print("\nSimulating 200 frames...")
+    for i in range(200):
+        # Add noise to mean
+        noise = np.random.randn(10) * 0.5
+        features = mean_features + noise
+
+        # Update tracker (update median every frame for testing)
+        tracker.update(features, update_median=True)
+
+        if i % 50 == 49:
+            median = tracker.get_median()
+            print(f"\nFrame {i+1}:")
+            print(f"  True mean:    {mean_features[:3]}...")
+            print(f"  Running median: {median[:3]}...")
+            print(f"  Error: {np.abs(median - mean_features).mean():.6f}")
+
+    print("\nTracker converges to true mean!")
+
+    # Test dual tracker
+    print("\n" + "="*80)
+    print("Dual Median Tracker - Test")
+    print("="*80)
+
+    dual_tracker = DualMedianTracker(hog_dim=4464, geom_dim=238)
+
+    print("\nUpdating with synthetic HOG and geometric features...")
+    for i in range(100):
+        hog_feat = np.random.randn(4464)
+        geom_feat = np.random.randn(238)
+        dual_tracker.update(hog_feat, geom_feat, update_median=(i % 2 == 0))
+
+    combined = dual_tracker.get_combined_median()
+    print(f"Combined median shape: {combined.shape}")
+    print(f"Expected: (4702,)")
+    assert combined.shape == (4702,)
+
+    print("\n" + "="*80)
+    print("All tests passed!")
+    print("="*80)
+
+
+if __name__ == "__main__":
+    test_running_median()

pyfaceau/processor.py

@@ -0,0 +1,270 @@
+"""
+OpenFace-compatible AU extraction processor.
+
+This module provides a drop-in replacement for OpenFace 3.0
+with the same API for easy integration into existing workflows
+like S1 Face Mirror.
+"""
+
+import cv2
+import csv
+import numpy as np
+from pathlib import Path
+from typing import Optional, Callable
+from .pipeline import FullPythonAUPipeline
+
+
+class OpenFaceProcessor:
+    """
+    OpenFace 2.2-compatible AU extraction processor.
+
+    Drop-in replacement for OpenFace 3.0 with the pyfaceau pipeline.
+    Designed for seamless integration with S1 Face Mirror and other
+    OpenFace-based applications.
+
+    Features:
+    - 17 Action Units (AU01-AU45)
+    - r > 0.92 correlation with OpenFace 2.2 C++
+    - CLNF landmark refinement
+    - Real-time capable (72 fps)
+    - 100% Python (no compilation)
+
+    Example:
+        ```python
+        processor = OpenFaceProcessor(
+            weights_dir='weights/',
+            use_clnf_refinement=True
+        )
+
+        processor.process_video(
+            'input.mp4',
+            'output.csv',
+            progress_callback=my_callback
+        )
+        ```
+    """
+
+    def __init__(
+        self,
+        device: Optional[str] = None,
+        weights_dir: Optional[str] = None,
+        use_clnf_refinement: bool = True,
+        num_threads: int = 6,
+        verbose: bool = False,
+        **kwargs
+    ):
+        """
+        Initialize OpenFace AU extraction processor.
+
+        Args:
+            device: Unused (kept for API compatibility). PyFaceAU auto-detects.
+            weights_dir: Path to weights directory (default: ./weights)
+            use_clnf_refinement: Enable CLNF landmark refinement (default: True)
+            num_threads: Unused (kept for API compatibility)
+            verbose: Enable verbose logging (default: False)
+            **kwargs: Additional arguments (ignored for compatibility)
+        """
+        self.verbose = verbose
+
+        # Determine weights directory
+        if weights_dir is None:
+            script_dir = Path(__file__).parent.parent
+            weights_dir = script_dir / 'weights'
+        else:
+            weights_dir = Path(weights_dir)
+
+        if self.verbose:
+            print("Initializing PyFaceAU (OpenFace 2.2 Python replacement)...")
+
+        # Initialize the PyFaceAU pipeline
+        self.pipeline = FullPythonAUPipeline(
+            retinaface_model=str(weights_dir / 'retinaface_mobilenet025_coreml.onnx'),
+            pfld_model=str(weights_dir / 'pfld_cunjian.onnx'),
+            pdm_file=str(weights_dir / 'In-the-wild_aligned_PDM_68.txt'),
+            au_models_dir=str(weights_dir / 'AU_predictors'),
+            triangulation_file=str(weights_dir / 'tris_68_full.txt'),
+            patch_expert_file=str(weights_dir / 'svr_patches_0.25_general.txt') if use_clnf_refinement else None,
+            use_clnf_refinement=use_clnf_refinement,
+            use_batched_predictor=True,
+            verbose=verbose
+        )
+
+        if self.verbose:
+            print(f"  PyFaceAU initialized")
+            print(f"  CLNF refinement: {'Enabled' if use_clnf_refinement else 'Disabled'}")
+            print(f"  Expected accuracy: r > 0.92 (OpenFace 2.2 correlation)")
+            print()
+
+    def process_video(
+        self,
+        video_path: str,
+        output_csv_path: str,
+        progress_callback: Optional[Callable[[int, int, float], None]] = None
+    ) -> int:
+        """
+        Process video and extract AUs.
+
+        Compatible with S1 Face Mirror integration and other OpenFace-based
+        applications.
+
+        Args:
+            video_path: Path to input video file
+            output_csv_path: Path to output CSV file
+            progress_callback: Optional callback function(current, total, fps)
+                             for progress updates
+
+        Returns:
+            Number of frames successfully processed
+        """
+        video_path = Path(video_path)
+        output_csv_path = Path(output_csv_path)
+
+        if self.verbose:
+            print(f"Processing: {video_path.name}")
+
+        # Ensure output directory exists
+        output_csv_path.parent.mkdir(parents=True, exist_ok=True)
+
+        # Process video through pipeline
+        try:
+            df = self.pipeline.process_video(
+                video_path=str(video_path),
+                output_csv=str(output_csv_path),
+                max_frames=None,
+                progress_callback=progress_callback
+            )
+
+            success_count = df['success'].sum()
+
+            if self.verbose:
+                total_frames = len(df)
+                print(f"  Processed {success_count}/{total_frames} frames successfully")
+                if success_count < total_frames:
+                    failed = total_frames - success_count
+                    print(f"  {failed} frames failed (no face detected)")
+                print(f"  Output: {output_csv_path}")
+
+            return int(success_count)
+
+        except Exception as e:
+            if self.verbose:
+                print(f"  Error processing video: {e}")
+            raise
+
+    def clear_cache(self):
+        """
+        Clear any internal caches to free memory.
+
+        Call this if processing many videos with different resolutions
+        and want to free up cache memory between batches.
+        """
+        # PyFaceAU pipeline doesn't currently use caching
+        # This method is kept for API compatibility
+        pass
+
+
+def process_videos(
+    directory_path: str,
+    specific_files: Optional[list] = None,
+    output_dir: Optional[str] = None,
+    **processor_kwargs
+) -> int:
+    """
+    Process multiple video files using OpenFaceProcessor.
+
+    This function provides batch processing capability compatible with
+    S1 Face Mirror workflows.
+
+    Args:
+        directory_path: Path to directory containing video files
+        specific_files: List of specific files to process (optional)
+        output_dir: Output directory for CSV files (optional)
+        **processor_kwargs: Additional arguments passed to OpenFaceProcessor
+
+    Returns:
+        Number of files successfully processed
+
+    Example:
+        ```python
+        # Process all mirrored videos in a directory
+        count = process_videos(
+            directory_path='/path/to/mirrored/videos',
+            output_dir='/path/to/output',
+            use_clnf_refinement=True
+        )
+        print(f"Processed {count} videos")
+        ```
+    """
+    directory_path = Path(directory_path)
+
+    # Check if directory exists
+    if not directory_path.is_dir():
+        print(f"Error: Directory '{directory_path}' does not exist.")
+        return 0
+
+    # Determine output directory
+    if output_dir is None:
+        # Default: S1O Processed Files/Combined Data/
+        s1o_base = directory_path.parent.parent / 'S1O Processed Files'
+        output_dir = s1o_base / 'Combined Data'
+    else:
+        output_dir = Path(output_dir)
+
+    output_dir.mkdir(parents=True, exist_ok=True)
+    print(f"Output directory: {output_dir}")
+
+    # Initialize processor
+    processor = OpenFaceProcessor(**processor_kwargs)
+
+    # Counter for processed files
+    processed_count = 0
+
+    # Define which files to process
+    files_to_process = []
+
+    if specific_files:
+        # Process only the specific files
+        files_to_process = [Path(f) for f in specific_files]
+        print(f"Processing {len(files_to_process)} specific files from current session.")
+    else:
+        # Process all eligible files in the directory
+        files_to_process = list(directory_path.iterdir())
+        print(f"Processing all eligible files in {directory_path}")
+
+    # Process each file
+    for file_path in files_to_process:
+        # Skip if not a file or doesn't exist
+        if not file_path.is_file():
+            print(f"Warning: {file_path} does not exist or is not a file. Skipping.")
+            continue
+
+        filename = file_path.name
+
+        # Skip files with 'debug' in the filename
+        if 'debug' in filename:
+            print(f"Skipping debug file: {filename}")
+            continue
+
+        # Process file with 'mirrored' in the filename
+        if 'mirrored' in filename:
+            # Generate output CSV filename
+            # Example: "video_left_mirrored.mp4" -> "video_left_mirrored.csv"
+            csv_filename = file_path.stem + '.csv'
+            output_csv_path = output_dir / csv_filename
+
+            try:
+                # Process video and extract AUs
+                frame_count = processor.process_video(file_path, output_csv_path)
+
+                if frame_count > 0:
+                    processed_count += 1
+                    print(f"Successfully processed: {filename}\n")
+                else:
+                    print(f"Failed to process: {filename}\n")
+
+            except Exception as e:
+                print(f"Error processing {filename}: {e}\n")
+
+    print(f"\nProcessing complete. {processed_count} files were processed.")
+
+    return processed_count

pyfaceau/refinement/__init__.py

@@ -0,0 +1,12 @@
+"""
+CLNF landmark refinement module for PyFaceAU
+
+This module implements targeted CLNF refinement to improve PFLD landmark accuracy,
+specifically for brow landmarks (17-26) and lip corners (48, 54) which are critical
+for AU01, AU02, and AU23 detection.
+"""
+
+from .svr_patch_expert import SVRPatchExpert, SVRPatchExpertLoader
+from .targeted_refiner import TargetedCLNFRefiner
+
+__all__ = ['SVRPatchExpert', 'SVRPatchExpertLoader', 'TargetedCLNFRefiner']