pyfaceau 1.0.6__cp310-cp310-macosx_11_0_arm64.whl

pyfaceau/prediction/model_parser.py

@@ -0,0 +1,337 @@
+#!/usr/bin/env python3
+"""
+OpenFace 2.2 SVR Model Parser
+
+Parses OpenFace 2.2's binary .dat files containing Linear SVR models for AU prediction.
+
+Binary Format (per model):
+1. means matrix: (rows, cols, dtype, data[rows*cols*8])
+2. support_vectors matrix: (rows, cols, dtype, data[rows*cols*8])
+3. bias: float64 (8 bytes)
+
+Usage:
+    parser = OF22ModelParser(models_dir)
+    models = parser.load_all_models()
+"""
+
+import numpy as np
+from pathlib import Path
+from typing import Dict, Tuple
+import struct
+
+
+class OF22ModelParser:
+    """
+    Parser for OpenFace 2.2 Linear SVR models stored in binary .dat format
+    """
+
+    def __init__(self, models_dir: str):
+        """
+        Initialize parser with path to OF2.2 models directory
+
+        Args:
+            models_dir: Path to AU_predictors directory (will use svr_combined by default)
+        """
+        self.models_dir = Path(models_dir)
+        if not self.models_dir.exists():
+            raise ValueError(f"Models directory not found: {models_dir}")
+
+        # All 17 AUs that OF2.2 supports with intensity estimation (SVR)
+        # AU28 is excluded - it only has occurrence detection (SVM), not intensity
+        # Based on AU_all_best.txt configuration
+        self.available_aus = [
+            'AU01_r', 'AU02_r', 'AU04_r', 'AU05_r', 'AU06_r', 'AU07_r',
+            'AU09_r', 'AU10_r', 'AU12_r', 'AU14_r', 'AU15_r', 'AU17_r',
+            'AU20_r', 'AU23_r', 'AU25_r', 'AU26_r', 'AU45_r'
+        ]
+
+        # Mapping of AU to recommended model type from AU_all_best.txt
+        self.recommended_model_types = {
+            'AU01_r': 'dynamic',
+            'AU02_r': 'dynamic',
+            'AU04_r': 'static',
+            'AU05_r': 'dynamic',
+            'AU06_r': 'static',
+            'AU07_r': 'static',
+            'AU09_r': 'dynamic',
+            'AU10_r': 'static',
+            'AU12_r': 'static',
+            'AU14_r': 'static',
+            'AU15_r': 'dynamic',
+            'AU17_r': 'dynamic',
+            'AU20_r': 'dynamic',
+            'AU23_r': 'dynamic',
+            'AU25_r': 'dynamic',
+            'AU26_r': 'dynamic',
+            'AU45_r': 'dynamic'
+        }
+
+    def parse_svr_model(self, dat_file_path: Path, is_dynamic: bool = True) -> Dict:
+        """
+        Parse a single SVR model from .dat file
+
+        Binary format for DYNAMIC models:
+        1. int32: marker (always 1)
+        2. float64: cutoff for person-specific calibration
+        3. means matrix: (rows, cols, dtype, data)
+        4. support_vectors matrix: (rows, cols, dtype, data)
+        5. bias: float64
+
+        Binary format for STATIC models:
+        1. means matrix: (rows, cols, dtype, data)
+        2. support_vectors matrix: (rows, cols, dtype, data)
+        3. bias: float64
+
+        Args:
+            dat_file_path: Path to .dat file
+            is_dynamic: True for dynamic models (has cutoff), False for static models
+
+        Returns:
+            dict with keys: 'cutoff', 'means', 'support_vectors', 'bias'
+        """
+        if not dat_file_path.exists():
+            raise FileNotFoundError(f"Model file not found: {dat_file_path}")
+
+        with open(dat_file_path, 'rb') as f:
+            # All models start with a marker (int32)
+            # marker = 1: dynamic model (has cutoff)
+            # marker = 0: static model (no cutoff)
+            marker = struct.unpack('<i', f.read(4))[0]
+
+            if marker == 1:
+                # Dynamic model - read cutoff value
+                cutoff = struct.unpack('<d', f.read(8))[0]  # float64
+                if not is_dynamic:
+                    print(f"Warning: File has dynamic marker but loaded as static")
+            elif marker == 0:
+                # Static model - no cutoff
+                cutoff = 0.0
+                if is_dynamic:
+                    print(f"Warning: File has static marker but loaded as dynamic")
+            else:
+                raise ValueError(f"Invalid marker: expected 0 or 1, got {marker}")
+
+            # Read means matrix header
+            means_rows = struct.unpack('<i', f.read(4))[0]
+            means_cols = struct.unpack('<i', f.read(4))[0]
+            means_dtype = struct.unpack('<i', f.read(4))[0]  # OpenCV type code
+
+            # Read means data (may be empty if cols=0)
+            if means_rows > 0 and means_cols > 0:
+                means_data = np.frombuffer(
+                    f.read(means_rows * means_cols * 8),
+                    dtype=np.float64
+                )
+                means = means_data.reshape((means_rows, means_cols))
+            else:
+                # Empty means matrix - no centering needed
+                means = np.array([], dtype=np.float64).reshape(means_rows, means_cols)
+
+            # Read support vectors matrix header
+            sv_rows = struct.unpack('<i', f.read(4))[0]
+            sv_cols = struct.unpack('<i', f.read(4))[0]
+            sv_dtype = struct.unpack('<i', f.read(4))[0]
+
+            # Read support vectors data
+            sv_data = np.frombuffer(
+                f.read(sv_rows * sv_cols * 8),
+                dtype=np.float64
+            )
+            support_vectors = sv_data.reshape((sv_rows, sv_cols))
+
+            # Read bias
+            bias = struct.unpack('<d', f.read(8))[0]  # double = float64
+
+        return {
+            'cutoff': cutoff,
+            'means': means,
+            'support_vectors': support_vectors,
+            'bias': bias,
+            'file': str(dat_file_path)
+        }
+
+    def load_au_model(self, au_name: str, use_dynamic: bool = None, use_combined: bool = True) -> Dict:
+        """
+        Load a specific AU model
+
+        Args:
+            au_name: AU name (e.g., 'AU01_r', 'AU12_r')
+            use_dynamic: If True, load dynamic model; if False, load static model;
+                        if None, use recommended type from AU_all_best.txt
+            use_combined: If True, prefer svr_combined models (default)
+
+        Returns:
+            Parsed model dict
+        """
+        # Convert AU name to file format (e.g., 'AU01_r' -> 'AU_1')
+        au_num = au_name.replace('AU', '').replace('_r', '').lstrip('0')
+
+        # Determine model type (dynamic vs static)
+        if use_dynamic is None:
+            # Use recommended type from configuration
+            model_type_str = self.recommended_model_types.get(au_name, 'dynamic')
+        else:
+            model_type_str = 'dynamic' if use_dynamic else 'static'
+
+        # Try different file naming patterns in svr_combined
+        base_dir = self.models_dir if not use_combined else (self.models_dir / 'svr_combined' if (self.models_dir / 'svr_combined').exists() else self.models_dir)
+
+        # Possible filename patterns (svr_combined uses inconsistent naming)
+        possible_files = [
+            f"AU_{au_num}_{model_type_str}_intensity_comb.dat",
+            f"AU_{au_num}_{model_type_str}_intensity.dat",
+        ]
+
+        dat_path = None
+        for filename in possible_files:
+            candidate = base_dir / filename
+            if candidate.exists():
+                dat_path = candidate
+                break
+
+        if dat_path is None:
+            raise FileNotFoundError(
+                f"Model not found for {au_name}. Tried: {possible_files} in {base_dir}"
+            )
+
+        # Parse the model with correct format (dynamic vs static)
+        is_dynamic = (model_type_str == 'dynamic')
+        model = self.parse_svr_model(dat_path, is_dynamic=is_dynamic)
+        model['au_name'] = au_name
+        model['model_type'] = model_type_str
+        model['filename'] = dat_path.name
+
+        return model
+
+    def load_all_models(self, use_recommended: bool = True, use_combined: bool = True) -> Dict[str, Dict]:
+        """
+        Load all available AU models
+
+        Args:
+            use_recommended: If True, use recommended static/dynamic type for each AU
+                            from AU_all_best.txt configuration
+            use_combined: If True, load from svr_combined (higher quality, trained on
+                         combined datasets)
+
+        Returns:
+            Dictionary mapping AU names to model dicts
+        """
+        models = {}
+        failed_aus = []
+
+        for au_name in self.available_aus:
+            try:
+                use_dynamic = None if use_recommended else True
+                model = self.load_au_model(au_name, use_dynamic=use_dynamic, use_combined=use_combined)
+                models[au_name] = model
+                print(f"Loaded {au_name} ({model['model_type']}): cutoff={model['cutoff']:.2f}, "
+                      f"means={model['means'].shape}, SV={model['support_vectors'].shape}, "
+                      f"bias={model['bias']:.4f} [{model['filename']}]")
+            except FileNotFoundError as e:
+                failed_aus.append(au_name)
+                print(f"✗ Failed to load {au_name}: {e}")
+
+        print(f"\nLoaded {len(models)}/{len(self.available_aus)} AU models")
+        if failed_aus:
+            print(f"Failed AUs: {failed_aus}")
+
+        return models
+
+    def predict_au(self, fhog_features: np.ndarray, model: Dict) -> float:
+        """
+        Predict AU intensity using Linear SVR
+
+        Prediction formula (from OF2.2 SVR_static_lin_regressors.cpp:98):
+        - preds = (fhog_descriptor - means) * support_vectors + bias
+
+        Note: The 'cutoff' value in dynamic models is for person-specific
+        calibration and is NOT used in the linear prediction itself.
+
+        Args:
+            fhog_features: FHOG feature vector (1D array)
+            model: Parsed SVR model dict
+
+        Returns:
+            Predicted AU intensity (float, typically 0-5 range)
+        """
+        # Ensure features are 2D (1, n_features)
+        if fhog_features.ndim == 1:
+            fhog_features = fhog_features.reshape(1, -1)
+
+        # Check dimensions against support vectors
+        # SV shape is (n_features, 1), so check against dimension 0
+        expected_dims = model['support_vectors'].shape[0]
+        if fhog_features.shape[1] != expected_dims:
+            raise ValueError(
+                f"Feature dimension mismatch: got {fhog_features.shape[1]}, "
+                f"expected {expected_dims}"
+            )
+
+        # Center features (if means matrix exists and is non-empty)
+        if model['means'].size > 0:
+            centered = fhog_features - model['means']
+        else:
+            centered = fhog_features
+
+        # Apply support vectors (matrix multiplication)
+        # centered: (1, 4702), support_vectors: (4702, 1) -> result: (1, 1)
+        prediction = np.dot(centered, model['support_vectors']) + model['bias']
+
+        return float(prediction[0, 0])
+
+
+def main():
+    """Test the model parser"""
+    import sys
+
+    models_dir = "/Users/johnwilsoniv/repo/fea_tool/external_libs/openFace/OpenFace/lib/local/FaceAnalyser/AU_predictors"
+
+    print("=" * 80)
+    print("OpenFace 2.2 SVR Model Parser - Test (All 17 AUs)")
+    print("=" * 80)
+    print(f"\nModels directory: {models_dir}\n")
+
+    # Test parser
+    parser = OF22ModelParser(models_dir)
+
+    # Load all models (using recommended static/dynamic types and svr_combined)
+    print("\nLoading all AU models (using AU_all_best.txt configuration)...")
+    print("-" * 80)
+    models = parser.load_all_models(use_recommended=True, use_combined=True)
+
+    # Summary
+    print("\n" + "=" * 80)
+    print("Model Loading Summary")
+    print("=" * 80)
+    print(f"Successfully loaded: {len(models)} AU models")
+    print(f"AU names: {list(models.keys())}")
+
+    # Test prediction with dummy features
+    if models:
+        print("\n" + "=" * 80)
+        print("Testing Prediction with Dummy Features")
+        print("=" * 80)
+
+        # Get first model
+        au_name = list(models.keys())[0]
+        model = models[au_name]
+
+        # Create dummy FHOG features (all zeros)
+        n_features = model['means'].shape[1]
+        dummy_features = np.zeros(n_features, dtype=np.float64)
+
+        print(f"\nTesting {au_name}:")
+        print(f"  Feature dimensions: {n_features}")
+        print(f"  Dummy features: all zeros")
+
+        prediction = parser.predict_au(dummy_features, model)
+        print(f"  Prediction: {prediction:.4f}")
+        print(f"  (Expected: close to bias={model['bias']:.4f} since features are zero)")
+
+    print("\n" + "=" * 80)
+    print("Test complete!")
+    print("=" * 80)
+
+
+if __name__ == "__main__":
+    main()

pyfaceau/prediction/running_median.py

@@ -0,0 +1,318 @@
+#!/usr/bin/env python3
+"""
+Histogram-Based Running Median Tracker
+
+Implements OpenFace 2.2's histogram-based running median algorithm for
+person-specific normalization. Matches the C++ implementation in
+FaceAnalyser::UpdateRunningMedian().
+
+Reference: OpenFace/lib/local/FaceAnalyser/src/FaceAnalyser.cpp:764-821
+"""
+
+import numpy as np
+
+
+class HistogramBasedMedianTracker:
+    """
+    Histogram-based running median tracker matching OpenFace 2.2's implementation
+
+    Uses binned histograms to efficiently compute running median without storing
+    all historical values. More memory-efficient than rolling window approach.
+    """
+
+    def __init__(self, feature_dim: int, num_bins: int = 200,
+                 min_val: float = -3.0, max_val: float = 5.0):
+        """
+        Initialize histogram-based median tracker
+
+        Args:
+            feature_dim: Dimensionality of feature vectors
+            num_bins: Number of histogram bins (OF2.2 uses 200)
+            min_val: Minimum value for histogram range
+            max_val: Maximum value for histogram range
+        """
+        self.feature_dim = feature_dim
+        self.num_bins = num_bins
+        self.min_val = min_val
+        self.max_val = max_val
+
+        # Histogram: (feature_dim, num_bins)
+        # Each row tracks the distribution of one feature dimension
+        self.histogram = np.zeros((feature_dim, num_bins), dtype=np.int32)
+
+        # Current median estimate
+        self.current_median = np.zeros(feature_dim, dtype=np.float64)
+
+        # Total count of updates
+        self.hist_count = 0
+
+        # Precompute bin width
+        self.length = max_val - min_val
+        self.bin_width = self.length / num_bins
+
+    def update(self, features: np.ndarray, update_histogram: bool = True) -> None:
+        """
+        Update tracker with new feature vector
+
+        Matches C++ UpdateRunningMedian() logic:
+        1. Bin each feature value into histogram
+        2. Update histogram counts
+        3. Recompute median from cumulative distribution
+
+        Args:
+            features: Feature vector (1D array)
+            update_histogram: Whether to update histogram (OF2.2 does this every 2nd frame)
+        """
+        # Ensure 1D
+        if features.ndim == 2:
+            features = features.flatten()
+
+        assert features.shape[0] == self.feature_dim, \
+            f"Expected {self.feature_dim} features, got {features.shape[0]}"
+
+        if update_histogram:
+            # Convert feature values to bin indices
+            # Formula from C++: (descriptor - min_val) * num_bins / (max_val - min_val)
+            converted = (features - self.min_val) * self.num_bins / self.length
+
+            # Cap values to [0, num_bins-1] BEFORE casting to int (matches C++)
+            # C++ does: setTo(num_bins-1, converted > num_bins-1) then setTo(0, converted < 0)
+            converted = np.clip(converted, 0.0, float(self.num_bins - 1))
+
+            # Cast to int (truncation, matches C++ (int) cast)
+            converted = converted.astype(np.int32)
+
+            # Update histogram counts
+            for i in range(self.feature_dim):
+                bin_idx = converted[i]
+                self.histogram[i, bin_idx] += 1
+
+            self.hist_count += 1
+
+        # Compute median (matches C++: always sets median, even on frame 0)
+        # On frame 0 (hist_count==0), C++ sets median=descriptor.clone()
+        # On frame 1+ with hist_count==1, C++ also uses descriptor directly
+        # Otherwise, compute from histogram
+        if self.hist_count == 0:
+            # Frame 0: histogram not updated yet, use descriptor directly
+            self.current_median = features.copy()
+        elif self.hist_count == 1:
+            # Frame 1: histogram updated once, still use descriptor directly (matches C++)
+            self.current_median = features.copy()
+        else:
+            # Frame 2+: compute from histogram
+            self._compute_median()
+
+    def _compute_median(self, first_descriptor: np.ndarray = None) -> None:
+        """
+        Compute median from histogram using cumulative sum
+
+        Matches C++ logic:
+        - If hist_count == 1: median = descriptor (special case)
+        - Otherwise: Find bin where cumulative sum reaches (hist_count+1)/2
+        - Convert bin index back to feature value
+
+        Args:
+            first_descriptor: If provided and hist_count==1, use directly as median
+        """
+        # Special case: First frame (matches C++ if(hist_count == 1) { median = descriptor.clone(); })
+        if self.hist_count == 1 and first_descriptor is not None:
+            self.current_median = first_descriptor.copy()
+            return
+
+        cutoff_point = (self.hist_count + 1) // 2
+
+        for i in range(self.feature_dim):
+            cumulative_sum = 0
+
+            for j in range(self.num_bins):
+                cumulative_sum += self.histogram[i, j]
+
+                if cumulative_sum >= cutoff_point:
+                    # Convert bin index back to value
+                    # Formula from C++: min_val + bin_idx * bin_width + 0.5 * bin_width
+                    self.current_median[i] = (
+                        self.min_val +
+                        j * self.bin_width +
+                        0.5 * self.bin_width
+                    )
+                    break
+
+    def get_median(self) -> np.ndarray:
+        """
+        Get current running median
+
+        Returns:
+            Median feature vector (1D array)
+        """
+        return self.current_median.copy()
+
+    def reset(self) -> None:
+        """Reset tracker (e.g., for new video)"""
+        self.histogram.fill(0)
+        self.current_median.fill(0.0)
+        self.hist_count = 0
+
+
+class DualHistogramMedianTracker:
+    """
+    Manages separate histogram-based running medians for HOG and geometric features
+
+    OF2.2 tracks HOG and geometric features separately with different histogram
+    parameters, then concatenates them when computing dynamic model predictions.
+    """
+
+    def __init__(self,
+                 hog_dim: int = 4464,
+                 geom_dim: int = 238,
+                 hog_bins: int = 200,
+                 hog_min: float = -3.0,
+                 hog_max: float = 5.0,
+                 geom_bins: int = 200,
+                 geom_min: float = -3.0,
+                 geom_max: float = 5.0):
+        """
+        Initialize dual histogram median tracker
+
+        Args:
+            hog_dim: HOG feature dimensionality
+            geom_dim: Geometric feature dimensionality
+            hog_bins: Number of bins for HOG histogram
+            hog_min: Minimum value for HOG histogram
+            hog_max: Maximum value for HOG histogram
+            geom_bins: Number of bins for geometric histogram
+            geom_min: Minimum value for geometric histogram
+            geom_max: Maximum value for geometric histogram
+        """
+        self.hog_tracker = HistogramBasedMedianTracker(
+            hog_dim, hog_bins, hog_min, hog_max
+        )
+        self.geom_tracker = HistogramBasedMedianTracker(
+            geom_dim, geom_bins, geom_min, geom_max
+        )
+
+    def update(self, hog_features: np.ndarray, geom_features: np.ndarray,
+               update_histogram: bool = True) -> None:
+        """
+        Update both trackers
+
+        Args:
+            hog_features: HOG feature vector
+            geom_features: Geometric feature vector
+            update_histogram: Whether to update histograms
+        """
+        # Both trackers handle first frame special case internally
+        self.hog_tracker.update(hog_features, update_histogram)
+
+        # CRITICAL: OpenFace clamps HOG median to >= 0 after update (line 405 in FaceAnalyser.cpp)
+        # this->hog_desc_median.setTo(0, this->hog_desc_median < 0);
+        self.hog_tracker.current_median[self.hog_tracker.current_median < 0] = 0.0
+
+        self.geom_tracker.update(geom_features, update_histogram)
+
+    def get_combined_median(self) -> np.ndarray:
+        """
+        Get concatenated [HOG_median, geom_median]
+
+        Returns:
+            Combined median vector (4702 dims)
+        """
+        hog_median = self.hog_tracker.get_median()
+        geom_median = self.geom_tracker.get_median()
+        return np.concatenate([hog_median, geom_median])
+
+    def get_hog_median(self) -> np.ndarray:
+        """Get HOG median only"""
+        return self.hog_tracker.get_median()
+
+    def get_geom_median(self) -> np.ndarray:
+        """Get geometric median only"""
+        return self.geom_tracker.get_median()
+
+    def reset(self) -> None:
+        """Reset both trackers"""
+        self.hog_tracker.reset()
+        self.geom_tracker.reset()
+
+
+def test_histogram_tracker():
+    """Test histogram-based median tracker"""
+    print("="*80)
+    print("Histogram-Based Median Tracker - Test")
+    print("="*80)
+
+    # Create tracker
+    tracker = HistogramBasedMedianTracker(
+        feature_dim=10,
+        num_bins=200,
+        min_val=-3.0,
+        max_val=5.0
+    )
+
+    # Generate synthetic data (random walk around mean)
+    np.random.seed(42)
+    mean_features = np.array([0.0, 0.5, 1.0, -0.5, -1.0, 2.0, -2.0, 1.5, -1.5, 0.8])
+
+    print(f"\nSimulating 500 frames...")
+    print(f"True mean: {mean_features}")
+
+    for i in range(500):
+        # Add noise to mean
+        noise = np.random.randn(10) * 0.3
+        features = mean_features + noise
+
+        # Ensure values in histogram range
+        features = np.clip(features, -3.0, 5.0)
+
+        # Update tracker (update histogram every frame for testing)
+        tracker.update(features, update_histogram=True)
+
+        if i in [49, 99, 199, 499]:
+            median = tracker.get_median()
+            error = np.abs(median - mean_features).mean()
+            print(f"\nFrame {i+1}:")
+            print(f"  True mean:      {mean_features[:3]}...")
+            print(f"  Running median: {median[:3]}...")
+            print(f"  MAE: {error:.6f}")
+
+    print("\nTracker converges to true mean!")
+
+    # Test dual tracker
+    print("\n" + "="*80)
+    print("Dual Histogram Median Tracker - Test")
+    print("="*80)
+
+    dual_tracker = DualHistogramMedianTracker(
+        hog_dim=4464,
+        geom_dim=238,
+        hog_bins=200,
+        hog_min=-3.0,
+        hog_max=5.0,
+        geom_bins=200,
+        geom_min=-3.0,
+        geom_max=5.0
+    )
+
+    print("\nUpdating with synthetic HOG and geometric features...")
+    for i in range(100):
+        hog_feat = np.random.randn(4464) * 0.5
+        geom_feat = np.random.randn(238) * 0.5
+
+        # Clip to histogram range
+        hog_feat = np.clip(hog_feat, -3.0, 5.0)
+        geom_feat = np.clip(geom_feat, -3.0, 5.0)
+
+        dual_tracker.update(hog_feat, geom_feat, update_histogram=(i % 2 == 0))
+
+    combined = dual_tracker.get_combined_median()
+    print(f"Combined median shape: {combined.shape}")
+    print(f"Expected: (4702,)")
+    assert combined.shape == (4702,), f"Expected (4702,), got {combined.shape}"
+
+    print("\n" + "="*80)
+    print("All tests passed!")
+    print("="*80)
+
+
+if __name__ == "__main__":
+    test_histogram_tracker()