pyfaceau 1.0.6__cp312-cp312-macosx_11_0_arm64.whl

pyfaceau/features/histogram_median_tracker.py

@@ -0,0 +1,335 @@
+#!/usr/bin/env python3
+"""
+Histogram-Based Running Median Tracker
+
+Implements OpenFace 2.2's histogram-based running median algorithm for
+person-specific normalization. Matches the C++ implementation in
+FaceAnalyser::UpdateRunningMedian().
+
+Reference: OpenFace/lib/local/FaceAnalyser/src/FaceAnalyser.cpp:764-821
+"""
+
+import numpy as np
+
+
+class HistogramBasedMedianTracker:
+    """
+    Histogram-based running median tracker matching OpenFace 2.2's implementation
+
+    Uses binned histograms to efficiently compute running median without storing
+    all historical values. More memory-efficient than rolling window approach.
+    """
+
+    def __init__(self, feature_dim: int, num_bins: int = 200,
+                 min_val: float = -3.0, max_val: float = 5.0):
+        """
+        Initialize histogram-based median tracker
+
+        Args:
+            feature_dim: Dimensionality of feature vectors
+            num_bins: Number of histogram bins (OF2.2 uses 200)
+            min_val: Minimum value for histogram range
+            max_val: Maximum value for histogram range
+        """
+        self.feature_dim = feature_dim
+        self.num_bins = num_bins
+        self.min_val = min_val
+        self.max_val = max_val
+
+        # Histogram: (feature_dim, num_bins)
+        # Each row tracks the distribution of one feature dimension
+        self.histogram = np.zeros((feature_dim, num_bins), dtype=np.int32)
+
+        # Current median estimate
+        self.current_median = np.zeros(feature_dim, dtype=np.float64)
+
+        # Total count of updates
+        self.hist_count = 0
+
+        # Precompute bin width
+        self.length = max_val - min_val
+        self.bin_width = self.length / num_bins
+
+    def update(self, features: np.ndarray, update_histogram: bool = True) -> None:
+        """
+        Update tracker with new feature vector
+
+        Matches C++ UpdateRunningMedian() logic:
+        1. Bin each feature value into histogram
+        2. Update histogram counts
+        3. Recompute median from cumulative distribution
+
+        Args:
+            features: Feature vector (1D array)
+            update_histogram: Whether to update histogram (OF2.2 does this every 2nd frame)
+        """
+        # Ensure 1D
+        if features.ndim == 2:
+            features = features.flatten()
+
+        assert features.shape[0] == self.feature_dim, \
+            f"Expected {self.feature_dim} features, got {features.shape[0]}"
+
+        if update_histogram:
+            # Convert feature values to bin indices
+            # Formula from C++: (descriptor - min_val) * num_bins / (max_val - min_val)
+            converted = (features - self.min_val) * self.num_bins / self.length
+
+            # Cap values to [0, num_bins-1] BEFORE casting to int (matches C++)
+            # C++ does: setTo(num_bins-1, converted > num_bins-1) then setTo(0, converted < 0)
+            converted = np.clip(converted, 0.0, float(self.num_bins - 1))
+
+            # Cast to int (truncation, matches C++ (int) cast)
+            converted = converted.astype(np.int32)
+
+            # Update histogram counts
+            for i in range(self.feature_dim):
+                bin_idx = converted[i]
+                self.histogram[i, bin_idx] += 1
+
+            self.hist_count += 1
+
+        # Compute median (matches C++: always sets median, even on frame 0)
+        # On frame 0 (hist_count==0), C++ sets median=descriptor.clone()
+        # On frame 1+ with hist_count==1, C++ also uses descriptor directly
+        # Otherwise, compute from histogram
+        if self.hist_count == 0:
+            # Frame 0: histogram not updated yet, use descriptor directly
+            self.current_median = features.copy()
+        elif self.hist_count == 1:
+            # Frame 1: histogram updated once, still use descriptor directly (matches C++)
+            self.current_median = features.copy()
+        else:
+            # Frame 2+: compute from histogram
+            self._compute_median()
+
+    def _compute_median(self, first_descriptor: np.ndarray = None) -> None:
+        """
+        Compute median from histogram using cumulative sum
+
+        Matches C++ logic:
+        - If hist_count == 1: median = descriptor (special case)
+        - Otherwise: Find bin where cumulative sum reaches (hist_count+1)/2
+        - Convert bin index back to feature value
+
+        Args:
+            first_descriptor: If provided and hist_count==1, use directly as median
+        """
+        # Special case: First frame (matches C++ if(hist_count == 1) { median = descriptor.clone(); })
+        if self.hist_count == 1 and first_descriptor is not None:
+            self.current_median = first_descriptor.copy()
+            return
+
+        cutoff_point = (self.hist_count + 1) // 2
+
+        for i in range(self.feature_dim):
+            cumulative_sum = 0
+
+            for j in range(self.num_bins):
+                cumulative_sum += self.histogram[i, j]
+
+                if cumulative_sum >= cutoff_point:
+                    # Convert bin index back to value
+                    # Formula from C++: min_val + bin_idx * bin_width + 0.5 * bin_width
+                    self.current_median[i] = (
+                        self.min_val +
+                        j * self.bin_width +
+                        0.5 * self.bin_width
+                    )
+                    break
+
+    def get_median(self) -> np.ndarray:
+        """
+        Get current running median
+
+        Returns:
+            Median feature vector (1D array)
+        """
+        return self.current_median.copy()
+
+    def reset(self) -> None:
+        """Reset tracker (e.g., for new video)"""
+        self.histogram.fill(0)
+        self.current_median.fill(0.0)
+        self.hist_count = 0
+
+
+class DualHistogramMedianTracker:
+    """
+    Manages separate histogram-based running medians for HOG and geometric features
+
+    OF2.2 tracks HOG and geometric features separately with different histogram
+    parameters, then concatenates them when computing dynamic model predictions.
+    """
+
+    def __init__(self,
+                 hog_dim: int = 4464,
+                 geom_dim: int = 238,
+                 hog_bins: int = 200,
+                 hog_min: float = -3.0,
+                 hog_max: float = 5.0,
+                 geom_bins: int = 200,
+                 geom_min: float = -3.0,
+                 geom_max: float = 5.0):
+        """
+        Initialize dual histogram median tracker
+
+        Args:
+            hog_dim: HOG feature dimensionality
+            geom_dim: Geometric feature dimensionality
+            hog_bins: Number of bins for HOG histogram
+            hog_min: Minimum value for HOG histogram
+            hog_max: Maximum value for HOG histogram
+            geom_bins: Number of bins for geometric histogram
+            geom_min: Minimum value for geometric histogram
+            geom_max: Maximum value for geometric histogram
+        """
+        self.hog_tracker = HistogramBasedMedianTracker(
+            hog_dim, hog_bins, hog_min, hog_max
+        )
+        self.geom_tracker = HistogramBasedMedianTracker(
+            geom_dim, geom_bins, geom_min, geom_max
+        )
+
+        # Track frame number to match C++ behavior (geometric updates every other frame)
+        self.frames_tracking = 0
+
+    def update(self, hog_features: np.ndarray, geom_features: np.ndarray,
+               update_histogram: bool = True) -> None:
+        """
+        Update both trackers
+
+        CRITICAL: Matches C++ OpenFace 2.2 behavior (FaceAnalyser.cpp:400-428)
+        - BOTH HOG and geometric medians: updated every OTHER frame (when frames_tracking % 2 == 1)
+        - This is done as "a small speedup" optimization in C++
+
+        Args:
+            hog_features: HOG feature vector
+            geom_features: Geometric feature vector
+            update_histogram: Whether to update histograms
+        """
+        # CRITICAL: C++ updates BOTH medians only on odd frames (line 400-428 in FaceAnalyser.cpp)
+        # if(frames_tracking % 2 == 1)
+        update_on_this_frame = update_histogram and (self.frames_tracking % 2 == 1)
+
+        # Update HOG tracker (only on odd frames)
+        self.hog_tracker.update(hog_features, update_on_this_frame)
+
+        # CRITICAL: OpenFace clamps HOG median to >= 0 after update (line 405 in FaceAnalyser.cpp)
+        # this->hog_desc_median.setTo(0, this->hog_desc_median < 0);
+        if update_on_this_frame:
+            self.hog_tracker.current_median[self.hog_tracker.current_median < 0] = 0.0
+
+        # Update geometric tracker (only on odd frames)
+        self.geom_tracker.update(geom_features, update_on_this_frame)
+
+        # Increment frame counter
+        self.frames_tracking += 1
+
+    def get_combined_median(self) -> np.ndarray:
+        """
+        Get concatenated [HOG_median, geom_median]
+
+        Returns:
+            Combined median vector (4702 dims)
+        """
+        hog_median = self.hog_tracker.get_median()
+        geom_median = self.geom_tracker.get_median()
+        return np.concatenate([hog_median, geom_median])
+
+    def get_hog_median(self) -> np.ndarray:
+        """Get HOG median only"""
+        return self.hog_tracker.get_median()
+
+    def get_geom_median(self) -> np.ndarray:
+        """Get geometric median only"""
+        return self.geom_tracker.get_median()
+
+    def reset(self) -> None:
+        """Reset both trackers"""
+        self.hog_tracker.reset()
+        self.geom_tracker.reset()
+        self.frames_tracking = 0
+
+
+def test_histogram_tracker():
+    """Test histogram-based median tracker"""
+    print("="*80)
+    print("Histogram-Based Median Tracker - Test")
+    print("="*80)
+
+    # Create tracker
+    tracker = HistogramBasedMedianTracker(
+        feature_dim=10,
+        num_bins=200,
+        min_val=-3.0,
+        max_val=5.0
+    )
+
+    # Generate synthetic data (random walk around mean)
+    np.random.seed(42)
+    mean_features = np.array([0.0, 0.5, 1.0, -0.5, -1.0, 2.0, -2.0, 1.5, -1.5, 0.8])
+
+    print(f"\nSimulating 500 frames...")
+    print(f"True mean: {mean_features}")
+
+    for i in range(500):
+        # Add noise to mean
+        noise = np.random.randn(10) * 0.3
+        features = mean_features + noise
+
+        # Ensure values in histogram range
+        features = np.clip(features, -3.0, 5.0)
+
+        # Update tracker (update histogram every frame for testing)
+        tracker.update(features, update_histogram=True)
+
+        if i in [49, 99, 199, 499]:
+            median = tracker.get_median()
+            error = np.abs(median - mean_features).mean()
+            print(f"\nFrame {i+1}:")
+            print(f"  True mean:      {mean_features[:3]}...")
+            print(f"  Running median: {median[:3]}...")
+            print(f"  MAE: {error:.6f}")
+
+    print("\nTracker converges to true mean!")
+
+    # Test dual tracker
+    print("\n" + "="*80)
+    print("Dual Histogram Median Tracker - Test")
+    print("="*80)
+
+    dual_tracker = DualHistogramMedianTracker(
+        hog_dim=4464,
+        geom_dim=238,
+        hog_bins=200,
+        hog_min=-3.0,
+        hog_max=5.0,
+        geom_bins=200,
+        geom_min=-3.0,
+        geom_max=5.0
+    )
+
+    print("\nUpdating with synthetic HOG and geometric features...")
+    for i in range(100):
+        hog_feat = np.random.randn(4464) * 0.5
+        geom_feat = np.random.randn(238) * 0.5
+
+        # Clip to histogram range
+        hog_feat = np.clip(hog_feat, -3.0, 5.0)
+        geom_feat = np.clip(geom_feat, -3.0, 5.0)
+
+        dual_tracker.update(hog_feat, geom_feat, update_histogram=(i % 2 == 0))
+
+    combined = dual_tracker.get_combined_median()
+    print(f"Combined median shape: {combined.shape}")
+    print(f"Expected: (4702,)")
+    assert combined.shape == (4702,), f"Expected (4702,), got {combined.shape}"
+
+    print("\n" + "="*80)
+    print("All tests passed!")
+    print("="*80)
+
+
+if __name__ == "__main__":
+    test_histogram_tracker()

pyfaceau/features/pdm.py

@@ -0,0 +1,269 @@
+#!/usr/bin/env python3
+"""
+OpenFace 2.2 PDM (Point Distribution Model) Parser
+
+Parses text-based PDM files containing mean landmark positions and
+principal components for facial shape reconstruction.
+
+Format:
+1. Mean values section: (204,) - average 3D landmark positions
+2. Principal components section: (204, 34) - eigenvectors for reconstruction
+
+Usage:
+    pdm = PDMParser("In-the-wild_aligned_PDM_68.txt")
+    reconstructed_landmarks = pdm.reconstruct_from_params(pdm_params)
+"""
+
+import numpy as np
+from pathlib import Path
+from typing import Tuple
+
+
+class PDMParser:
+    """Parser for OpenFace 2.2 PDM files"""
+
+    def __init__(self, pdm_file_path: str):
+        """
+        Initialize parser and load PDM
+
+        Args:
+            pdm_file_path: Path to PDM .txt file
+        """
+        self.pdm_file_path = Path(pdm_file_path)
+        if not self.pdm_file_path.exists():
+            raise FileNotFoundError(f"PDM file not found: {pdm_file_path}")
+
+        # Load PDM components
+        self.mean_shape, self.princ_comp, self.eigen_values = self._parse_pdm()
+
+        print(f"Loaded PDM from {self.pdm_file_path.name}")
+        print(f"  Mean shape: {self.mean_shape.shape}")
+        print(f"  Principal components: {self.princ_comp.shape}")
+        print(f"  Eigenvalues: {self.eigen_values.shape}")
+
+    def _parse_pdm(self) -> Tuple[np.ndarray, np.ndarray, np.ndarray]:
+        """
+        Parse PDM file
+
+        Returns:
+            Tuple of (mean_shape, principal_components, eigen_values)
+            - mean_shape: (204, 1) array of mean landmark positions
+            - principal_components: (204, 34) matrix of eigenvectors
+            - eigen_values: (34,) array of eigenvalues (variances)
+        """
+        with open(self.pdm_file_path, 'r') as f:
+            lines = f.readlines()
+
+        # Parse mean values section
+        mean_shape = self._parse_matrix_section(lines, 0)
+
+        # Find principal components section
+        # Look for the comment line that starts the PC section
+        pc_start = None
+        for i, line in enumerate(lines):
+            if 'principal components' in line.lower() and 'eigenvectors' in line.lower():
+                pc_start = i
+                break
+
+        if pc_start is None:
+            raise ValueError("Principal components section not found in PDM file")
+
+        # Parse principal components section
+        princ_comp = self._parse_matrix_section(lines, pc_start)
+
+        # Find eigenvalues section
+        # Look for the comment line that starts the eigenvalues section
+        eigen_start = None
+        for i, line in enumerate(lines):
+            if 'eigenvalues' in line.lower() and 'variances' in line.lower():
+                eigen_start = i
+                break
+
+        if eigen_start is None:
+            raise ValueError("Eigenvalues section not found in PDM file")
+
+        # Parse eigenvalues section
+        eigen_values = self._parse_matrix_section(lines, eigen_start)
+        eigen_values = eigen_values.flatten()  # Convert to 1D array
+
+        return mean_shape, princ_comp, eigen_values
+
+    def _parse_matrix_section(self, lines, start_idx):
+        """
+        Parse a matrix section starting at given line index
+
+        Args:
+            lines: All lines from file
+            start_idx: Index of comment line before matrix
+
+        Returns:
+            numpy array with the matrix data
+        """
+        # Skip comment line
+        idx = start_idx + 1
+
+        # Read dimensions
+        rows = int(lines[idx].strip())
+        idx += 1
+
+        cols = int(lines[idx].strip())
+        idx += 1
+
+        # Skip dtype line
+        idx += 1
+
+        # Read matrix data
+        data = []
+        values_read = 0
+        total_values = rows * cols
+
+        while values_read < total_values and idx < len(lines):
+            line = lines[idx].strip()
+            if line and not line.startswith('#'):
+                # Split on whitespace and convert to float
+                values = [float(x) for x in line.split()]
+                data.extend(values)
+                values_read += len(values)
+            idx += 1
+
+        if len(data) != total_values:
+            raise ValueError(
+                f"Expected {total_values} values, got {len(data)}"
+            )
+
+        # Reshape to matrix
+        matrix = np.array(data, dtype=np.float64).reshape(rows, cols)
+
+        return matrix
+
+    def reconstruct_from_params(self, pdm_params: np.ndarray) -> np.ndarray:
+        """
+        Reconstruct 3D landmarks from PDM parameters
+
+        This matches OpenFace 2.2's reconstruction:
+        reconstructed_landmarks = princ_comp × pdm_params
+
+        Args:
+            pdm_params: (34,) array of PDM parameters from CSV (p_0...p_33)
+
+        Returns:
+            (204,) array of reconstructed 3D landmarks
+            Format: [X_0...X_67, Y_0...Y_67, Z_0...Z_67]
+        """
+        if pdm_params.ndim == 2:
+            pdm_params = pdm_params.flatten()
+
+        if pdm_params.shape[0] != self.princ_comp.shape[1]:
+            raise ValueError(
+                f"Expected {self.princ_comp.shape[1]} PDM params, "
+                f"got {pdm_params.shape[0]}"
+            )
+
+        # Reconstruct: princ_comp (204, 34) × params (34,) = (204,)
+        reconstructed = np.dot(self.princ_comp, pdm_params)
+
+        return reconstructed
+
+    def extract_geometric_features(self, pdm_params: np.ndarray) -> np.ndarray:
+        """
+        Extract geometric features for AU prediction
+
+        Matches OpenFace 2.2's geom_descriptor_frame construction:
+        1. Reconstruct landmarks from PDM
+        2. Concatenate: [reconstructed_landmarks, pdm_params]
+
+        Args:
+            pdm_params: (34,) array of PDM parameters
+
+        Returns:
+            Geometric feature vector matching OF2.2 format
+        """
+        # Reconstruct landmarks
+        reconstructed_landmarks = self.reconstruct_from_params(pdm_params)
+
+        # Concatenate: [landmarks, params]
+        geom_features = np.concatenate([reconstructed_landmarks, pdm_params])
+
+        return geom_features
+
+
+def test_pdm_parser():
+    """Test PDM parser"""
+    import pandas as pd
+
+    print("="*80)
+    print("PDM Parser - Test")
+    print("="*80)
+
+    # Path to PDM file
+    pdm_file = "/Users/johnwilsoniv/repo/fea_tool/external_libs/openFace/OpenFace/lib/local/FaceAnalyser/AU_predictors/In-the-wild_aligned_PDM_68.txt"
+
+    # Load PDM
+    print("\nLoading PDM...")
+    pdm = PDMParser(pdm_file)
+
+    # Load CSV to get PDM params
+    csv_file = "/Users/johnwilsoniv/Documents/SplitFace Open3/S1 Face Mirror/of22_validation/IMG_0942_left_mirrored.csv"
+    print(f"\nLoading CSV: {csv_file}")
+    df = pd.read_csv(csv_file)
+
+    # Extract PDM params from first frame
+    pdm_cols = [f'p_{i}' for i in range(34)]
+    pdm_params = df.iloc[0][pdm_cols].values
+
+    print(f"\nPDM params from frame 0: {pdm_params.shape}")
+    print(f"  First 5 values: {pdm_params[:5]}")
+
+    # Reconstruct landmarks
+    print("\nReconstructing landmarks...")
+    reconstructed = pdm.reconstruct_from_params(pdm_params)
+    print(f"Reconstructed landmarks: {reconstructed.shape}")
+    print(f"  First 5 values: {reconstructed[:5]}")
+
+    # Check range
+    print(f"\nReconstructed landmark statistics:")
+    print(f"  Min: {reconstructed.min():.2f}")
+    print(f"  Max: {reconstructed.max():.2f}")
+    print(f"  Mean: {reconstructed.mean():.2f}")
+    print(f"  Std: {reconstructed.std():.2f}")
+
+    # Compare to raw landmarks
+    X_cols = [f'X_{i}' for i in range(68)]
+    Y_cols = [f'Y_{i}' for i in range(68)]
+    Z_cols = [f'Z_{i}' for i in range(68)]
+
+    raw_landmarks = np.concatenate([
+        df.iloc[0][X_cols].values,
+        df.iloc[0][Y_cols].values,
+        df.iloc[0][Z_cols].values
+    ])
+
+    print(f"\nRaw landmark statistics (from CSV):")
+    print(f"  Min: {raw_landmarks.min():.2f}")
+    print(f"  Max: {raw_landmarks.max():.2f}")
+    print(f"  Mean: {raw_landmarks.mean():.2f}")
+    print(f"  Std: {raw_landmarks.std():.2f}")
+
+    # Extract geometric features
+    print("\nExtracting geometric features...")
+    geom_features = pdm.extract_geometric_features(pdm_params)
+    print(f"Geometric features: {geom_features.shape}")
+    print(f"  Expected: (238,) = 204 (landmarks) + 34 (params)")
+
+    # Check if reconstructed landmarks fit in histogram range [-60, 60]
+    in_range = np.sum((reconstructed >= -60) & (reconstructed <= 60))
+    total = reconstructed.size
+    pct = 100.0 * in_range / total
+
+    print(f"\nHistogram range check [-60, 60]:")
+    print(f"  Values in range: {in_range}/{total} ({pct:.2f}%)")
+    print(f"  Values below -60: {np.sum(reconstructed < -60)}")
+    print(f"  Values above 60: {np.sum(reconstructed > 60)}")
+
+    print("\n" + "="*80)
+    print("Test complete!")
+    print("="*80)
+
+
+if __name__ == "__main__":
+    test_pdm_parser()

pyfaceau/features/triangulation.py

@@ -0,0 +1,64 @@
+#!/usr/bin/env python3
+"""
+Parse OpenFace triangulation file for face masking
+"""
+
+import numpy as np
+
+class TriangulationParser:
+    """Parser for OpenFace tris_68.txt triangulation data"""
+
+    def __init__(self, tris_file: str):
+        """
+        Load triangulation from OpenFace format file
+
+        Args:
+            tris_file: Path to tris_68.txt file
+        """
+        with open(tris_file, 'r') as f:
+            lines = [line.strip() for line in f if line.strip() and not line.startswith('#')]
+
+        # Parse file format:
+        # Line 0: Total number of triangle lines (111)
+        # Line 1: Number of triangulation sets (3)
+        # Line 2: Dimension (4)
+        # Line 3+: Triangle definitions (vertex indices)
+
+        total_triangles = int(lines[0])
+
+        # Read all triangles starting from line 3
+        triangles = []
+        for i in range(3, len(lines)):  # Start from line 3, skip header
+            tri = list(map(int, lines[i].split()))
+            if len(tri) == 3:  # Valid triangle
+                triangles.append(tri)
+
+        self.triangles = np.array(triangles, dtype=np.int32)
+
+        print(f"Loaded {len(self.triangles)} triangles from {tris_file}")
+
+    def create_face_mask(self, landmarks: np.ndarray, img_width: int, img_height: int) -> np.ndarray:
+        """
+        Create binary mask for face region using triangulation
+
+        Args:
+            landmarks: (68, 2) array of facial landmark coordinates
+            img_width: Mask width in pixels
+            img_height: Mask height in pixels
+
+        Returns:
+            (height, width) binary mask (0=background, 255=face)
+        """
+        import cv2
+
+        mask = np.zeros((img_height, img_width), dtype=np.uint8)
+
+        # Fill each triangle
+        for tri in self.triangles:
+            # Get the three vertices for this triangle
+            pts = landmarks[tri].astype(np.int32)
+
+            # Fill the triangle
+            cv2.fillConvexPoly(mask, pts, 255)
+
+        return mask