pyfaceau 1.0.1__cp310-cp310-macosx_10_9_x86_64.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pyfaceau/__init__.py +19 -0
- pyfaceau/alignment/__init__.py +0 -0
- pyfaceau/alignment/calc_params.py +671 -0
- pyfaceau/alignment/face_aligner.py +352 -0
- pyfaceau/alignment/numba_calcparams_accelerator.py +244 -0
- pyfaceau/cython_histogram_median.cpython-310-darwin.so +0 -0
- pyfaceau/cython_rotation_update.cpython-310-darwin.so +0 -0
- pyfaceau/detectors/__init__.py +0 -0
- pyfaceau/detectors/pfld.py +128 -0
- pyfaceau/detectors/retinaface.py +352 -0
- pyfaceau/download_weights.py +134 -0
- pyfaceau/features/__init__.py +0 -0
- pyfaceau/features/histogram_median_tracker.py +335 -0
- pyfaceau/features/pdm.py +269 -0
- pyfaceau/features/triangulation.py +64 -0
- pyfaceau/parallel_pipeline.py +462 -0
- pyfaceau/pipeline.py +1083 -0
- pyfaceau/prediction/__init__.py +0 -0
- pyfaceau/prediction/au_predictor.py +434 -0
- pyfaceau/prediction/batched_au_predictor.py +269 -0
- pyfaceau/prediction/model_parser.py +337 -0
- pyfaceau/prediction/running_median.py +318 -0
- pyfaceau/prediction/running_median_fallback.py +200 -0
- pyfaceau/processor.py +270 -0
- pyfaceau/refinement/__init__.py +12 -0
- pyfaceau/refinement/svr_patch_expert.py +361 -0
- pyfaceau/refinement/targeted_refiner.py +362 -0
- pyfaceau/utils/__init__.py +0 -0
- pyfaceau/utils/cython_extensions/cython_histogram_median.c +35391 -0
- pyfaceau/utils/cython_extensions/cython_histogram_median.pyx +316 -0
- pyfaceau/utils/cython_extensions/cython_rotation_update.c +32262 -0
- pyfaceau/utils/cython_extensions/cython_rotation_update.pyx +211 -0
- pyfaceau/utils/cython_extensions/setup.py +47 -0
- pyfaceau-1.0.1.data/scripts/pyfaceau_gui.py +302 -0
- pyfaceau-1.0.1.dist-info/METADATA +466 -0
- pyfaceau-1.0.1.dist-info/RECORD +40 -0
- pyfaceau-1.0.1.dist-info/WHEEL +5 -0
- pyfaceau-1.0.1.dist-info/entry_points.txt +3 -0
- pyfaceau-1.0.1.dist-info/licenses/LICENSE +40 -0
- pyfaceau-1.0.1.dist-info/top_level.txt +1 -0
pyfaceau/__init__.py
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
"""
|
|
2
|
+
pyfaceau - Pure Python OpenFace 2.2 AU Extraction
|
|
3
|
+
|
|
4
|
+
A complete Python implementation of OpenFace 2.2's AU extraction pipeline
|
|
5
|
+
with high-performance parallel processing support and CLNF landmark refinement.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
__version__ = "1.0.0"
|
|
9
|
+
|
|
10
|
+
from .pipeline import FullPythonAUPipeline
|
|
11
|
+
from .parallel_pipeline import ParallelAUPipeline
|
|
12
|
+
from .processor import OpenFaceProcessor, process_videos
|
|
13
|
+
|
|
14
|
+
__all__ = [
|
|
15
|
+
'FullPythonAUPipeline',
|
|
16
|
+
'ParallelAUPipeline',
|
|
17
|
+
'OpenFaceProcessor',
|
|
18
|
+
'process_videos'
|
|
19
|
+
]
|
|
File without changes
|
|
@@ -0,0 +1,671 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""
|
|
3
|
+
Python Implementation of OpenFace 2.2 CalcParams Function
|
|
4
|
+
|
|
5
|
+
Replicates the PDM::CalcParams() function from OpenFace 2.2's PDM.cpp
|
|
6
|
+
for calculating optimal 3D pose parameters (6 global + 34 local) from 2D landmarks.
|
|
7
|
+
|
|
8
|
+
Reference: OpenFace/lib/local/LandmarkDetector/src/PDM.cpp lines 508-705
|
|
9
|
+
|
|
10
|
+
This implements iterative optimization using Jacobian and Hessian to fit
|
|
11
|
+
a 3D Point Distribution Model (PDM) to detected 2D facial landmarks.
|
|
12
|
+
|
|
13
|
+
Author: Replicated from OpenFace 2.2 C++ implementation
|
|
14
|
+
Date: 2025-10-29
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
import numpy as np
|
|
18
|
+
from scipy import linalg
|
|
19
|
+
import cv2
|
|
20
|
+
|
|
21
|
+
# Try to import Cython-optimized rotation update for 99.9% accuracy
|
|
22
|
+
try:
|
|
23
|
+
import sys
|
|
24
|
+
from pathlib import Path
|
|
25
|
+
# Add parent directory to path so we can import cython extensions
|
|
26
|
+
sys.path.insert(0, str(Path(__file__).parent.parent))
|
|
27
|
+
from cython_rotation_update import update_rotation_cython
|
|
28
|
+
CYTHON_AVAILABLE = True
|
|
29
|
+
print("Cython rotation update module loaded - targeting 99.9% accuracy")
|
|
30
|
+
except ImportError:
|
|
31
|
+
CYTHON_AVAILABLE = False
|
|
32
|
+
print("Warning: Cython rotation update not available - using Python (99.45% accuracy)")
|
|
33
|
+
|
|
34
|
+
# Try to import Numba JIT-accelerated CalcParams functions for 2-5x speedup
|
|
35
|
+
try:
|
|
36
|
+
from pyfaceau.alignment.numba_calcparams_accelerator import (
|
|
37
|
+
compute_jacobian_accelerated,
|
|
38
|
+
project_shape_to_2d_jit,
|
|
39
|
+
euler_to_rotation_matrix_jit
|
|
40
|
+
)
|
|
41
|
+
NUMBA_AVAILABLE = True
|
|
42
|
+
except ImportError:
|
|
43
|
+
NUMBA_AVAILABLE = False
|
|
44
|
+
print("Warning: Numba JIT accelerator not available - using standard Python (slower)")
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
class CalcParams:
|
|
48
|
+
"""
|
|
49
|
+
Calculate optimal PDM parameters from 2D landmarks
|
|
50
|
+
|
|
51
|
+
Implements the CalcParams algorithm from OpenFace 2.2 which optimizes:
|
|
52
|
+
- 6 global parameters: scale, rx, ry, rz, tx, ty
|
|
53
|
+
- 34 local parameters: PCA shape coefficients
|
|
54
|
+
|
|
55
|
+
Uses iterative Gauss-Newton optimization with Jacobian and Hessian.
|
|
56
|
+
"""
|
|
57
|
+
|
|
58
|
+
def __init__(self, pdm_parser):
|
|
59
|
+
"""
|
|
60
|
+
Initialize CalcParams with PDM model
|
|
61
|
+
|
|
62
|
+
Args:
|
|
63
|
+
pdm_parser: PDMParser instance containing mean_shape, princ_comp, eigen_values
|
|
64
|
+
"""
|
|
65
|
+
self.pdm = pdm_parser
|
|
66
|
+
self.mean_shape = pdm_parser.mean_shape # (204, 1)
|
|
67
|
+
self.princ_comp = pdm_parser.princ_comp # (204, 34)
|
|
68
|
+
self.eigen_values = pdm_parser.eigen_values # (34,)
|
|
69
|
+
|
|
70
|
+
@staticmethod
|
|
71
|
+
def euler_to_rotation_matrix(euler_angles):
|
|
72
|
+
"""
|
|
73
|
+
Convert Euler angles to 3x3 rotation matrix
|
|
74
|
+
|
|
75
|
+
Uses XYZ convention: R = Rx * Ry * Rz (left-handed positive sign)
|
|
76
|
+
Matches Utilities::Euler2RotationMatrix() from RotationHelpers.h
|
|
77
|
+
|
|
78
|
+
Args:
|
|
79
|
+
euler_angles: (rx, ry, rz) in radians
|
|
80
|
+
|
|
81
|
+
Returns:
|
|
82
|
+
3x3 rotation matrix
|
|
83
|
+
"""
|
|
84
|
+
rx, ry, rz = euler_angles
|
|
85
|
+
|
|
86
|
+
s1, s2, s3 = np.sin(rx), np.sin(ry), np.sin(rz)
|
|
87
|
+
c1, c2, c3 = np.cos(rx), np.cos(ry), np.cos(rz)
|
|
88
|
+
|
|
89
|
+
R = np.array([
|
|
90
|
+
[c2 * c3, -c2 * s3, s2],
|
|
91
|
+
[c1 * s3 + c3 * s1 * s2, c1 * c3 - s1 * s2 * s3, -c2 * s1],
|
|
92
|
+
[s1 * s3 - c1 * c3 * s2, c3 * s1 + c1 * s2 * s3, c1 * c2]
|
|
93
|
+
], dtype=np.float32)
|
|
94
|
+
|
|
95
|
+
return R
|
|
96
|
+
|
|
97
|
+
@staticmethod
|
|
98
|
+
def rotation_matrix_to_euler(R):
|
|
99
|
+
"""
|
|
100
|
+
Convert 3x3 rotation matrix to Euler angles using robust quaternion extraction
|
|
101
|
+
|
|
102
|
+
Matches RotationMatrix2Euler() from RotationHelpers.h
|
|
103
|
+
Uses Shepperd's method for robust quaternion extraction (handles all cases)
|
|
104
|
+
|
|
105
|
+
Args:
|
|
106
|
+
R: 3x3 rotation matrix
|
|
107
|
+
|
|
108
|
+
Returns:
|
|
109
|
+
(rx, ry, rz) Euler angles in radians
|
|
110
|
+
"""
|
|
111
|
+
# Robust quaternion extraction using Shepperd's method
|
|
112
|
+
# This handles all rotation cases without singularities
|
|
113
|
+
trace = R[0,0] + R[1,1] + R[2,2]
|
|
114
|
+
|
|
115
|
+
if trace > 0:
|
|
116
|
+
# Standard case: trace is positive
|
|
117
|
+
s = np.sqrt(trace + 1.0) * 2.0 # s = 4*q0
|
|
118
|
+
q0 = 0.25 * s
|
|
119
|
+
q1 = (R[2,1] - R[1,2]) / s
|
|
120
|
+
q2 = (R[0,2] - R[2,0]) / s
|
|
121
|
+
q3 = (R[1,0] - R[0,1]) / s
|
|
122
|
+
elif (R[0,0] > R[1,1]) and (R[0,0] > R[2,2]):
|
|
123
|
+
# q1 is largest component
|
|
124
|
+
s = np.sqrt(1.0 + R[0,0] - R[1,1] - R[2,2]) * 2.0 # s = 4*q1
|
|
125
|
+
q0 = (R[2,1] - R[1,2]) / s
|
|
126
|
+
q1 = 0.25 * s
|
|
127
|
+
q2 = (R[0,1] + R[1,0]) / s
|
|
128
|
+
q3 = (R[0,2] + R[2,0]) / s
|
|
129
|
+
elif R[1,1] > R[2,2]:
|
|
130
|
+
# q2 is largest component
|
|
131
|
+
s = np.sqrt(1.0 + R[1,1] - R[0,0] - R[2,2]) * 2.0 # s = 4*q2
|
|
132
|
+
q0 = (R[0,2] - R[2,0]) / s
|
|
133
|
+
q1 = (R[0,1] + R[1,0]) / s
|
|
134
|
+
q2 = 0.25 * s
|
|
135
|
+
q3 = (R[1,2] + R[2,1]) / s
|
|
136
|
+
else:
|
|
137
|
+
# q3 is largest component
|
|
138
|
+
s = np.sqrt(1.0 + R[2,2] - R[0,0] - R[1,1]) * 2.0 # s = 4*q3
|
|
139
|
+
q0 = (R[1,0] - R[0,1]) / s
|
|
140
|
+
q1 = (R[0,2] + R[2,0]) / s
|
|
141
|
+
q2 = (R[1,2] + R[2,1]) / s
|
|
142
|
+
q3 = 0.25 * s
|
|
143
|
+
|
|
144
|
+
# Quaternion to Euler angles
|
|
145
|
+
t1 = 2.0 * (q0*q2 + q1*q3)
|
|
146
|
+
t1 = np.clip(t1, -1.0, 1.0) # Handle precision issues
|
|
147
|
+
|
|
148
|
+
yaw = np.arcsin(t1)
|
|
149
|
+
pitch = np.arctan2(2.0 * (q0*q1 - q2*q3), q0*q0 - q1*q1 - q2*q2 + q3*q3)
|
|
150
|
+
roll = np.arctan2(2.0 * (q0*q3 - q1*q2), q0*q0 + q1*q1 - q2*q2 - q3*q3)
|
|
151
|
+
|
|
152
|
+
return np.array([pitch, yaw, roll], dtype=np.float32)
|
|
153
|
+
|
|
154
|
+
@staticmethod
|
|
155
|
+
def rotation_matrix_to_axis_angle(R):
|
|
156
|
+
"""Convert rotation matrix to axis-angle representation using Rodrigues"""
|
|
157
|
+
axis_angle, _ = cv2.Rodrigues(R)
|
|
158
|
+
return axis_angle.flatten()
|
|
159
|
+
|
|
160
|
+
@staticmethod
|
|
161
|
+
def axis_angle_to_euler(axis_angle):
|
|
162
|
+
"""Convert axis-angle to Euler angles via rotation matrix"""
|
|
163
|
+
R, _ = cv2.Rodrigues(axis_angle)
|
|
164
|
+
return CalcParams.rotation_matrix_to_euler(R)
|
|
165
|
+
|
|
166
|
+
@staticmethod
|
|
167
|
+
def orthonormalise(R):
|
|
168
|
+
"""
|
|
169
|
+
Ensure rotation matrix is orthonormal
|
|
170
|
+
|
|
171
|
+
Matches PDM::Orthonormalise() from PDM.cpp lines 59-76
|
|
172
|
+
Uses SVD to project matrix onto SO(3)
|
|
173
|
+
|
|
174
|
+
Args:
|
|
175
|
+
R: 3x3 matrix (possibly not perfectly orthonormal)
|
|
176
|
+
|
|
177
|
+
Returns:
|
|
178
|
+
3x3 orthonormal rotation matrix
|
|
179
|
+
"""
|
|
180
|
+
U, S, Vt = np.linalg.svd(R)
|
|
181
|
+
|
|
182
|
+
# Ensure no reflection (determinant = 1, not -1)
|
|
183
|
+
W = np.eye(3, dtype=np.float32)
|
|
184
|
+
d = np.linalg.det(U @ Vt)
|
|
185
|
+
W[2, 2] = d
|
|
186
|
+
|
|
187
|
+
R_ortho = U @ W @ Vt
|
|
188
|
+
return R_ortho.astype(np.float32)
|
|
189
|
+
|
|
190
|
+
def calc_shape_3d(self, params_local):
|
|
191
|
+
"""
|
|
192
|
+
Calculate 3D shape from local parameters
|
|
193
|
+
|
|
194
|
+
Matches PDM::CalcShape3D()
|
|
195
|
+
|
|
196
|
+
Args:
|
|
197
|
+
params_local: (34,) PCA coefficients
|
|
198
|
+
|
|
199
|
+
Returns:
|
|
200
|
+
shape_3d: (204,) 3D coordinates [X0...X67, Y0...Y67, Z0...Z67]
|
|
201
|
+
"""
|
|
202
|
+
params_local = params_local.reshape(-1, 1)
|
|
203
|
+
shape_3d = self.mean_shape + self.princ_comp @ params_local
|
|
204
|
+
return shape_3d.flatten()
|
|
205
|
+
|
|
206
|
+
def extract_bounding_box(self, landmarks_2d):
|
|
207
|
+
"""
|
|
208
|
+
Extract bounding box from 2D landmarks
|
|
209
|
+
|
|
210
|
+
Args:
|
|
211
|
+
landmarks_2d: (n*2,) array [X0...Xn, Y0...Yn]
|
|
212
|
+
|
|
213
|
+
Returns:
|
|
214
|
+
(min_x, max_x, min_y, max_y)
|
|
215
|
+
"""
|
|
216
|
+
n = len(landmarks_2d) // 2
|
|
217
|
+
x_coords = landmarks_2d[:n]
|
|
218
|
+
y_coords = landmarks_2d[n:]
|
|
219
|
+
|
|
220
|
+
return np.min(x_coords), np.max(x_coords), np.min(y_coords), np.max(y_coords)
|
|
221
|
+
|
|
222
|
+
def calc_bounding_box_model(self):
|
|
223
|
+
"""
|
|
224
|
+
Calculate bounding box of mean PDM shape at identity pose
|
|
225
|
+
|
|
226
|
+
Returns:
|
|
227
|
+
(width, height) of model bounding box
|
|
228
|
+
"""
|
|
229
|
+
# Get mean shape at identity pose (scale=1, rotation=0, translation=0)
|
|
230
|
+
mean_3d = self.mean_shape.flatten() # (204,)
|
|
231
|
+
n = len(mean_3d) // 3
|
|
232
|
+
|
|
233
|
+
# Project to 2D with identity transformation (just take X, Y)
|
|
234
|
+
x_coords = mean_3d[:n]
|
|
235
|
+
y_coords = mean_3d[n:2*n]
|
|
236
|
+
|
|
237
|
+
width = np.max(x_coords) - np.min(x_coords)
|
|
238
|
+
height = np.max(y_coords) - np.min(y_coords)
|
|
239
|
+
|
|
240
|
+
return width, height
|
|
241
|
+
|
|
242
|
+
def compute_jacobian(self, params_local, params_global, weight_matrix):
|
|
243
|
+
"""
|
|
244
|
+
Compute Jacobian matrix for optimization
|
|
245
|
+
|
|
246
|
+
Matches PDM::ComputeJacobian() from PDM.cpp lines 346-449
|
|
247
|
+
|
|
248
|
+
Jacobian has shape (n*2, 6+m) where:
|
|
249
|
+
- n = number of landmarks
|
|
250
|
+
- 6 = global params (scale, rx, ry, rz, tx, ty)
|
|
251
|
+
- m = number of local params (34 PCA coefficients)
|
|
252
|
+
|
|
253
|
+
Args:
|
|
254
|
+
params_local: (34,) local parameters
|
|
255
|
+
params_global: (6,) global parameters [scale, rx, ry, rz, tx, ty]
|
|
256
|
+
weight_matrix: (n*2, n*2) diagonal weight matrix
|
|
257
|
+
|
|
258
|
+
Returns:
|
|
259
|
+
J: (n*2, 6+m) Jacobian matrix
|
|
260
|
+
J_w_t: (6+m, n*2) weighted Jacobian transpose
|
|
261
|
+
"""
|
|
262
|
+
# Use Numba JIT-accelerated version if available (2-5x speedup!)
|
|
263
|
+
if NUMBA_AVAILABLE:
|
|
264
|
+
n_vis = self.mean_shape.shape[0] // 3
|
|
265
|
+
m = 34
|
|
266
|
+
return compute_jacobian_accelerated(
|
|
267
|
+
params_local, params_global, self.princ_comp,
|
|
268
|
+
self.mean_shape, weight_matrix, n_vis, m
|
|
269
|
+
)
|
|
270
|
+
|
|
271
|
+
# Fall back to Python implementation (slower)
|
|
272
|
+
n = 68 # number of landmarks
|
|
273
|
+
m = 34 # number of PCA modes
|
|
274
|
+
|
|
275
|
+
# Extract global parameters
|
|
276
|
+
s = params_global[0] # scale
|
|
277
|
+
euler = params_global[1:4] # rotation (rx, ry, rz)
|
|
278
|
+
|
|
279
|
+
# Get 3D shape from current local parameters
|
|
280
|
+
shape_3d = self.calc_shape_3d(params_local) # (204,)
|
|
281
|
+
shape_3d = shape_3d.reshape(3, n).astype(np.float32) # (3, 68) -> [X, Y, Z] rows
|
|
282
|
+
|
|
283
|
+
# Get rotation matrix (force float32 for consistency with C++)
|
|
284
|
+
R = self.euler_to_rotation_matrix(euler).astype(np.float32) # (3, 3)
|
|
285
|
+
r11, r12, r13 = R[0, :]
|
|
286
|
+
r21, r22, r23 = R[1, :]
|
|
287
|
+
r31, r32, r33 = R[2, :]
|
|
288
|
+
|
|
289
|
+
# Initialize Jacobian
|
|
290
|
+
J = np.zeros((n * 2, 6 + m), dtype=np.float32)
|
|
291
|
+
|
|
292
|
+
# Iterate over each landmark
|
|
293
|
+
for i in range(n):
|
|
294
|
+
X, Y, Z = shape_3d[:, i]
|
|
295
|
+
|
|
296
|
+
# Row for x coordinate
|
|
297
|
+
row_x = i
|
|
298
|
+
# Row for y coordinate
|
|
299
|
+
row_y = i + n
|
|
300
|
+
|
|
301
|
+
# --- Global parameter derivatives ---
|
|
302
|
+
|
|
303
|
+
# Scaling term (column 0)
|
|
304
|
+
J[row_x, 0] = X * r11 + Y * r12 + Z * r13
|
|
305
|
+
J[row_y, 0] = X * r21 + Y * r22 + Z * r23
|
|
306
|
+
|
|
307
|
+
# Rotation terms (columns 1-3: rx, ry, rz)
|
|
308
|
+
# Using small angle approximation for rotation Jacobian
|
|
309
|
+
J[row_x, 1] = s * (Y * r13 - Z * r12)
|
|
310
|
+
J[row_y, 1] = s * (Y * r23 - Z * r22)
|
|
311
|
+
|
|
312
|
+
J[row_x, 2] = -s * (X * r13 - Z * r11)
|
|
313
|
+
J[row_y, 2] = -s * (X * r23 - Z * r21)
|
|
314
|
+
|
|
315
|
+
J[row_x, 3] = s * (X * r12 - Y * r11)
|
|
316
|
+
J[row_y, 3] = s * (X * r22 - Y * r21)
|
|
317
|
+
|
|
318
|
+
# Translation terms (columns 4-5: tx, ty)
|
|
319
|
+
J[row_x, 4] = 1.0
|
|
320
|
+
J[row_y, 4] = 0.0
|
|
321
|
+
|
|
322
|
+
J[row_x, 5] = 0.0
|
|
323
|
+
J[row_y, 5] = 1.0
|
|
324
|
+
|
|
325
|
+
# --- Local parameter derivatives ---
|
|
326
|
+
# Columns 6 to 6+m-1
|
|
327
|
+
for j in range(m):
|
|
328
|
+
# Get principal component for this mode at this landmark
|
|
329
|
+
Vx = self.princ_comp[i, j] # princ_comp is (204, 34), row i corresponds to X coordinate
|
|
330
|
+
Vy = self.princ_comp[i + n, j] # row i+n corresponds to Y coordinate
|
|
331
|
+
Vz = self.princ_comp[i + 2*n, j] # row i+2*n corresponds to Z coordinate
|
|
332
|
+
|
|
333
|
+
# Derivative of 2D projection w.r.t. local parameter j
|
|
334
|
+
J[row_x, 6 + j] = s * (r11*Vx + r12*Vy + r13*Vz)
|
|
335
|
+
J[row_y, 6 + j] = s * (r21*Vx + r22*Vy + r23*Vz)
|
|
336
|
+
|
|
337
|
+
# Apply weighting (matches PDM.cpp lines 422-445)
|
|
338
|
+
if not np.allclose(np.diag(weight_matrix), 1.0):
|
|
339
|
+
J_weighted = J.copy()
|
|
340
|
+
for i in range(n):
|
|
341
|
+
w_x = weight_matrix[i, i]
|
|
342
|
+
w_y = weight_matrix[i + n, i + n]
|
|
343
|
+
J_weighted[i, :] *= w_x
|
|
344
|
+
J_weighted[i + n, :] *= w_y
|
|
345
|
+
J_w_t = J_weighted.T
|
|
346
|
+
else:
|
|
347
|
+
J_w_t = J.T
|
|
348
|
+
|
|
349
|
+
return J, J_w_t
|
|
350
|
+
|
|
351
|
+
def update_model_parameters(self, delta_p, params_local, params_global):
|
|
352
|
+
"""
|
|
353
|
+
Update model parameters using computed delta
|
|
354
|
+
|
|
355
|
+
Matches PDM::UpdateModelParameters() from PDM.cpp lines 454-505
|
|
356
|
+
|
|
357
|
+
Args:
|
|
358
|
+
delta_p: (6+m,) parameter update
|
|
359
|
+
params_local: (34,) current local parameters
|
|
360
|
+
params_global: (6,) current global parameters
|
|
361
|
+
|
|
362
|
+
Returns:
|
|
363
|
+
updated_local: (34,) updated local parameters
|
|
364
|
+
updated_global: (6,) updated global parameters
|
|
365
|
+
"""
|
|
366
|
+
# Clone parameters
|
|
367
|
+
updated_global = params_global.copy()
|
|
368
|
+
updated_local = params_local.copy()
|
|
369
|
+
|
|
370
|
+
# Scaling and translation can be directly added
|
|
371
|
+
updated_global[0] += delta_p[0] # scale
|
|
372
|
+
updated_global[4] += delta_p[4] # tx
|
|
373
|
+
updated_global[5] += delta_p[5] # ty
|
|
374
|
+
|
|
375
|
+
# Rotation update is more complex (uses rotation composition)
|
|
376
|
+
if CYTHON_AVAILABLE:
|
|
377
|
+
# Use Cython-optimized rotation update for 99.9% accuracy
|
|
378
|
+
# C-level math guarantees bit-for-bit precision
|
|
379
|
+
euler_current = params_global[1:4].astype(np.float32)
|
|
380
|
+
delta_rotation = delta_p[1:4].astype(np.float32)
|
|
381
|
+
euler_new = update_rotation_cython(euler_current, delta_rotation)
|
|
382
|
+
updated_global[1:4] = euler_new
|
|
383
|
+
else:
|
|
384
|
+
# Fallback to Python implementation (99.45% accuracy)
|
|
385
|
+
# Get current rotation matrix
|
|
386
|
+
euler_current = params_global[1:4]
|
|
387
|
+
R1 = self.euler_to_rotation_matrix(euler_current)
|
|
388
|
+
|
|
389
|
+
# Construct incremental rotation matrix R'
|
|
390
|
+
# R' = [1, -wz, wy ]
|
|
391
|
+
# [wz, 1, -wx ]
|
|
392
|
+
# [-wy, wx, 1 ]
|
|
393
|
+
R2 = np.eye(3, dtype=np.float32)
|
|
394
|
+
R2[1, 2] = -delta_p[1] # -wx
|
|
395
|
+
R2[2, 1] = delta_p[1] # wx
|
|
396
|
+
R2[2, 0] = -delta_p[2] # -wy
|
|
397
|
+
R2[0, 2] = delta_p[2] # wy
|
|
398
|
+
R2[0, 1] = -delta_p[3] # -wz
|
|
399
|
+
R2[1, 0] = delta_p[3] # wz
|
|
400
|
+
|
|
401
|
+
# Orthonormalise R2
|
|
402
|
+
R2 = self.orthonormalise(R2)
|
|
403
|
+
|
|
404
|
+
# Combine rotations
|
|
405
|
+
R3 = R1 @ R2
|
|
406
|
+
|
|
407
|
+
# Convert back to Euler angles using quaternion (matching C++ RotationHelpers.h)
|
|
408
|
+
# C++ uses: RotationMatrix2AxisAngle then AxisAngle2Euler (via quaternion)
|
|
409
|
+
# Direct quaternion conversion matches C++ better than axis-angle via OpenCV
|
|
410
|
+
euler_new = self.rotation_matrix_to_euler(R3)
|
|
411
|
+
|
|
412
|
+
# Handle numerical instability
|
|
413
|
+
if np.any(np.isnan(euler_new)):
|
|
414
|
+
euler_new = np.zeros(3, dtype=np.float32)
|
|
415
|
+
|
|
416
|
+
updated_global[1:4] = euler_new
|
|
417
|
+
|
|
418
|
+
# Update local parameters (simple addition)
|
|
419
|
+
if len(delta_p) > 6:
|
|
420
|
+
updated_local += delta_p[6:]
|
|
421
|
+
|
|
422
|
+
return updated_local, updated_global
|
|
423
|
+
|
|
424
|
+
def calc_params(self, landmarks_2d, rotation_init=None):
|
|
425
|
+
"""
|
|
426
|
+
Calculate optimal PDM parameters from 2D landmarks
|
|
427
|
+
|
|
428
|
+
Main function matching PDM::CalcParams() from PDM.cpp lines 508-705
|
|
429
|
+
|
|
430
|
+
Args:
|
|
431
|
+
landmarks_2d: (136,) array of 2D landmarks [X0...X67, Y0...Y67]
|
|
432
|
+
OR (68, 2) array of landmarks
|
|
433
|
+
rotation_init: Initial rotation (rx, ry, rz). Defaults to (0, 0, 0)
|
|
434
|
+
|
|
435
|
+
Returns:
|
|
436
|
+
params_global: (6,) optimized global parameters [scale, rx, ry, rz, tx, ty]
|
|
437
|
+
params_local: (34,) optimized local parameters
|
|
438
|
+
"""
|
|
439
|
+
# Convert landmarks to (136,) format if needed
|
|
440
|
+
if landmarks_2d.shape == (68, 2):
|
|
441
|
+
landmarks_2d = np.concatenate([landmarks_2d[:, 0], landmarks_2d[:, 1]])
|
|
442
|
+
|
|
443
|
+
n = 68 # number of landmarks
|
|
444
|
+
m = 34 # number of PCA modes
|
|
445
|
+
|
|
446
|
+
if rotation_init is None:
|
|
447
|
+
rotation_init = np.zeros(3, dtype=np.float32)
|
|
448
|
+
|
|
449
|
+
# Filter invisible landmarks (x coordinate == 0)
|
|
450
|
+
visi_ind_2d = np.ones(n * 2, dtype=bool)
|
|
451
|
+
visi_ind_3d = np.ones(n * 3, dtype=bool)
|
|
452
|
+
|
|
453
|
+
for i in range(n):
|
|
454
|
+
if landmarks_2d[i] == 0: # invisible landmark
|
|
455
|
+
visi_ind_2d[i] = False
|
|
456
|
+
visi_ind_2d[i + n] = False
|
|
457
|
+
visi_ind_3d[i] = False
|
|
458
|
+
visi_ind_3d[i + n] = False
|
|
459
|
+
visi_ind_3d[i + 2*n] = False
|
|
460
|
+
|
|
461
|
+
# Subsample mean shape and principal components for visible landmarks
|
|
462
|
+
# Use .copy() to prevent modifying shared PDM state
|
|
463
|
+
M = self.mean_shape[visi_ind_3d].reshape(-1, 1).copy()
|
|
464
|
+
V = self.princ_comp[visi_ind_3d, :].copy()
|
|
465
|
+
|
|
466
|
+
# Extract visible landmarks
|
|
467
|
+
landmarks_vis = landmarks_2d[visi_ind_2d]
|
|
468
|
+
|
|
469
|
+
# Number of visible points
|
|
470
|
+
n_vis = np.sum(visi_ind_3d) // 3
|
|
471
|
+
|
|
472
|
+
# Compute initial global parameters from bounding box
|
|
473
|
+
min_x, max_x, min_y, max_y = self.extract_bounding_box(landmarks_vis)
|
|
474
|
+
width = abs(max_x - min_x)
|
|
475
|
+
height = abs(max_y - min_y)
|
|
476
|
+
|
|
477
|
+
model_width, model_height = self.calc_bounding_box_model()
|
|
478
|
+
|
|
479
|
+
scaling = ((width / model_width) + (height / model_height)) / 2.0
|
|
480
|
+
|
|
481
|
+
R = self.euler_to_rotation_matrix(rotation_init)
|
|
482
|
+
translation = np.array([(min_x + max_x) / 2.0, (min_y + max_y) / 2.0], dtype=np.float32)
|
|
483
|
+
|
|
484
|
+
# Initialize parameters
|
|
485
|
+
params_local = np.zeros(m, dtype=np.float32)
|
|
486
|
+
params_global = np.array([
|
|
487
|
+
scaling,
|
|
488
|
+
rotation_init[0],
|
|
489
|
+
rotation_init[1],
|
|
490
|
+
rotation_init[2],
|
|
491
|
+
translation[0],
|
|
492
|
+
translation[1]
|
|
493
|
+
], dtype=np.float32)
|
|
494
|
+
|
|
495
|
+
# CRITICAL FIX: Use isolated PDM copies instead of modifying shared state
|
|
496
|
+
# This prevents corruption of geometric features that read PDM concurrently
|
|
497
|
+
# Save original PDM state
|
|
498
|
+
mean_shape_original = self.mean_shape
|
|
499
|
+
princ_comp_original = self.princ_comp
|
|
500
|
+
|
|
501
|
+
# Temporarily update PDM to use subsampled matrices (isolated copies)
|
|
502
|
+
self.mean_shape = M
|
|
503
|
+
self.princ_comp = V
|
|
504
|
+
|
|
505
|
+
try:
|
|
506
|
+
# Initial projection error
|
|
507
|
+
shape_3d = M.flatten() + (V @ params_local.reshape(-1, 1)).flatten()
|
|
508
|
+
shape_3d = shape_3d.reshape(3, n_vis).astype(np.float32)
|
|
509
|
+
|
|
510
|
+
# Use Numba JIT for 2D projection if available
|
|
511
|
+
if NUMBA_AVAILABLE:
|
|
512
|
+
curr_shape_2d = project_shape_to_2d_jit(shape_3d, R, scaling, translation[0], translation[1], n_vis)
|
|
513
|
+
else:
|
|
514
|
+
curr_shape_2d = np.zeros(n_vis * 2, dtype=np.float32)
|
|
515
|
+
for i in range(n_vis):
|
|
516
|
+
X, Y, Z = shape_3d[:, i]
|
|
517
|
+
curr_shape_2d[i] = scaling * (R[0,0]*X + R[0,1]*Y + R[0,2]*Z) + translation[0]
|
|
518
|
+
curr_shape_2d[i + n_vis] = scaling * (R[1,0]*X + R[1,1]*Y + R[1,2]*Z) + translation[1]
|
|
519
|
+
|
|
520
|
+
curr_error = np.linalg.norm(curr_shape_2d - landmarks_vis)
|
|
521
|
+
|
|
522
|
+
# Regularization (inverse of eigenvalues)
|
|
523
|
+
# Using 1.0 to match C++ baseline (was 10.0 - too much!)
|
|
524
|
+
reg_factor = 1.0
|
|
525
|
+
regularisation = np.zeros(6 + m, dtype=np.float32)
|
|
526
|
+
regularisation[6:] = reg_factor / self.eigen_values
|
|
527
|
+
regularisation = np.diag(regularisation)
|
|
528
|
+
|
|
529
|
+
# Weight matrix (identity for now)
|
|
530
|
+
weight_matrix = np.eye(n_vis * 2, dtype=np.float32)
|
|
531
|
+
|
|
532
|
+
# Iterative optimization (up to 1000 iterations)
|
|
533
|
+
not_improved_in = 0
|
|
534
|
+
max_iterations = 1000
|
|
535
|
+
|
|
536
|
+
for iteration in range(max_iterations):
|
|
537
|
+
# Get current 3D shape
|
|
538
|
+
shape_3d = self.calc_shape_3d(params_local)
|
|
539
|
+
shape_3d = shape_3d.reshape(3, n_vis).astype(np.float32)
|
|
540
|
+
|
|
541
|
+
# Get current rotation
|
|
542
|
+
R = self.euler_to_rotation_matrix(params_global[1:4]).astype(np.float32)
|
|
543
|
+
s = params_global[0]
|
|
544
|
+
t = params_global[4:6]
|
|
545
|
+
|
|
546
|
+
# Project to 2D using Numba JIT if available
|
|
547
|
+
if NUMBA_AVAILABLE:
|
|
548
|
+
curr_shape_2d = project_shape_to_2d_jit(shape_3d, R, s, t[0], t[1], n_vis)
|
|
549
|
+
else:
|
|
550
|
+
curr_shape_2d = np.zeros(n_vis * 2, dtype=np.float32)
|
|
551
|
+
for i in range(n_vis):
|
|
552
|
+
X, Y, Z = shape_3d[:, i]
|
|
553
|
+
curr_shape_2d[i] = s * (R[0,0]*X + R[0,1]*Y + R[0,2]*Z) + t[0]
|
|
554
|
+
curr_shape_2d[i + n_vis] = s * (R[1,0]*X + R[1,1]*Y + R[1,2]*Z) + t[1]
|
|
555
|
+
|
|
556
|
+
# Compute error residual
|
|
557
|
+
error_resid = (landmarks_vis - curr_shape_2d).astype(np.float32)
|
|
558
|
+
|
|
559
|
+
# Compute Jacobian
|
|
560
|
+
J, J_w_t = self.compute_jacobian(params_local, params_global, weight_matrix)
|
|
561
|
+
|
|
562
|
+
# Compute J_w_t * error_resid
|
|
563
|
+
J_w_t_m = J_w_t @ error_resid
|
|
564
|
+
|
|
565
|
+
# Add regularization term for local parameters
|
|
566
|
+
J_w_t_m[6:] = J_w_t_m[6:] - (regularisation[6:, 6:] @ params_local.reshape(-1, 1)).flatten()
|
|
567
|
+
|
|
568
|
+
# Compute Hessian: J_w_t * J + regularisation
|
|
569
|
+
Hessian = J_w_t @ J + regularisation
|
|
570
|
+
|
|
571
|
+
# Adaptive Tikhonov regularization for ill-conditioned matrices
|
|
572
|
+
# Use OpenCV's Cholesky solver to match C++ exactly (cv::solve with DECOMP_CHOLESKY)
|
|
573
|
+
try:
|
|
574
|
+
# Convert to OpenCV format (needs float64 for numerical stability)
|
|
575
|
+
Hessian_cv = Hessian.astype(np.float64)
|
|
576
|
+
J_w_t_m_cv = J_w_t_m.reshape(-1, 1).astype(np.float64)
|
|
577
|
+
|
|
578
|
+
# Try OpenCV's Cholesky first (matches C++ line 657 exactly)
|
|
579
|
+
success, param_update_cv = cv2.solve(
|
|
580
|
+
Hessian_cv,
|
|
581
|
+
J_w_t_m_cv,
|
|
582
|
+
flags=cv2.DECOMP_CHOLESKY
|
|
583
|
+
)
|
|
584
|
+
|
|
585
|
+
if success:
|
|
586
|
+
param_update = param_update_cv.flatten().astype(np.float32)
|
|
587
|
+
else:
|
|
588
|
+
# If OpenCV Cholesky fails, add adaptive Tikhonov and retry
|
|
589
|
+
tikhonov_lambda = 1e-6 * np.mean(np.diag(Hessian_cv))
|
|
590
|
+
Hessian_stable = Hessian_cv + np.eye(Hessian_cv.shape[0]) * tikhonov_lambda
|
|
591
|
+
|
|
592
|
+
success, param_update_cv = cv2.solve(
|
|
593
|
+
Hessian_stable,
|
|
594
|
+
J_w_t_m_cv,
|
|
595
|
+
flags=cv2.DECOMP_CHOLESKY
|
|
596
|
+
)
|
|
597
|
+
|
|
598
|
+
if success:
|
|
599
|
+
param_update = param_update_cv.flatten().astype(np.float32)
|
|
600
|
+
else:
|
|
601
|
+
# Last resort: scipy lstsq
|
|
602
|
+
param_update = np.linalg.lstsq(Hessian, J_w_t_m, rcond=1e-6)[0].astype(np.float32)
|
|
603
|
+
|
|
604
|
+
except Exception as e:
|
|
605
|
+
# Fallback to scipy if OpenCV fails for any reason
|
|
606
|
+
try:
|
|
607
|
+
param_update = linalg.solve(Hessian, J_w_t_m, assume_a='pos')
|
|
608
|
+
except np.linalg.LinAlgError:
|
|
609
|
+
param_update = np.linalg.lstsq(Hessian, J_w_t_m, rcond=1e-6)[0].astype(np.float32)
|
|
610
|
+
|
|
611
|
+
# Reduce step size to avoid overshoot
|
|
612
|
+
param_update *= 0.75
|
|
613
|
+
|
|
614
|
+
# Update parameters
|
|
615
|
+
params_local, params_global = self.update_model_parameters(
|
|
616
|
+
param_update, params_local, params_global
|
|
617
|
+
)
|
|
618
|
+
|
|
619
|
+
# Compute new error
|
|
620
|
+
shape_3d = self.calc_shape_3d(params_local)
|
|
621
|
+
shape_3d = shape_3d.reshape(3, n_vis)
|
|
622
|
+
R = self.euler_to_rotation_matrix(params_global[1:4])
|
|
623
|
+
s = params_global[0]
|
|
624
|
+
t = params_global[4:6]
|
|
625
|
+
|
|
626
|
+
curr_shape_2d = np.zeros(n_vis * 2, dtype=np.float32)
|
|
627
|
+
for i in range(n_vis):
|
|
628
|
+
X, Y, Z = shape_3d[:, i]
|
|
629
|
+
curr_shape_2d[i] = s * (R[0,0]*X + R[0,1]*Y + R[0,2]*Z) + t[0]
|
|
630
|
+
curr_shape_2d[i + n_vis] = s * (R[1,0]*X + R[1,1]*Y + R[1,2]*Z) + t[1]
|
|
631
|
+
|
|
632
|
+
new_error = np.linalg.norm(curr_shape_2d - landmarks_vis)
|
|
633
|
+
|
|
634
|
+
# Check for improvement
|
|
635
|
+
if 0.999 * curr_error < new_error:
|
|
636
|
+
not_improved_in += 1
|
|
637
|
+
if not_improved_in == 3:
|
|
638
|
+
break
|
|
639
|
+
else:
|
|
640
|
+
not_improved_in = 0
|
|
641
|
+
|
|
642
|
+
curr_error = new_error
|
|
643
|
+
|
|
644
|
+
finally:
|
|
645
|
+
# Restore original PDM matrices (critical for thread safety)
|
|
646
|
+
self.mean_shape = mean_shape_original
|
|
647
|
+
self.princ_comp = princ_comp_original
|
|
648
|
+
|
|
649
|
+
# DO NOT normalize - C++ OpenFace outputs unnormalized params!
|
|
650
|
+
# (Previous assumption was wrong - C++ does NOT divide by sqrt(eigenvalue))
|
|
651
|
+
# params_local_normalized = params_local / np.sqrt(self.eigen_values)
|
|
652
|
+
|
|
653
|
+
return params_global, params_local
|
|
654
|
+
|
|
655
|
+
|
|
656
|
+
def test_calc_params():
|
|
657
|
+
"""Test CalcParams implementation"""
|
|
658
|
+
print("=" * 80)
|
|
659
|
+
print("CalcParams Implementation Test")
|
|
660
|
+
print("=" * 80)
|
|
661
|
+
|
|
662
|
+
# This would require loading a PDM and testing
|
|
663
|
+
# For now, just verify the module loads
|
|
664
|
+
print("\nCalcParams module loaded successfully")
|
|
665
|
+
print("Ready to use with PDMParser")
|
|
666
|
+
|
|
667
|
+
print("\n" + "=" * 80)
|
|
668
|
+
|
|
669
|
+
|
|
670
|
+
if __name__ == "__main__":
|
|
671
|
+
test_calc_params()
|