pyfaceau 1.0.6__tar.gz → 1.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pyfaceau-1.3.0/LICENSE +47 -0
- {pyfaceau-1.0.6/pyfaceau.egg-info → pyfaceau-1.3.0}/PKG-INFO +27 -15
- {pyfaceau-1.0.6 → pyfaceau-1.3.0}/README.md +21 -9
- {pyfaceau-1.0.6 → pyfaceau-1.3.0}/pyfaceau/alignment/face_aligner.py +57 -27
- pyfaceau-1.3.0/pyfaceau/alignment/paw.py +285 -0
- pyfaceau-1.3.0/pyfaceau/data/__init__.py +19 -0
- pyfaceau-1.3.0/pyfaceau/data/hdf5_dataset.py +508 -0
- pyfaceau-1.3.0/pyfaceau/data/quality_filter.py +277 -0
- pyfaceau-1.3.0/pyfaceau/data/training_data_generator.py +548 -0
- pyfaceau-1.3.0/pyfaceau/detectors/__init__.py +20 -0
- pyfaceau-1.3.0/pyfaceau/detectors/extract_mtcnn_weights.py +191 -0
- pyfaceau-1.3.0/pyfaceau/detectors/openface_mtcnn.py +786 -0
- pyfaceau-1.3.0/pyfaceau/detectors/pymtcnn_detector.py +243 -0
- {pyfaceau-1.0.6 → pyfaceau-1.3.0}/pyfaceau/download_weights.py +3 -3
- {pyfaceau-1.0.6 → pyfaceau-1.3.0}/pyfaceau/features/histogram_median_tracker.py +14 -26
- {pyfaceau-1.0.6 → pyfaceau-1.3.0}/pyfaceau/features/pdm.py +56 -10
- pyfaceau-1.3.0/pyfaceau/nn/__init__.py +88 -0
- pyfaceau-1.3.0/pyfaceau/nn/au_prediction_inference.py +447 -0
- pyfaceau-1.3.0/pyfaceau/nn/au_prediction_net.py +501 -0
- pyfaceau-1.3.0/pyfaceau/nn/landmark_pose_inference.py +536 -0
- pyfaceau-1.3.0/pyfaceau/nn/landmark_pose_net.py +497 -0
- pyfaceau-1.3.0/pyfaceau/nn/train_au_prediction.py +521 -0
- pyfaceau-1.3.0/pyfaceau/nn/train_landmark_pose.py +508 -0
- {pyfaceau-1.0.6 → pyfaceau-1.3.0}/pyfaceau/parallel_pipeline.py +5 -6
- {pyfaceau-1.0.6 → pyfaceau-1.3.0}/pyfaceau/pipeline.py +255 -300
- {pyfaceau-1.0.6 → pyfaceau-1.3.0}/pyfaceau/prediction/model_parser.py +12 -8
- {pyfaceau-1.0.6 → pyfaceau-1.3.0}/pyfaceau/processor.py +25 -13
- {pyfaceau-1.0.6 → pyfaceau-1.3.0}/pyfaceau/refinement/__init__.py +2 -1
- pyfaceau-1.3.0/pyfaceau/refinement/pdm.py +286 -0
- {pyfaceau-1.0.6 → pyfaceau-1.3.0}/pyfaceau/refinement/targeted_refiner.py +6 -16
- {pyfaceau-1.0.6 → pyfaceau-1.3.0/pyfaceau.egg-info}/PKG-INFO +27 -15
- {pyfaceau-1.0.6 → pyfaceau-1.3.0}/pyfaceau.egg-info/SOURCES.txt +16 -3
- {pyfaceau-1.0.6 → pyfaceau-1.3.0}/pyfaceau.egg-info/requires.txt +1 -1
- {pyfaceau-1.0.6 → pyfaceau-1.3.0}/pyproject.toml +7 -7
- {pyfaceau-1.0.6 → pyfaceau-1.3.0}/setup.py +20 -43
- pyfaceau-1.0.6/COMMERCIAL-LICENSE.md +0 -109
- pyfaceau-1.0.6/LICENSE +0 -40
- pyfaceau-1.0.6/pyfaceau/utils/__init__.py +0 -0
- pyfaceau-1.0.6/pyfaceau/utils/cython_extensions/cython_histogram_median.c +0 -35391
- pyfaceau-1.0.6/pyfaceau/utils/cython_extensions/cython_rotation_update.c +0 -32262
- {pyfaceau-1.0.6 → pyfaceau-1.3.0}/MANIFEST.in +0 -0
- {pyfaceau-1.0.6 → pyfaceau-1.3.0}/pyfaceau/__init__.py +0 -0
- {pyfaceau-1.0.6 → pyfaceau-1.3.0}/pyfaceau/alignment/__init__.py +0 -0
- {pyfaceau-1.0.6 → pyfaceau-1.3.0}/pyfaceau/alignment/calc_params.py +0 -0
- {pyfaceau-1.0.6 → pyfaceau-1.3.0}/pyfaceau/alignment/numba_calcparams_accelerator.py +0 -0
- {pyfaceau-1.0.6 → pyfaceau-1.3.0}/pyfaceau/detectors/pfld.py +0 -0
- {pyfaceau-1.0.6 → pyfaceau-1.3.0}/pyfaceau/detectors/retinaface.py +0 -0
- {pyfaceau-1.0.6/pyfaceau/detectors → pyfaceau-1.3.0/pyfaceau/features}/__init__.py +0 -0
- {pyfaceau-1.0.6 → pyfaceau-1.3.0}/pyfaceau/features/triangulation.py +0 -0
- {pyfaceau-1.0.6/pyfaceau/features → pyfaceau-1.3.0/pyfaceau/prediction}/__init__.py +0 -0
- {pyfaceau-1.0.6 → pyfaceau-1.3.0}/pyfaceau/prediction/au_predictor.py +0 -0
- {pyfaceau-1.0.6 → pyfaceau-1.3.0}/pyfaceau/prediction/batched_au_predictor.py +0 -0
- {pyfaceau-1.0.6 → pyfaceau-1.3.0}/pyfaceau/prediction/running_median.py +0 -0
- {pyfaceau-1.0.6 → pyfaceau-1.3.0}/pyfaceau/prediction/running_median_fallback.py +0 -0
- {pyfaceau-1.0.6 → pyfaceau-1.3.0}/pyfaceau/refinement/svr_patch_expert.py +0 -0
- {pyfaceau-1.0.6/pyfaceau/prediction → pyfaceau-1.3.0/pyfaceau/utils}/__init__.py +0 -0
- {pyfaceau-1.0.6 → pyfaceau-1.3.0}/pyfaceau/utils/cython_extensions/cython_histogram_median.pyx +0 -0
- {pyfaceau-1.0.6 → pyfaceau-1.3.0}/pyfaceau/utils/cython_extensions/cython_rotation_update.pyx +0 -0
- {pyfaceau-1.0.6 → pyfaceau-1.3.0}/pyfaceau/utils/cython_extensions/setup.py +0 -0
- {pyfaceau-1.0.6 → pyfaceau-1.3.0}/pyfaceau.egg-info/dependency_links.txt +0 -0
- {pyfaceau-1.0.6 → pyfaceau-1.3.0}/pyfaceau.egg-info/entry_points.txt +0 -0
- {pyfaceau-1.0.6 → pyfaceau-1.3.0}/pyfaceau.egg-info/not-zip-safe +0 -0
- {pyfaceau-1.0.6 → pyfaceau-1.3.0}/pyfaceau.egg-info/top_level.txt +0 -0
- {pyfaceau-1.0.6 → pyfaceau-1.3.0}/pyfaceau_gui.py +0 -0
- {pyfaceau-1.0.6 → pyfaceau-1.3.0}/requirements.txt +0 -0
- {pyfaceau-1.0.6 → pyfaceau-1.3.0}/setup.cfg +0 -0
pyfaceau-1.3.0/LICENSE
ADDED
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
Creative Commons Attribution-NonCommercial 4.0 International (CC BY-NC 4.0)
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2025 John Wilson IV, MD
|
|
4
|
+
|
|
5
|
+
This work is licensed under the Creative Commons Attribution-NonCommercial 4.0
|
|
6
|
+
International License. To view a copy of this license, visit
|
|
7
|
+
http://creativecommons.org/licenses/by-nc/4.0/ or send a letter to
|
|
8
|
+
Creative Commons, PO Box 1866, Mountain View, CA 94042, USA.
|
|
9
|
+
|
|
10
|
+
================================================================================
|
|
11
|
+
|
|
12
|
+
You are free to:
|
|
13
|
+
|
|
14
|
+
* Share — copy and redistribute the material in any medium or format
|
|
15
|
+
* Adapt — remix, transform, and build upon the material
|
|
16
|
+
|
|
17
|
+
The licensor cannot revoke these freedoms as long as you follow the license terms.
|
|
18
|
+
|
|
19
|
+
================================================================================
|
|
20
|
+
|
|
21
|
+
Under the following terms:
|
|
22
|
+
|
|
23
|
+
* Attribution — You must give appropriate credit, provide a link to the
|
|
24
|
+
license, and indicate if changes were made. You may do so in any reasonable
|
|
25
|
+
manner, but not in any way that suggests the licensor endorses you or your use.
|
|
26
|
+
|
|
27
|
+
* NonCommercial — You may not use the material for commercial purposes.
|
|
28
|
+
|
|
29
|
+
* No additional restrictions — You may not apply legal terms or technological
|
|
30
|
+
measures that legally restrict others from doing anything the license permits.
|
|
31
|
+
|
|
32
|
+
================================================================================
|
|
33
|
+
|
|
34
|
+
Notices:
|
|
35
|
+
|
|
36
|
+
You do not have to comply with the license for elements of the material in the
|
|
37
|
+
public domain or where your use is permitted by an applicable exception or
|
|
38
|
+
limitation.
|
|
39
|
+
|
|
40
|
+
No warranties are given. The license may not give you all of the permissions
|
|
41
|
+
necessary for your intended use. For example, other rights such as publicity,
|
|
42
|
+
privacy, or moral rights may limit how you use the material.
|
|
43
|
+
|
|
44
|
+
================================================================================
|
|
45
|
+
|
|
46
|
+
Full legal code available at:
|
|
47
|
+
https://creativecommons.org/licenses/by-nc/4.0/legalcode
|
|
@@ -1,15 +1,15 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: pyfaceau
|
|
3
|
-
Version: 1.0
|
|
3
|
+
Version: 1.3.0
|
|
4
4
|
Summary: Pure Python OpenFace 2.2 AU extraction with CLNF landmark refinement
|
|
5
5
|
Home-page: https://github.com/johnwilsoniv/face-analysis
|
|
6
6
|
Author: John Wilson
|
|
7
7
|
Author-email:
|
|
8
8
|
License: CC BY-NC 4.0
|
|
9
|
-
Project-URL: Homepage, https://github.com/johnwilsoniv/
|
|
10
|
-
Project-URL: Documentation, https://github.com/johnwilsoniv/
|
|
11
|
-
Project-URL: Repository, https://github.com/johnwilsoniv/
|
|
12
|
-
Project-URL: Bug Tracker, https://github.com/johnwilsoniv/
|
|
9
|
+
Project-URL: Homepage, https://github.com/johnwilsoniv/pyfaceau
|
|
10
|
+
Project-URL: Documentation, https://github.com/johnwilsoniv/pyfaceau
|
|
11
|
+
Project-URL: Repository, https://github.com/johnwilsoniv/pyfaceau
|
|
12
|
+
Project-URL: Bug Tracker, https://github.com/johnwilsoniv/pyfaceau/issues
|
|
13
13
|
Keywords: facial-action-units,openface,computer-vision,facial-analysis,emotion-recognition
|
|
14
14
|
Classifier: Development Status :: 5 - Production/Stable
|
|
15
15
|
Classifier: Intended Audience :: Science/Research
|
|
@@ -32,7 +32,7 @@ Requires-Dist: scipy>=1.7.0
|
|
|
32
32
|
Requires-Dist: scikit-learn>=1.0.0
|
|
33
33
|
Requires-Dist: tqdm>=4.62.0
|
|
34
34
|
Requires-Dist: pyfhog>=0.1.0
|
|
35
|
-
Requires-Dist:
|
|
35
|
+
Requires-Dist: pyclnf>=0.2.0
|
|
36
36
|
Provides-Extra: dev
|
|
37
37
|
Requires-Dist: pytest>=7.0.0; extra == "dev"
|
|
38
38
|
Requires-Dist: black>=22.0.0; extra == "dev"
|
|
@@ -74,8 +74,14 @@ pyfaceau is a Python reimplementation of the [OpenFace 2.2](https://github.com/T
|
|
|
74
74
|
#### Option 1: Install from PyPI (Recommended)
|
|
75
75
|
|
|
76
76
|
```bash
|
|
77
|
-
#
|
|
78
|
-
pip install pyfaceau
|
|
77
|
+
# For NVIDIA GPU (CUDA):
|
|
78
|
+
pip install pyfaceau[cuda]
|
|
79
|
+
|
|
80
|
+
# For Apple Silicon (M1/M2/M3):
|
|
81
|
+
pip install pyfaceau[coreml]
|
|
82
|
+
|
|
83
|
+
# For CPU-only:
|
|
84
|
+
pip install pyfaceau[cpu]
|
|
79
85
|
|
|
80
86
|
# Download model weights (14MB)
|
|
81
87
|
python -m pyfaceau.download_weights
|
|
@@ -84,12 +90,14 @@ python -m pyfaceau.download_weights
|
|
|
84
90
|
# https://github.com/johnwilsoniv/face-analysis/tree/main/S0%20PyfaceAU/weights
|
|
85
91
|
```
|
|
86
92
|
|
|
93
|
+
**Note:** PyFaceAU v1.1.0+ uses [PyMTCNN](https://pypi.org/project/pymtcnn/) for cross-platform face detection with CUDA/CoreML/CPU support.
|
|
94
|
+
|
|
87
95
|
#### Option 2: Install from Source
|
|
88
96
|
|
|
89
97
|
```bash
|
|
90
98
|
# Clone repository
|
|
91
99
|
git clone https://github.com/johnwilsoniv/face-analysis.git
|
|
92
|
-
cd "face-analysis/
|
|
100
|
+
cd "face-analysis/pyfaceau"
|
|
93
101
|
|
|
94
102
|
# Install in development mode
|
|
95
103
|
pip install -e .
|
|
@@ -104,13 +112,13 @@ pip install -e .
|
|
|
104
112
|
```python
|
|
105
113
|
from pyfaceau import ParallelAUPipeline
|
|
106
114
|
|
|
107
|
-
# Initialize parallel pipeline
|
|
115
|
+
# Initialize parallel pipeline
|
|
108
116
|
pipeline = ParallelAUPipeline(
|
|
109
|
-
retinaface_model='weights/retinaface_mobilenet025_coreml.onnx',
|
|
110
117
|
pfld_model='weights/pfld_cunjian.onnx',
|
|
111
118
|
pdm_file='weights/In-the-wild_aligned_PDM_68.txt',
|
|
112
119
|
au_models_dir='path/to/AU_predictors',
|
|
113
120
|
triangulation_file='weights/tris_68_full.txt',
|
|
121
|
+
mtcnn_backend='auto', # or 'cuda', 'coreml', 'cpu'
|
|
114
122
|
num_workers=6, # Adjust based on CPU cores
|
|
115
123
|
batch_size=30
|
|
116
124
|
)
|
|
@@ -125,20 +133,19 @@ print(f"Processed {len(results)} frames")
|
|
|
125
133
|
# Typical output: ~28-50 FPS depending on CPU cores
|
|
126
134
|
```
|
|
127
135
|
|
|
128
|
-
#### Standard Mode (
|
|
136
|
+
#### Standard Mode (5-35 FPS depending on backend)
|
|
129
137
|
|
|
130
138
|
```python
|
|
131
139
|
from pyfaceau import FullPythonAUPipeline
|
|
132
140
|
|
|
133
141
|
# Initialize standard pipeline
|
|
134
142
|
pipeline = FullPythonAUPipeline(
|
|
135
|
-
retinaface_model='weights/retinaface_mobilenet025_coreml.onnx',
|
|
136
143
|
pfld_model='weights/pfld_cunjian.onnx',
|
|
137
144
|
pdm_file='weights/In-the-wild_aligned_PDM_68.txt',
|
|
138
145
|
au_models_dir='path/to/AU_predictors',
|
|
139
146
|
triangulation_file='weights/tris_68_full.txt',
|
|
147
|
+
mtcnn_backend='auto', # Automatically selects best backend
|
|
140
148
|
use_calc_params=True,
|
|
141
|
-
use_coreml=True, # macOS only
|
|
142
149
|
verbose=False
|
|
143
150
|
)
|
|
144
151
|
|
|
@@ -149,6 +156,11 @@ results = pipeline.process_video(
|
|
|
149
156
|
)
|
|
150
157
|
```
|
|
151
158
|
|
|
159
|
+
**Performance by backend:**
|
|
160
|
+
- CUDA (NVIDIA GPU): ~35 FPS
|
|
161
|
+
- CoreML (Apple Silicon): ~20-25 FPS
|
|
162
|
+
- CPU: ~5-10 FPS
|
|
163
|
+
|
|
152
164
|
### Example Output
|
|
153
165
|
|
|
154
166
|
```csv
|
|
@@ -166,7 +178,7 @@ pyfaceau replicates the complete OpenFace 2.2 AU extraction pipeline:
|
|
|
166
178
|
```
|
|
167
179
|
Video Input
|
|
168
180
|
↓
|
|
169
|
-
Face Detection (
|
|
181
|
+
Face Detection (PyMTCNN - CUDA/CoreML/CPU)
|
|
170
182
|
↓
|
|
171
183
|
Landmark Detection (PFLD 68-point)
|
|
172
184
|
↓
|
|
@@ -29,8 +29,14 @@ pyfaceau is a Python reimplementation of the [OpenFace 2.2](https://github.com/T
|
|
|
29
29
|
#### Option 1: Install from PyPI (Recommended)
|
|
30
30
|
|
|
31
31
|
```bash
|
|
32
|
-
#
|
|
33
|
-
pip install pyfaceau
|
|
32
|
+
# For NVIDIA GPU (CUDA):
|
|
33
|
+
pip install pyfaceau[cuda]
|
|
34
|
+
|
|
35
|
+
# For Apple Silicon (M1/M2/M3):
|
|
36
|
+
pip install pyfaceau[coreml]
|
|
37
|
+
|
|
38
|
+
# For CPU-only:
|
|
39
|
+
pip install pyfaceau[cpu]
|
|
34
40
|
|
|
35
41
|
# Download model weights (14MB)
|
|
36
42
|
python -m pyfaceau.download_weights
|
|
@@ -39,12 +45,14 @@ python -m pyfaceau.download_weights
|
|
|
39
45
|
# https://github.com/johnwilsoniv/face-analysis/tree/main/S0%20PyfaceAU/weights
|
|
40
46
|
```
|
|
41
47
|
|
|
48
|
+
**Note:** PyFaceAU v1.1.0+ uses [PyMTCNN](https://pypi.org/project/pymtcnn/) for cross-platform face detection with CUDA/CoreML/CPU support.
|
|
49
|
+
|
|
42
50
|
#### Option 2: Install from Source
|
|
43
51
|
|
|
44
52
|
```bash
|
|
45
53
|
# Clone repository
|
|
46
54
|
git clone https://github.com/johnwilsoniv/face-analysis.git
|
|
47
|
-
cd "face-analysis/
|
|
55
|
+
cd "face-analysis/pyfaceau"
|
|
48
56
|
|
|
49
57
|
# Install in development mode
|
|
50
58
|
pip install -e .
|
|
@@ -59,13 +67,13 @@ pip install -e .
|
|
|
59
67
|
```python
|
|
60
68
|
from pyfaceau import ParallelAUPipeline
|
|
61
69
|
|
|
62
|
-
# Initialize parallel pipeline
|
|
70
|
+
# Initialize parallel pipeline
|
|
63
71
|
pipeline = ParallelAUPipeline(
|
|
64
|
-
retinaface_model='weights/retinaface_mobilenet025_coreml.onnx',
|
|
65
72
|
pfld_model='weights/pfld_cunjian.onnx',
|
|
66
73
|
pdm_file='weights/In-the-wild_aligned_PDM_68.txt',
|
|
67
74
|
au_models_dir='path/to/AU_predictors',
|
|
68
75
|
triangulation_file='weights/tris_68_full.txt',
|
|
76
|
+
mtcnn_backend='auto', # or 'cuda', 'coreml', 'cpu'
|
|
69
77
|
num_workers=6, # Adjust based on CPU cores
|
|
70
78
|
batch_size=30
|
|
71
79
|
)
|
|
@@ -80,20 +88,19 @@ print(f"Processed {len(results)} frames")
|
|
|
80
88
|
# Typical output: ~28-50 FPS depending on CPU cores
|
|
81
89
|
```
|
|
82
90
|
|
|
83
|
-
#### Standard Mode (
|
|
91
|
+
#### Standard Mode (5-35 FPS depending on backend)
|
|
84
92
|
|
|
85
93
|
```python
|
|
86
94
|
from pyfaceau import FullPythonAUPipeline
|
|
87
95
|
|
|
88
96
|
# Initialize standard pipeline
|
|
89
97
|
pipeline = FullPythonAUPipeline(
|
|
90
|
-
retinaface_model='weights/retinaface_mobilenet025_coreml.onnx',
|
|
91
98
|
pfld_model='weights/pfld_cunjian.onnx',
|
|
92
99
|
pdm_file='weights/In-the-wild_aligned_PDM_68.txt',
|
|
93
100
|
au_models_dir='path/to/AU_predictors',
|
|
94
101
|
triangulation_file='weights/tris_68_full.txt',
|
|
102
|
+
mtcnn_backend='auto', # Automatically selects best backend
|
|
95
103
|
use_calc_params=True,
|
|
96
|
-
use_coreml=True, # macOS only
|
|
97
104
|
verbose=False
|
|
98
105
|
)
|
|
99
106
|
|
|
@@ -104,6 +111,11 @@ results = pipeline.process_video(
|
|
|
104
111
|
)
|
|
105
112
|
```
|
|
106
113
|
|
|
114
|
+
**Performance by backend:**
|
|
115
|
+
- CUDA (NVIDIA GPU): ~35 FPS
|
|
116
|
+
- CoreML (Apple Silicon): ~20-25 FPS
|
|
117
|
+
- CPU: ~5-10 FPS
|
|
118
|
+
|
|
107
119
|
### Example Output
|
|
108
120
|
|
|
109
121
|
```csv
|
|
@@ -121,7 +133,7 @@ pyfaceau replicates the complete OpenFace 2.2 AU extraction pipeline:
|
|
|
121
133
|
```
|
|
122
134
|
Video Input
|
|
123
135
|
↓
|
|
124
|
-
Face Detection (
|
|
136
|
+
Face Detection (PyMTCNN - CUDA/CoreML/CPU)
|
|
125
137
|
↓
|
|
126
138
|
Landmark Detection (PFLD 68-point)
|
|
127
139
|
↓
|
|
@@ -100,12 +100,12 @@ class OpenFace22FaceAligner:
|
|
|
100
100
|
source_rigid = self._extract_rigid_points(landmarks_68)
|
|
101
101
|
dest_rigid = self._extract_rigid_points(self.reference_shape)
|
|
102
102
|
|
|
103
|
-
# Compute scale (no rotation from Kabsch)
|
|
104
|
-
scale_identity = self.
|
|
105
|
-
scale = scale_identity
|
|
103
|
+
# Compute scale (no rotation from Kabsch) - matching working commit approach
|
|
104
|
+
scale_identity = self._compute_scale_only(source_rigid, dest_rigid)
|
|
105
|
+
scale = scale_identity
|
|
106
106
|
|
|
107
|
-
# Apply INVERSE of
|
|
108
|
-
#
|
|
107
|
+
# Apply INVERSE of p_rz rotation
|
|
108
|
+
# p_rz describes rotation FROM canonical TO tilted
|
|
109
109
|
# We need rotation FROM tilted TO canonical, which is -p_rz
|
|
110
110
|
angle = -p_rz
|
|
111
111
|
cos_a = np.cos(angle)
|
|
@@ -117,7 +117,7 @@ class OpenFace22FaceAligner:
|
|
|
117
117
|
# Combine scale and rotation
|
|
118
118
|
scale_rot_matrix = scale * R
|
|
119
119
|
|
|
120
|
-
# Build 2×3 affine warp matrix
|
|
120
|
+
# Build 2×3 affine warp matrix using pose translation
|
|
121
121
|
warp_matrix = self._build_warp_matrix(scale_rot_matrix, pose_tx, pose_ty)
|
|
122
122
|
|
|
123
123
|
# Apply affine transformation
|
|
@@ -239,6 +239,37 @@ class OpenFace22FaceAligner:
|
|
|
239
239
|
|
|
240
240
|
return warp_matrix
|
|
241
241
|
|
|
242
|
+
def _build_warp_matrix_centroid(self, scale_rot: np.ndarray, src_centroid: np.ndarray, dst_centroid: np.ndarray) -> np.ndarray:
|
|
243
|
+
"""
|
|
244
|
+
Build 2×3 affine warp matrix using source and destination centroids
|
|
245
|
+
|
|
246
|
+
This is the corrected version that uses rigid point centroids instead of
|
|
247
|
+
pose translation parameters, which gives better alignment with C++ OpenFace.
|
|
248
|
+
|
|
249
|
+
Args:
|
|
250
|
+
scale_rot: (2, 2) similarity transform matrix (scale × rotation)
|
|
251
|
+
src_centroid: (2,) centroid of source rigid points
|
|
252
|
+
dst_centroid: (2,) centroid of destination rigid points
|
|
253
|
+
|
|
254
|
+
Returns:
|
|
255
|
+
(2, 3) affine warp matrix for cv2.warpAffine
|
|
256
|
+
"""
|
|
257
|
+
# Initialize 2×3 warp matrix
|
|
258
|
+
warp_matrix = np.zeros((2, 3), dtype=np.float32)
|
|
259
|
+
|
|
260
|
+
# Copy scale-rotation to first 2×2 block
|
|
261
|
+
warp_matrix[:2, :2] = scale_rot
|
|
262
|
+
|
|
263
|
+
# Transform source centroid through scale-rotation
|
|
264
|
+
T_src = scale_rot @ src_centroid
|
|
265
|
+
|
|
266
|
+
# Translation: map src_centroid to dst_centroid, then center in output
|
|
267
|
+
# dst_centroid is in PDM space (centered around 0), so add output_center
|
|
268
|
+
warp_matrix[0, 2] = dst_centroid[0] - T_src[0] + self.output_width / 2
|
|
269
|
+
warp_matrix[1, 2] = dst_centroid[1] - T_src[1] + self.output_height / 2
|
|
270
|
+
|
|
271
|
+
return warp_matrix
|
|
272
|
+
|
|
242
273
|
def _extract_rigid_points(self, landmarks: np.ndarray) -> np.ndarray:
|
|
243
274
|
"""
|
|
244
275
|
Extract 24 rigid points from 68 landmarks
|
|
@@ -286,37 +317,34 @@ class OpenFace22FaceAligner:
|
|
|
286
317
|
|
|
287
318
|
# Rotation matrix: R = V^T × corr × U^T
|
|
288
319
|
# OpenFace C++ uses: R = svd.vt.t() * corr * svd.u.t()
|
|
320
|
+
# But we need to transpose to match C++ behavior
|
|
321
|
+
# Testing showed R.T gives correct rotation direction (+18° vs -18°)
|
|
289
322
|
R = Vt.T @ corr @ U.T
|
|
290
323
|
|
|
291
|
-
return R
|
|
324
|
+
return R.T # Transpose to match C++ rotation direction
|
|
292
325
|
|
|
293
|
-
def
|
|
326
|
+
def _align_shapes_with_scale_and_rotation(self, src: np.ndarray, dst: np.ndarray) -> np.ndarray:
|
|
294
327
|
"""
|
|
295
|
-
Compute similarity transform (scale
|
|
328
|
+
Compute similarity transform (scale + rotation via Kabsch) between two point sets
|
|
296
329
|
|
|
297
|
-
|
|
298
|
-
they are already in canonical orientation. We only need scale + translation,
|
|
299
|
-
NOT rotation via Kabsch.
|
|
330
|
+
This matches C++ AlignShapesWithScale in RotationHelpers.h lines 195-241.
|
|
300
331
|
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
2. On reconstructed landmarks → params_global₂ → AlignFace
|
|
332
|
+
CRITICAL: p_rz is NOT used for alignment! C++ computes rotation from landmarks
|
|
333
|
+
using Kabsch algorithm.
|
|
304
334
|
|
|
305
|
-
|
|
306
|
-
are already canonical. Our Python uses CSV landmarks (already canonical), so we
|
|
307
|
-
skip rotation computation entirely.
|
|
308
|
-
|
|
309
|
-
Algorithm:
|
|
335
|
+
Algorithm (matching C++):
|
|
310
336
|
1. Mean-normalize both src and dst
|
|
311
337
|
2. Compute RMS scale for each
|
|
312
|
-
3.
|
|
338
|
+
3. Normalize by scale
|
|
339
|
+
4. Compute rotation via Kabsch2D
|
|
340
|
+
5. Return: (s_dst / s_src) × R_kabsch
|
|
313
341
|
|
|
314
342
|
Args:
|
|
315
343
|
src: (N, 2) source points (detected landmarks)
|
|
316
344
|
dst: (N, 2) destination points (reference shape)
|
|
317
345
|
|
|
318
346
|
Returns:
|
|
319
|
-
(2, 2) similarity transform matrix (scale ×
|
|
347
|
+
(2, 2) similarity transform matrix (scale × rotation)
|
|
320
348
|
"""
|
|
321
349
|
n = src.shape[0]
|
|
322
350
|
|
|
@@ -335,18 +363,20 @@ class OpenFace22FaceAligner:
|
|
|
335
363
|
dst_mean_normed[:, 1] -= mean_dst_y
|
|
336
364
|
|
|
337
365
|
# 2. Compute RMS scale for each point set
|
|
338
|
-
#
|
|
366
|
+
# C++ RotationHelpers.h line 221-222
|
|
339
367
|
src_sq = src_mean_normed ** 2
|
|
340
368
|
dst_sq = dst_mean_normed ** 2
|
|
341
369
|
|
|
342
370
|
s_src = np.sqrt(np.sum(src_sq) / n)
|
|
343
371
|
s_dst = np.sqrt(np.sum(dst_sq) / n)
|
|
344
372
|
|
|
345
|
-
# 3. Normalize by scale
|
|
373
|
+
# 3. Normalize by scale (C++ line 224-225)
|
|
346
374
|
src_norm = src_mean_normed / s_src
|
|
347
375
|
dst_norm = dst_mean_normed / s_dst
|
|
348
376
|
|
|
349
|
-
#
|
|
350
|
-
|
|
377
|
+
# 4. Get rotation via Kabsch2D (C++ line 230)
|
|
378
|
+
R = self._align_shapes_kabsch_2d(src_norm, dst_norm)
|
|
379
|
+
|
|
380
|
+
# 5. Return scale * rotation (C++ line 233)
|
|
351
381
|
scale = s_dst / s_src
|
|
352
|
-
return scale *
|
|
382
|
+
return scale * R
|
|
@@ -0,0 +1,285 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Piecewise Affine Warp (PAW) for face alignment.
|
|
3
|
+
|
|
4
|
+
Based on OpenFace implementation:
|
|
5
|
+
- lib/local/LandmarkDetector/src/PAW.cpp
|
|
6
|
+
- Active Appearance Models Revisited (Matthews & Baker, IJCV 2004)
|
|
7
|
+
|
|
8
|
+
This implementation matches the C++ PAW algorithm for pixel-perfect alignment.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
import numpy as np
|
|
12
|
+
import cv2
|
|
13
|
+
from typing import Tuple, Optional
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class PAW:
|
|
17
|
+
"""
|
|
18
|
+
Piecewise Affine Warp using triangulation.
|
|
19
|
+
|
|
20
|
+
Warps faces by applying independent affine transforms to each triangle,
|
|
21
|
+
allowing for complex non-affine deformations.
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
def __init__(self, destination_landmarks: np.ndarray, triangulation: np.ndarray,
|
|
25
|
+
min_x: Optional[float] = None, min_y: Optional[float] = None,
|
|
26
|
+
max_x: Optional[float] = None, max_y: Optional[float] = None):
|
|
27
|
+
"""
|
|
28
|
+
Initialize PAW with destination shape and triangulation.
|
|
29
|
+
|
|
30
|
+
Args:
|
|
31
|
+
destination_landmarks: (2*N,) array with [x0...xN, y0...yN] format
|
|
32
|
+
triangulation: (M, 3) array of triangle vertex indices
|
|
33
|
+
min_x, min_y, max_x, max_y: Optional bounds for output image
|
|
34
|
+
"""
|
|
35
|
+
self.destination_landmarks = destination_landmarks.copy()
|
|
36
|
+
self.triangulation = triangulation.copy()
|
|
37
|
+
|
|
38
|
+
num_points = len(destination_landmarks) // 2
|
|
39
|
+
num_tris = len(triangulation)
|
|
40
|
+
|
|
41
|
+
# Extract x and y coordinates
|
|
42
|
+
xs = destination_landmarks[:num_points]
|
|
43
|
+
ys = destination_landmarks[num_points:]
|
|
44
|
+
|
|
45
|
+
# Pre-compute alpha and beta coefficients for each triangle
|
|
46
|
+
self.alpha = np.zeros((num_tris, 3), dtype=np.float32)
|
|
47
|
+
self.beta = np.zeros((num_tris, 3), dtype=np.float32)
|
|
48
|
+
|
|
49
|
+
# Store triangle bounding boxes for optimization
|
|
50
|
+
self.triangle_bounds = []
|
|
51
|
+
|
|
52
|
+
for tri_idx in range(num_tris):
|
|
53
|
+
j, k, l = triangulation[tri_idx]
|
|
54
|
+
|
|
55
|
+
# Compute coefficients (from PAW.cpp lines 83-96)
|
|
56
|
+
c1 = ys[l] - ys[j]
|
|
57
|
+
c2 = xs[l] - xs[j]
|
|
58
|
+
c4 = ys[k] - ys[j]
|
|
59
|
+
c3 = xs[k] - xs[j]
|
|
60
|
+
c5 = c3 * c1 - c2 * c4
|
|
61
|
+
|
|
62
|
+
if abs(c5) < 1e-10:
|
|
63
|
+
# Degenerate triangle, skip
|
|
64
|
+
continue
|
|
65
|
+
|
|
66
|
+
self.alpha[tri_idx, 0] = (ys[j] * c2 - xs[j] * c1) / c5
|
|
67
|
+
self.alpha[tri_idx, 1] = c1 / c5
|
|
68
|
+
self.alpha[tri_idx, 2] = -c2 / c5
|
|
69
|
+
|
|
70
|
+
self.beta[tri_idx, 0] = (xs[j] * c4 - ys[j] * c3) / c5
|
|
71
|
+
self.beta[tri_idx, 1] = -c4 / c5
|
|
72
|
+
self.beta[tri_idx, 2] = c3 / c5
|
|
73
|
+
|
|
74
|
+
# Store triangle vertices and bounding box for point-in-triangle tests
|
|
75
|
+
tri_xs = [xs[j], xs[k], xs[l]]
|
|
76
|
+
tri_ys = [ys[j], ys[k], ys[l]]
|
|
77
|
+
self.triangle_bounds.append({
|
|
78
|
+
'vertices': [(tri_xs[i], tri_ys[i]) for i in range(3)],
|
|
79
|
+
'min_x': min(tri_xs),
|
|
80
|
+
'max_x': max(tri_xs),
|
|
81
|
+
'min_y': min(tri_ys),
|
|
82
|
+
'max_y': max(tri_ys)
|
|
83
|
+
})
|
|
84
|
+
|
|
85
|
+
# Determine output image bounds
|
|
86
|
+
if min_x is None:
|
|
87
|
+
min_x = float(np.min(xs))
|
|
88
|
+
min_y = float(np.min(ys))
|
|
89
|
+
max_x = float(np.max(xs))
|
|
90
|
+
max_y = float(np.max(ys))
|
|
91
|
+
|
|
92
|
+
self.min_x = min_x
|
|
93
|
+
self.min_y = min_y
|
|
94
|
+
|
|
95
|
+
width = int(max_x - min_x + 1.5)
|
|
96
|
+
height = int(max_y - min_y + 1.5)
|
|
97
|
+
|
|
98
|
+
# Create pixel mask and triangle ID map
|
|
99
|
+
self.pixel_mask = np.zeros((height, width), dtype=np.uint8)
|
|
100
|
+
self.triangle_id = np.full((height, width), -1, dtype=np.int32)
|
|
101
|
+
|
|
102
|
+
# Determine which triangle each pixel belongs to
|
|
103
|
+
curr_tri = -1
|
|
104
|
+
for y in range(height):
|
|
105
|
+
for x in range(width):
|
|
106
|
+
px = x + min_x
|
|
107
|
+
py = y + min_y
|
|
108
|
+
curr_tri = self._find_triangle(px, py, curr_tri)
|
|
109
|
+
if curr_tri != -1:
|
|
110
|
+
self.triangle_id[y, x] = curr_tri
|
|
111
|
+
self.pixel_mask[y, x] = 1
|
|
112
|
+
|
|
113
|
+
# Pre-allocate arrays
|
|
114
|
+
self.coefficients = np.zeros((num_tris, 6), dtype=np.float32)
|
|
115
|
+
self.map_x = np.zeros((height, width), dtype=np.float32)
|
|
116
|
+
self.map_y = np.zeros((height, width), dtype=np.float32)
|
|
117
|
+
|
|
118
|
+
def warp(self, image: np.ndarray, source_landmarks: np.ndarray) -> np.ndarray:
|
|
119
|
+
"""
|
|
120
|
+
Warp image using source landmarks to destination landmarks.
|
|
121
|
+
|
|
122
|
+
Args:
|
|
123
|
+
image: Source image to warp
|
|
124
|
+
source_landmarks: (2*N,) array with [x0...xN, y0...yN] format
|
|
125
|
+
|
|
126
|
+
Returns:
|
|
127
|
+
Warped image matching destination shape
|
|
128
|
+
"""
|
|
129
|
+
# Compute warp coefficients from source landmarks
|
|
130
|
+
self._calc_coeff(source_landmarks)
|
|
131
|
+
|
|
132
|
+
# Compute pixel mapping (where to sample from)
|
|
133
|
+
self._warp_region()
|
|
134
|
+
|
|
135
|
+
# Apply warp using OpenCV remap with bilinear interpolation
|
|
136
|
+
warped = cv2.remap(image, self.map_x, self.map_y, cv2.INTER_LINEAR)
|
|
137
|
+
|
|
138
|
+
return warped
|
|
139
|
+
|
|
140
|
+
def _calc_coeff(self, source_landmarks: np.ndarray):
|
|
141
|
+
"""
|
|
142
|
+
Calculate warping coefficients from source landmarks.
|
|
143
|
+
Matches PAW::CalcCoeff() in PAW.cpp lines 338-370.
|
|
144
|
+
"""
|
|
145
|
+
num_points = len(source_landmarks) // 2
|
|
146
|
+
|
|
147
|
+
for tri_idx in range(len(self.triangulation)):
|
|
148
|
+
i, j, k = self.triangulation[tri_idx]
|
|
149
|
+
|
|
150
|
+
# Extract source coordinates for triangle vertices
|
|
151
|
+
c1 = source_landmarks[i]
|
|
152
|
+
c2 = source_landmarks[j] - c1
|
|
153
|
+
c3 = source_landmarks[k] - c1
|
|
154
|
+
c4 = source_landmarks[i + num_points]
|
|
155
|
+
c5 = source_landmarks[j + num_points] - c4
|
|
156
|
+
c6 = source_landmarks[k + num_points] - c4
|
|
157
|
+
|
|
158
|
+
# Get precomputed alpha and beta
|
|
159
|
+
alpha = self.alpha[tri_idx]
|
|
160
|
+
beta = self.beta[tri_idx]
|
|
161
|
+
|
|
162
|
+
# Compute 6 coefficients for affine transform
|
|
163
|
+
self.coefficients[tri_idx, 0] = c1 + c2 * alpha[0] + c3 * beta[0]
|
|
164
|
+
self.coefficients[tri_idx, 1] = c2 * alpha[1] + c3 * beta[1]
|
|
165
|
+
self.coefficients[tri_idx, 2] = c2 * alpha[2] + c3 * beta[2]
|
|
166
|
+
self.coefficients[tri_idx, 3] = c4 + c5 * alpha[0] + c6 * beta[0]
|
|
167
|
+
self.coefficients[tri_idx, 4] = c5 * alpha[1] + c6 * beta[1]
|
|
168
|
+
self.coefficients[tri_idx, 5] = c5 * alpha[2] + c6 * beta[2]
|
|
169
|
+
|
|
170
|
+
def _warp_region(self):
|
|
171
|
+
"""
|
|
172
|
+
Compute source pixel coordinates for each destination pixel.
|
|
173
|
+
Matches PAW::WarpRegion() in PAW.cpp lines 374-436.
|
|
174
|
+
"""
|
|
175
|
+
height, width = self.pixel_mask.shape
|
|
176
|
+
|
|
177
|
+
for y in range(height):
|
|
178
|
+
yi = float(y) + self.min_y
|
|
179
|
+
|
|
180
|
+
for x in range(width):
|
|
181
|
+
xi = float(x) + self.min_x
|
|
182
|
+
|
|
183
|
+
if self.pixel_mask[y, x] == 0:
|
|
184
|
+
# Outside face region
|
|
185
|
+
self.map_x[y, x] = -1
|
|
186
|
+
self.map_y[y, x] = -1
|
|
187
|
+
else:
|
|
188
|
+
# Get triangle for this pixel
|
|
189
|
+
tri_idx = self.triangle_id[y, x]
|
|
190
|
+
coeff = self.coefficients[tri_idx]
|
|
191
|
+
|
|
192
|
+
# Apply affine transform: x_src = coeff[0] + coeff[1]*xi + coeff[2]*yi
|
|
193
|
+
self.map_x[y, x] = coeff[0] + coeff[1] * xi + coeff[2] * yi
|
|
194
|
+
self.map_y[y, x] = coeff[3] + coeff[4] * xi + coeff[5] * yi
|
|
195
|
+
|
|
196
|
+
@staticmethod
|
|
197
|
+
def _same_side(x0: float, y0: float, x1: float, y1: float,
|
|
198
|
+
x2: float, y2: float, x3: float, y3: float) -> bool:
|
|
199
|
+
"""
|
|
200
|
+
Check if point (x0,y0) is on same side of line (x2,y2)-(x3,y3) as point (x1,y1).
|
|
201
|
+
Matches PAW::sameSide() in PAW.cpp lines 443-451.
|
|
202
|
+
"""
|
|
203
|
+
x = (x3 - x2) * (y0 - y2) - (x0 - x2) * (y3 - y2)
|
|
204
|
+
y = (x3 - x2) * (y1 - y2) - (x1 - x2) * (y3 - y2)
|
|
205
|
+
return x * y >= 0
|
|
206
|
+
|
|
207
|
+
@staticmethod
|
|
208
|
+
def _point_in_triangle(x0: float, y0: float, x1: float, y1: float,
|
|
209
|
+
x2: float, y2: float, x3: float, y3: float) -> bool:
|
|
210
|
+
"""
|
|
211
|
+
Check if point (x0,y0) is inside triangle (x1,y1)-(x2,y2)-(x3,y3).
|
|
212
|
+
Matches PAW::pointInTriangle() in PAW.cpp lines 454-461.
|
|
213
|
+
"""
|
|
214
|
+
same_1 = PAW._same_side(x0, y0, x1, y1, x2, y2, x3, y3)
|
|
215
|
+
same_2 = PAW._same_side(x0, y0, x2, y2, x1, y1, x3, y3)
|
|
216
|
+
same_3 = PAW._same_side(x0, y0, x3, y3, x1, y1, x2, y2)
|
|
217
|
+
return same_1 and same_2 and same_3
|
|
218
|
+
|
|
219
|
+
def _find_triangle(self, x: float, y: float, guess: int = -1) -> int:
|
|
220
|
+
"""
|
|
221
|
+
Find which triangle contains point (x, y).
|
|
222
|
+
Matches PAW::findTriangle() in PAW.cpp lines 465-515.
|
|
223
|
+
|
|
224
|
+
Args:
|
|
225
|
+
x, y: Point coordinates
|
|
226
|
+
guess: Previous triangle index for optimization
|
|
227
|
+
|
|
228
|
+
Returns:
|
|
229
|
+
Triangle index or -1 if point is outside all triangles
|
|
230
|
+
"""
|
|
231
|
+
# Try guess first for speed
|
|
232
|
+
if guess != -1:
|
|
233
|
+
bounds = self.triangle_bounds[guess]
|
|
234
|
+
vertices = bounds['vertices']
|
|
235
|
+
if self._point_in_triangle(x, y, vertices[0][0], vertices[0][1],
|
|
236
|
+
vertices[1][0], vertices[1][1],
|
|
237
|
+
vertices[2][0], vertices[2][1]):
|
|
238
|
+
return guess
|
|
239
|
+
|
|
240
|
+
# Search all triangles
|
|
241
|
+
for tri_idx, bounds in enumerate(self.triangle_bounds):
|
|
242
|
+
# Quick bounding box check
|
|
243
|
+
if (x < bounds['min_x'] or x > bounds['max_x'] or
|
|
244
|
+
y < bounds['min_y'] or y > bounds['max_y']):
|
|
245
|
+
continue
|
|
246
|
+
|
|
247
|
+
# Precise point-in-triangle test
|
|
248
|
+
vertices = bounds['vertices']
|
|
249
|
+
if self._point_in_triangle(x, y, vertices[0][0], vertices[0][1],
|
|
250
|
+
vertices[1][0], vertices[1][1],
|
|
251
|
+
vertices[2][0], vertices[2][1]):
|
|
252
|
+
return tri_idx
|
|
253
|
+
|
|
254
|
+
return -1
|
|
255
|
+
|
|
256
|
+
|
|
257
|
+
def load_triangulation(filepath: str) -> np.ndarray:
|
|
258
|
+
"""
|
|
259
|
+
Load triangulation file in OpenFace format.
|
|
260
|
+
|
|
261
|
+
Format:
|
|
262
|
+
Line 1: Number of triangles
|
|
263
|
+
Line 2: Number of columns (always 3)
|
|
264
|
+
Lines 3+: Triangle vertex indices (3 per line)
|
|
265
|
+
|
|
266
|
+
Args:
|
|
267
|
+
filepath: Path to triangulation file (e.g., tris_68_full.txt)
|
|
268
|
+
|
|
269
|
+
Returns:
|
|
270
|
+
(M, 3) array of triangle vertex indices
|
|
271
|
+
"""
|
|
272
|
+
with open(filepath, 'r') as f:
|
|
273
|
+
lines = f.readlines()
|
|
274
|
+
|
|
275
|
+
num_tris = int(lines[0].strip())
|
|
276
|
+
num_cols = int(lines[1].strip())
|
|
277
|
+
|
|
278
|
+
assert num_cols == 3, f"Expected 3 columns, got {num_cols}"
|
|
279
|
+
|
|
280
|
+
triangulation = np.zeros((num_tris, 3), dtype=np.int32)
|
|
281
|
+
for i in range(num_tris):
|
|
282
|
+
parts = lines[i + 2].strip().split()
|
|
283
|
+
triangulation[i] = [int(parts[j]) for j in range(3)]
|
|
284
|
+
|
|
285
|
+
return triangulation
|