openface3 0.1.0__tar.gz

openface3-0.1.0/.gitignore

@@ -0,0 +1,49 @@
+# Python
+.venv/
+venv/
+ENV/
+env/
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+*.egg-info/
+dist/
+build/
+
+# IDEs
+.vscode/
+.idea/
+*.swp
+*.swo
+*~
+
+# OS
+.DS_Store
+Thumbs.db
+
+# Project-specific
+weights/
+outputs/
+.of3mini_star/
+output*.txt
+*.npy
+*.pkl
+
+# Jupyter
+.ipynb_checkpoints/
+
+# VS Code debugging
+.debugpy/
+
+# Temporary files
+*.tmp
+*.log
+logs
+assets
+datasets
+*.gz
+vis_output
+CLAUDE.md
+certs

openface3-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Haneesh Byreddy
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

openface3-0.1.0/PKG-INFO

@@ -0,0 +1,92 @@
+Metadata-Version: 2.4
+Name: openface3
+Version: 0.1.0
+Summary: Facial analysis pipeline: detection, 98-point alignment, and multitask emotion/gaze/AU inference
+Project-URL: Homepage, https://github.com/haneeshbyreddy/openface3
+Project-URL: Issues, https://github.com/haneeshbyreddy/openface3/issues
+Project-URL: HuggingFace, https://huggingface.co/haneeshbyreddy/openface3-weights
+License: MIT License
+        
+        Copyright (c) 2026 Haneesh Byreddy
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+        SOFTWARE.
+License-File: LICENSE
+Keywords: action-units,emotion,face,facial-analysis,gaze
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.9
+Requires-Dist: albumentations
+Requires-Dist: huggingface-hub
+Requires-Dist: iglovikov-helper-functions
+Requires-Dist: numpy
+Requires-Dist: opencv-python
+Requires-Dist: timm
+Requires-Dist: torch>=2.0
+Requires-Dist: torchvision>=0.15
+Description-Content-Type: text/markdown
+
+# openface3
+
+Facial analysis pipeline: face detection, 98-point landmark alignment, and multitask emotion / gaze / action unit inference.
+
+## Install
+
+```bash
+pip install openface3
+```
+
+Weights are downloaded automatically from [Hugging Face](https://huggingface.co/haneeshbyreddy/openface3-weights) on first use and cached via the standard HF cache (`~/.cache/huggingface/hub/`).
+
+## Usage
+
+```python
+import cv2
+from openface3 import Pipeline
+
+pipeline = Pipeline(device='cuda')  # or 'cpu'
+
+image = cv2.cvtColor(cv2.imread('image.jpg'), cv2.COLOR_BGR2RGB)
+results, timings = pipeline.run(image)
+
+for face in results:
+    print(face['emotion'])   # (1, 8) logits — softmax to get probabilities
+    print(face['gaze'])      # (1, 2) [horizontal_deg, vertical_deg]
+    print(face['au'])        # (1, 8) AU activations in [0, 1]
+    print(face['landmarks']) # (98, 2) WFLW keypoints in image coords
+```
+
+Serialise a frame to JSON:
+
+```python
+payload = Pipeline.serialize(results, timings, frame_n=0, ts=0.0)
+```
+
+## Labels
+
+```python
+from openface3 import EMOTION_LABELS, AU_IDS
+# EMOTION_LABELS: ['Neutral','Happy','Sad','Surprise','Fear','Disgust','Anger','Contempt']
+# AU_IDS:         ['AU01','AU02','AU04','AU06','AU09','AU12','AU25','AU26']
+```
+
+## License
+
+MIT

openface3-0.1.0/README.md

@@ -0,0 +1,47 @@
+# openface3
+
+Facial analysis pipeline: face detection, 98-point landmark alignment, and multitask emotion / gaze / action unit inference.
+
+## Install
+
+```bash
+pip install openface3
+```
+
+Weights are downloaded automatically from [Hugging Face](https://huggingface.co/haneeshbyreddy/openface3-weights) on first use and cached via the standard HF cache (`~/.cache/huggingface/hub/`).
+
+## Usage
+
+```python
+import cv2
+from openface3 import Pipeline
+
+pipeline = Pipeline(device='cuda')  # or 'cpu'
+
+image = cv2.cvtColor(cv2.imread('image.jpg'), cv2.COLOR_BGR2RGB)
+results, timings = pipeline.run(image)
+
+for face in results:
+    print(face['emotion'])   # (1, 8) logits — softmax to get probabilities
+    print(face['gaze'])      # (1, 2) [horizontal_deg, vertical_deg]
+    print(face['au'])        # (1, 8) AU activations in [0, 1]
+    print(face['landmarks']) # (98, 2) WFLW keypoints in image coords
+```
+
+Serialise a frame to JSON:
+
+```python
+payload = Pipeline.serialize(results, timings, frame_n=0, ts=0.0)
+```
+
+## Labels
+
+```python
+from openface3 import EMOTION_LABELS, AU_IDS
+# EMOTION_LABELS: ['Neutral','Happy','Sad','Surprise','Fear','Disgust','Anger','Contempt']
+# AU_IDS:         ['AU01','AU02','AU04','AU06','AU09','AU12','AU25','AU26']
+```
+
+## License
+
+MIT

openface3-0.1.0/openface3/__init__.py

@@ -0,0 +1 @@
+from .pipeline import Pipeline, EMOTION_LABELS, AU_IDS

openface3-0.1.0/openface3/alignment.py

@@ -0,0 +1,118 @@
+import cv2
+import numpy as np
+import torch
+
+from .models.landmark import StackedHGNetV1
+
+
+# ── Model config ──────────────────────────────────────────────────────────────
+
+class Config:
+    width        = 256
+    height       = 256
+    use_AAM      = True
+    classes_num  = [98, 9, 98]
+    nstack       = 4
+    add_coord    = True
+    decoder_type = 'default'
+    edge_info    = (
+        (False, (0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32)),
+        (True,  (33,34,35,36,37,38,39,40,41)),
+        (True,  (42,43,44,45,46,47,48,49,50)),
+        (False, (51,52,53,54)),
+        (False, (55,56,57,58,59)),
+        (True,  (60,61,62,63,64,65,66,67)),
+        (True,  (68,69,70,71,72,73,74,75)),
+        (True,  (76,77,78,79,80,81,82,83,84,85,86,87)),
+        (True,  (88,89,90,91,92,93,94,95)),
+    )
+
+
+# ── Crop helpers (also used directly in eval scripts) ─────────────────────────
+
+class GetCropMatrix:
+    def __init__(self, image_size, target_face_scale, align_corners=False):
+        self.image_size        = image_size
+        self.target_face_scale = target_face_scale
+        self.align_corners     = align_corners
+
+    def process(self, scale, center_w, center_h):
+        to_w = (self.image_size - 1) if self.align_corners else self.image_size
+        to_h = (self.image_size - 1) if self.align_corners else self.image_size
+        scale_mu = self.image_size / (scale * self.target_face_scale * 200.0)
+        cosv, sinv = 1.0, 0.0
+        acos, asin = scale_mu * cosv, scale_mu * sinv
+        fx, fy = center_w, center_h
+        tx, ty = to_w / 2.0, to_h / 2.0
+        return np.array([
+            [acos, -asin, tx - acos * fx + asin * fy],
+            [asin,  acos, ty - asin * fx - acos * fy],
+            [0.0,   0.0,  1.0]], dtype=np.float32)
+
+
+class TransformPerspective:
+    def __init__(self, image_size):
+        self.image_size = image_size
+
+    def process(self, image, matrix):
+        return cv2.warpPerspective(image, matrix,
+                                   dsize=(self.image_size, self.image_size),
+                                   flags=cv2.INTER_LINEAR, borderValue=0)
+
+
+# ── Aligner ───────────────────────────────────────────────────────────────────
+
+class Aligner:
+    def __init__(self, model_path: str, device='cpu'):
+        self.device     = torch.device(device)
+        self.input_size = 256
+        cfg = Config()
+        net = StackedHGNetV1(config=cfg, classes_num=cfg.classes_num,
+                             edge_info=cfg.edge_info, nstack=cfg.nstack,
+                             add_coord=cfg.add_coord, decoder_type=cfg.decoder_type)
+        ckpt = torch.load(model_path, map_location=self.device)
+        net.load_state_dict(ckpt['net'], strict=False)
+        self.model = net.to(self.device).eval()
+
+        self._crop_matrix = GetCropMatrix(image_size=self.input_size,
+                                          target_face_scale=1.0, align_corners=True)
+        self._warp        = TransformPerspective(image_size=self.input_size)
+
+    # ── preprocess ────────────────────────────────────────────────────────────
+
+    def preprocess(self, image: np.ndarray, bbox):
+        """Bbox-based crop (GetCropMatrix approach)."""
+        x1, y1, x2, y2 = bbox
+        cw     = (x1 + x2) / 2
+        ch     = (y1 + y2) / 2
+        scale  = max(x2 - x1, y2 - y1) / 200.0
+        matrix = self._crop_matrix.process(scale, cw, ch)
+        crop   = self._warp.process(image, matrix)
+        return crop, matrix
+
+    # ── infer ─────────────────────────────────────────────────────────────────
+
+    def infer(self, crop: np.ndarray) -> torch.Tensor:
+        """(256,256,3) uint8 RGB → raw coords tensor (98,2) in [-1,1]."""
+        t = torch.from_numpy(crop[np.newaxis]).float().permute(0, 3, 1, 2)
+        t = (t / 255.0) * 2.0 - 1.0
+        with torch.inference_mode():
+            out = self.model(t.to(self.device))
+        return out[-1][0]   # (98, 2)
+
+    # ── postprocess ───────────────────────────────────────────────────────────
+
+    def postprocess(self, raw_coords: torch.Tensor, matrix: np.ndarray) -> np.ndarray:
+        """Denorm + backproject to original image space → (98,2)."""
+        pts = ((raw_coords + 1) * self.input_size - 1) / 2
+        pts = pts.cpu().numpy()
+        inv = np.linalg.inv(matrix)
+        out = np.empty_like(pts)
+        out[:, 0] = inv[0, 0]*pts[:, 0] + inv[0, 1]*pts[:, 1] + inv[0, 2]
+        out[:, 1] = inv[1, 0]*pts[:, 0] + inv[1, 1]*pts[:, 1] + inv[1, 2]
+        return out
+
+    def __call__(self, image: np.ndarray, bbox) -> np.ndarray:
+        """bbox → (98,2) landmarks in original image space."""
+        crop, matrix = self.preprocess(image, bbox)
+        return self.postprocess(self.infer(crop), matrix)

openface3-0.1.0/openface3/analysis.py

@@ -0,0 +1,40 @@
+import numpy as np
+import torch
+from torchvision import transforms
+
+from .models.multitask import MTL
+
+
+class Analyzer:
+    def __init__(self, model_path: str, device='cpu'):
+        self.device = torch.device(device)
+        self.model  = MTL().to(self.device)
+        state_dict  = torch.load(model_path, map_location=self.device)
+        self.model.load_state_dict(state_dict)
+        self.model.eval()
+
+        self.transform = transforms.Compose([
+            transforms.ToTensor(),
+            transforms.Resize((224, 224)),
+            transforms.Normalize(mean=[0.485, 0.456, 0.406],
+                                  std=[0.229, 0.224, 0.225]),
+        ])
+
+    def preprocess(self, face_crop: np.ndarray) -> torch.Tensor:
+        """(H,W,3) uint8 RGB → normalised tensor on device."""
+        return self.transform(face_crop).unsqueeze(0).to(self.device)
+
+    def infer(self, tensor: torch.Tensor):
+        with torch.inference_mode():
+            return self.model(tensor)
+
+    def postprocess(self, raw) -> dict:
+        emotion_raw, gaze_raw, au_raw = raw
+        return {
+            'emotion': emotion_raw.cpu().numpy(),
+            'gaze':    gaze_raw.cpu().numpy(),
+            'au':      au_raw.cpu().numpy(),
+        }
+
+    def __call__(self, face_crop: np.ndarray) -> dict:
+        return self.postprocess(self.infer(self.preprocess(face_crop)))

openface3-0.1.0/openface3/detection.py

@@ -0,0 +1,130 @@
+from itertools import product
+from math import ceil
+import albumentations as A
+import numpy as np
+import torch
+import torch.nn.functional as F
+from iglovikov_helper_functions.utils.image_utils import pad_to_size, unpad_from_size
+from torch.utils import model_zoo
+from torchvision.ops import nms
+
+from .models.retinaface import RetinaFace
+
+
+# ── Prior box / decode utilities ─────────────────────────────────────────────
+
+def priorbox(min_sizes, steps, clip, image_size):
+    feature_maps = [[ceil(image_size[0] / s), ceil(image_size[1] / s)] for s in steps]
+    anchors = []
+    for k, f in enumerate(feature_maps):
+        for i, j in product(range(f[0]), range(f[1])):
+            for ms in min_sizes[k]:
+                cx = (j + 0.5) * steps[k] / image_size[1]
+                cy = (i + 0.5) * steps[k] / image_size[0]
+                anchors += [cx, cy, ms / image_size[1], ms / image_size[0]]
+    out = torch.Tensor(anchors).view(-1, 4)
+    if clip:
+        out.clamp_(max=1, min=0)
+    return out
+
+
+def decode(loc, priors, variances):
+    boxes = torch.cat((
+        priors[:, :2] + loc[:, :2] * variances[0] * priors[:, 2:],
+        priors[:, 2:] * torch.exp(loc[:, 2:] * variances[1])), 1)
+    boxes[:, :2] -= boxes[:, 2:] / 2
+    boxes[:, 2:] += boxes[:, :2]
+    return boxes
+
+
+def decode_landm(pre, priors, variances):
+    return torch.cat([
+        priors[:, :2] + pre[:, 2*i:2*i+2] * variances[0] * priors[:, 2:]
+        for i in range(5)], dim=1)
+
+
+# ── Detector ──────────────────────────────────────────────────────────────────
+
+WEIGHTS_URL = "https://github.com/ternaus/retinaface/releases/download/0.01/retinaface_resnet50_2020-07-20-f168fae3c.zip"
+
+class Detector:
+    def __init__(self, device='cpu', max_size=640):
+        self.device   = device
+        self.max_size = max_size
+        self.model = RetinaFace(
+            name='Resnet50', pretrained=False,
+            return_layers={'layer2': 1, 'layer3': 2, 'layer4': 3},
+            in_channels=256, out_channels=256).to(device)
+        self.transform  = A.Compose([A.LongestMaxSize(max_size=max_size, p=1), A.Normalize(p=1)])
+        self.prior_box  = priorbox(min_sizes=[[16, 32], [64, 128], [256, 512]],
+                                   steps=[8, 16, 32], clip=False,
+                                   image_size=(max_size, max_size)).to(device)
+        self.variance       = [0.1, 0.2]
+        self._scale_bbox    = torch.tensor([max_size, max_size, max_size, max_size],
+                                           dtype=torch.float32).to(device)
+        self._scale_landm   = torch.tensor([max_size, max_size] * 5,
+                                           dtype=torch.float32).to(device)
+
+    def load_weights(self):
+        state_dict = model_zoo.load_url(WEIGHTS_URL, progress=True, map_location='cpu')
+        self.model.load_state_dict(state_dict)
+
+    def eval(self):
+        self.model.eval()
+
+    def preprocess(self, image: np.ndarray):
+        h, w = image.shape[:2]
+        transformed = self.transform(image=image)['image']
+        paded       = pad_to_size(target_size=(self.max_size, self.max_size), image=transformed)
+        tensor      = torch.from_numpy(np.transpose(paded['image'], (2, 0, 1))).to(self.device)
+        meta = {'pads': paded['pads'], 'resize_coeff': max(h, w) / self.max_size,
+                'original_size': (h, w)}
+        return tensor.unsqueeze(0), meta
+
+    def infer(self, tensor: torch.Tensor):
+        with torch.inference_mode():
+            return self.model(tensor)
+
+    def postprocess(self, raw, meta, confidence_threshold=0.7, nms_threshold=0.4):
+        loc, conf, land = raw
+        conf      = F.softmax(conf, dim=-1)
+        boxes     = decode(loc[0], self.prior_box, self.variance) * self._scale_bbox
+        scores    = conf[0][:, 1]
+        landmarks = decode_landm(land[0], self.prior_box, self.variance) * self._scale_landm
+
+        valid  = torch.where(scores > confidence_threshold)[0]
+        boxes, landmarks, scores = boxes[valid], landmarks[valid], scores[valid]
+        order  = scores.argsort(descending=True)
+        boxes, landmarks, scores = boxes[order], landmarks[order], scores[order]
+
+        keep      = nms(boxes, scores, nms_threshold)
+        boxes     = boxes[keep].int()
+        if boxes.shape[0] == 0:
+            return []
+
+        landmarks = landmarks[keep].cpu().numpy().reshape(-1, 2)
+        scores    = scores[keep].cpu().numpy().astype(np.float64)
+        boxes     = boxes.cpu().numpy()
+
+        unpadded     = unpad_from_size(meta['pads'], bboxes=boxes, keypoints=landmarks)
+        rc           = meta['resize_coeff']
+        boxes        = (unpadded['bboxes'] * rc).astype(int)
+        landmarks    = (unpadded['keypoints'].reshape(-1, 10) * rc).astype(int)
+        oh, ow       = meta['original_size']
+
+        results = []
+        for i, bbox in enumerate(boxes):
+            x1 = int(np.clip(bbox[0], 0, ow - 1))
+            x2 = int(np.clip(bbox[2], x1 + 1, ow - 1))
+            y1 = int(np.clip(bbox[1], 0, oh - 1))
+            y2 = int(np.clip(bbox[3], y1 + 1, oh - 1))
+            if x1 >= x2 or y1 >= y2:
+                continue
+            results.append({'bbox':      [x1, y1, x2, y2],
+                            'score':     float(scores[i]),
+                            'landmarks': landmarks[i].reshape(-1, 2).tolist()})
+        return results
+
+    def __call__(self, image: np.ndarray, confidence_threshold=0.7, nms_threshold=0.4):
+        tensor, meta = self.preprocess(image)
+        return self.postprocess(self.infer(tensor), meta, confidence_threshold, nms_threshold)

openface3-0.1.0/openface3/models/landmark.py

@@ -0,0 +1,287 @@
+import numpy as np
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+
+# ── CoordConv ─────────────────────────────────────────────────────────────────
+
+class AddCoordsTh(nn.Module):
+    def __init__(self, x_dim, y_dim, with_r=False, with_boundary=False):
+        super().__init__()
+        self.x_dim = x_dim
+        self.y_dim = y_dim
+        self.with_r = with_r
+        self.with_boundary = with_boundary
+
+    def forward(self, input_tensor, heatmap=None):
+        batch = input_tensor.shape[0]
+
+        xx_ones  = torch.ones([1, self.y_dim], dtype=torch.int32).to(input_tensor)
+        xx_ones  = xx_ones.unsqueeze(-1)                                               # (1, y_dim, 1)
+        xx_range = torch.arange(self.x_dim, dtype=torch.int32).unsqueeze(0).to(input_tensor)
+        xx_range = xx_range.unsqueeze(1)                                               # (1, 1, x_dim)
+        xx_channel = torch.matmul(xx_ones.float(), xx_range.float()).unsqueeze(-1)    # (1, y_dim, x_dim, 1)
+
+        yy_ones  = torch.ones([1, self.x_dim], dtype=torch.int32).to(input_tensor)
+        yy_ones  = yy_ones.unsqueeze(1)                                                # (1, 1, x_dim)
+        yy_range = torch.arange(self.y_dim, dtype=torch.int32).unsqueeze(0).to(input_tensor)
+        yy_range = yy_range.unsqueeze(-1)                                              # (1, y_dim, 1)
+        yy_channel = torch.matmul(yy_range.float(), yy_ones.float()).unsqueeze(-1)    # (1, y_dim, x_dim, 1)
+
+        xx_channel = xx_channel.permute(0, 3, 2, 1) / (self.x_dim - 1) * 2 - 1
+        yy_channel = yy_channel.permute(0, 3, 2, 1) / (self.y_dim - 1) * 2 - 1
+
+        xx_channel = xx_channel.repeat(batch, 1, 1, 1)
+        yy_channel = yy_channel.repeat(batch, 1, 1, 1)
+
+        if self.with_boundary and heatmap is not None:
+            boundary    = torch.clamp(heatmap[:, -1:, :, :], 0.0, 1.0)
+            zero        = torch.zeros_like(xx_channel)
+            xx_boundary = torch.where(boundary > 0.05, xx_channel, zero)
+            yy_boundary = torch.where(boundary > 0.05, yy_channel, zero)
+
+        ret = torch.cat([input_tensor, xx_channel, yy_channel], dim=1)
+        if self.with_r:
+            rr  = torch.sqrt(xx_channel ** 2 + yy_channel ** 2)
+            rr  = rr / torch.max(rr)
+            ret = torch.cat([ret, rr], dim=1)
+        if self.with_boundary and heatmap is not None:
+            ret = torch.cat([ret, xx_boundary, yy_boundary], dim=1)
+        return ret
+
+
+class CoordConvTh(nn.Module):
+    def __init__(self, x_dim, y_dim, with_r, with_boundary,
+                 in_channels, out_channels, first_one=False,
+                 relu=False, bn=False, *args, **kwargs):
+        super().__init__()
+        self.addcoords = AddCoordsTh(x_dim=x_dim, y_dim=y_dim,
+                                     with_r=with_r, with_boundary=with_boundary)
+        in_channels += 2
+        if with_r:
+            in_channels += 1
+        if with_boundary and not first_one:
+            in_channels += 2
+        self.conv = nn.Conv2d(in_channels=in_channels, out_channels=out_channels, *args, **kwargs)
+        self.relu = nn.ReLU() if relu else None
+        self.bn   = nn.BatchNorm2d(out_channels) if bn else None
+        self.with_boundary = with_boundary
+        self.first_one = first_one
+
+    def forward(self, input_tensor, heatmap=None):
+        assert (self.with_boundary and not self.first_one) == (heatmap is not None)
+        ret = self.addcoords(input_tensor, heatmap)
+        ret = self.conv(ret)
+        if self.bn   is not None: ret = self.bn(ret)
+        if self.relu is not None: ret = self.relu(ret)
+        return ret
+
+
+# ── Decoder ───────────────────────────────────────────────────────────────────
+
+class _DecoderDefault:
+    def __init__(self, weight=1, use_weight_map=False):
+        self.weight = weight
+        self.use_weight_map = use_weight_map
+
+    def _make_grid(self, h, w):
+        yy, xx = torch.meshgrid(
+            torch.arange(h).float() / (h - 1) * 2 - 1,
+            torch.arange(w).float() / (w - 1) * 2 - 1)
+        return yy, xx
+
+    def get_coords_from_heatmap(self, heatmap):
+        batch, npoints, h, w = heatmap.shape
+        if self.use_weight_map:
+            heatmap = heatmap * self.weight
+        yy, xx = self._make_grid(h, w)
+        yy = yy.view(1, 1, h, w).to(heatmap)
+        xx = xx.view(1, 1, h, w).to(heatmap)
+        heatmap_sum = torch.clamp(heatmap.sum([2, 3]), min=1e-6)
+        yy_coord = (yy * heatmap).sum([2, 3]) / heatmap_sum
+        xx_coord = (xx * heatmap).sum([2, 3]) / heatmap_sum
+        return torch.stack([xx_coord, yy_coord], dim=-1)
+
+
+def get_decoder(decoder_type='default'):
+    if decoder_type == 'default':
+        return _DecoderDefault()
+    raise NotImplementedError(f"Unknown decoder: {decoder_type}")
+
+
+# ── Building blocks ───────────────────────────────────────────────────────────
+
+class Activation(nn.Module):
+    def __init__(self, kind='relu', channel=None):
+        super().__init__()
+        self.kind = kind
+        norm_str, act_str = kind.split('+') if '+' in kind else ('none', kind)
+        self.norm_fn = {'in': F.instance_norm,
+                        'bn': nn.BatchNorm2d(channel),
+                        'bn_noaffine': nn.BatchNorm2d(channel, affine=False, track_running_stats=True),
+                        'none': None}[norm_str]
+        self.act_fn  = {'relu': F.relu, 'softplus': nn.Softplus(), 'exp': torch.exp,
+                        'sigmoid': torch.sigmoid, 'tanh': torch.tanh, 'none': None}[act_str]
+
+    def forward(self, x):
+        if self.norm_fn is not None: x = self.norm_fn(x)
+        if self.act_fn  is not None: x = self.act_fn(x)
+        return x
+
+
+class ConvBlock(nn.Module):
+    def __init__(self, inp_dim, out_dim, kernel_size=3, stride=1, bn=False, relu=True, groups=1):
+        super().__init__()
+        self.conv = nn.Conv2d(inp_dim, out_dim, kernel_size, stride,
+                              padding=(kernel_size - 1) // 2, groups=groups, bias=True)
+        self.relu = nn.ReLU() if relu else None
+        self.bn   = nn.BatchNorm2d(out_dim) if bn else None
+
+    def forward(self, x):
+        x = self.conv(x)
+        if self.bn   is not None: x = self.bn(x)
+        if self.relu is not None: x = self.relu(x)
+        return x
+
+
+class ResBlock(nn.Module):
+    def __init__(self, inp_dim, out_dim, mid_dim=None):
+        super().__init__()
+        mid_dim = mid_dim or out_dim // 2
+        self.relu  = nn.ReLU()
+        self.bn1   = nn.BatchNorm2d(inp_dim)
+        self.conv1 = ConvBlock(inp_dim, mid_dim, 1, relu=False)
+        self.bn2   = nn.BatchNorm2d(mid_dim)
+        self.conv2 = ConvBlock(mid_dim, mid_dim, 3, relu=False)
+        self.bn3   = nn.BatchNorm2d(mid_dim)
+        self.conv3 = ConvBlock(mid_dim, out_dim, 1, relu=False)
+        self.need_skip  = inp_dim != out_dim
+        self.skip_layer = ConvBlock(inp_dim, out_dim, 1, relu=False) if self.need_skip else None
+
+    def forward(self, x):
+        residual = self.skip_layer(x) if self.need_skip else x
+        out = self.conv3(self.relu(self.bn3(self.conv2(self.relu(self.bn2(self.conv1(self.relu(self.bn1(x)))))))))
+        return out + residual
+
+
+class Hourglass(nn.Module):
+    def __init__(self, n, f, increase=0, up_mode='nearest',
+                 add_coord=False, first_one=False, x_dim=64, y_dim=64):
+        super().__init__()
+        nf = f + increase
+        self.coordconv = CoordConvTh(x_dim=x_dim, y_dim=y_dim, with_r=True, with_boundary=True,
+                                     relu=False, bn=False, in_channels=f, out_channels=f,
+                                     first_one=first_one, kernel_size=1, stride=1, padding=0
+                                     ) if add_coord else None
+        self.up1   = ResBlock(f, f)
+        self.pool1 = nn.MaxPool2d(kernel_size=2, stride=2)
+        self.low1  = ResBlock(f, nf)
+        self.n     = n
+        self.low2  = Hourglass(n=n-1, f=nf, increase=increase, up_mode=up_mode,
+                                add_coord=False) if n > 1 else ResBlock(nf, nf)
+        self.low3  = ResBlock(nf, f)
+        self.up2   = nn.Upsample(scale_factor=2, mode=up_mode)
+
+    def forward(self, x, heatmap=None):
+        if self.coordconv is not None:
+            x = self.coordconv(x, heatmap)
+        return self.up1(x) + self.up2(self.low3(self.low2(self.low1(self.pool1(x)))))
+
+
+class E2HTransform(nn.Module):
+    def __init__(self, edge_info, num_points, num_edges):
+        super().__init__()
+        e2h = np.zeros([num_points, num_edges])
+        for eid, (_, indices) in enumerate(edge_info):
+            for pid in indices:
+                e2h[pid, eid] = 1
+        e2h = torch.from_numpy(e2h).float()
+        self.register_buffer('weight', e2h.view(e2h.size(0), e2h.size(1), 1, 1))
+        bias = ((e2h @ torch.ones(e2h.size(1))) < 0.5).to(e2h)
+        self.register_buffer('bias', bias)
+
+    def forward(self, edgemaps):
+        return F.conv2d(edgemaps, weight=self.weight, bias=self.bias)
+
+
+# ── StackedHGNetV1 ────────────────────────────────────────────────────────────
+
+class StackedHGNetV1(nn.Module):
+    def __init__(self, config, classes_num, edge_info,
+                 nstack=4, nlevels=4, in_channel=256, increase=0,
+                 add_coord=True, decoder_type='default'):
+        super().__init__()
+        self.cfg        = config
+        self.decoder    = get_decoder(decoder_type)
+        self.nstack     = nstack
+        self.add_coord  = add_coord
+        self.num_heats  = classes_num[0]
+
+        first_conv = CoordConvTh(x_dim=config.width, y_dim=config.height,
+                                 with_r=True, with_boundary=False,
+                                 relu=True, bn=True, in_channels=3, out_channels=64,
+                                 kernel_size=7, stride=2, padding=3
+                                 ) if add_coord else ConvBlock(3, 64, 7, 2, bn=True, relu=True)
+
+        self.pre = nn.Sequential(first_conv, ResBlock(64, 128),
+                                 nn.MaxPool2d(2, 2), ResBlock(128, 128), ResBlock(128, in_channel))
+
+        self.hgs = nn.ModuleList([
+            Hourglass(n=nlevels, f=in_channel, increase=increase, add_coord=add_coord,
+                      first_one=(i == 0),
+                      x_dim=config.width  // nstack,
+                      y_dim=config.height // nstack)
+            for i in range(nstack)])
+
+        self.features = nn.ModuleList([
+            nn.Sequential(ResBlock(in_channel, in_channel),
+                          ConvBlock(in_channel, in_channel, 1, bn=True, relu=True))
+            for _ in range(nstack)])
+
+        self.out_heatmaps   = nn.ModuleList([ConvBlock(in_channel, self.num_heats, 1, relu=False, bn=False) for _ in range(nstack)])
+        self.merge_features = nn.ModuleList([ConvBlock(in_channel, in_channel, 1, relu=False, bn=False) for _ in range(nstack - 1)])
+        self.merge_heatmaps = nn.ModuleList([ConvBlock(self.num_heats, in_channel, 1, relu=False, bn=False) for _ in range(nstack - 1)])
+        self.heatmap_act    = Activation("in+relu", self.num_heats)
+
+        if config.use_AAM:
+            self.num_edges  = classes_num[1]
+            self.num_points = classes_num[2]
+            self.e2h        = E2HTransform(edge_info, self.num_points, self.num_edges)
+            self.out_edgemaps   = nn.ModuleList([ConvBlock(in_channel, self.num_edges,  1, relu=False, bn=False) for _ in range(nstack)])
+            self.out_pointmaps  = nn.ModuleList([ConvBlock(in_channel, self.num_points, 1, relu=False, bn=False) for _ in range(nstack)])
+            self.merge_edgemaps  = nn.ModuleList([ConvBlock(self.num_edges,  in_channel, 1, relu=False, bn=False) for _ in range(nstack - 1)])
+            self.merge_pointmaps = nn.ModuleList([ConvBlock(self.num_points, in_channel, 1, relu=False, bn=False) for _ in range(nstack - 1)])
+            self.edgemap_act  = Activation("sigmoid", self.num_edges)
+            self.pointmap_act = Activation("sigmoid", self.num_points)
+
+    def forward(self, x):
+        x = self.pre(x)
+        y, fusionmaps, heatmaps = [], [], None
+
+        for i in range(self.nstack):
+            hg      = self.hgs[i](x, heatmap=heatmaps)
+            feature = self.features[i](hg)
+            heatmaps = self.heatmap_act(self.out_heatmaps[i](feature))
+
+            if self.cfg.use_AAM:
+                pointmaps = self.pointmap_act(self.out_pointmaps[i](feature))
+                edgemaps  = self.edgemap_act(self.out_edgemaps[i](feature))
+                fusion    = self.e2h(edgemaps) * pointmaps * heatmaps
+            else:
+                fusion = heatmaps
+
+            landmarks = self.decoder.get_coords_from_heatmap(fusion)
+
+            if i < self.nstack - 1:
+                x = x + self.merge_features[i](feature) + self.merge_heatmaps[i](heatmaps)
+                if self.cfg.use_AAM:
+                    x += self.merge_pointmaps[i](pointmaps) + self.merge_edgemaps[i](edgemaps)
+
+            y.append(landmarks)
+            if self.cfg.use_AAM:
+                y.append(pointmaps)
+                y.append(edgemaps)
+            fusionmaps.append(fusion)
+
+        return y, fusionmaps, landmarks

openface3-0.1.0/openface3/models/multitask.py

@@ -0,0 +1,93 @@
+import math
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import timm
+
+
+# ── Graph Neural Network (AU head) ────────────────────────────────────────────
+
+def _normalize_digraph(A):
+    b, n, _ = A.shape
+    degs_inv_sqrt = A.detach().sum(dim=-1) ** -0.5
+    norm = torch.eye(n, device=A.device).view(1, n, n) * degs_inv_sqrt.view(b, n, 1)
+    return torch.bmm(torch.bmm(norm, A), norm)
+
+
+class GNN(nn.Module):
+    def __init__(self, in_channels, num_classes, neighbor_num=4, metric='dots'):
+        super().__init__()
+        self.metric       = metric
+        self.neighbor_num = neighbor_num
+        self.U   = nn.Linear(in_channels, in_channels)
+        self.V   = nn.Linear(in_channels, in_channels)
+        self.bnv = nn.BatchNorm1d(num_classes)
+        self.relu = nn.ReLU()
+        nn.init.normal_(self.U.weight, 0, math.sqrt(2. / in_channels))
+        nn.init.normal_(self.V.weight, 0, math.sqrt(2. / in_channels))
+        self.bnv.weight.data.fill_(1)
+        self.bnv.bias.data.zero_()
+
+    def forward(self, x):
+        b, n, c = x.shape
+        si = x.detach()
+        if self.metric == 'dots':
+            si = torch.einsum('bij,bjk->bik', si, si.transpose(1, 2))
+        elif self.metric == 'cosine':
+            si = F.normalize(si, p=2, dim=-1)
+            si = torch.einsum('bij,bjk->bik', si, si.transpose(1, 2))
+        elif self.metric == 'l1':
+            si = torch.abs(si.unsqueeze(2) - si.unsqueeze(1)).sum(dim=-1)
+        else:
+            raise ValueError(f"Unknown metric: {self.metric}")
+
+        largest = self.metric != 'l1'
+        threshold = si.topk(k=self.neighbor_num, dim=-1, largest=largest)[0][:, :, -1].view(b, n, 1)
+        adj = (si >= threshold).float() if largest else (si <= threshold).float()
+        A   = _normalize_digraph(adj)
+        agg = torch.einsum('bij,bjk->bik', A, self.V(x))
+        return self.relu(x + self.bnv(agg + self.U(x)))
+
+
+class AUHead(nn.Module):
+    def __init__(self, in_channels, num_classes, neighbor_num=4, metric='dots'):
+        super().__init__()
+        self.class_linears = nn.ModuleList([nn.Linear(in_channels, in_channels) for _ in range(num_classes)])
+        self.gnn  = GNN(in_channels, num_classes, neighbor_num=neighbor_num, metric=metric)
+        self.sc   = nn.Parameter(torch.zeros(num_classes, in_channels))
+        self.relu = nn.ReLU()
+        nn.init.xavier_uniform_(self.sc)
+
+    def forward(self, x):
+        f_u = torch.cat([layer(x).unsqueeze(1) for layer in self.class_linears], dim=1)
+        f_v = self.gnn(f_u)
+        sc  = F.normalize(self.relu(self.sc), p=2, dim=-1)
+        cl  = F.normalize(f_v, p=2, dim=-1)
+        return (cl * sc.unsqueeze(0)).sum(dim=-1)
+
+
+# ── MTL Model ─────────────────────────────────────────────────────────────────
+
+class MTL(nn.Module):
+    def __init__(self, base_model_name='tf_efficientnet_b0_ns', expr_classes=8, au_numbers=8):
+        super().__init__()
+        self.base_model = timm.create_model(base_model_name, pretrained=False)
+        self.base_model.classifier = nn.Identity()
+        d = self.base_model.num_features
+
+        self.relu = nn.ReLU()
+        self.fc_emotion = nn.Linear(d, d)
+        self.fc_gaze    = nn.Linear(d, d)
+        self.fc_au      = nn.Linear(d, d)
+
+        self.emotion_classifier = nn.Linear(d, expr_classes)
+        self.gaze_regressor     = nn.Linear(d, 2)
+        self.au_regressor       = AUHead(in_channels=d, num_classes=au_numbers, neighbor_num=4, metric='dots')
+
+    def forward(self, x):
+        feat = self.base_model(x)
+        emotion = self.emotion_classifier(self.relu(self.fc_emotion(feat)))
+        gaze    = self.gaze_regressor(self.relu(self.fc_gaze(feat)))
+        au      = self.au_regressor(self.relu(self.fc_au(feat)))
+        return emotion, gaze, au

openface3-0.1.0/openface3/models/retinaface.py

@@ -0,0 +1,121 @@
+from typing import Dict, List, Tuple
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from torchvision import models
+from torchvision.models import _utils
+
+
+# ── FPN / SSH building blocks ─────────────────────────────────────────────────
+
+def _conv_bn(inp, oup, stride=1, leaky=0):
+    return nn.Sequential(
+        nn.Conv2d(inp, oup, 3, stride, 1, bias=False),
+        nn.BatchNorm2d(oup),
+        nn.LeakyReLU(negative_slope=leaky, inplace=True))
+
+def _conv_bn_no_relu(inp, oup, stride):
+    return nn.Sequential(
+        nn.Conv2d(inp, oup, 3, stride, 1, bias=False),
+        nn.BatchNorm2d(oup))
+
+def _conv_bn1x1(inp, oup, stride, leaky=0):
+    return nn.Sequential(
+        nn.Conv2d(inp, oup, 1, stride, padding=0, bias=False),
+        nn.BatchNorm2d(oup),
+        nn.LeakyReLU(negative_slope=leaky, inplace=True))
+
+
+class SSH(nn.Module):
+    def __init__(self, in_channel, out_channel):
+        super().__init__()
+        if out_channel % 4 != 0:
+            raise ValueError(f"out_channel must be divisible by 4, got {out_channel}")
+        leaky = 0.1 if out_channel <= 64 else 0
+        self.conv3X3   = _conv_bn_no_relu(in_channel, out_channel // 2, stride=1)
+        self.conv5X5_1 = _conv_bn(in_channel, out_channel // 4, stride=1, leaky=leaky)
+        self.conv5X5_2 = _conv_bn_no_relu(out_channel // 4, out_channel // 4, stride=1)
+        self.conv7X7_2 = _conv_bn(out_channel // 4, out_channel // 4, stride=1, leaky=leaky)
+        self.conv7x7_3 = _conv_bn_no_relu(out_channel // 4, out_channel // 4, stride=1)
+
+    def forward(self, x):
+        c3   = self.conv3X3(x)
+        c5_1 = self.conv5X5_1(x)
+        c5   = self.conv5X5_2(c5_1)
+        c7   = self.conv7x7_3(self.conv7X7_2(c5_1))
+        return F.relu(torch.cat([c3, c5, c7], dim=1))
+
+
+class FPN(nn.Module):
+    def __init__(self, in_channels_list, out_channels):
+        super().__init__()
+        leaky = 0.1 if out_channels <= 64 else 0
+        self.output1 = _conv_bn1x1(in_channels_list[0], out_channels, stride=1, leaky=leaky)
+        self.output2 = _conv_bn1x1(in_channels_list[1], out_channels, stride=1, leaky=leaky)
+        self.output3 = _conv_bn1x1(in_channels_list[2], out_channels, stride=1, leaky=leaky)
+        self.merge1  = _conv_bn(out_channels, out_channels, leaky=leaky)
+        self.merge2  = _conv_bn(out_channels, out_channels, leaky=leaky)
+
+    def forward(self, x: Dict[str, torch.Tensor]) -> List[torch.Tensor]:
+        y = list(x.values())
+        o1, o2, o3 = self.output1(y[0]), self.output2(y[1]), self.output3(y[2])
+        o2 = self.merge2(o2 + F.interpolate(o3, size=o2.shape[2:], mode='nearest'))
+        o1 = self.merge1(o1 + F.interpolate(o2, size=o1.shape[2:], mode='nearest'))
+        return [o1, o2, o3]
+
+
+# ── Prediction heads ──────────────────────────────────────────────────────────
+
+class _ClassHead(nn.Module):
+    def __init__(self, in_channels=512, num_anchors=2):
+        super().__init__()
+        self.conv1x1 = nn.Conv2d(in_channels, num_anchors * 2, 1)
+
+    def forward(self, x):
+        return self.conv1x1(x).permute(0, 2, 3, 1).contiguous().view(x.shape[0], -1, 2)
+
+
+class _BboxHead(nn.Module):
+    def __init__(self, in_channels=512, num_anchors=2):
+        super().__init__()
+        self.conv1x1 = nn.Conv2d(in_channels, num_anchors * 4, 1)
+
+    def forward(self, x):
+        return self.conv1x1(x).permute(0, 2, 3, 1).contiguous().view(x.shape[0], -1, 4)
+
+
+class _LandmarkHead(nn.Module):
+    def __init__(self, in_channels=512, num_anchors=2):
+        super().__init__()
+        self.conv1x1 = nn.Conv2d(in_channels, num_anchors * 10, 1)
+
+    def forward(self, x):
+        return self.conv1x1(x).permute(0, 2, 3, 1).contiguous().view(x.shape[0], -1, 10)
+
+
+# ── RetinaFace ────────────────────────────────────────────────────────────────
+
+class RetinaFace(nn.Module):
+    def __init__(self, name, pretrained, in_channels, return_layers, out_channels):
+        super().__init__()
+        if name != "Resnet50":
+            raise NotImplementedError(f"Only Resnet50 supported, got {name}")
+        backbone  = models.resnet50(pretrained=pretrained)
+        self.body = _utils.IntermediateLayerGetter(backbone, return_layers)
+        c = in_channels
+        self.fpn  = FPN([c * 2, c * 4, c * 8], out_channels)
+        self.ssh1 = SSH(out_channels, out_channels)
+        self.ssh2 = SSH(out_channels, out_channels)
+        self.ssh3 = SSH(out_channels, out_channels)
+        self.ClassHead    = nn.ModuleList([_ClassHead(out_channels)    for _ in range(3)])
+        self.BboxHead     = nn.ModuleList([_BboxHead(out_channels)     for _ in range(3)])
+        self.LandmarkHead = nn.ModuleList([_LandmarkHead(out_channels) for _ in range(3)])
+
+    def forward(self, x) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
+        fpn_out  = self.fpn(self.body(x))
+        features = [self.ssh1(fpn_out[0]), self.ssh2(fpn_out[1]), self.ssh3(fpn_out[2])]
+        bbox = torch.cat([self.BboxHead[i](f)     for i, f in enumerate(features)], dim=1)
+        cls  = torch.cat([self.ClassHead[i](f)    for i, f in enumerate(features)], dim=1)
+        ldm  = torch.cat([self.LandmarkHead[i](f) for i, f in enumerate(features)], dim=1)
+        return bbox, cls, ldm

openface3-0.1.0/openface3/pipeline.py

@@ -0,0 +1,139 @@
+import time
+
+import numpy as np
+import torch
+from huggingface_hub import hf_hub_download
+
+from .detection import Detector
+from .alignment import Aligner
+from .analysis  import Analyzer
+
+HF_REPO_ID = "haneeshbyreddy/openface3-weights"
+
+EMOTION_LABELS = ['Neutral', 'Happy', 'Sad', 'Surprise', 'Fear', 'Disgust', 'Anger', 'Contempt']
+AU_IDS         = ['AU01', 'AU02', 'AU04', 'AU06', 'AU09', 'AU12', 'AU25', 'AU26']
+
+
+class Pipeline:
+    def __init__(self, device='cpu'):
+        self.detector = Detector(device=device)
+        self.detector.load_weights()
+        self.detector.eval()
+
+        landmark_path  = hf_hub_download(HF_REPO_ID, 'aligner.pkl')
+        multitask_path = hf_hub_download(HF_REPO_ID, 'analyzer.pth')
+        self.aligner   = Aligner(model_path=landmark_path,  device=device)
+        self.analyzer  = Analyzer(model_path=multitask_path, device=device)
+        self._warmup()
+
+    def _warmup(self):
+        dummy_det  = torch.zeros(1, 3, self.detector.max_size, self.detector.max_size,
+                                 device=self.detector.device)
+        self.detector.infer(dummy_det)
+
+        dummy_crop = np.zeros((256, 256, 3), dtype=np.uint8)
+        self.aligner.infer(dummy_crop)
+
+        dummy_face = self.analyzer.preprocess(np.zeros((224, 224, 3), dtype=np.uint8))
+        self.analyzer.infer(dummy_face)
+
+    def run(self, image: np.ndarray) -> tuple:
+        """image: (H,W,3) uint8 RGB. Returns (results, timings)."""
+        t_total = time.perf_counter()
+
+        t0              = time.perf_counter()
+        tensor, meta    = self.detector.preprocess(image)
+        t1              = time.perf_counter()
+        raw_det         = self.detector.infer(tensor)
+        detect_infer_ms = (time.perf_counter() - t1) * 1000.0
+        detections      = self.detector.postprocess(raw_det, meta, confidence_threshold=0.95)
+        detect_ms       = (time.perf_counter() - t0) * 1000.0
+
+        results          = []
+        align_ms         = 0.0
+        align_infer_ms   = 0.0
+        analyze_ms       = 0.0
+        analyze_infer_ms = 0.0
+
+        for det in detections:
+            t0           = time.perf_counter()
+            crop, matrix = self.aligner.preprocess(image, det['bbox'])
+            t1           = time.perf_counter()
+            raw_lm       = self.aligner.infer(crop)
+            align_infer_ms += (time.perf_counter() - t1) * 1000.0
+            landmarks    = self.aligner.postprocess(raw_lm, matrix)
+            align_ms    += (time.perf_counter() - t0) * 1000.0
+
+            x1, y1, x2, y2 = det['bbox']
+            face_crop = image[y1:y2, x1:x2]
+
+            if face_crop.size > 0:
+                t0               = time.perf_counter()
+                tensor_face      = self.analyzer.preprocess(face_crop)
+                t1               = time.perf_counter()
+                raw_ana          = self.analyzer.infer(tensor_face)
+                analyze_infer_ms += (time.perf_counter() - t1) * 1000.0
+                analysis         = self.analyzer.postprocess(raw_ana)
+                analyze_ms      += (time.perf_counter() - t0) * 1000.0
+            else:
+                analysis = {}
+
+            results.append({**det, 'landmarks': landmarks, **analysis})
+
+        total_ms = (time.perf_counter() - t_total) * 1000.0
+        timings  = {
+            'detect_ms':        detect_ms,
+            'detect_infer_ms':  detect_infer_ms,
+            'align_ms':         align_ms,
+            'align_infer_ms':   align_infer_ms,
+            'analyze_ms':       analyze_ms,
+            'analyze_infer_ms': analyze_infer_ms,
+            'total_ms':         total_ms,
+        }
+        return results, timings
+
+    @staticmethod
+    def serialize(results: list, timings: dict, frame_n: int = 0, ts: float = 0.0) -> dict:
+        """
+        Convert pipeline output to a JSON-ready dict.
+        Both main.py (JSONL recording) and the backend API use this format.
+        """
+        faces = []
+        for r in results:
+            em_logits = r['emotion'].flatten().tolist() if r.get('emotion') is not None else []
+            top_em    = int(np.argmax(em_logits)) if em_logits else 0
+            gaze      = r['gaze'].flatten().tolist() if r.get('gaze') is not None else [0.0, 0.0]
+            au_vals   = r['au'].flatten().tolist()   if r.get('au')   is not None else []
+
+            faces.append({
+                'bbox':       r['bbox'],
+                'confidence': round(float(r['score']), 4),
+                'landmarks':  r['landmarks'].tolist(),
+                'emotion': {
+                    'label':  EMOTION_LABELS[top_em] if em_logits else None,
+                    'logits': [round(v, 4) for v in em_logits],
+                },
+                'gaze': {
+                    'horizontal_deg': round(gaze[0], 2),
+                    'vertical_deg':   round(gaze[1], 2),
+                },
+                'action_units': {
+                    au_id: round(float(np.clip(v, 0.0, 1.0)), 4)
+                    for au_id, v in zip(AU_IDS, au_vals)
+                },
+            })
+
+        return {
+            'frame': frame_n,
+            'ts':    round(ts, 4),
+            'faces': faces,
+            'timings': {
+                'detect':  {'total_ms': round(timings['detect_ms'],       2),
+                            'infer_ms': round(timings['detect_infer_ms'], 2)},
+                'align':   {'total_ms': round(timings['align_ms'],        2),
+                            'infer_ms': round(timings['align_infer_ms'],  2)},
+                'analyze': {'total_ms': round(timings['analyze_ms'],      2),
+                            'infer_ms': round(timings['analyze_infer_ms'],2)},
+                'total_ms': round(timings['total_ms'], 2),
+            },
+        }

openface3-0.1.0/pyproject.toml

@@ -0,0 +1,39 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "openface3"
+version = "0.1.0"
+description = "Facial analysis pipeline: detection, 98-point alignment, and multitask emotion/gaze/AU inference"
+readme = "README.md"
+requires-python = ">=3.9"
+license = { file = "LICENSE" }
+keywords = ["face", "emotion", "gaze", "action-units", "facial-analysis"]
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "License :: OSI Approved :: MIT License",
+    "Operating System :: OS Independent",
+    "Topic :: Scientific/Engineering :: Artificial Intelligence",
+]
+dependencies = [
+    "torch>=2.0",
+    "torchvision>=0.15",
+    "numpy",
+    "opencv-python",
+    "albumentations",
+    "iglovikov-helper-functions",
+    "timm",
+    "huggingface-hub",
+]
+
+[project.urls]
+Homepage  = "https://github.com/haneeshbyreddy/openface3"
+Issues    = "https://github.com/haneeshbyreddy/openface3/issues"
+HuggingFace = "https://huggingface.co/haneeshbyreddy/openface3-weights"
+
+[tool.hatch.build.targets.wheel]
+packages = ["openface3"]
+
+[tool.hatch.build.targets.sdist]
+include = ["openface3/"]