syncnet-python 0.1.0__py3-none-any.whl

syncnet_python/detectors/s3fd/box_utils.py

@@ -0,0 +1,233 @@
+from itertools import product as product
+
+import numpy as np
+import torch
+
+
+def nms_(dets, thresh):
+    """
+    Courtesy of Ross Girshick
+    [https://github.com/rbgirshick/py-faster-rcnn/blob/master/lib/nms/py_cpu_nms.py]
+    """
+    x1 = dets[:, 0]
+    y1 = dets[:, 1]
+    x2 = dets[:, 2]
+    y2 = dets[:, 3]
+    scores = dets[:, 4]
+
+    areas = (x2 - x1) * (y2 - y1)
+    order = scores.argsort()[::-1]
+
+    keep = []
+    while order.size > 0:
+        i = order[0]
+        keep.append(int(i))
+        xx1 = np.maximum(x1[i], x1[order[1:]])
+        yy1 = np.maximum(y1[i], y1[order[1:]])
+        xx2 = np.minimum(x2[i], x2[order[1:]])
+        yy2 = np.minimum(y2[i], y2[order[1:]])
+
+        w = np.maximum(0.0, xx2 - xx1)
+        h = np.maximum(0.0, yy2 - yy1)
+        inter = w * h
+        ovr = inter / (areas[i] + areas[order[1:]] - inter)
+
+        inds = np.where(ovr <= thresh)[0]
+        order = order[inds + 1]
+
+    return np.array(keep).astype(int)
+
+
+def decode(loc, priors, variances):
+    """Decode locations from predictions using priors to undo
+    the encoding we did for offset regression at train time.
+    Args:
+        loc (tensor): location predictions for loc layers,
+            Shape: [num_priors,4]
+        priors (tensor): Prior boxes in center-offset form.
+            Shape: [num_priors,4].
+        variances: (list[float]) Variances of priorboxes
+    Return:
+        decoded bounding box predictions
+    """
+
+    boxes = torch.cat(
+        (
+            priors[:, :2] + loc[:, :2] * variances[0] * priors[:, 2:],
+            priors[:, 2:] * torch.exp(loc[:, 2:] * variances[1]),
+        ),
+        1,
+    )
+    boxes[:, :2] -= boxes[:, 2:] / 2
+    boxes[:, 2:] += boxes[:, :2]
+    return boxes
+
+
+def nms(boxes, scores, overlap=0.5, top_k=200):
+    """Apply non-maximum suppression at test time to avoid detecting too many
+    overlapping bounding boxes for a given object.
+    Args:
+        boxes: (tensor) The location preds for the img, Shape: [num_priors,4].
+        scores: (tensor) The class predscores for the img, Shape:[num_priors].
+        overlap: (float) The overlap thresh for suppressing unnecessary boxes.
+        top_k: (int) The Maximum number of box preds to consider.
+    Return:
+        The indices of the kept boxes with respect to num_priors.
+    """
+
+    keep = scores.new(scores.size(0)).zero_().long()
+    if boxes.numel() == 0:
+        return keep, 0
+    x1 = boxes[:, 0]
+    y1 = boxes[:, 1]
+    x2 = boxes[:, 2]
+    y2 = boxes[:, 3]
+    area = torch.mul(x2 - x1, y2 - y1)
+    v, idx = scores.sort(0)  # sort in ascending order
+    # I = I[v >= 0.01]
+    idx = idx[-top_k:]  # indices of the top-k largest vals
+    xx1 = boxes.new()
+    yy1 = boxes.new()
+    xx2 = boxes.new()
+    yy2 = boxes.new()
+    w = boxes.new()
+    h = boxes.new()
+
+    # keep = torch.Tensor()
+    count = 0
+    while idx.numel() > 0:
+        i = idx[-1]  # index of current largest val
+        # keep.append(i)
+        keep[count] = i
+        count += 1
+        if idx.size(0) == 1:
+            break
+        idx = idx[:-1]  # remove kept element from view
+        # load bboxes of next highest vals
+        xx1.resize_(idx.size(0))
+        yy1.resize_(idx.size(0))
+        xx2.resize_(idx.size(0))
+        yy2.resize_(idx.size(0))
+        
+        torch.index_select(x1, 0, idx, out=xx1)
+        torch.index_select(y1, 0, idx, out=yy1)
+        torch.index_select(x2, 0, idx, out=xx2)
+        torch.index_select(y2, 0, idx, out=yy2)
+        # store element-wise max with next highest score
+        xx1 = torch.clamp(xx1, min=x1[i])
+        yy1 = torch.clamp(yy1, min=y1[i])
+        xx2 = torch.clamp(xx2, max=x2[i])
+        yy2 = torch.clamp(yy2, max=y2[i])
+        w.resize_as_(xx2)
+        h.resize_as_(yy2)
+        w = xx2 - xx1
+        h = yy2 - yy1
+        # check sizes of xx1 and xx2.. after each iteration
+        w = torch.clamp(w, min=0.0)
+        h = torch.clamp(h, min=0.0)
+        inter = w * h
+        # IoU = i / (area(a) + area(b) - i)
+        rem_areas = torch.index_select(area, 0, idx)  # load remaining areas)
+        union = (rem_areas - inter) + area[i]
+        IoU = inter / union  # store result in iou
+        # keep only elements with an IoU <= overlap
+        idx = idx[IoU.le(overlap)]
+    return keep, count
+
+
+class Detect(object):
+    def __init__(
+        self,
+        num_classes=2,
+        top_k=750,
+        nms_thresh=0.3,
+        conf_thresh=0.05,
+        variance=[0.1, 0.2],
+        nms_top_k=5000,
+    ):
+        self.num_classes = num_classes
+        self.top_k = top_k
+        self.nms_thresh = nms_thresh
+        self.conf_thresh = conf_thresh
+        self.variance = variance
+        self.nms_top_k = nms_top_k
+
+    def forward(self, loc_data, conf_data, prior_data):
+        num = loc_data.size(0)
+        num_priors = prior_data.size(0)
+
+        conf_preds = conf_data.view(num, num_priors, self.num_classes).transpose(2, 1)
+        batch_priors = prior_data.view(-1, num_priors, 4).expand(num, num_priors, 4)
+        batch_priors = batch_priors.contiguous().view(-1, 4)
+
+        decoded_boxes = decode(loc_data.view(-1, 4), batch_priors, self.variance)
+        decoded_boxes = decoded_boxes.view(num, num_priors, 4)
+
+        output = torch.zeros(num, self.num_classes, self.top_k, 5)
+
+        for i in range(num):
+            boxes = decoded_boxes[i].clone()
+            conf_scores = conf_preds[i].clone()
+
+            for cl in range(1, self.num_classes):
+                c_mask = conf_scores[cl].gt(self.conf_thresh)
+                scores = conf_scores[cl][c_mask]
+
+                if scores.dim() == 0:
+                    continue
+                l_mask = c_mask.unsqueeze(1).expand_as(boxes)
+                boxes_ = boxes[l_mask].view(-1, 4)
+                ids, count = nms(boxes_, scores, self.nms_thresh, self.nms_top_k)
+                count = count if count < self.top_k else self.top_k
+
+                output[i, cl, :count] = torch.cat(
+                    (scores[ids[:count]].unsqueeze(1), boxes_[ids[:count]]), 1
+                )
+
+        return output
+
+
+class PriorBox(object):
+    def __init__(
+        self,
+        input_size,
+        feature_maps,
+        variance=[0.1, 0.2],
+        min_sizes=[16, 32, 64, 128, 256, 512],
+        steps=[4, 8, 16, 32, 64, 128],
+        clip=False,
+    ):
+        super(PriorBox, self).__init__()
+
+        self.imh = input_size[0]
+        self.imw = input_size[1]
+        self.feature_maps = feature_maps
+
+        self.variance = variance
+        self.min_sizes = min_sizes
+        self.steps = steps
+        self.clip = clip
+
+    def forward(self):
+        mean = []
+        for k, fmap in enumerate(self.feature_maps):
+            feath = fmap[0]
+            featw = fmap[1]
+            for i, j in product(range(feath), range(featw)):
+                f_kw = self.imw / self.steps[k]
+                f_kh = self.imh / self.steps[k]
+
+                cx = (j + 0.5) / f_kw
+                cy = (i + 0.5) / f_kh
+
+                s_kw = self.min_sizes[k] / self.imw
+                s_kh = self.min_sizes[k] / self.imh
+
+                mean += [cx, cy, s_kw, s_kh]
+
+        output = torch.FloatTensor(mean).view(-1, 4)
+
+        if self.clip:
+            output.clamp_(max=1, min=0)
+
+        return output

syncnet_python/detectors/s3fd/nets.py

@@ -0,0 +1,177 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import torch.nn.init as init
+
+from .box_utils import Detect, PriorBox
+
+
+class L2Norm(nn.Module):
+    def __init__(self, n_channels, scale):
+        super(L2Norm, self).__init__()
+        self.n_channels = n_channels
+        self.gamma = scale or None
+        self.eps = 1e-10
+        self.weight = nn.Parameter(torch.Tensor(self.n_channels))
+        self.reset_parameters()
+
+    def reset_parameters(self):
+        init.constant_(self.weight, self.gamma)
+
+    def forward(self, x):
+        norm = x.pow(2).sum(dim=1, keepdim=True).sqrt() + self.eps
+        x = torch.div(x, norm)
+        out = self.weight.unsqueeze(0).unsqueeze(2).unsqueeze(3).expand_as(x) * x
+        return out
+
+
+class S3FDNet(nn.Module):
+    def __init__(self, device="cuda"):
+        super(S3FDNet, self).__init__()
+        self.device = device
+
+        self.vgg = nn.ModuleList(
+            [
+                nn.Conv2d(3, 64, 3, 1, padding=1),
+                nn.ReLU(inplace=True),
+                nn.Conv2d(64, 64, 3, 1, padding=1),
+                nn.ReLU(inplace=True),
+                nn.MaxPool2d(2, 2),
+                nn.Conv2d(64, 128, 3, 1, padding=1),
+                nn.ReLU(inplace=True),
+                nn.Conv2d(128, 128, 3, 1, padding=1),
+                nn.ReLU(inplace=True),
+                nn.MaxPool2d(2, 2),
+                nn.Conv2d(128, 256, 3, 1, padding=1),
+                nn.ReLU(inplace=True),
+                nn.Conv2d(256, 256, 3, 1, padding=1),
+                nn.ReLU(inplace=True),
+                nn.Conv2d(256, 256, 3, 1, padding=1),
+                nn.ReLU(inplace=True),
+                nn.MaxPool2d(2, 2, ceil_mode=True),
+                nn.Conv2d(256, 512, 3, 1, padding=1),
+                nn.ReLU(inplace=True),
+                nn.Conv2d(512, 512, 3, 1, padding=1),
+                nn.ReLU(inplace=True),
+                nn.Conv2d(512, 512, 3, 1, padding=1),
+                nn.ReLU(inplace=True),
+                nn.MaxPool2d(2, 2),
+                nn.Conv2d(512, 512, 3, 1, padding=1),
+                nn.ReLU(inplace=True),
+                nn.Conv2d(512, 512, 3, 1, padding=1),
+                nn.ReLU(inplace=True),
+                nn.Conv2d(512, 512, 3, 1, padding=1),
+                nn.ReLU(inplace=True),
+                nn.MaxPool2d(2, 2),
+                nn.Conv2d(512, 1024, 3, 1, padding=6, dilation=6),
+                nn.ReLU(inplace=True),
+                nn.Conv2d(1024, 1024, 1, 1),
+                nn.ReLU(inplace=True),
+            ]
+        )
+
+        self.L2Norm3_3 = L2Norm(256, 10)
+        self.L2Norm4_3 = L2Norm(512, 8)
+        self.L2Norm5_3 = L2Norm(512, 5)
+
+        self.extras = nn.ModuleList(
+            [
+                nn.Conv2d(1024, 256, 1, 1),
+                nn.Conv2d(256, 512, 3, 2, padding=1),
+                nn.Conv2d(512, 128, 1, 1),
+                nn.Conv2d(128, 256, 3, 2, padding=1),
+            ]
+        )
+
+        self.loc = nn.ModuleList(
+            [
+                nn.Conv2d(256, 4, 3, 1, padding=1),
+                nn.Conv2d(512, 4, 3, 1, padding=1),
+                nn.Conv2d(512, 4, 3, 1, padding=1),
+                nn.Conv2d(1024, 4, 3, 1, padding=1),
+                nn.Conv2d(512, 4, 3, 1, padding=1),
+                nn.Conv2d(256, 4, 3, 1, padding=1),
+            ]
+        )
+
+        self.conf = nn.ModuleList(
+            [
+                nn.Conv2d(256, 4, 3, 1, padding=1),
+                nn.Conv2d(512, 2, 3, 1, padding=1),
+                nn.Conv2d(512, 2, 3, 1, padding=1),
+                nn.Conv2d(1024, 2, 3, 1, padding=1),
+                nn.Conv2d(512, 2, 3, 1, padding=1),
+                nn.Conv2d(256, 2, 3, 1, padding=1),
+            ]
+        )
+
+        self.softmax = nn.Softmax(dim=-1)
+        self.detect = Detect()
+
+    def forward(self, x):
+        x = x.to(self.device)
+        size = x.size()[2:]
+        sources = list()
+        loc = list()
+        conf = list()
+
+        for k in range(16):
+            x = self.vgg[k](x)
+        s = self.L2Norm3_3(x)
+        sources.append(s)
+
+        for k in range(16, 23):
+            x = self.vgg[k](x)
+        s = self.L2Norm4_3(x)
+        sources.append(s)
+
+        for k in range(23, 30):
+            x = self.vgg[k](x)
+        s = self.L2Norm5_3(x)
+        sources.append(s)
+
+        for k in range(30, len(self.vgg)):
+            x = self.vgg[k](x)
+        sources.append(x)
+
+        # apply extra layers and cache source layer outputs
+        for k, v in enumerate(self.extras):
+            x = F.relu(v(x), inplace=True)
+            if k % 2 == 1:
+                sources.append(x)
+
+        # apply multibox head to source layers
+        loc_x = self.loc[0](sources[0])
+        conf_x = self.conf[0](sources[0])
+
+        max_conf, _ = torch.max(conf_x[:, 0:3, :, :], dim=1, keepdim=True)
+        conf_x = torch.cat((max_conf, conf_x[:, 3:, :, :]), dim=1)
+
+        loc.append(loc_x.permute(0, 2, 3, 1).contiguous())
+        conf.append(conf_x.permute(0, 2, 3, 1).contiguous())
+
+        for i in range(1, len(sources)):
+            x = sources[i]
+            conf.append(self.conf[i](x).permute(0, 2, 3, 1).contiguous())
+            loc.append(self.loc[i](x).permute(0, 2, 3, 1).contiguous())
+
+        features_maps = []
+        for i in range(len(loc)):
+            feat = []
+            feat += [loc[i].size(1), loc[i].size(2)]
+            features_maps += [feat]
+
+        loc = torch.cat([o.view(o.size(0), -1) for o in loc], 1)
+        conf = torch.cat([o.view(o.size(0), -1) for o in conf], 1)
+
+        with torch.no_grad():
+            self.priorbox = PriorBox(size, features_maps)
+            self.priors = self.priorbox.forward()
+
+        output = self.detect.forward(
+            loc.view(loc.size(0), -1, 4),
+            self.softmax(conf.view(conf.size(0), -1, 2)),
+            self.priors.type(type(x.data)).to(self.device),
+        )
+
+        return output

syncnet_python/run_syncnet_pipeline_on_1example.py

@@ -0,0 +1,28 @@
+from syncnet_pipeline import SyncNetPipeline   # file you just saved
+import logging
+logging.basicConfig(
+    level=logging.INFO,  # Change to DEBUG if you want more detail
+    format="%(asctime)s [%(levelname)s] %(message)s"
+)
+
+# 1.  Initialise the pipeline  (put your weight files in the same folder or give absolute paths)
+pipe = SyncNetPipeline(
+    {
+        "s3fd_weights":  "../weights/sfd_face.pth",
+        "syncnet_weights": "../weights/syncnet_v2.model",
+    },
+    device="cuda",          # or "cpu"
+)
+
+# 2.  Run inference on one clip
+results = pipe.inference(
+    video_path="../example/video.avi",   # RGB video
+    audio_path="../example/speech.wav",   # speech track (any ffmpeg-readable format)
+    cache_dir="../example/cache",    # optional; omit to auto-cleanup intermediates
+)
+
+# 3.  Inspect outputs
+offsets, confs, dists, max_conf, min_dist, s3fd_json, has_face = results
+print("best-confidence   :", max_conf)
+print("lowest distance   :", min_dist)
+print("per-crop offsets  :", offsets)

syncnet_python/run_syncnet_pipeline_on_mocha_generation_on_mocha_bench.py

@@ -0,0 +1,157 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+Batch SyncNet evaluation for MoChaBench.
+"""
+
+import json
+import csv
+from pathlib import Path
+import time
+from collections import defaultdict
+
+import pandas as pd
+from syncnet_pipeline import SyncNetPipeline
+
+# ------------------------------------------------------------------ #
+# 1)  paths & config                                                 #
+# ------------------------------------------------------------------ #
+
+# folder that contains *this* script
+ROOT = Path(__file__).resolve().parents[2] # ROOT = Path(r"/full/path/to/MoChaBench") <<< fill with your own repo path/MoChaBench
+
+# ---- adapt these if you move folders around ---------------------- #
+BASE_VIDEO = ROOT / "mocha-generation"       # e.g. …/MoChaBench/mocha-generation
+BASE_BENCHMARK = ROOT / "benchmark"              # parent folder that contains 'speeches/'
+CSV_FILE   = BASE_BENCHMARK / "benchmark.csv"
+# ------------------------------------------------------------------ #
+
+OUT_CSV  = ROOT / "eval-lipsync" / "mocha-eval-results" / "sync_scores.csv"
+OUT_JSON = ROOT / "eval-lipsync"/  "mocha-eval-results" / "sync_scores.json"
+
+pipe = SyncNetPipeline(
+    {
+        "s3fd_weights":  ROOT / "eval-lipsync" / "weights" / "sfd_face.pth",
+        "syncnet_weights": ROOT / "eval-lipsync" /"weights" / "syncnet_v2.model",
+    },
+    device="cuda",
+)
+
+# category buckets for your extra means
+ENGLISH_1P_CATEGORIES = {
+    "1p_closeup_facingcamera",
+    "1p_camera_movement",
+    "1p_emotion",
+    "1p_mediumshot_actioncontrol",
+    "2p_1clip_1talk",
+    "1p_protrait",
+}
+TURNTALK_CATEGORY = {"2p_2clip_2talk"}
+
+# ------------------------------------------------------------------ #
+# 2)  helper: run one sample                                         #
+# ------------------------------------------------------------------ #
+def run_sample(row):
+    idx = int(row["idx_in_category"])
+    cat = row["category"].strip()
+    base_name = row["context_id"].strip()  # e.g. 1_man_bag_of_gold
+    id = f"{cat}_{base_name}"
+    video_fp = BASE_VIDEO / cat / f"{base_name}.mp4"
+    audio_fp = BASE_BENCHMARK / "speeches" / cat / f"{base_name}_speech.wav"
+
+    if not video_fp.exists():
+        raise FileNotFoundError(f"Video not found: {video_fp}")
+    if not audio_fp.exists():
+        raise FileNotFoundError(f"Audio not found: {audio_fp}")
+
+    t0 = time.time()
+    off, confs, dists, best_conf, min_dist, _, has_face = pipe.inference(
+        video_path=str(video_fp),
+        audio_path=str(audio_fp),
+        cache_dir= ROOT / "eval-lipsync"/  "mocha-eval-results" / "cache" / id
+    )
+    return {
+        "idx": idx,
+        "category": cat,
+        "video": str(Path(cat) / f"{base_name}.mp4"),
+        "audio": str(Path(cat) / f"{base_name}_speech.wav"),
+        "offsets": [int(o) for o in off],
+        "best_conf": float(best_conf),
+        "min_dist": float(min_dist),
+        "has_face": has_face,
+        "runtime_s": round(time.time() - t0, 2),
+    }
+
+
+# ------------------------------------------------------------------ #
+# 3)  main loop                                                      #
+# ------------------------------------------------------------------ #
+def main():
+    df = pd.read_csv(CSV_FILE)
+    results = []
+    for _, row in df.iterrows():
+        try:
+            res = run_sample(row)
+            results.append(res)
+            print(
+                f"[{res['idx']}] {res['category']}  "
+                f"Δ={res['offsets']}  conf={res['best_conf']:.3f}  "
+                f"dist={res['min_dist']:.3f}"
+            )
+        except Exception as e:
+            print(f"[{row['idx']}] ERROR – {e}")
+
+    # ------------------------------------------------------------------ #
+    # 4)  save full table                                                #
+    # ------------------------------------------------------------------ #
+    pd.DataFrame(results).to_csv(OUT_CSV, index=False)
+    with open(OUT_JSON, "w") as f:
+        json.dump(results, f, indent=2)
+
+    # ------------------------------------------------------------------ #
+    # 5)  category aggregates                                            #
+    # ------------------------------------------------------------------ #
+    # Separate stats for each category (only when face is detected)
+    cat_dists = defaultdict(list)
+    cat_confs = defaultdict(list)
+
+    for r in results:
+        # !!!! sometimes SyncNetPipeline fails to detect faces, we should not inlcude those
+        if r["has_face"]:
+            cat_dists[r["category"]].append(r["min_dist"])
+            cat_confs[r["category"]].append(r["best_conf"])
+
+    def mean(vals):
+        return sum(vals) / len(vals) if vals else float("nan")
+
+    print("\n=== per-category averages (only if face detected) ===")
+    for cat in sorted(set(cat_dists.keys()) | set(cat_confs.keys())):
+        avg_dist = mean(cat_dists[cat])
+        avg_conf = mean(cat_confs[cat])
+        print(f"{cat:30s}:  dist={avg_dist:.3f}   conf={avg_conf:.3f}   (n={len(cat_dists[cat])})")
+
+
+    #  super-sets
+    english_dists = [
+        d for c, v in cat_dists.items() if c in ENGLISH_1P_CATEGORIES for d in v
+    ]
+    english_confs = [
+        c for cat, v in cat_confs.items() if cat in ENGLISH_1P_CATEGORIES for c in v
+    ]
+
+    dialog_dists = [
+        d for c, v in cat_dists.items() if c in TURNTALK_CATEGORY for d in v
+    ]
+    dialog_confs = [
+        c for cat, v in cat_confs.items() if cat in TURNTALK_CATEGORY for c in v
+    ]
+    print("\n--- aggregate groups (only face-detected entries) ---")
+    print(f"single-character English (1p*): dist={mean(english_dists):.3f}  conf={mean(english_confs):.3f}")
+    print(f"turn-based dialogue English (2p_2clip): dist={mean(dialog_dists):.3f}  conf={mean(dialog_confs):.3f}")
+
+    print(f"\nSaved detailed table → {OUT_CSV}")
+    print(f"Saved JSON dump      → {OUT_JSON}")
+
+
+if __name__ == "__main__":
+    main()