syncnet-python 0.1.0__py3-none-any.whl

syncnet_python/SyncNetInstance.py

@@ -0,0 +1,210 @@
+import glob
+import math
+import os
+import subprocess
+import time
+
+import cv2
+import numpy
+import python_speech_features
+import torch
+
+from scipy import signal
+from scipy.io import wavfile
+
+
+# ==================== Get OFFSET ====================
+
+
+def calc_pdist(feat1, feat2, vshift=10):
+    win_size = vshift * 2 + 1
+
+    feat2p = torch.nn.functional.pad(feat2, (0, 0, vshift, vshift))
+
+    dists = []
+
+    for i in range(0, len(feat1)):
+        dists.append(
+            torch.nn.functional.pairwise_distance(
+                feat1[[i], :].repeat(win_size, 1), feat2p[i : i + win_size, :]
+            )
+        )
+
+    return dists
+
+
+# ==================== MAIN DEF ====================
+
+
+class SyncNetInstance(torch.nn.Module):
+    def __init__(
+        self,
+        net: torch.nn.Module,
+        device: str = "cuda",
+        dropout: float = 0,
+        num_layers_in_fc_layers: int = 1024,
+    ):
+        super(SyncNetInstance, self).__init__()
+        self.__S__ = net
+        self.device = device
+
+    def evaluate(self, opt):
+        self.__S__.to(self.device)
+        self.__S__.eval()
+
+        # ========== ==========
+        # Load video
+        # ========== ==========
+
+        images = []
+
+        flist = glob.glob(os.path.join(opt.tmp_dir, "*.jpg"))
+        flist.sort()
+
+        for fname in flist:
+            images.append(cv2.imread(fname))
+
+        im = numpy.stack(images, axis=3)
+        im = numpy.expand_dims(im, axis=0)
+        im = numpy.transpose(im, (0, 3, 4, 1, 2))
+
+        imtv = torch.autograd.Variable(torch.from_numpy(im.astype(float)).float())
+
+        # ========== ==========
+        # Load audio
+        # ========== ==========
+
+        sample_rate, audio = wavfile.read(os.path.join(opt.tmp_dir, "audio.wav"))
+        mfcc = zip(*python_speech_features.mfcc(audio, sample_rate))
+        mfcc = numpy.stack([numpy.array(i) for i in mfcc])
+
+        cc = numpy.expand_dims(numpy.expand_dims(mfcc, axis=0), axis=0)
+        cct = torch.autograd.Variable(torch.from_numpy(cc.astype(float)).float())
+
+        # ========== ==========
+        # Check audio and video input length
+        # ========== ==========
+
+        if (float(len(audio)) / 16000) != (float(len(images)) / 25):
+            print(
+                "WARNING: Audio (%.4fs) and video (%.4fs) lengths are different."
+                % (float(len(audio)) / 16000, float(len(images)) / 25)
+            )
+
+        min_length = min(len(images), math.floor(len(audio) / 640))
+
+        # ========== ==========
+        # Generate video and audio feats
+        # ========== ==========
+
+        lastframe = min_length - 5
+        im_feat = []
+        cc_feat = []
+
+        tS = time.time()
+        for i in range(0, lastframe, opt.batch_size):
+            im_batch = [
+                imtv[:, :, vframe : vframe + 5, :, :]
+                for vframe in range(i, min(lastframe, i + opt.batch_size))
+            ]
+            im_in = torch.cat(im_batch, 0).to(self.device)
+            im_out = self.__S__.forward_lip(im_in)
+            im_feat.append(im_out.data.cpu())
+
+            cc_batch = [
+                cct[:, :, :, vframe * 4 : vframe * 4 + 20]
+                for vframe in range(i, min(lastframe, i + opt.batch_size))
+            ]
+            cc_in = torch.cat(cc_batch, 0).to(self.device)
+            cc_out = self.__S__.forward_aud(cc_in)
+            cc_feat.append(cc_out.data.cpu())
+
+        im_feat = torch.cat(im_feat, 0)
+        cc_feat = torch.cat(cc_feat, 0)
+
+        # ========== ==========
+        # Compute offset
+        # ========== ==========
+
+        print("Compute time %.3f sec." % (time.time() - tS))
+
+        dists = calc_pdist(im_feat, cc_feat, vshift=opt.vshift)
+        mdist = torch.mean(torch.stack(dists, 1), 1)
+
+        minval, minidx = torch.min(mdist, 0)
+
+        offset = opt.vshift - minidx
+        conf = torch.median(mdist) - minval
+
+        fdist = numpy.stack([dist[minidx].numpy() for dist in dists])
+        # fdist   = numpy.pad(fdist, (3,3), 'constant', constant_values=15)
+        fconf = torch.median(mdist).numpy() - fdist
+        fconfm = signal.medfilt(fconf, kernel_size=9)
+
+        numpy.set_printoptions(formatter={"float": "{: 0.3f}".format})
+        print("Framewise conf: ")
+        print(fconfm)
+        print(
+            "AV offset: \t%d \nMin dist: \t%.3f\nConfidence: \t%.3f"
+            % (offset, minval, conf)
+        )
+
+        # dists_npy = numpy.array([dist.numpy() for dist in dists])
+        # return offset.numpy(), conf.numpy(), minval.numpy()
+        return int(offset), float(conf), float(minval)
+
+    def extract_feature(self, opt, videofile):
+        self.__S__.eval()
+        # ========== ==========
+        # Load video
+        # ========== ==========
+        cap = cv2.VideoCapture(videofile)
+
+        frame_num = 1
+        images = []
+        while frame_num:
+            frame_num += 1
+            ret, image = cap.read()
+            if ret == 0:
+                break
+
+            images.append(image)
+
+        im = numpy.stack(images, axis=3)
+        im = numpy.expand_dims(im, axis=0)
+        im = numpy.transpose(im, (0, 3, 4, 1, 2))
+
+        imtv = torch.autograd.Variable(torch.from_numpy(im.astype(float)).float())
+
+        # ========== ==========
+        # Generate video feats
+        # ========== ==========
+
+        lastframe = len(images) - 4
+        im_feat = []
+
+        tS = time.time()
+        for i in range(0, lastframe, opt.batch_size):
+            im_batch = [
+                imtv[:, :, vframe : vframe + 5, :, :]
+                for vframe in range(i, min(lastframe, i + opt.batch_size))
+            ]
+            im_in = torch.cat(im_batch, 0).to(self.device)
+            im_out = self.__S__.forward_lipfeat(im_in)
+            im_feat.append(im_out.data.cpu())
+
+        im_feat = torch.cat(im_feat, 0)
+
+        # ========== ==========
+        # Compute offset
+        # ========== ==========
+
+        print("Compute time %.3f sec." % (time.time() - tS))
+
+        return im_feat
+
+    def loadParameters(self, path):
+        loaded_state = torch.load(path, map_location=lambda storage, loc: storage)
+        self_state = self.__S__.state_dict()
+        for name, param in loaded_state.items():
+            self_state[name].copy_(param)

syncnet_python/SyncNetModel.py

@@ -0,0 +1,99 @@
+import torch
+import torch.nn as nn
+
+
+def save(model, filename):
+    with open(filename, "wb") as f:
+        torch.save(model, f)
+        print("%s saved." % filename)
+
+
+def load(filename):
+    net = torch.load(filename)
+    return net
+
+
+class S(nn.Module):
+    def __init__(self, num_layers_in_fc_layers=1024):
+        super(S, self).__init__()
+        self.__nFeatures__ = 24
+        self.__nChs__ = 32
+        self.__midChs__ = 32
+        self.netcnnaud = nn.Sequential(
+            nn.Conv2d(1, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)),
+            nn.BatchNorm2d(64),
+            nn.ReLU(inplace=True),
+            nn.MaxPool2d(kernel_size=(1, 1), stride=(1, 1)),
+            nn.Conv2d(64, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)),
+            nn.BatchNorm2d(192),
+            nn.ReLU(inplace=True),
+            nn.MaxPool2d(kernel_size=(3, 3), stride=(1, 2)),
+            nn.Conv2d(192, 384, kernel_size=(3, 3), padding=(1, 1)),
+            nn.BatchNorm2d(384),
+            nn.ReLU(inplace=True),
+            nn.Conv2d(384, 256, kernel_size=(3, 3), padding=(1, 1)),
+            nn.BatchNorm2d(256),
+            nn.ReLU(inplace=True),
+            nn.Conv2d(256, 256, kernel_size=(3, 3), padding=(1, 1)),
+            nn.BatchNorm2d(256),
+            nn.ReLU(inplace=True),
+            nn.MaxPool2d(kernel_size=(3, 3), stride=(2, 2)),
+            nn.Conv2d(256, 512, kernel_size=(5, 4), padding=(0, 0)),
+            nn.BatchNorm2d(512),
+            nn.ReLU(),
+        )
+        self.netfcaud = nn.Sequential(
+            nn.Linear(512, 512),
+            nn.BatchNorm1d(512),
+            nn.ReLU(),
+            nn.Linear(512, num_layers_in_fc_layers),
+        )
+        self.netfclip = nn.Sequential(
+            nn.Linear(512, 512),
+            nn.BatchNorm1d(512),
+            nn.ReLU(),
+            nn.Linear(512, num_layers_in_fc_layers),
+        )
+        self.netcnnlip = nn.Sequential(
+            nn.Conv3d(3, 96, kernel_size=(5, 7, 7), stride=(1, 2, 2), padding=0),
+            nn.BatchNorm3d(96),
+            nn.ReLU(inplace=True),
+            nn.MaxPool3d(kernel_size=(1, 3, 3), stride=(1, 2, 2)),
+            nn.Conv3d(
+                96, 256, kernel_size=(1, 5, 5), stride=(1, 2, 2), padding=(0, 1, 1)
+            ),
+            nn.BatchNorm3d(256),
+            nn.ReLU(inplace=True),
+            nn.MaxPool3d(kernel_size=(1, 3, 3), stride=(1, 2, 2), padding=(0, 1, 1)),
+            nn.Conv3d(256, 256, kernel_size=(1, 3, 3), padding=(0, 1, 1)),
+            nn.BatchNorm3d(256),
+            nn.ReLU(inplace=True),
+            nn.Conv3d(256, 256, kernel_size=(1, 3, 3), padding=(0, 1, 1)),
+            nn.BatchNorm3d(256),
+            nn.ReLU(inplace=True),
+            nn.Conv3d(256, 256, kernel_size=(1, 3, 3), padding=(0, 1, 1)),
+            nn.BatchNorm3d(256),
+            nn.ReLU(inplace=True),
+            nn.MaxPool3d(kernel_size=(1, 3, 3), stride=(1, 2, 2)),
+            nn.Conv3d(256, 512, kernel_size=(1, 6, 6), padding=0),
+            nn.BatchNorm3d(512),
+            nn.ReLU(inplace=True),
+        )
+
+    def forward_aud(self, x):
+        mid = self.netcnnaud(x)  # N x ch x 24 x M
+        mid = mid.view((mid.size()[0], -1))  # N x (ch x 24)
+        out = self.netfcaud(mid)
+        return out
+
+    def forward_lip(self, x):
+        mid = self.netcnnlip(x)
+        mid = mid.view((mid.size()[0], -1))  # N x (ch x 24)
+        out = self.netfclip(mid)
+        return out
+
+    def forward_lipfeat(self, x):
+        mid = self.netcnnlip(x)
+        out = mid.view((mid.size()[0], -1))  # N x (ch x 24)
+
+        return out

syncnet_python/__init__.py

@@ -0,0 +1,29 @@
+"""SyncNet Python: Audio-visual synchronization detection.
+
+This package provides a PyTorch implementation of SyncNet for detecting
+synchronization between audio and video in multimedia content.
+"""
+
+__version__ = "0.1.0"
+
+# Import main components
+try:
+    from .syncnet_pipeline import SyncNetPipeline
+    from .SyncNetModel import S as SyncNetModel
+    from .SyncNetInstance import SyncNetInstance
+except ImportError:
+    # Fallback for development
+    SyncNetPipeline = None
+    SyncNetModel = None
+    SyncNetInstance = None
+
+__all__ = [
+    "SyncNetPipeline",
+    "SyncNetModel", 
+    "SyncNetInstance",
+    "__version__"
+]
+
+def get_version():
+    """Get package version."""
+    return __version__

syncnet_python/cli.py

@@ -0,0 +1,127 @@
+#!/usr/bin/env python3
+"""Command-line interface for SyncNet Python."""
+
+import argparse
+import json
+import sys
+from pathlib import Path
+
+from .syncnet_pipeline import SyncNetPipeline
+
+
+def main():
+    """Main CLI entry point."""
+    parser = argparse.ArgumentParser(
+        description="SyncNet: Audio-visual synchronization detection",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog="""
+Examples:
+  # Process a single video
+  syncnet-python video.mp4
+  
+  # Process multiple videos
+  syncnet-python video1.mp4 video2.mp4 --output results.json
+  
+  # Use CPU instead of GPU
+  syncnet-python video.mp4 --device cpu
+        """
+    )
+    
+    parser.add_argument(
+        'videos',
+        nargs='+',
+        help='Video file(s) to process'
+    )
+    
+    parser.add_argument(
+        '-o', '--output',
+        help='Output JSON file for results'
+    )
+    
+    parser.add_argument(
+        '--device',
+        default='cuda',
+        choices=['cuda', 'cpu'],
+        help='Device to run models on (default: cuda)'
+    )
+    
+    parser.add_argument(
+        '--s3fd-weights',
+        default='weights/sfd_face.pth',
+        help='Path to S3FD weights'
+    )
+    
+    parser.add_argument(
+        '--syncnet-weights',
+        default='weights/syncnet_v2.model',
+        help='Path to SyncNet weights'
+    )
+    
+    parser.add_argument(
+        '--cache-dir',
+        help='Directory for temporary files'
+    )
+    
+    args = parser.parse_args()
+    
+    # Initialize pipeline
+    try:
+        pipeline = SyncNetPipeline(
+            {
+                "s3fd_weights": args.s3fd_weights,
+                "syncnet_weights": args.syncnet_weights,
+            },
+            device=args.device,
+        )
+    except Exception as e:
+        print(f"Error initializing pipeline: {e}", file=sys.stderr)
+        return 1
+    
+    results_all = []
+    
+    # Process videos
+    for video_path in args.videos:
+        print(f"Processing: {video_path}")
+        
+        try:
+            results = pipeline.inference(
+                video_path=video_path,
+                audio_path=None,
+                cache_dir=args.cache_dir
+            )
+            
+            offsets, confs, dists, max_conf, min_dist, s3fd_json, has_face = results
+            
+            result_dict = {
+                "video": video_path,
+                "status": "success",
+                "offset": int(offsets[0]) if offsets else None,
+                "confidence": float(max_conf),
+                "min_distance": float(min_dist),
+                "has_face": has_face
+            }
+            
+            print(f"  Offset: {offsets[0] if offsets else 'N/A'} frames")
+            print(f"  Confidence: {max_conf:.3f}")
+            
+        except Exception as e:
+            result_dict = {
+                "video": video_path,
+                "status": "error",
+                "error": str(e)
+            }
+            print(f"  Error: {e}")
+        
+        results_all.append(result_dict)
+    
+    # Output results
+    if args.output:
+        with open(args.output, 'w') as f:
+            json.dump(results_all, f, indent=2)
+        print(f"\nResults saved to: {args.output}")
+    
+    return 0
+
+
+if __name__ == '__main__':
+    sys.exit(main())

syncnet_python/detectors/__init__.py

@@ -0,0 +1 @@
+from .s3fd import S3FD

syncnet_python/detectors/s3fd/__init__.py

@@ -0,0 +1,66 @@
+import logging
+import time
+
+import cv2
+import numpy as np
+import torch
+
+from .box_utils import nms_
+from .nets import S3FDNet
+
+img_mean = np.array([104.0, 117.0, 123.0])[:, np.newaxis, np.newaxis].astype("float32")
+
+
+class S3FD:
+    def __init__(self, net: S3FDNet, device="cuda"):
+        """
+        We now accept an *already-initialized* S3FDNet as `net`,
+        instead of loading weights here.
+        """
+        tstamp = time.time()
+        self.device = device
+        self.net = net.to(self.device)
+        self.net.eval()
+        logging.info(
+            f"[S3FD] S3FDNet instance is ready (initialized in {time.time()-tstamp:.4f} sec)."
+        )
+
+    def detect_faces(self, image, conf_th=0.8, scales=[1]):
+        """
+        Same detection code as before, but we no longer load the model here.
+        """
+        self.net.to(self.device)
+        self.net.eval()
+        w, h = image.shape[1], image.shape[0]
+        bboxes = np.empty(shape=(0, 5))
+
+        with torch.no_grad():
+            for s in scales:
+                scaled_img = cv2.resize(
+                    image, dsize=(0, 0), fx=s, fy=s, interpolation=cv2.INTER_LINEAR
+                )
+                scaled_img = np.swapaxes(scaled_img, 1, 2)
+                scaled_img = np.swapaxes(scaled_img, 1, 0)
+                scaled_img = scaled_img[[2, 1, 0], :, :]
+                scaled_img = scaled_img.astype("float32")
+                scaled_img -= img_mean
+                scaled_img = scaled_img[[2, 1, 0], :, :]
+                x = torch.from_numpy(scaled_img).unsqueeze(0).to(self.device)
+
+                y = self.net(x)  # forward pass
+                detections = y.data.to(self.device)
+                scale_tensor = torch.Tensor([w, h, w, h]).to(self.device)
+
+                for i in range(detections.size(1)):
+                    j = 0
+                    while detections[0, i, j, 0] > conf_th:
+                        score = detections[0, i, j, 0].item()
+                        pt = (detections[0, i, j, 1:] * scale_tensor).cpu().numpy()
+                        bbox = (pt[0], pt[1], pt[2], pt[3], score)
+                        bboxes = np.vstack((bboxes, bbox))
+                        j += 1
+
+            # NMS, etc. (unchanged)
+            keep = nms_(bboxes, 0.1)
+            bboxes = bboxes[keep]
+        return bboxes