syncnet-python 0.1.0__tar.gz

syncnet_python-0.1.0/CLAUDE.md

@@ -0,0 +1,79 @@
+# CLAUDE.md
+
+This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
+
+## Project Overview
+
+SyncNet_py313 is a Python 3.13 implementation of SyncNet, a neural network model for audio-visual synchronization detection. It evaluates lip-sync quality in videos by computing synchronization confidence scores between audio and visual features.
+
+## Common Commands
+
+### Setup and Dependencies
+```bash
+# Install dependencies
+pip install -r requirements.txt
+
+# Ensure ffmpeg is installed on the system
+# Ubuntu/Debian: sudo apt-get install ffmpeg
+# macOS: brew install ffmpeg
+```
+
+### Running SyncNet
+
+```bash
+# Run on a single example
+cd script
+python run_syncnet_pipeline_on_1example.py
+
+# Run on MoChaBench evaluation set
+python run_syncnet_pipeline_on_mocha_generation_on_mocha_bench.py
+
+# Run on custom model results
+python run_syncnet_pipeline_on_your_own_model_results.py
+```
+
+## Architecture Overview
+
+### Core Components
+
+1. **SyncNetModel** (`script/SyncNetModel.py`): The neural network architecture
+   - Audio encoder: 1D convolutions + FC layers for audio feature extraction
+   - Visual encoder: 3D convolutions + FC layers for video feature extraction
+   - Outputs 1024-dimensional embeddings for both modalities
+
+2. **SyncNetInstance** (`script/SyncNetInstance.py`): Inference wrapper
+   - Loads pre-trained weights from `weights/syncnet_v2.model`
+   - Calculates synchronization scores using sliding window approach
+   - Returns offset values and confidence scores
+
+3. **Pipeline** (`script/syncnet_pipeline.py`): End-to-end processing
+   - Face detection using S3FD detector
+   - Audio extraction and MFCC feature computation
+   - Video frame extraction and face cropping
+   - Synchronization score calculation
+
+4. **Face Detector** (`script/detectors/s3fd/`): S3FD implementation
+   - Uses pre-trained weights from `weights/sfd_face.pth`
+   - Handles multi-scale face detection
+
+### Data Flow
+
+1. Input: Video file (e.g., `.avi`, `.mp4`)
+2. Face detection → Face tracking → Face cropping
+3. Audio extraction → MFCC features (13 coefficients)
+4. Parallel processing of audio and visual streams
+5. Output: Synchronization scores and offset values
+
+### Key Implementation Details
+
+- Audio: 16kHz sampling, 25ms windows, 10ms stride
+- Video: 25 FPS, 224x224 face crops, 5-frame sequences
+- Embeddings: 1024-dimensional for both modalities
+- Scoring: Cosine similarity between audio/visual embeddings
+
+## Important Notes
+
+- The model requires pre-trained weights in `weights/` directory
+- GPU is strongly recommended for inference (falls back to CPU if unavailable)
+- Face detection is critical - videos without detectable faces will fail
+- The pipeline creates temporary directories for intermediate processing

syncnet_python-0.1.0/INSTALL.md

@@ -0,0 +1,73 @@
+# Installation Guide
+
+## Prerequisites
+
+- Python 3.13 or higher
+- CUDA-capable GPU (optional, but recommended)
+- FFmpeg
+
+## Installation Steps
+
+### 1. Clone the repository
+
+```bash
+git clone https://github.com/yourusername/SyncNet_py313.git
+cd SyncNet_py313
+```
+
+### 2. Create virtual environment
+
+```bash
+python -m venv venv
+source venv/bin/activate  # On Windows: venv\Scripts\activate
+```
+
+### 3. Install dependencies
+
+```bash
+pip install -r requirements.txt
+```
+
+Or install in development mode:
+
+```bash
+pip install -e .
+```
+
+### 4. Download model weights
+
+Place the following files in the `weights/` directory:
+- `sfd_face.pth` - S3FD face detector weights
+- `syncnet_v2.model` - SyncNet model weights
+
+### 5. Verify installation
+
+```bash
+python scripts/run_example.py
+```
+
+## Troubleshooting
+
+### CUDA not available
+
+If you see CUDA-related errors, ensure:
+1. NVIDIA drivers are installed
+2. CUDA toolkit is installed
+3. PyTorch is installed with CUDA support:
+   ```bash
+   pip install torch torchvision --index-url https://download.pytorch.org/whl/cu121
+   ```
+
+### FFmpeg not found
+
+Install FFmpeg:
+- Ubuntu/Debian: `sudo apt-get install ffmpeg`
+- macOS: `brew install ffmpeg`
+- Windows: Download from https://ffmpeg.org/download.html
+
+### Missing dependencies
+
+If you encounter import errors:
+```bash
+pip install --upgrade -r requirements.txt
+```

syncnet_python-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 SyncNet Python 3.13 Contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

syncnet_python-0.1.0/MANIFEST.in

@@ -0,0 +1,33 @@
+include README.md
+include LICENSE
+include INSTALL.md
+include CLAUDE.md
+include requirements.txt
+
+# Include example files
+include example/*.wav
+include example/*.avi
+
+# Include scripts
+recursive-include scripts *.py
+
+# Include package data
+recursive-include syncnet *.py
+recursive-include script *.py
+
+# Exclude unnecessary files
+global-exclude __pycache__
+global-exclude *.py[co]
+global-exclude .DS_Store
+global-exclude *.swp
+global-exclude *.swo
+
+# Exclude weights (too large for PyPI)
+exclude weights/*.pth
+exclude weights/*.model
+
+# Exclude test and cache directories
+prune temp_test
+prune example/cache
+prune .git
+prune .claude

syncnet_python-0.1.0/PKG-INFO

@@ -0,0 +1,150 @@
+Metadata-Version: 2.4
+Name: syncnet-python
+Version: 0.1.0
+Summary: SyncNet: Audio-visual synchronization detection using deep learning
+Author: SyncNet Python Contributors
+Maintainer: SyncNet Python Contributors
+License: MIT
+Project-URL: Homepage, https://github.com/nawta/SyncNet_py313
+Project-URL: Bug Reports, https://github.com/nawta/SyncNet_py313/issues
+Project-URL: Source, https://github.com/nawta/SyncNet_py313
+Keywords: audio-visual,synchronization,deep-learning,pytorch,lip-sync,video-processing,computer-vision
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Science/Research
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Multimedia :: Video
+Classifier: Topic :: Scientific/Engineering :: Image Processing
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: torch>=2.0.0
+Requires-Dist: torchvision>=0.15.0
+Requires-Dist: numpy>=1.24.0
+Requires-Dist: scipy>=1.10.0
+Requires-Dist: pandas>=2.0.0
+Requires-Dist: scenedetect[opencv]>=0.6.0
+Requires-Dist: opencv-contrib-python>=4.8.0
+Requires-Dist: python-speech-features>=0.6
+Requires-Dist: ffmpeg-python>=0.2.0
+Provides-Extra: dev
+Requires-Dist: pytest>=7.4.0; extra == "dev"
+Requires-Dist: pytest-asyncio>=0.21.0; extra == "dev"
+Requires-Dist: black>=23.0.0; extra == "dev"
+Requires-Dist: ruff>=0.1.0; extra == "dev"
+Requires-Dist: mypy>=1.7.0; extra == "dev"
+Requires-Dist: types-opencv-python; extra == "dev"
+Requires-Dist: types-scipy; extra == "dev"
+Dynamic: license-file
+
+# SyncNet Python
+
+Audio-visual synchronization detection using deep learning.
+
+## Overview
+
+SyncNet Python is a PyTorch implementation of the SyncNet model, which detects audio-visual synchronization in videos. It can identify lip-sync errors by analyzing the correspondence between mouth movements and spoken audio.
+
+## Features
+
+- 🎥 **Audio-Visual Sync Detection**: Accurately detect synchronization between audio and video
+- 🔍 **Face Detection**: Automatic face detection and tracking using S3FD
+- 🚀 **Batch Processing**: Process multiple videos efficiently
+- 🐍 **Python API**: Easy-to-use Python interface
+- 📊 **Confidence Scores**: Get confidence metrics for sync quality
+
+## Installation
+
+```bash
+pip install syncnet-python
+```
+
+### Additional Requirements
+
+1. **FFmpeg**: Required for video processing
+   ```bash
+   # Ubuntu/Debian
+   sudo apt-get install ffmpeg
+   
+   # macOS
+   brew install ffmpeg
+   ```
+
+2. **Model Weights**: Download pre-trained weights
+   - Download `sfd_face.pth` and `syncnet_v2.model`
+   - Place them in a `weights/` directory
+
+## Quick Start
+
+```python
+from syncnet_python import SyncNetPipeline
+
+# Initialize pipeline
+pipeline = SyncNetPipeline(
+    s3fd_weights="weights/sfd_face.pth",
+    syncnet_weights="weights/syncnet_v2.model",
+    device="cuda"  # or "cpu"
+)
+
+# Process video
+results = pipeline.inference(
+    video_path="video.mp4",
+    audio_path=None  # Extract from video
+)
+
+# Get results
+offset, confidence = results['offset'], results['confidence']
+print(f"AV Offset: {offset} frames")
+print(f"Confidence: {confidence:.3f}")
+```
+
+## Command Line Usage
+
+```bash
+# Process single video
+syncnet-python video.mp4
+
+# Process multiple videos
+syncnet-python video1.mp4 video2.mp4 --output results.json
+
+# Use CPU instead of GPU
+syncnet-python video.mp4 --device cpu
+```
+
+## Requirements
+
+- Python 3.9+
+- PyTorch 2.0+
+- CUDA (optional but recommended)
+- FFmpeg
+
+## Citation
+
+If you use this code in your research, please cite:
+
+```bibtex
+@inproceedings{chung2016out,
+  title={Out of time: automated lip sync in the wild},
+  author={Chung, Joon Son and Zisserman, Andrew},
+  booktitle={Asian Conference on Computer Vision},
+  year={2016}
+}
+```
+
+## License
+
+MIT License - see LICENSE file for details.
+
+## Links
+
+- GitHub: https://github.com/yourusername/syncnet-python
+- Documentation: https://syncnet-python.readthedocs.io
+- Issues: https://github.com/yourusername/syncnet-python/issues

syncnet_python-0.1.0/README.md

@@ -0,0 +1,103 @@
+# SyncNet Python
+
+Audio-visual synchronization detection using deep learning.
+
+## Overview
+
+SyncNet Python is a PyTorch implementation of the SyncNet model, which detects audio-visual synchronization in videos. It can identify lip-sync errors by analyzing the correspondence between mouth movements and spoken audio.
+
+## Features
+
+- 🎥 **Audio-Visual Sync Detection**: Accurately detect synchronization between audio and video
+- 🔍 **Face Detection**: Automatic face detection and tracking using S3FD
+- 🚀 **Batch Processing**: Process multiple videos efficiently
+- 🐍 **Python API**: Easy-to-use Python interface
+- 📊 **Confidence Scores**: Get confidence metrics for sync quality
+
+## Installation
+
+```bash
+pip install syncnet-python
+```
+
+### Additional Requirements
+
+1. **FFmpeg**: Required for video processing
+   ```bash
+   # Ubuntu/Debian
+   sudo apt-get install ffmpeg
+   
+   # macOS
+   brew install ffmpeg
+   ```
+
+2. **Model Weights**: Download pre-trained weights
+   - Download `sfd_face.pth` and `syncnet_v2.model`
+   - Place them in a `weights/` directory
+
+## Quick Start
+
+```python
+from syncnet_python import SyncNetPipeline
+
+# Initialize pipeline
+pipeline = SyncNetPipeline(
+    s3fd_weights="weights/sfd_face.pth",
+    syncnet_weights="weights/syncnet_v2.model",
+    device="cuda"  # or "cpu"
+)
+
+# Process video
+results = pipeline.inference(
+    video_path="video.mp4",
+    audio_path=None  # Extract from video
+)
+
+# Get results
+offset, confidence = results['offset'], results['confidence']
+print(f"AV Offset: {offset} frames")
+print(f"Confidence: {confidence:.3f}")
+```
+
+## Command Line Usage
+
+```bash
+# Process single video
+syncnet-python video.mp4
+
+# Process multiple videos
+syncnet-python video1.mp4 video2.mp4 --output results.json
+
+# Use CPU instead of GPU
+syncnet-python video.mp4 --device cpu
+```
+
+## Requirements
+
+- Python 3.9+
+- PyTorch 2.0+
+- CUDA (optional but recommended)
+- FFmpeg
+
+## Citation
+
+If you use this code in your research, please cite:
+
+```bibtex
+@inproceedings{chung2016out,
+  title={Out of time: automated lip sync in the wild},
+  author={Chung, Joon Son and Zisserman, Andrew},
+  booktitle={Asian Conference on Computer Vision},
+  year={2016}
+}
+```
+
+## License
+
+MIT License - see LICENSE file for details.
+
+## Links
+
+- GitHub: https://github.com/yourusername/syncnet-python
+- Documentation: https://syncnet-python.readthedocs.io
+- Issues: https://github.com/yourusername/syncnet-python/issues

syncnet_python-0.1.0/pyproject.toml

@@ -0,0 +1,102 @@
+[build-system]
+requires = ["setuptools>=65", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "syncnet-python"
+version = "0.1.0"
+description = "SyncNet: Audio-visual synchronization detection using deep learning"
+readme = "README.md"
+requires-python = ">=3.9"
+license = {text = "MIT"}
+authors = [
+    {name = "SyncNet Python Contributors"},
+]
+maintainers = [
+    {name = "SyncNet Python Contributors"},
+]
+keywords = ["audio-visual", "synchronization", "deep-learning", "pytorch", "lip-sync", "video-processing", "computer-vision"]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Science/Research",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: MIT License",
+    "Operating System :: OS Independent",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Topic :: Scientific/Engineering :: Artificial Intelligence",
+    "Topic :: Multimedia :: Video",
+    "Topic :: Scientific/Engineering :: Image Processing",
+]
+dependencies = [
+    "torch>=2.0.0",
+    "torchvision>=0.15.0",
+    "numpy>=1.24.0",
+    "scipy>=1.10.0",
+    "pandas>=2.0.0",
+    "scenedetect[opencv]>=0.6.0",
+    "opencv-contrib-python>=4.8.0",
+    "python-speech-features>=0.6",
+    "ffmpeg-python>=0.2.0",
+]
+
+[project.urls]
+"Homepage" = "https://github.com/nawta/SyncNet_py313"
+"Bug Reports" = "https://github.com/nawta/SyncNet_py313/issues"
+"Source" = "https://github.com/nawta/SyncNet_py313"
+
+[project.optional-dependencies]
+dev = [
+    "pytest>=7.4.0",
+    "pytest-asyncio>=0.21.0",
+    "black>=23.0.0",
+    "ruff>=0.1.0",
+    "mypy>=1.7.0",
+    "types-opencv-python",
+    "types-scipy",
+]
+
+[project.scripts]
+syncnet-python = "syncnet_python.cli:main"
+
+[tool.setuptools.packages.find]
+where = ["."]
+include = ["syncnet_python*"]
+exclude = ["tests*", "docs*", "examples*"]
+
+[tool.setuptools.package-data]
+syncnet_python = ["*.py", "detectors/*.py", "detectors/s3fd/*.py"]
+
+[tool.black]
+line-length = 88
+target-version = ['py313']
+
+[tool.ruff]
+target-version = "py313"
+line-length = 88
+select = [
+    "E",  # pycodestyle errors
+    "W",  # pycodestyle warnings
+    "F",  # pyflakes
+    "I",  # isort
+    "B",  # flake8-bugbear
+    "C4", # flake8-comprehensions
+    "UP", # pyupgrade
+]
+
+[tool.mypy]
+python_version = "3.13"
+warn_return_any = true
+warn_unused_configs = true
+disallow_untyped_defs = true
+disallow_incomplete_defs = true
+check_untyped_defs = true
+no_implicit_optional = true
+warn_redundant_casts = true
+warn_unused_ignores = true
+warn_no_return = true
+strict_equality = true