video-duplicate 1.0.0__tar.gz

video_duplicate-1.0.0/PKG-INFO

@@ -0,0 +1,59 @@
+Metadata-Version: 2.3
+Name: video-duplicate
+Version: 1.0.0
+Summary: Find duplicate videos across folders
+Author: ritonun
+Author-email: ritonun <you@example.com>
+Requires-Dist: pillow>=12.3.0
+Requires-Dist: pyacoustid>=1.3.1
+Requires-Dist: videohash>=3.0.1
+Requires-Python: >=3.13
+Description-Content-Type: text/markdown
+
+# Video Duplicate
+
+Tool to detect duplicate videos.
+
+## Install
+Dependencies:
+```bash
+sudo apt install libchromaprint-tools
+```
+Python dependencies:
+```bash
+pip install pyacoustid videohash pillow
+```
+
+## Usage
+```python
+folders = [Path("data/dup"), Path("data/test")]
+duplicates = load_probable_duplicate(folders, tolerance=2.0)
+print(f"Found {len(duplicates)} duplicates")
+
+for duplicate in duplicates:
+    v1, v2 = duplicate
+    if check_if_video_duplicate_quick_hash(v1, v2, size=1024 * 1024):
+        # check if the first 1MB (size) has the same hash between the two videos
+        print(f"{v1} & {v2} are duplicate.")
+        continue
+
+    # compare audio between the two video
+    similarity = acoustic_compare_two_videos(v1, v2)
+    if isinstance(similarity, str):
+        print(f"error: {similarity}")
+        continue
+
+    if similarity > 0.9:
+        print(f"{v1} & {v2} are duplicate.")
+        continue
+    elif similarity > 0.8:
+        print(f"{v1} & {v2} are likely duplicate.")
+        continue
+
+    if check_if_duplicate_videos_phash(v1, v2):
+        # best method but very long/computer intensive
+        print(f"{v1} & {v2} are duplicate.")
+
+    else:
+        print(f"{v1} & {v2} are NOT duplicate.")
+```

video_duplicate-1.0.0/README.md

@@ -0,0 +1,47 @@
+# Video Duplicate
+
+Tool to detect duplicate videos.
+
+## Install
+Dependencies:
+```bash
+sudo apt install libchromaprint-tools
+```
+Python dependencies:
+```bash
+pip install pyacoustid videohash pillow
+```
+
+## Usage
+```python
+folders = [Path("data/dup"), Path("data/test")]
+duplicates = load_probable_duplicate(folders, tolerance=2.0)
+print(f"Found {len(duplicates)} duplicates")
+
+for duplicate in duplicates:
+    v1, v2 = duplicate
+    if check_if_video_duplicate_quick_hash(v1, v2, size=1024 * 1024):
+        # check if the first 1MB (size) has the same hash between the two videos
+        print(f"{v1} & {v2} are duplicate.")
+        continue
+
+    # compare audio between the two video
+    similarity = acoustic_compare_two_videos(v1, v2)
+    if isinstance(similarity, str):
+        print(f"error: {similarity}")
+        continue
+
+    if similarity > 0.9:
+        print(f"{v1} & {v2} are duplicate.")
+        continue
+    elif similarity > 0.8:
+        print(f"{v1} & {v2} are likely duplicate.")
+        continue
+
+    if check_if_duplicate_videos_phash(v1, v2):
+        # best method but very long/computer intensive
+        print(f"{v1} & {v2} are duplicate.")
+
+    else:
+        print(f"{v1} & {v2} are NOT duplicate.")
+```

video_duplicate-1.0.0/pyproject.toml

@@ -0,0 +1,22 @@
+[project]
+name = "video-duplicate"
+version = "1.0.0"
+description = "Find duplicate videos across folders"
+readme = "README.md"
+requires-python = ">=3.13"
+dependencies = [
+    "pillow>=12.3.0",
+    "pyacoustid>=1.3.1",
+    "videohash>=3.0.1",
+]
+
+[[project.authors]]
+name = "ritonun"
+email = "you@example.com"
+
+[project.scripts]
+video-duplicate = "video_duplicate:main"
+
+[build-system]
+requires = ["uv_build>=0.12.19,<0.13.0"]
+build-backend = "uv_build"

video_duplicate-1.0.0/pyproject.toml.orig

@@ -0,0 +1,15 @@
+[project]
+name = "video-duplicate"
+version = "1.0.0"
+description = "Find duplicate videos across folders"
+readme = "README.md"
+authors = [{ name = "ritonun", email = "you@example.com" }]
+requires-python = ">=3.13"
+dependencies = ["pillow>=12.3.0", "pyacoustid>=1.3.1", "videohash>=3.0.1"]
+
+[project.scripts]
+video-duplicate = "video_duplicate:main"
+
+[build-system]
+requires = ["uv_build>=0.12.19,<0.13.0"]
+build-backend = "uv_build"

video_duplicate-1.0.0/src/video_duplicate/__init__.py

@@ -0,0 +1,8 @@
+# __init__.py
+
+from .audio import acoustic_compare_two_videos
+from .file import load_probable_duplicate, load_videos_from_folder
+from .hashing import (
+    check_if_duplicate_videos_phash,
+    check_if_video_duplicate_quick_hash,
+)

video_duplicate-1.0.0/src/video_duplicate/audio.py

@@ -0,0 +1,68 @@
+# audio .py
+from functools import lru_cache
+from pathlib import Path
+
+import acoustid
+
+type Fingerprint = tuple[float, bytes]
+
+
+@lru_cache(maxsize=1024)
+def acoustic_fingerprint(
+    path: Path | str, fingerprint_length: int = 60
+) -> Fingerprint | None:
+    """
+    Generate a Chromaprint fingerprint locally.
+
+    Returns: (duration, fingerprint) || None if fingerprinting failed
+    """
+
+    try:
+        duration, fingerprint = acoustid.fingerprint_file(
+            str(path),
+            maxlength=fingerprint_length,
+            force_fpcalc=True,
+        )
+    except Exception:
+        return None
+
+    if fingerprint is None:
+        return None
+
+    return float(duration), fingerprint
+
+
+def compare_acoustic_fingerprints(
+    fingerprint1: Fingerprint | None,
+    fingerprint2: Fingerprint | None,
+) -> float | str:
+    """Compare two locally generated Chromaprint fingerprints."""
+
+    if fingerprint1 is None:
+        return "Could not fingerprint video 1"
+
+    if fingerprint2 is None:
+        return "Could not fingerprint video 2."
+
+    try:
+        similarity = acoustid.compare_fingerprints(
+            fingerprint1,
+            fingerprint2,
+        )
+    except Exception as exc:
+        return f"Fingerprint comparison failed: {exc}"
+
+    return similarity
+
+
+def acoustic_compare_two_videos(
+    path1: Path, path2: Path, fingerprint_length: int = 60
+) -> float | str:
+    """
+    Fingerprint two videos and compare their audio.
+    """
+
+    fingerprint1 = acoustic_fingerprint(path1, fingerprint_length=fingerprint_length)
+    fingerprint2 = acoustic_fingerprint(path2, fingerprint_length=fingerprint_length)
+
+    return compare_acoustic_fingerprints(fingerprint1, fingerprint2)

video_duplicate-1.0.0/src/video_duplicate/file.py

@@ -0,0 +1,79 @@
+# file.py
+
+from itertools import combinations
+from pathlib import Path
+
+from .utils import get_duration
+
+
+def load_probable_duplicate(folders: list[Path], tolerance=2.0):
+    """Find probable duplicate videos across multiple folders"""
+
+    videos_by_folder = [load_videos_from_folder(folder) for folder in folders]
+    durations = {
+        video: get_duration(video) for videos in videos_by_folder for video in videos
+    }
+
+    duplicates = []
+
+    for videos1, videos2 in combinations(videos_by_folder, 2):
+        for video1 in videos1:
+            duration1 = durations[video1]
+
+            for video2 in videos2:
+                duration2 = durations[video2]
+
+                if abs(duration1 - duration2) <= tolerance:
+                    duplicates.append([video1, video2])
+
+    return duplicates
+
+
+def load_videos_from_folder(path: Path):
+    """From a folder load all videos"""
+
+    videos = []
+    files = list(path.iterdir())
+    for file in files:
+        p = Path(file)
+        if p.suffix.lower() in VIDEO_EXTENSIONS:
+            videos.append(file)
+    return videos
+
+
+VIDEO_EXTENSIONS = {
+    ".3g2",
+    ".3gp",
+    ".asf",
+    ".avi",
+    ".divx",
+    ".dv",
+    ".f4v",
+    ".flv",
+    ".m1v",
+    ".m2p",
+    ".m2t",
+    ".m2ts",
+    ".m2v",
+    ".m4v",
+    ".mkv",
+    ".mod",
+    ".mov",
+    ".mp2",
+    ".mp4",
+    ".mpe",
+    ".mpeg",
+    ".mpg",
+    ".mts",
+    ".mxf",
+    ".ogm",
+    ".ogv",
+    ".qt",
+    ".rm",
+    ".rmvb",
+    ".tod",
+    ".ts",
+    ".vob",
+    ".webm",
+    ".wmv",
+}

video_duplicate-1.0.0/src/video_duplicate/hashing.py

@@ -0,0 +1,75 @@
+# hashing.py
+import hashlib
+from pathlib import Path
+
+from PIL import Image
+
+# Compatibility for libraries still using Image.ANTIALIAS
+if not hasattr(Image, "ANTIALIAS"):
+    Image.ANTIALIAS = Image.Resampling.LANCZOS  # type: ignore
+
+
+from videohash import VideoHash
+
+
+def hash_first_mb_comparison(path, size=1024 * 1024):
+    h = hashlib.sha256()
+    with open(path, "rb") as f:
+        h.update(f.read(size))
+    return h.digest()
+
+
+def get_video_phash(filepath: Path | str, frame_interval=2):
+    """
+    Safely generates a perceptual hash for a video file.
+    Returns the hexadecimal hash string, or None if it fails.
+    """
+    try:
+        # The library creates temporary directories for frame extraction.
+        # It usually cleans them up, but we wrap in a try-except to be safe.
+        vh = VideoHash(str(filepath), frame_interval=frame_interval)
+        return vh.hash_hex
+    except Exception as e:
+        print(f"[!] Error hashing {filepath}: {e}")
+        return None
+
+
+def are_videos_duplicates(hash_hex_1, hash_hex_2, tolerance=4):
+    """
+    Compares two hexadecimal hashes.
+    Returns True if the hamming distance is <= tolerance.
+    """
+    if not hash_hex_1 or not hash_hex_2:
+        return False
+
+    # Convert hex strings to integers
+    int1 = int(hash_hex_1, 16)
+    int2 = int(hash_hex_2, 16)
+
+    # XOR the two integers, then count the '1' bits to get the Hamming distance
+    hamming_distance = bin(int1 ^ int2).count("1")
+
+    return hamming_distance <= tolerance
+
+
+def check_if_duplicate_videos_phash(
+    path1: Path | str, path2: Path | str, tolerance=4, frame_interval=2
+):
+    """
+    Takes two video paths, generates their perceptual hashes,
+    and returns True if they are duplicates within the given tolerance.
+    """
+    # 1. Generate hashes for both videos
+    hash1 = get_video_phash(path1, frame_interval=frame_interval)
+    hash2 = get_video_phash(path2, frame_interval=frame_interval)
+
+    # 2. Compare the hex strings using the existing function
+    return are_videos_duplicates(hash1, hash2, tolerance)
+
+
+def check_if_video_duplicate_quick_hash(
+    path1: Path | str, path2: Path | str, size=1024 * 1024
+):
+    ha = hash_first_mb_comparison(path1, size=size)
+    hb = hash_first_mb_comparison(path2, size=size)
+    return ha == hb

video_duplicate-1.0.0/src/video_duplicate/utils.py

@@ -0,0 +1,22 @@
+import subprocess
+from pathlib import Path
+
+
+def get_duration(path: Path | str):
+    result = subprocess.run(
+        [
+            "ffprobe",
+            "-v",
+            "error",
+            "-show_entries",
+            "format=duration",
+            "-of",
+            "default=noprint_wrappers=1:nokey=1",
+            str(path),
+        ],
+        capture_output=True,
+        text=True,
+        check=True,
+    )
+
+    return float(result.stdout.strip())