deepclip 0.1.0__tar.gz

deepclip-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 seoyunje
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

deepclip-0.1.0/PKG-INFO

@@ -0,0 +1,36 @@
+Metadata-Version: 2.4
+Name: deepclip
+Version: 0.1.0
+Summary: CLIP for deepfakes: face forgery detection aligned with CLIP text embeddings
+Author-email: seoyunje <seoyunje2001@gmail.com>
+License-Expression: MIT
+Project-URL: Homepage, https://github.com/SEOYUNJE/DeepClip
+Project-URL: Bug Tracker, https://github.com/SEOYUNJE/DeepClip/issues
+Project-URL: Source Code, https://github.com/SEOYUNJE/DeepClip
+Project-URL: Releases, https://github.com/SEOYUNJE/DeepClip/releases
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Operating System :: OS Independent
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: torch>=2.4.0
+Requires-Dist: timm>=1.0.0
+Requires-Dist: torchmetrics>=1.4.0
+Requires-Dist: albumentations>=2.0.8
+Requires-Dist: opencv-python>=5.0.0.93
+Requires-Dist: numpy<3.0.0,>=1.23.0
+Requires-Dist: pandas>=2.2.3
+Requires-Dist: scikit-learn>=1.6.1
+Requires-Dist: regex>=2023.0.0
+Dynamic: license-file
+
+# DeepClip
+CLIP for deepfakes
+
+## Dataset
+
+[`preprocess/`](preprocess/README.md) builds the frame-level FaceForensics++ dataset and its forgery annotation
+(face crops, 68 facial landmarks, forgery masks and forgery features) used to train DeepClip.

deepclip-0.1.0/README.md

@@ -0,0 +1,7 @@
+# DeepClip
+CLIP for deepfakes
+
+## Dataset
+
+[`preprocess/`](preprocess/README.md) builds the frame-level FaceForensics++ dataset and its forgery annotation
+(face crops, 68 facial landmarks, forgery masks and forgery features) used to train DeepClip.

deepclip-0.1.0/deepclip/__init__.py

@@ -0,0 +1,31 @@
+import warnings
+import logging
+from importlib.metadata import version, PackageNotFoundError
+
+try:
+    __version__ = version("deepclip") # pyproject.toml
+except PackageNotFoundError: # Not installed (e.g. run from the repository)
+    __version__ = "0.0.0"
+
+# ------- 로그 및 경고 제어 ---------
+warnings.filterwarnings("ignore") # 일반 경고 숨기기
+logging.getLogger("timm").setLevel(logging.ERROR) # timm 내부 로그 숨기기
+logging.getLogger("torch").setLevel(logging.ERROR) # torch 관련 로그 숨기기
+
+from .models import image_encoder, text_encoder, deepclip
+from .models.image_encoder import DeepImageEncoder
+from .models.text_encoder import DeepTextEncoder
+from .models.deepclip import (
+    DeepClip,
+    deepclip_b0,
+    deepclip_b5,
+)
+
+__all__ = [
+    "__version__",
+    "DeepImageEncoder",
+    "DeepTextEncoder",
+    "DeepClip",
+    "deepclip_b0",
+    "deepclip_b5",
+]

deepclip-0.1.0/deepclip/config/deepclip_b0.yaml

@@ -0,0 +1,65 @@
+dataset: "FaceForensics++"
+
+model:
+      model_name: "tf_efficientnet_b0.ns_jft_in1k"
+      img_size: [224,224]
+
+      # ===== Feature Extraction Blocks =====
+      l_block_idx: 1 # Low-level feature stage index
+      h_block_idx: 6 # High-level feature stage index
+
+      # ===== Transformer Dimensions =====
+      l_dim: 24 # Dimension for low-block transformer
+      h_dim: 256 # Dimension for high-block transformer
+      l_depths: [2,2,4,2] # Number of transformer layers (Low)
+      h_depths: [4] # Number of transformer layers (High)
+
+      # ===== Attention Settings =====
+      l_windows: [7,7,14,7] # Window Size for Attention (Low)
+      h_windows: [7] # Window Size for Attention (High)
+      l_heads: [1,2,4,8] # Multi Head Attention (Low)
+      h_heads: [4] # Multi Head Attention (High)
+      l_ratio: [4,4,4,4] # MLP expansion ratio (Low)
+      h_ratio: [4] # MLP expansion ratio (High)
+
+      # ===== Regularization (Dropout) =====
+      l_drop: 0  # Pos dropout, Mlp dropout, Proj dropout (Low)
+      h_drop: 0.05 # Pos dropout, Mlp dropout, Proj dropout (High)
+      l_attn_drop: 0.05 # Attention Dropout (Low)
+      h_attn_drop: 0 # Attention Dropout (High)
+      l_drop_path: 0.1 # Stochastic depth rate (Low)
+      h_drop_path: 0.05 # Stochastic depth rate (High)
+
+      # ===== Norm (GCViT + image projection) =====
+      image_norm: "rmsnorm" # rmsnorm or layernorm
+
+      # ===== Text Encoder (frozen) =====
+      clip_name: "ViT-B/16"
+      embed_dim: 512 # Text embedding dimension of clip_name (checked)
+
+text:
+      type_sampling_alpha: 1.0 # Type weight = freq^-alpha
+      region_sampling_beta: 1.0 # Region weight = region_freq^-beta
+
+loss:
+      real_text_weight: 1.0 # Weight of the REAL_TEXT column of the loss
+      fake_text_weight: 1.0 # Weight of the FAKE caption columns of the loss
+
+train:
+      epochs: 10
+      use_amp: True # Mixed Precision Training
+      n_accumulate: 1 # Gradient Accumulation steps
+      optimizer: "AdamW"
+      scheduler: "CosineAnnealingLR"
+
+      # DataLoader
+      train_bs: 16
+      valid_bs: 32
+
+      # Optimizer (decay / no_decay groups, same lr)
+      lr: 2e-4
+      weight_decay: 1e-3
+
+      # Early Stopping (valid_loss)
+      patience: 3
+      min_delta: 0.001

deepclip-0.1.0/deepclip/config/deepclip_b5.yaml

@@ -0,0 +1,65 @@
+dataset: "FaceForensics++"
+
+model:
+      model_name: "tf_efficientnet_b5.ns_jft_in1k"
+      img_size: [384,384]
+
+      # ===== Feature Extraction Blocks =====
+      l_block_idx: 1 # Low-level feature stage index
+      h_block_idx: 6 # High-level feature stage index
+
+      # ===== Transformer Dimensions =====
+      l_dim: 48 # Dimension for low-block transformer
+      h_dim: 512 # Dimension for high-block transformer
+      l_depths: [2,2,6,2] # Number of transformer layers (Low)
+      h_depths: [6] # Number of transformer layers (High)
+
+      # ===== Attention Settings =====
+      l_windows: [12,12,24,12] # Window Size for Attention (Low)
+      h_windows: [12] # Window Size for Attention (High)
+      l_heads: [2,4,8,16] # Multi Head Attention (Low)
+      h_heads: [16] # Multi Head Attention (High)
+      l_ratio: [3,3,3,3] # MLP expansion ratio (Low)
+      h_ratio: [3] # MLP expansion ratio (High)
+
+      # ===== Regularization (Dropout) =====
+      l_drop: 0.0  # Pos dropout, Mlp dropout, Proj dropout (Low)
+      h_drop: 0.1 # Pos dropout, Mlp dropout, Proj dropout (High)
+      l_attn_drop: 0.1 # Attention Dropout (Low)
+      h_attn_drop: 0.0 # Attention Dropout (High)
+      l_drop_path: 0.15 # Stochastic depth rate (Low)
+      h_drop_path: 0.1 # Stochastic depth rate (High)
+
+      # ===== Norm (GCViT + image projection) =====
+      image_norm: "rmsnorm" # rmsnorm or layernorm
+
+      # ===== Text Encoder (frozen) =====
+      clip_name: "ViT-L/14"
+      embed_dim: 768 # Text embedding dimension of clip_name (checked)
+
+text:
+      type_sampling_alpha: 1.0 # Type weight = freq^-alpha
+      region_sampling_beta: 1.0 # Region weight = region_freq^-beta
+
+loss:
+      real_text_weight: 1.0 # Weight of the REAL_TEXT column of the loss
+      fake_text_weight: 1.0 # Weight of the FAKE caption columns of the loss
+
+train:
+      epochs: 10
+      use_amp: True # Mixed Precision Training
+      n_accumulate: 1 # Gradient Accumulation steps
+      optimizer: "AdamW"
+      scheduler: "CosineAnnealingLR"
+
+      # DataLoader
+      train_bs: 16
+      valid_bs: 32
+
+      # Optimizer (decay / no_decay groups, same lr)
+      lr: 5e-5
+      weight_decay: 2e-4
+
+      # Early Stopping (valid_loss)
+      patience: 3
+      min_delta: 0.001

deepclip-0.1.0/deepclip/data/__init__.py

@@ -0,0 +1,9 @@
+from .dataset import DeepClipDataset
+from .collate import DeepClipCollate
+from .transforms import (
+    get_train_transforms,
+    get_valid_transforms,
+    get_test_transforms,
+)
+from .handle_imbalance import class_imbalance_handle
+from .split_data import split_data

deepclip-0.1.0/deepclip/data/collate.py

@@ -0,0 +1,104 @@
+import numpy as np
+import torch
+from torch.utils.data import get_worker_info
+from deepclip.data.text import REAL_TEXT, parse_pair_label, sample_selection, render_caption
+
+class DeepClipCollate:
+    """
+    Makes the captions of a batch and the targets of the loss.
+
+    For every image: REAL -> REAL_TEXT, FAKE -> the caption of the pair chosen by sample_selection
+    (a new choice at every step). The B captions are reduced to the U unique captions (in order).
+
+    Returns a dict:
+        images       (B,3,H,W)
+        labels       (B,)    0: REAL, 1: FAKE
+        text_idx     (U,)    rows of text_table of the unique captions
+        Y            (B,U)   1 if caption j is true for image i
+                             (REAL_TEXT column: image i is REAL, FAKE column: the pair of caption j is on in image i)
+        W            (B,U)   real_text_weight for the REAL_TEXT column, fake_text_weight for the FAKE columns
+        pair_targets (B,20)  pair_label of every image (all 0 for REAL), target of map and the AP
+    """
+
+    def __init__(
+            self,
+            caption_to_idx: dict,
+            type_weights: np.ndarray,
+            region_weights: np.ndarray,
+            real_text_weight: float = 1.0,
+            fake_text_weight: float = 1.0,
+            seed: int | None = None,
+            ):
+        """
+        Args:
+            caption_to_idx: caption -> row of text_table (DeepClip.caption_to_idx).
+            type_weights: (4,5) weights of the types (type_weights of the train FAKE frames).
+            region_weights: (4,) weights of the regions (region_weights of the train FAKE frames).
+            real_text_weight, fake_text_weight: Weights of the REAL_TEXT column and the FAKE columns of the loss.
+            seed: Fixed seed for the same captions at every epoch (valid). None for new captions at every epoch (train).
+        """
+
+        self.caption_to_idx = caption_to_idx
+        self.type_weights = type_weights
+        self.region_weights = region_weights
+        self.real_text_weight = real_text_weight
+        self.fake_text_weight = fake_text_weight
+        self.seed = seed
+        self.reset()
+
+    def reset(self):
+        """Restarts the random captions (with seed: the same captions as the previous epoch when num_workers=0)"""
+        self.rng = np.random.default_rng(self.seed if self.seed is not None else np.random.randint(2**31))
+        self._worker_id = None
+
+    def _update_rng(self):
+        # Every DataLoader worker gets a copy of this object, so the rng is set again in each worker
+        info = get_worker_info()
+
+        if info is not None and self._worker_id != info.id:
+            self._worker_id = info.id
+            self.rng = np.random.default_rng([self.seed, info.id] if self.seed is not None else info.seed)
+
+    def __call__(self, batch):
+        self._update_rng()
+
+        imgs, labels, pair_labels = zip(*batch)
+        images = torch.stack(imgs) # (B,3,H,W)
+        labels = torch.stack(labels).float() # (B,)
+
+        lab = np.stack([parse_pair_label(p) for p in pair_labels]) # (B,4,5)
+        is_fake = labels.numpy() == 1
+
+        # ====== Caption of every image ======
+        captions, selections = [], []
+        for i in range(len(batch)):
+            if is_fake[i]:
+                sel = sample_selection(lab[i], self.type_weights, self.region_weights, self.rng)
+                captions.append(render_caption(sel))
+            else:
+                sel = np.zeros_like(lab[i])
+                captions.append(REAL_TEXT)
+            selections.append(sel)
+
+        # ====== Unique captions (in order) ======
+        uniq = list(dict.fromkeys(captions))
+        first = [captions.index(c) for c in uniq]
+        text_idx = torch.tensor([self.caption_to_idx[c] for c in uniq], dtype=torch.long) # (U,)
+
+        # ====== Targets Y and weights W ======
+        is_real_col = np.array([c == REAL_TEXT for c in uniq]) # (U,)
+        sel_cols = np.stack([selections[i] for i in first]) # (U,4,5)
+
+        # the pair of caption j is on in image i
+        subset = ~(sel_cols[None] & ~lab[:, None]).any(axis=(2, 3)) # (B,U)
+        Y = np.where(is_real_col[None], ~is_fake[:, None], is_fake[:, None] & subset)
+        W = np.where(is_real_col, self.real_text_weight, self.fake_text_weight)
+
+        return {
+            "images": images,
+            "labels": labels,
+            "text_idx": text_idx,
+            "Y": torch.from_numpy(Y).float(),
+            "W": torch.from_numpy(np.broadcast_to(W, Y.shape).copy()).float(),
+            "pair_targets": torch.from_numpy(lab.reshape(len(batch), -1)).long(),
+        }

deepclip-0.1.0/deepclip/data/dataset.py

@@ -0,0 +1,67 @@
+import pandas as pd
+from typing import List
+import cv2
+import torch
+from torch.utils.data import Dataset
+import albumentations as A
+
+class DeepClipDataset(Dataset):
+    """
+    Dataset class for DeepClip, based on DeepFakeDataset of DeepGuard.
+    Reads image paths, labels and pair labels from a pandas DataFrame, loads images,
+    and applies specified transformations. The caption is made by DeepClipCollate.
+
+    Attributes:
+        meta_df (pd.DataFrame): Dataframe containing image paths, labels and pair labels.
+        transforms (A.Compose): Albumentations transformation pipeline.
+        img_size (List[int]): [Height, Width] of the image.
+        img_col (str): Name of the column containing image file paths.
+        label_col (str): Name of the column containing labels (0: REAL, 1: FAKE).
+        pair_col (str): Name of the column containing pair labels (20 characters of 0 and 1).
+        return_label (bool): Whether to return (image, label, pair_label) or just image.
+    """
+    def __init__(
+                self,
+                meta_df: pd.DataFrame,
+                img_size: List[int],
+                transforms: A.Compose,
+                img_col: str = "img_path",
+                label_col: str = "label",
+                pair_col: str = "pair_label",
+                return_label: bool = True,
+                ):
+        self.meta_df = meta_df
+        self.img_size = img_size
+        self.transforms = transforms(img_size)
+        self.img_col = img_col
+        self.label_col = label_col
+        self.pair_col = pair_col
+        self.return_label = return_label
+
+    def __len__(self):
+        return len(self.meta_df)
+
+    def _load_image(self, path: str):
+        img_bgr = cv2.imread(path)
+        if img_bgr is None:
+            raise FileNotFoundError(f"Cannot read image: {path}")
+
+        img_rgb = cv2.cvtColor(img_bgr, cv2.COLOR_BGR2RGB)
+
+        return img_rgb
+
+    def __getitem__(self, idx):
+        row = self.meta_df.iloc[idx]
+        img_path = str(row[self.img_col])
+
+        img = self._load_image(img_path)
+        img = self.transforms(image=img)['image']
+
+        if not self.return_label:
+            return img
+        else:
+            label = row[self.label_col]
+            return img, torch.tensor(label, dtype=torch.float32), str(row[self.pair_col])
+
+    def __str__(self):
+        return f"DeepClipDataset with {len(self.meta_df)} samples"

deepclip-0.1.0/deepclip/data/handle_imbalance.py

@@ -0,0 +1,44 @@
+import pandas as pd
+from typing import Optional
+
+def class_imbalance_handle(
+    real_df: pd.DataFrame,
+    fake_df: pd.DataFrame,
+    k: int,
+    seed: int = 2025):
+    """
+    Constructs a balanced training dataset by combining real data with a specific slice of fake data.
+
+    This function implements a sliding-window approach to select synthetic (fake) samples 
+    to match the count of real samples. It ensures a 1:1 class distribution for 
+    iterative training or cross-validation scenarios.
+
+    Args:
+        real_df (pd.DataFrame): The original/minority class dataframe.
+        fake_df (pd.DataFrame): The synthetic/majority class dataframe to sample from.
+        k (int): The fold index used to determine the starting position of the fake data slice.
+        seed (int, optional): Random seed for reproducibility during sampling and shuffling. Defaults to 2025.
+
+    """
+
+    n_real = len(real_df)
+    start_idx = k * n_real
+    end_idx = start_idx + n_real
+
+    # Handle wrap-around if the slice exceeds the number of fake samples
+    if end_idx > len(fake_df):
+        remaining = fake_df[start_idx:]
+        n_remaining = len(remaining)
+
+        # Sample additional fake data from the start to match the required slice
+        extra = fake_df[:start_idx].sample(n=n_real - n_remaining, replace=False, random_state=seed)
+        fake_slice = pd.concat([remaining, extra], axis=0).reset_index(drop=True)
+
+    else:
+        fake_slice = fake_df[start_idx:end_idx].reset_index(drop=True)
+
+    # Combine real and fake samples, then shuffle
+    train_df = pd.concat([real_df, fake_slice], ignore_index=True)
+    train_df = train_df.sample(frac=1, random_state=seed).reset_index(drop=True)
+
+    return train_df

deepclip-0.1.0/deepclip/data/split_data.py

@@ -0,0 +1,83 @@
+import pandas as pd
+from sklearn.model_selection import train_test_split, GroupKFold
+
+def split_data(
+                meta_df: pd.DataFrame,
+                seed: int = 2025,
+                label_col: str = 'label',
+                ori_vid: str = 'ori_vid', 
+                dataset: str = 'kodf',
+                group_col: str = "pid",
+                test_size: float = 0.2,
+                balance_val: bool = True,
+                debug: bool = False,
+                debug_ratio: float = 0.1,
+) :
+    """Splits the dataset into training and validation sets based on original video IDs.
+
+    This function ensures that all fake videos derived from the same original video 
+    stay within the same split (Train or Val) to prevent data leakage. It first 
+    splits the real videos and then maps the corresponding fake videos.
+
+    Args:
+        meta_df: The complete metadata DataFrame containing labels and video IDs.
+        seed: Random seed for reproducibility. Defaults to 2025.
+        label_col: Name of the column containing labels (0 for REAL, 1 for FAKE).
+        ori_vid: Name of the column identifying the original video source.
+        test_size: Proportion of the dataset to include in the validation split.
+        balance_val: If True, samples the fake validation set to match the 
+            length of the real validation set.
+        debug: If True, downsamples the final datasets for quick testing.
+        debug_ratio: The ratio of data to keep when debug mode is enabled.
+
+    Returns:
+        tuple[pd.DataFrame, pd.DataFrame]: A tuple containing (train_df, valid_df).
+    """
+    
+    # Split real and fake(0: REAL, 1: FAKE)
+    
+    assert meta_df[label_col].dtype in [int, float], "You need to label encoding {'FAKE': 1, 'REAL': 0}"
+    
+    df_real = meta_df[meta_df[label_col] == 0].reset_index(drop=True)
+    df_fake = meta_df[meta_df[label_col] == 1].reset_index(drop=True)
+
+    # Split real Images
+    if dataset.lower() != "kodf":
+        real_train, real_val = train_test_split(df_real, test_size=test_size, random_state=seed, shuffle=True)
+    else: 
+        df_real_shuffled = df_real.sample(frac=1, random_state=seed).reset_index(drop=True)
+        
+        n_splits = int(1 / test_size)
+        gkf = GroupKFold(n_splits=n_splits)
+        
+        splits = list(gkf.split(df_real_shuffled, groups=df_real_shuffled[group_col]))
+        train_idx, val_idx = splits[0]
+        
+        real_train = df_real_shuffled.iloc[train_idx]
+        real_val = df_real_shuffled.iloc[val_idx]
+    
+    # Match fake images to corresponding real train/val
+    fake_train = df_fake[df_fake[ori_vid].isin(real_train[ori_vid].unique())].reset_index(drop=True)
+    fake_val = df_fake[df_fake[ori_vid].isin(real_val[ori_vid].unique())].reset_index(drop=True)
+
+    # Sample fake_val to match real_val length
+    if balance_val:
+        fake_val = fake_val.sample(len(real_val), random_state=seed)
+
+    # Concatenate real + fake
+    train_df = pd.concat([real_train, fake_train], axis=0)
+    valid_df = pd.concat([real_val, fake_val], axis=0)
+
+    # Debug mode: downsample dataset
+    if debug:
+        n_train = max(1, int(len(train_df) * debug_ratio))
+        n_valid = max(1, int(len(valid_df) * debug_ratio))
+        train_df = train_df.sample(n=n_train, random_state=seed).reset_index(drop=True)
+        valid_df = valid_df.sample(n=n_valid, random_state=seed).reset_index(drop=True)
+
+
+    # Shuffle final datasets
+    train_df = train_df.sample(frac=1, random_state=seed).reset_index(drop=True)
+    valid_df = valid_df.sample(frac=1, random_state=seed).reset_index(drop=True)
+
+    return train_df, valid_df

deepclip-0.1.0/deepclip/data/text/__init__.py

@@ -0,0 +1,16 @@
+from .template import (
+    REGIONS,
+    TYPES,
+    PAIR_ORDER,
+    REAL_TEXT,
+    parse_pair_label,
+    pair_frequency,
+    region_frequency,
+    type_weights,
+    region_weights,
+    sample_selection,
+    render_caption,
+    all_captions,
+    single_pair_caption,
+)
+from .tokenizer import tokenize